Skip to content

[wasi] Bundle runtimeconfig in the single file bundle #95377

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
radical merged 5 commits into from
Nov 29, 2023
Merged

[wasi] Bundle runtimeconfig in the single file bundle #95377

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
179a7c4
[wasi] WasiAppBuilder: error out for single file bundles, if there ..
radical Nov 29, 2023
4c0a041
[wasi] driver.c: bundle runtimeconfig.bin, and use that for the singl…
radical Nov 29, 2023
b89f55d
[wasi] Don't add '--dir' for single file bundles
radical Nov 29, 2023
67745cb
[wasi] wbt: Add tests for invariantglobalization
radical Nov 29, 2023
e26cde3
Use ENABLE_AOT only if we will be AOT'ing
radical Nov 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 68 additions & 35 deletions src/mono/wasi/Wasi.Build.Tests/WasiTemplateTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,40 +81,89 @@ public void ConsoleBuildThenPublish(string config, bool aot)
UseCache: false));
}

public static TheoryData<string, bool, bool> TestDataForConsolePublishAndRun()
public static TheoryData<string, bool> TestDataForConsolePublishAndRunForSingleFileBundle(bool propertyValue)
{
var data = new TheoryData<string, bool, bool>();
data.Add("Debug", false, false);
data.Add("Debug", true, true);
data.Add("Release", false, false); // Release relinks by default
var data = new TheoryData<string, bool>();
data.Add("Debug", propertyValue);
data.Add("Release", propertyValue);
return data;
}

[ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))]
[MemberData(nameof(TestDataForConsolePublishAndRun))]
public void ConsolePublishAndRunForSingleFileBundle(string config, bool relinking, bool invariantTimezone)
[MemberData(nameof(TestDataForConsolePublishAndRunForSingleFileBundle), parameters: false)]
[MemberData(nameof(TestDataForConsolePublishAndRunForSingleFileBundle), parameters: true)]
public void ConsolePublishAndRunForSingleFileBundle_InvariantTimeZone(string config, bool invariantTimezone)
{
string mainWithTzTest = """
using System;

Console.WriteLine("Hello, Wasi Console!");
for (int i = 0; i < args.Length; i ++)
Console.WriteLine($"args[{i}] = {args[i]}");

try
{
TimeZoneInfo tst = TimeZoneInfo.FindSystemTimeZoneById("Asia/Tokyo");
Console.WriteLine($"{tst.DisplayName} BaseUtcOffset is {tst.BaseUtcOffset}");
}
catch (TimeZoneNotFoundException tznfe)
{
Console.WriteLine($"Could not find Asia/Tokyo: {tznfe.Message}");
}

return 42;
""";

string extraProperties = invariantTimezone ? "<InvariantTimezone>true</InvariantTimezone>" : "";
CommandResult res = ConsolePublishAndRunForSingleFileBundleInternal(config, mainWithTzTest, extraProperties: extraProperties);
if(invariantTimezone)
Assert.Contains("Could not find Asia/Tokyo", res.Output);
else
Assert.Contains("Asia/Tokyo BaseUtcOffset is 09:00:00", res.Output);
}

[ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))]
[MemberData(nameof(TestDataForConsolePublishAndRunForSingleFileBundle), parameters: false)]
[MemberData(nameof(TestDataForConsolePublishAndRunForSingleFileBundle), parameters: true)]
public void ConsolePublishAndRunForSingleFileBundle_InvariantGlobalization(string config, bool invariantGlobalization)
{
string mainWithGlobalizationTest = """
using System;
using System.Globalization;

Console.WriteLine("Hello, Wasi Console!");
for (int i = 0; i < args.Length; i ++)
Console.WriteLine($"args[{i}] = {args[i]}");

Console.WriteLine($"Number: {int.Parse("1", CultureInfo.InvariantCulture)}");
return 42;
""";

string extraProperties = invariantGlobalization ? "<InvariantGlobalization>true</InvariantGlobalization>" : "";
CommandResult res = ConsolePublishAndRunForSingleFileBundleInternal(config, mainWithGlobalizationTest, extraProperties: extraProperties);
Assert.Contains("Number: 1", res.Output);
}

private CommandResult ConsolePublishAndRunForSingleFileBundleInternal(string config, string programContents, string extraProperties = "", bool aot = false)
{
if (programContents.Length == 0)
throw new ArgumentException("Cannot be empty", nameof(programContents));

string id = $"{config}_{GetRandomId()}";
string projectFile = CreateWasmTemplateProject(id, "wasiconsole");
string projectName = Path.GetFileNameWithoutExtension(projectFile);
File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_simpleMainWithArgs);

string extraProperties = "<WasmSingleFileBundle>true</WasmSingleFileBundle>";
if (relinking)
extraProperties += "<WasmBuildNative>true</WasmBuildNative>";
if (invariantTimezone)
extraProperties += "<InvariantTimezone>true</InvariantTimezone>";
File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programContents);

extraProperties += "<WasmSingleFileBundle>true</WasmSingleFileBundle>";
AddItemsPropertiesToProject(projectFile, extraProperties);

var buildArgs = new BuildArgs(projectName, config, /*aot*/false, id, null);
var buildArgs = new BuildArgs(projectName, config, aot, id, null);
buildArgs = ExpandBuildArgs(buildArgs);

bool expectRelinking = config == "Release" || relinking;
BuildProject(buildArgs,
id: id,
new BuildProjectOptions(
DotnetWasmFromRuntimePack: !expectRelinking,
DotnetWasmFromRuntimePack: false, // singlefilebundle will always relink
CreateProject: false,
Publish: true,
TargetFramework: BuildTestBase.DefaultTargetFramework,
Expand All @@ -133,15 +182,8 @@ public void ConsolePublishAndRunForSingleFileBundle(string config, bool relinkin
Assert.Contains("args[0] = x", res.Output);
Assert.Contains("args[1] = y", res.Output);
Assert.Contains("args[2] = z", res.Output);
if(invariantTimezone)
{
Assert.Contains("Could not find Asia/Tokyo", res.Output);
}
else
{
Assert.Contains("Asia/Tokyo BaseUtcOffset is 09:00:00", res.Output);
}

return res;
}

[Theory]
Expand Down Expand Up @@ -187,21 +229,12 @@ public void ConsoleBuildAndRunForDifferentOutputPaths(string config, bool append

private static readonly string s_simpleMainWithArgs = """
using System;
using System.Globalization;

Console.WriteLine("Hello, Wasi Console!");
for (int i = 0; i < args.Length; i ++)
Console.WriteLine($"args[{i}] = {args[i]}");

try
{
TimeZoneInfo tst = TimeZoneInfo.FindSystemTimeZoneById("Asia/Tokyo");
Console.WriteLine($"{tst.DisplayName} BaseUtcOffset is {tst.BaseUtcOffset}");
}
catch (TimeZoneNotFoundException tznfe)
{
Console.WriteLine($"Could not find Asia/Tokyo: {tznfe.Message}");
}

return 42;
""";
}
22 changes: 19 additions & 3 deletions src/mono/wasi/build/WasiApp.Native.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@
<!--<_WasiClangCommonFlags Include="-msimd128" Condition="'$(WasmEnableSIMD)' == 'true'" />-->

<_WasmCommonCFlags Include="-DGEN_PINVOKE=1" />
<_WasmCommonCFlags Condition="'$(RunAOTCompilation)' == 'true'" Include="-DENABLE_AOT=1" />
<_WasmCommonCFlags Condition="'$(_WasmShouldAOT)' == 'true'" Include="-DENABLE_AOT=1" />
<_WasmCommonCFlags Condition="'$(_DriverGenCNeeded)' == 'true'" Include="-DDRIVER_GEN=1" />
<_WasmCommonCFlags Condition="'$(WasmSingleFileBundle)' == 'true'" Include="-DWASM_SINGLE_FILE=1" />
<_WasmCommonCFlags Condition="'$(InvariantGlobalization)' == 'true'" Include="-DINVARIANT_GLOBALIZATION=1" />
Expand Down Expand Up @@ -367,6 +367,7 @@
<PropertyGroup>
<_WasmAssembliesBundleObjectFile>wasi_bundled_assemblies.o</_WasmAssembliesBundleObjectFile>
<_WasmIcuBundleObjectFile>wasi_bundled_icu.o</_WasmIcuBundleObjectFile>
<_WasmRuntimeConfigBundleFile>wasi_runtime_config_bin.o</_WasmRuntimeConfigBundleFile>
</PropertyGroup>
<!-- TODO make this incremental compilation -->
<EmitBundleObjectFiles
Expand Down Expand Up @@ -403,13 +404,28 @@
<ItemGroup Condition="'$(InvariantGlobalization)' != 'true'">
<_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmIcuBundleObjectFile)" />
<_WasiObjectFilesForBundle Include="%(BundledWasmIcu.DestinationFile)" />
</ItemGroup>

<ItemGroup Condition="'$(InvariantGlobalization)' != 'true'">
<WasmBundleFileToDelete Remove="$(_WasmIntermediateOutputPath)$(_WasmIcuBundleObjectFile)" />
<WasmBundleFileToDelete Remove="%(BundledWasmIcu.DestinationFile)" />
</ItemGroup>

<!-- runtimeconfig.bin -->
<EmitBundleObjectFiles
FilesToBundle="$(_ParsedRuntimeConfigFilePath)"
ClangExecutable="$(WasiClang)"
BundleRegistrationFunctionName="mono_register_runtimeconfig_bin"
BundleFile="$(_WasmRuntimeConfigBundleFile)"
OutputDirectory="$(_WasmIntermediateOutputPath)">
<Output TaskParameter="BundledResources" ItemName="_BundledRuntimeConfigBin" />
</EmitBundleObjectFiles>

<ItemGroup>
<_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmRuntimeConfigBundleFile)" />
<_WasiObjectFilesForBundle Include="%(_BundledRuntimeConfigBin.DestinationFile)" />

<WasmBundleFileToDelete Remove="$(_WasmIntermediateOutputPath)$(_WasmRuntimeConfigBundleFile)" />
<WasmBundleFileToDelete Remove="%(_BundledRuntimeConfigBin.DestinationFile)" />
</ItemGroup>
<Delete Files="@(WasmBundleFileToDelete)" />
</Target>

Expand Down
6 changes: 3 additions & 3 deletions src/mono/wasi/build/WasiApp.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,8 @@
<RunCommand Condition="'$(RunCommand)' == ''">dotnet</RunCommand>

<_RuntimeConfigJsonPath>$([MSBuild]::NormalizePath($(_AppBundleDirForRunCommand), '$(AssemblyName).runtimeconfig.json'))</_RuntimeConfigJsonPath>
<RunArguments Condition="'$(RunArguments)' == ''">exec &quot;$([MSBuild]::NormalizePath($(WasmAppHostDir), 'WasmAppHost.dll'))&quot; --runtime-config &quot;$(_RuntimeConfigJsonPath)&quot; $(WasmHostArguments)</RunArguments>
<RunArguments Condition="'$(RunArguments)' == '' and '$(WasmSingleFileBundle)' != 'true'">exec &quot;$([MSBuild]::NormalizePath($(WasmAppHostDir), 'WasmAppHost.dll'))&quot; --runtime-config &quot;$(_RuntimeConfigJsonPath)&quot; $(WasmHostArguments)</RunArguments>
Copy link
Member

@maraf maraf Nov 29, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I would prefer concatenating the arguments one by one and having a condition on the dir one rather than duplicating the whole line

Copy link
Member Author

@radical radical Nov 29, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's simple enough right now, so I'll keep it as-is. But I wanted to take the same approach as you suggested, but it would make a little more complicated, tracking to make sure that we don't add the extra argument when RunArguments was set before these lines.

<RunArguments Condition="'$(RunArguments)' == '' and '$(WasmSingleFileBundle)' == 'true'">exec &quot;$([MSBuild]::NormalizePath($(WasmAppHostDir), 'WasmAppHost.dll'))&quot; $(WasmHostArguments)</RunArguments>
<RunWorkingDirectory Condition="'$(RunWorkingDirectory)' == ''">$(_AppBundleDirForRunCommand)</RunWorkingDirectory>
</PropertyGroup>

Expand Down Expand Up @@ -334,7 +335,7 @@
</RuntimeConfigParserTask>

<ItemGroup>
<WasmFilesToIncludeInFileSystem Include="$(_ParsedRuntimeConfigFilePath)" />
<WasmFilesToIncludeInFileSystem Condition="'$(WasmSingleFileBundle)' != 'true'" Include="$(_ParsedRuntimeConfigFilePath)" />
</ItemGroup>
</Target>

Expand All @@ -352,7 +353,6 @@
<ItemGroup Condition="'$(WasmBuildNative)' != 'true'">
<!-- Add the default ones when we don't compile one -->
<WasmNativeAsset Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)dotnet.wasm"/>
<WasmFilesToIncludeInFileSystem Include="@(WasmNativeAsset)" Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.js.symbols'" />
</ItemGroup>

<ItemGroup Condition="'$(InvariantGlobalization)' != 'true' and '$(WasmSingleFileBundle)' != 'true'">
Expand Down
59 changes: 41 additions & 18 deletions src/mono/wasi/runtime/driver.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,9 @@ extern void mono_register_timezones_bundle (void);
#endif /* INVARIANT_TIMEZONE */
#ifdef WASM_SINGLE_FILE
extern void mono_register_assemblies_bundle (void);
#ifndef INVARIANT_GLOBALIZATION
extern void mono_register_runtimeconfig_bin (void);
extern bool mono_bundled_resources_get_data_resource_values (const char *id, const uint8_t **data_out, uint32_t *size_out);
#ifndef INVARIANT_GLOBALIZATION
extern void mono_register_icu_bundle (void);
#endif /* INVARIANT_GLOBALIZATION */
#endif /* WASM_SINGLE_FILE */
Expand Down Expand Up @@ -398,6 +399,44 @@ cleanup_runtime_config (MonovmRuntimeConfigArguments *args, void *user_data)
free (user_data);
}

static void
load_runtimeconfig()
{
#ifdef WASM_SINGLE_FILE
mono_register_runtimeconfig_bin ();

const uint8_t *buffer = NULL;
uint32_t data_len = 0;
if (!mono_bundled_resources_get_data_resource_values (RUNTIMECONFIG_BIN_FILE, &buffer, &data_len)) {
printf("Could not load " RUNTIMECONFIG_BIN_FILE " from the bundle\n");
assert(buffer);
}

MonovmRuntimeConfigArguments *arg = (MonovmRuntimeConfigArguments *)malloc (sizeof (MonovmRuntimeConfigArguments));
arg->kind = 1; // kind: image pointer
arg->runtimeconfig.data.data = buffer;
arg->runtimeconfig.data.data_len = data_len;
monovm_runtimeconfig_initialize (arg, cleanup_runtime_config, NULL);
#else
const char *file_name = RUNTIMECONFIG_BIN_FILE;
int str_len = strlen (file_name) + 1; // +1 is for the "/"
char *file_path = (char *)malloc (sizeof (char) * (str_len +1)); // +1 is for the terminating null character
int num_char = snprintf (file_path, (str_len + 1), "/%s", file_name);
struct stat buffer;

assert (num_char > 0 && num_char == str_len);

if (stat (file_path, &buffer) == 0) {
MonovmRuntimeConfigArguments *arg = (MonovmRuntimeConfigArguments *)malloc (sizeof (MonovmRuntimeConfigArguments));
arg->kind = 0;
arg->runtimeconfig.name.path = file_path;
monovm_runtimeconfig_initialize (arg, cleanup_runtime_config, file_path);
} else {
free (file_path);
}
#endif
}

void
mono_wasm_load_runtime (const char *unused, int debug_level)
{
Expand Down Expand Up @@ -432,23 +471,7 @@ mono_wasm_load_runtime (const char *unused, int debug_level)
appctx_values [0] = "/";
appctx_values [1] = "wasi-wasm";

const char *file_name = RUNTIMECONFIG_BIN_FILE;
int str_len = strlen (file_name) + 1; // +1 is for the "/"
char *file_path = (char *)malloc (sizeof (char) * (str_len +1)); // +1 is for the terminating null character
int num_char = snprintf (file_path, (str_len + 1), "/%s", file_name);
struct stat buffer;

assert (num_char > 0 && num_char == str_len);

if (stat (file_path, &buffer) == 0) {
MonovmRuntimeConfigArguments *arg = (MonovmRuntimeConfigArguments *)malloc (sizeof (MonovmRuntimeConfigArguments));
arg->kind = 0;
arg->runtimeconfig.name.path = file_path;
monovm_runtimeconfig_initialize (arg, cleanup_runtime_config, file_path);
} else {
free (file_path);
}

load_runtimeconfig();
monovm_initialize (2, appctx_keys, appctx_values);

mini_parse_debug_option ("top-runtime-invoke-unhandled");
Expand Down
8 changes: 4 additions & 4 deletions src/mono/wasm/host/wasi/WasiEngineHost.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@ private async Task<int> RunAsync()
// runtimeArguments: _args.CommonConfig.RuntimeArguments);
// runArgsJson.Save(Path.Combine(_args.CommonConfig.AppPath, "runArgs.json"));

var args = new List<string>()
List<string> args = new() { "run" };

if (!_args.IsSingleFileBundle)
{
"run",
"--dir",
"."
args.AddRange(["--dir", "."]);
};

args.AddRange(engineArgs);
Expand Down
17 changes: 15 additions & 2 deletions src/tasks/WasmAppBuilder/wasi/WasiAppBuilder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json.Nodes;
using Microsoft.Build.Framework;

Expand All @@ -21,9 +22,21 @@ protected override bool ValidateArguments()
throw new LogAsErrorException($"{nameof(IcuDataFileNames)} property shouldn't be empty when {nameof(InvariantGlobalization)}=false");

if (Assemblies.Length == 0 && !IsSingleFileBundle)
throw new LogAsErrorException("Cannot build Wasm app without any assemblies");

if (IsSingleFileBundle)
{
Log.LogError("Cannot build Wasm app without any assemblies");
return false;
if (ExtraFilesToDeploy.Length > 0)
{
throw new LogAsErrorException($"$({nameof(ExtraFilesToDeploy)}) is not supported for single file bundles. " +
$"Value: {string.Join(",", ExtraFilesToDeploy.Select(e => e.GetMetadata("FullPath")))}");
}

if (FilesToIncludeInFileSystem.Length > 0)
{
throw new LogAsErrorException($"$({nameof(FilesToIncludeInFileSystem)}) is not supported for single file bundles. " +
$"Value: {string.Join(",", FilesToIncludeInFileSystem.Select(e => e.ItemSpec))}");
}
}

return true;
Expand Down