ICU integration and asset loading overhaul

eng/Version.Details.xml

@@ -4,9 +4,9 @@
       <Uri>https://github.com/dotnet/standard</Uri>
       <Sha>cfe95a23647c7de1fe1a349343115bd7720d6949</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.NETCore.Runtime.ICU.Transport" Version="5.0.0-preview.8.20364.1">
+    <Dependency Name="Microsoft.NETCore.Runtime.ICU.Transport" Version="5.0.0-preview.8.20365.1">
       <Uri>https://github.com/dotnet/icu</Uri>
-      <Sha>bf5a3a643815a8a46693d618d08dbc96f353ca9e</Sha>
+      <Sha>7247fa0d9e8faee2cceee6f04856b2c447f41bca</Sha>
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>

eng/Versions.props

@@ -118,7 +118,7 @@
     <!-- ILLink -->
     <MicrosoftNETILLinkTasksVersion>5.0.0-preview.3.20363.5</MicrosoftNETILLinkTasksVersion>
     <!-- ICU -->
-    <MicrosoftNETCoreRuntimeICUTransportVersion>5.0.0-preview.8.20364.1</MicrosoftNETCoreRuntimeICUTransportVersion>
+    <MicrosoftNETCoreRuntimeICUTransportVersion>5.0.0-preview.8.20365.1</MicrosoftNETCoreRuntimeICUTransportVersion>
     <!-- Mono LLVM -->
     <runtimelinuxarm64MicrosoftNETCoreRuntimeMonoLLVMSdkVersion>9.0.1-alpha.1.20356.1</runtimelinuxarm64MicrosoftNETCoreRuntimeMonoLLVMSdkVersion>
     <runtimelinuxarm64MicrosoftNETCoreRuntimeMonoLLVMToolsVersion>9.0.1-alpha.1.20356.1</runtimelinuxarm64MicrosoftNETCoreRuntimeMonoLLVMToolsVersion>

eng/liveBuilds.targets

@@ -191,7 +191,8 @@
                              Include="
         $(LibrariesNativeArtifactsPath)dotnet.js;
         $(LibrariesNativeArtifactsPath)dotnet.wasm;
-        $(LibrariesNativeArtifactsPath)dotnet.timezones.blat;"
+        $(LibrariesNativeArtifactsPath)dotnet.timezones.blat;
+        $(LibrariesNativeArtifactsPath)icudt.dat;"
         IsNative="true" />
     </ItemGroup>
 

eng/testing/tests.mobile.targets

@@ -121,10 +121,17 @@
              AssemblyFile="$(WasmAppBuilderTasksAssemblyPath)" />
 
   <Target Condition="'$(TargetOS)' == 'Browser'" Name="BundleTestWasmApp">
+    <ItemGroup>
+      <WasmSatelliteAssemblies Include="$(PublishDir)*\*.resources.dll" />
+      <WasmSatelliteAssemblies>
+        <CultureName>$([System.IO.Directory]::GetParent('%(Identity)').Name)</CultureName>
+      </WasmSatelliteAssemblies>
+    </ItemGroup>
     <ItemGroup>
       <AssemblySearchPaths Include="$(PublishDir)"/>
       <WasmFilesToIncludeInFileSystem Include="@(ContentWithTargetPath)" />
-      <WasmFilesToIncludeInFileSystem Include="@(ReferenceCopyLocalPaths)" Condition="'%(ReferenceCopyLocalPaths.BuildReference)' == 'true'" />
+      <WasmFilesToIncludeInFileSystem Include="@(ReferenceCopyLocalPaths)" Condition="'%(ReferenceCopyLocalPaths.BuildReference)' == 'true' and !$([System.String]::new('%(ReferenceCopyLocalPaths.Identity)').EndsWith('.resources.dll'))" />
+      <WasmFilesToIncludeInFileSystem Include="@(WasmSatelliteAssemblies)" TargetPath="%(WasmSatelliteAssemblies.CultureName)\%(WasmSatelliteAssemblies.Filename)%(WasmSatelliteAssemblies.Extension)" />
       <ExtraAssemblies Include="$(PublishDir)$(AssemblyName).dll" />
       <!-- we need to preserve these facades for BinaryFormatter tests -->
       <ExtraAssemblies Include="$(PublishDir)mscorlib.dll" />
@@ -140,7 +147,6 @@
       <ExtraAssemblies Include="$(PublishDir)System.Xml.dll" />
       <ExtraAssemblies Include="$(PublishDir)WindowsBase.dll" />
     </ItemGroup>
-
     <Error Condition="!Exists('$(MicrosoftNetCoreAppRuntimePackRidDir)')" Text="MicrosoftNetCoreAppRuntimePackRidDir=$(MicrosoftNetCoreAppRuntimePackRidDir) doesn't exist" />
     <WasmAppBuilder
       AppDir="$(BundleDir)"
@@ -149,7 +155,7 @@
       MainJS="$(MonoProjectRoot)\wasm\runtime-test.js"
       ExtraAssemblies="@(ExtraAssemblies)"
       FilesToIncludeInFileSystem="@(WasmFilesToIncludeInFileSystem)"
-      AssemblySearchPaths="@(AssemblySearchPaths)" />
+      AssemblySearchPaths="@(AssemblySearchPaths)"/>
   </Target>
 
   <Target Name="AddTestRunnersToPublishedFiles"

src/libraries/Common/src/Interop/Interop.Calendar.cs

@@ -9,8 +9,8 @@ internal static partial class Interop
 {
     internal static partial class Globalization
     {
-        internal delegate void EnumCalendarInfoCallback(
-           [MarshalAs(UnmanagedType.LPWStr)] string calendarString,
+        internal unsafe delegate void EnumCalendarInfoCallback(
+           char* calendarString,
            IntPtr context);
 
         [DllImport(Libraries.GlobalizationNative, CharSet = CharSet.Unicode, EntryPoint = "GlobalizationNative_GetCalendars")]
@@ -19,8 +19,15 @@ internal delegate void EnumCalendarInfoCallback(
         [DllImport(Libraries.GlobalizationNative, CharSet = CharSet.Unicode, EntryPoint = "GlobalizationNative_GetCalendarInfo")]
         internal static extern unsafe ResultCode GetCalendarInfo(string localeName, CalendarId calendarId, CalendarDataType calendarDataType, char* result, int resultCapacity);
 
+#if TARGET_BROWSER
+        // Temp workaround for pinvoke callbacks for Mono-Wasm-Interpreter
+        // https://github.com/dotnet/runtime/issues/39100
+        [DllImport(Libraries.GlobalizationNative, CharSet = CharSet.Unicode, EntryPoint = "GlobalizationNative_EnumCalendarInfo")]
+        internal static extern bool EnumCalendarInfo(IntPtr callback, string localeName, CalendarId calendarId, CalendarDataType calendarDataType, IntPtr context);
+#else
         [DllImport(Libraries.GlobalizationNative, CharSet = CharSet.Unicode, EntryPoint = "GlobalizationNative_EnumCalendarInfo")]
         internal static extern bool EnumCalendarInfo(EnumCalendarInfoCallback callback, string localeName, CalendarId calendarId, CalendarDataType calendarDataType, IntPtr context);
+#endif
 
         [DllImport(Libraries.GlobalizationNative, EntryPoint = "GlobalizationNative_GetLatestJapaneseEra")]
         internal static extern int GetLatestJapaneseEra();

src/libraries/Common/src/Interop/Interop.TimeZoneDisplayNameType.cs

@@ -0,0 +1,16 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+internal static partial class Interop
+{
+    internal static partial class Globalization
+    {
+        // needs to be kept in sync with TimeZoneDisplayNameType in System.Globalization.Native
+        internal enum TimeZoneDisplayNameType
+        {
+            Generic = 0,
+            Standard = 1,
+            DaylightSavings = 2,
+        }
+    }
+}

src/libraries/Common/src/Interop/Interop.TimeZoneInfo.cs

@@ -7,14 +7,6 @@ internal static partial class Interop
 {
     internal static partial class Globalization
     {
-        // needs to be kept in sync with TimeZoneDisplayNameType in System.Globalization.Native
-        internal enum TimeZoneDisplayNameType
-        {
-            Generic = 0,
-            Standard = 1,
-            DaylightSavings = 2,
-        }
-
         [DllImport(Libraries.GlobalizationNative, CharSet = CharSet.Unicode, EntryPoint = "GlobalizationNative_GetTimeZoneDisplayName")]
         internal static extern unsafe ResultCode GetTimeZoneDisplayName(
             string localeName,

src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_internal.h

@@ -19,11 +19,13 @@
 
 // All ICU headers need to be included here so that all function prototypes are
 // available before the function pointers are declared below.
+#include <unicode/uclean.h>
 #include <unicode/ucurr.h>
 #include <unicode/ucal.h>
 #include <unicode/uchar.h>
 #include <unicode/ucol.h>
 #include <unicode/udat.h>
+#include <unicode/udata.h>
 #include <unicode/udatpg.h>
 #include <unicode/uenum.h>
 #include <unicode/uidna.h>
@@ -55,6 +57,7 @@
 
 #include "pal_compiler.h"
 
+#if !defined(STATIC_ICU)
 // List of all functions from the ICU libraries that are used in the System.Globalization.Native.so
 #define FOR_ALL_UNCONDITIONAL_ICU_FUNCTIONS \
     PER_FUNCTION_BLOCK(u_charsToUChars, libicuuc) \
@@ -279,3 +282,5 @@ FOR_ALL_ICU_FUNCTIONS
 #define usearch_getMatchedLength(...) usearch_getMatchedLength_ptr(__VA_ARGS__)
 #define usearch_last(...) usearch_last_ptr(__VA_ARGS__)
 #define usearch_openFromCollator(...) usearch_openFromCollator_ptr(__VA_ARGS__)
+
+#endif // !defined(STATIC_ICU)

src/libraries/Native/Unix/System.Globalization.Native/pal_icushim_static.c

@@ -0,0 +1,83 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+//
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "pal_icushim_internal.h"
+#include "pal_icushim.h"
+#include <unicode/putil.h>
+#include <unicode/uversion.h>
+#include <unicode/localpointer.h>
+#include <unicode/utrace.h>
+
+static void log_icu_error(const char* name, UErrorCode status)
+{
+    const char * statusText = u_errorName(status);
+    fprintf(stderr, "ICU call %s failed with error #%d '%s'.\n", name, status, statusText);
+}
+
+static void U_CALLCONV icu_trace_data(const void* context, int32_t fnNumber, int32_t level, const char* fmt, va_list args)
+{
+    char buf[1000];
+    utrace_vformat(buf, sizeof(buf), 0, fmt, args);
+    printf("[ICUDT] %s: %s\n", utrace_functionName(fnNumber), buf);
+}
+
+#ifdef __EMSCRIPTEN__
+#include <emscripten.h>
+
+EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void * pData);
+
+EMSCRIPTEN_KEEPALIVE int32_t mono_wasm_load_icu_data(void * pData)
+{
+    UErrorCode status = 0;
+    udata_setCommonData(pData, &status);
+
+    if (U_FAILURE(status)) {
+        log_icu_error("udata_setCommonData", status);
+        return 0;
+    } else {
+        //// Uncomment to enable ICU tracing,
+        //// see https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/tracing.md
+        // utrace_setFunctions(0, 0, 0, icu_trace_data);
+        // utrace_setLevel(UTRACE_VERBOSE);
+        return 1;
+    }
+}
+#endif
+
+int32_t GlobalizationNative_LoadICU(void)
+{
+    const char* icudir = getenv("DOTNET_ICU_DIR");
+    if (icudir)
+        u_setDataDirectory(icudir);
+    else
+        ; // default ICU search path behavior will be used, see http://userguide.icu-project.org/icudata
+
+    UErrorCode status = 0;
+    UVersionInfo version;
+    // Request the CLDR version to perform basic ICU initialization and find out
+    // whether it worked.
+    ulocdata_getCLDRVersion(version, &status);
+
+    if (U_FAILURE(status)) {
+        log_icu_error("ulocdata_getCLDRVersion", status);
+        return 0;
+    }
+
+    return 1;
+}
+
+void GlobalizationNative_InitICUFunctions(void* icuuc, void* icuin, const char* version, const char* suffix)
+{
+    // no-op for static
+}
+
+int32_t GlobalizationNative_GetICUVersion(void)
+{
+    UVersionInfo versionInfo;
+    u_getVersion(versionInfo);
+
+    return (versionInfo[0] << 24) + (versionInfo[1] << 16) + (versionInfo[2] << 8) + versionInfo[3];
+}

src/libraries/Native/native-binplace.proj

@@ -25,6 +25,7 @@
       <BinPlaceItem Condition="'$(TargetOS)' == 'Browser'" Include="$(NativeBinDir)dotnet.js" />
       <BinPlaceItem Condition="'$(TargetOS)' == 'Browser'" Include="$(NativeBinDir)dotnet.wasm" />
       <BinPlaceItem Condition="'$(TargetOS)' == 'Browser'" Include="$(NativeBinDir)dotnet.timezones.blat" />
+      <BinPlaceItem Condition="'$(TargetOS)' == 'Browser'" Include="$(NativeBinDir)icudt.dat" />
       <FileWrites Include="@(BinPlaceItem)" />
     </ItemGroup>
   </Target>

src/libraries/System.Globalization.Calendars/tests/TaiwanCalendar/TaiwanCalendarDaysAndMonths.cs

@@ -9,6 +9,7 @@ namespace System.Globalization.Tests
     public class TaiwanCalendarDaysAndMonths
     {
         [Fact]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/39285", TestPlatforms.Browser)]
         public void DayNames_MonthNames()
         {
             string[] expectedDayNames =

src/libraries/System.Globalization/tests/DateTimeFormatInfo/DateTimeFormatInfoCalendarWeekRule.cs

@@ -12,7 +12,16 @@ public static IEnumerable<object[]> CalendarWeekRule_Get_TestData()
         {
             yield return new object[] { DateTimeFormatInfo.InvariantInfo, CalendarWeekRule.FirstDay };
             yield return new object[] { new CultureInfo("en-US").DateTimeFormat, CalendarWeekRule.FirstDay };
-            yield return new object[] { new CultureInfo("br-FR").DateTimeFormat, DateTimeFormatInfoData.BrFRCalendarWeekRule() };
+
+            if (PlatformDetection.IsNotBrowser)
+            {
+                yield return new object[] { new CultureInfo("br-FR").DateTimeFormat, DateTimeFormatInfoData.BrFRCalendarWeekRule() };
+            }
+            else
+            {
+                // "br-FR" is not presented in Browser's ICU. Let's test ru-RU instead.
+                yield return new object[] { new CultureInfo("ru-RU").DateTimeFormat, CalendarWeekRule.FirstFourDayWeek };
+            }
         }
 
         [Theory]

src/libraries/System.Globalization/tests/NumberFormatInfo/NumberFormatInfoCurrencyGroupSizes.cs

@@ -13,7 +13,7 @@ public static IEnumerable<object[]> CurrencyGroupSizes_TestData()
             yield return new object[] { NumberFormatInfo.InvariantInfo, new int[] { 3 } };
             yield return new object[] { CultureInfo.GetCultureInfo("en-US").NumberFormat, new int[] { 3 } };
 
-            if (!PlatformDetection.IsUbuntu && !PlatformDetection.IsWindows7 && !PlatformDetection.IsWindows8x && !PlatformDetection.IsFedora)
+            if (PlatformDetection.IsNotBrowser && !PlatformDetection.IsUbuntu && !PlatformDetection.IsWindows7 && !PlatformDetection.IsWindows8x && !PlatformDetection.IsFedora)
             {
                 yield return new object[] { CultureInfo.GetCultureInfo("ur-IN").NumberFormat, new int[] { 3, 2 } };
             }

src/libraries/System.Globalization/tests/NumberFormatInfo/NumberFormatInfoCurrencyNegativePattern.cs

@@ -22,14 +22,24 @@ public void CurrencyNegativePattern_Get_ReturnsExpected(NumberFormatInfo format,
             Assert.Contains(format.CurrencyNegativePattern, acceptablePatterns);
         }
 
+        public static IEnumerable<object[]> CurrencyNegativePatternTestLocales()
+        {
+            yield return new object[] { "en-US" };
+            yield return new object[] { "en-CA" };
+            yield return new object[] { "fa-IR" };
+            yield return new object[] { "fr-CD" };
+            yield return new object[] { "fr-CA" };
+
+            if (PlatformDetection.IsNotBrowser)
+            {
+                // Browser's ICU doesn't contain these locales
+                yield return new object[] { "as" };
+                yield return new object[] { "es-BO" };
+            }
+        }
+
         [Theory]
-        [InlineData("en-US")]
-        [InlineData("en-CA")]
-        [InlineData("fa-IR")]
-        [InlineData("fr-CD")]
-        [InlineData("as")]
-        [InlineData("es-BO")]
-        [InlineData("fr-CA")]
+        [MemberData(nameof(CurrencyNegativePatternTestLocales))]
         public void CurrencyNegativePattern_Get_ReturnsExpected_ByLocale(string locale)
         {
             CultureInfo culture;

src/libraries/System.Globalization/tests/System/Globalization/TextInfoTests.cs

@@ -183,6 +183,19 @@ public static IEnumerable<object[]> ToLower_TestData_netcore()
             }
         }
 
+        public static IEnumerable<string> GetTestLocales()
+        {
+            yield return "tr";
+            yield return "tr-TR";
+
+            if (PlatformDetection.IsNotBrowser)
+            {
+                // Browser's ICU doesn't contain these locales
+                yield return "az";
+                yield return "az-Latn-AZ";
+            }
+        }
+
         public static IEnumerable<object[]> ToLower_TestData()
         {
             foreach (string cultureName in s_cultureNames)
@@ -226,7 +239,7 @@ public static IEnumerable<object[]> ToLower_TestData()
                 yield return new object[] { cultureName, "\u03A3", "\u03C3" };
             }
 
-            foreach (string cultureName in new string[] { "tr", "tr-TR", "az", "az-Latn-AZ" })
+            foreach (string cultureName in GetTestLocales())
             {
                 yield return new object[] { cultureName, "\u0130", "i" };
                 yield return new object[] { cultureName, "i", "i" };
@@ -349,7 +362,7 @@ public static IEnumerable<object[]> ToUpper_TestData()
             }
 
             // Turkish i
-            foreach (string cultureName in new string[] { "tr", "tr-TR", "az", "az-Latn-AZ" })
+            foreach (string cultureName in GetTestLocales())
             {
                 yield return new object[] { cultureName, "i", "\u0130" };
                 yield return new object[] { cultureName, "\u0130", "\u0130" };

src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems

@@ -1071,7 +1071,10 @@
     <Compile Include="$(CommonPath)Interop\Interop.ResultCode.cs">
       <Link>Common\Interop\Interop.ResultCode.cs</Link>
     </Compile>
-    <Compile Include="$(CommonPath)Interop\Interop.TimeZoneInfo.cs">
+    <Compile Include="$(CommonPath)Interop\Interop.TimeZoneDisplayNameType.cs">
+      <Link>Common\Interop\Interop.TimeZoneDisplayNameType.cs</Link>
+    </Compile>
+    <Compile Include="$(CommonPath)Interop\Interop.TimeZoneInfo.cs" Condition="'$(TargetsBrowser)' != 'true'">
       <Link>Common\Interop\Interop.TimeZoneInfo.cs</Link>
     </Compile>
     <Compile Include="$(CommonPath)Interop\Interop.Utils.cs">
@@ -1824,9 +1827,11 @@
   </ItemGroup>
   <ItemGroup Condition="'$(TargetsUnix)' == 'true'">
     <Compile Include="$(MSBuildThisFileDirectory)System\Environment.Unix.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\TimeZoneInfo.GetDisplayName.cs" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetsBrowser)' == 'true'">
     <Compile Include="$(MSBuildThisFileDirectory)System\Environment.Browser.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\TimeZoneInfo.GetDisplayName.Invariant.cs" />
   </ItemGroup>
   <ItemGroup Condition="'$(IsOSXLike)' == 'true'">
     <Compile Include="$(CommonPath)Interop\OSX\Interop.libobjc.cs">