[release/9.0] Bump OSX.12 helix queues to OSX.13 (#119814)

* [release/9.0] Bump OSX.12 helix queues to OSX.13

* Do superpmi too

* Incorporate #112647

* Backport #119273 for msquic and more test excludes

* Exclude msquic package in tests

Author: steveisok

eng/Versions.props

@@ -231,7 +231,6 @@
     <MicrosoftNETCoreRuntimeICUTransportVersion>9.0.0-rtm.24466.4</MicrosoftNETCoreRuntimeICUTransportVersion>
     <!-- MsQuic -->
     <MicrosoftNativeQuicMsQuicSchannelVersion>2.4.8</MicrosoftNativeQuicMsQuicSchannelVersion>
-    <SystemNetMsQuicTransportVersion>9.0.0-alpha.1.24167.3</SystemNetMsQuicTransportVersion>
     <!-- Mono LLVM -->
     <runtimelinuxarm64MicrosoftNETCoreRuntimeMonoLLVMSdkVersion>19.1.0-alpha.1.25466.4</runtimelinuxarm64MicrosoftNETCoreRuntimeMonoLLVMSdkVersion>
     <runtimelinuxarm64MicrosoftNETCoreRuntimeMonoLLVMToolsVersion>19.1.0-alpha.1.25466.4</runtimelinuxarm64MicrosoftNETCoreRuntimeMonoLLVMToolsVersion>

eng/pipelines/coreclr/templates/helix-queues-setup.yml

@@ -105,16 +105,16 @@ jobs:
     # OSX arm64
     - ${{ if eq(parameters.platform, 'osx_arm64') }}:
       - ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        - OSX.1200.ARM64.Open
+        - OSX.13.ARM64.Open
       - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        - OSX.1200.ARM64
+        - OSX.13.ARM64
 
     # OSX x64
     - ${{ if eq(parameters.platform, 'osx_x64') }}:
       - ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        - OSX.1200.Amd64.Open
+        - OSX.13.Amd64.Open
       - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        - OSX.1200.Amd64
+        - OSX.13.Amd64
 
     # windows x64
     - ${{ if eq(parameters.platform, 'windows_x64') }}:

eng/pipelines/libraries/helix-queues-setup.yml

@@ -73,11 +73,11 @@ jobs:
 
     # OSX arm64
     - ${{ if eq(parameters.platform, 'osx_arm64') }}:
-      - OSX.1200.ARM64.Open
+      - OSX.13.ARM64.Open
 
     # OSX x64
     - ${{ if eq(parameters.platform, 'osx_x64') }}:
-      - OSX.1200.Amd64.Open
+      - OSX.13.Amd64.Open
 
     # Android
     - ${{ if in(parameters.platform, 'android_x86', 'android_x64', 'linux_bionic_x64') }}:

src/coreclr/scripts/superpmi_collect_setup.py

@@ -465,7 +465,7 @@ def main(main_args):
         else:
             helix_queue = "Ubuntu.2204.Amd64"
     elif platform_name == "osx":
-        helix_queue = "OSX.1200.ARM64" if arch == "arm64" else "OSX.1200.Amd64"
+        helix_queue = "OSX.13.ARM64" if arch == "arm64" else "OSX.13.Amd64"
 
     # Copy the superpmi scripts
 

src/libraries/System.Net.Quic/readme.md

@@ -77,15 +77,19 @@ Testing on Linux is done with the help of docker images whose definition can be
 
 To consume a release version of the package, the docker image definition will contain:
 ```docker
-RUN curl -LO https://packages.microsoft.com/keys/microsoft.asc && \
-    echo 2cfd20a306b2fa5e25522d78f2ef50a1f429d35fd30bd983e2ebffc2b80944fa microsoft.asc | sha256sum --check - && \
-    apt-key add microsoft.asc && \
-    rm microsoft.asc && \
-    apt-add-repository https://packages.microsoft.com/debian/11/prod && \
-    apt-get update && \
-    apt-get install -y libmsquic
+RUN apt-get update \
+    && apt-get upgrade -y \
+    && apt-get install -y \
+        apt-transport-https \
+        curl \
+        software-properties-common \
+    && curl -sL https://packages.microsoft.com/config/debian/12/packages-microsoft-prod.deb -o packages-microsoft-prod.deb \
+    && dpkg -i packages-microsoft-prod.deb \
+    && rm packages-microsoft-prod.deb \
+    && apt-get update \
+    && apt-get install -y libmsquic
 ```
-Source: https://github.com/dotnet/dotnet-buildtools-prereqs-docker/blob/efbcd1079edef4698ada1676a5e33c4c9672f85a/src/debian/11/helix/amd64/Dockerfile#L44-L52
+Source: https://github.com/dotnet/dotnet-buildtools-prereqs-docker/blob/7f86167248bcbda898a6b7f17ed01dce3adff2dd/src/debian/12/helix/amd64/Dockerfile#L22-L31
 
 To consume the current main branch of msquic, we pull code from [dotnet/msquic](https://github.com/dotnet/msquic) and build it locally in our docker image:
 ```docker
@@ -94,13 +98,13 @@ RUN apt-get update -y && \
     apt-get upgrade -y && \
     apt-get install -y cmake clang ruby-dev gem lttng-tools libssl-dev && \
     gem install fpm
-RUN git clone --recursive https://github.com/dotnet/msquic
-RUN cd msquic/src/msquic && \
+RUN git clone --depth 1 --single-branch --branch main --recursive https://github.com/microsoft/msquic
+RUN cd msquic/ && \
     mkdir build && \
-    cmake -B build -DCMAKE_BUILD_TYPE=Release -DQUIC_ENABLE_LOGGING=false -DQUIC_USE_SYSTEM_LIBCRYPTO=true -DQUIC_BUILD_TOOLS=off -DQUIC_BUILD_TEST=off -DQUIC_BUILD_PERF=off && \
+    cmake -B build -DCMAKE_BUILD_TYPE=Debug -DQUIC_ENABLE_LOGGING=false -DQUIC_USE_SYSTEM_LIBCRYPTO=true -DQUIC_BUILD_TOOLS=off -DQUIC_BUILD_TEST=off -DQUIC_BUILD_PERF=off -DQUIC_TLS_LIB=quictls -DQUIC_ENABLE_SANITIZERS=on && \
     cd build && \
-    cmake --build . --config Release
-RUN cd msquic/src/msquic/build/bin/Release && \
+    cmake --build . --config Debug
+RUN cd msquic/build/bin/Debug && \
     rm libmsquic.so && \
     fpm -f -s dir -t deb -n libmsquic -v $( find -type f | cut -d "." -f 4- ) \
     --license MIT --url https://github.com/microsoft/msquic --log error \
@@ -109,15 +113,10 @@ RUN cd msquic/src/msquic/build/bin/Release && \
 ```
 
 Source:
-https://github.com/dotnet/runtime/blob/bd540938a4830ee91dec5ee2d39545b2f69a19d5/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Dockerfile#L4-L21
+https://github.com/dotnet/runtime/blob/c6566fb0bcc539c523be9796ba5af681bf65a904/src/libraries/System.Net.Http/tests/StressTests/HttpStress/Dockerfile#L4-L21
 
 Note that to propagate newest sources / package to the docker image used for the test runs, it must be rebuilt by [dotnet-buildtools-prereqs-docker-all](https://dev.azure.com/dnceng/internal/_build?definitionId=1183&_a=summary) pipeline with `noCache = true` variable. And since [#76630](https://github.com/dotnet/runtime/pull/76630), the newest image will get automatically picked up by the dotnet/runtime infra.
 
 ### Windows
 
-Officially released `msquic.dll` is published to NuGet.org, see [Microsoft.Native.Quic.MsQuic.Schannel](https://www.nuget.org/packages/Microsoft.Native.Quic.MsQuic.Schannel).
-
-To consume MsQuic from the current main branch, we use [dotnet/msquic](https://github.com/dotnet/msquic) repository which will build and publish `msquic.dll` to the transport feed, e.g. [dotnet8-transport](https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet8-transport). And from there, it'll get flown into this repository via [Darc subscription](https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md). See https://github.com/dotnet/runtime/blob/bd540938a4830ee91dec5ee2d39545b2f69a19d5/eng/Version.Details.xml#L7-L10 and maestro-bot PR: https://github.com/dotnet/runtime/pull/71900.
-
-
-System.Net.Quic [project file](https://github.com/dotnet/runtime/blob/0304f1f5157a8280fa093bdfc7cfb8d9f62e016f/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj) allows switching between those two options with [`UseQuicTransportPackage` property](https://github.com/dotnet/runtime/blob/0304f1f5157a8280fa093bdfc7cfb8d9f62e016f/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj#L15).
+Officially released `msquic.dll` is published to NuGet.org, see [Microsoft.Native.Quic.MsQuic.Schannel](https://www.nuget.org/packages/Microsoft.Native.Quic.MsQuic.Schannel).

src/libraries/System.Net.Quic/src/System.Net.Quic.csproj

@@ -15,9 +15,6 @@
     <DefineConstants Condition="'$(TargetPlatformIdentifier)' == 'windows'">$(DefineConstants);TARGET_WINDOWS</DefineConstants>
     <GeneratePlatformNotSupportedAssemblyMessage Condition="'$(TargetPlatformIdentifier)' == ''">SR.SystemNetQuic_PlatformNotSupported</GeneratePlatformNotSupportedAssemblyMessage>
     <ApiExclusionListPath Condition="'$(TargetPlatformIdentifier)' == ''">ExcludeApiList.PNSE.txt</ApiExclusionListPath>
-    <!-- This controls if we consume official binaries from MsQuic or if we use binaries published from dotnet/msquic repo.
-         Release branches should generally consume MsQuic release code, transport allows us to consume and test pre-released versions -->
-    <UseQuicTransportPackage  Condition="'$(UseQuicTransportPackage)' == ''">false</UseQuicTransportPackage>
   </PropertyGroup>
 
   <!-- Source files -->
@@ -159,41 +156,15 @@
                         '$(DotNetBuildSourceOnly)' != 'true'">
 
     <!-- Project references -->
-    <PackageReference Include="runtime.win-$(TargetArchitecture).runtime.native.System.Net.MsQuic.Transport"
-                      Version="$(SystemNetMsQuicTransportVersion)"
-                      PrivateAssets="all"
-                      Condition="'$(UseQuicTransportPackage)' == 'true'"
-                      GeneratePathProperty="true" />
-
     <PackageReference Include="Microsoft.Native.Quic.MsQuic.Schannel"
                       Version="$(MicrosoftNativeQuicMsQuicSchannelVersion)"
                       PrivateAssets="all"
                       Condition="'$(UseQuicTransportPackage)' != 'true'"
                       GeneratePathProperty="true" />
 
-    <NativeBinPlaceItem Include="$(PkgMicrosoft_Native_Quic_MsQuic_Schannel)\build\native\bin\$(TargetArchitecture)\*"
-                        Condition="'$(UseQuicTransportPackage)' != 'true'" />
-    <NativeBinPlaceItem Include="$(PkgRuntime_win-x64_runtime_native_System_Net_MsQuic_Transport)\runtimes\win-$(TargetArchitecture)\native\*"
-                        Condition="'$(UseQuicTransportPackage)' == 'true'" />
-    <NativeBinPlaceItem Include="$(PkgRuntime_win-x86_runtime_native_System_Net_MsQuic_Transport)\runtimes\win-$(TargetArchitecture)\native\*"
-                        Condition="'$(UseQuicTransportPackage)' == 'true'" />
-    <NativeBinPlaceItem Include="$(PkgRuntime_win-arm64_runtime_native_System_Net_MsQuic_Transport)\runtimes\win-$(TargetArchitecture)\native\*"
-                        Condition="'$(UseQuicTransportPackage)' == 'true'" />
+    <NativeBinPlaceItem Include="$(PkgMicrosoft_Native_Quic_MsQuic_Schannel)\build\native\bin\$(TargetArchitecture)\*" />
   </ItemGroup>
 
-  <!-- Include msquic in debug version for macOS so we can test it without packages. -->
-  <ItemGroup Condition="'$(TargetPlatformIdentifier)' == 'osx' and
-                          '$(TargetOS)' == 'osx' and
-                          '$(TargetArchitecture)' == 'x64' and
-                          '$(Configuration)' == 'Debug' and
-                          '$(DotNetBuildSourceOnly)' != 'true'">
-      <!-- MsQuic packages do not have macOS binaries. Our transport package is only one source. -->
-      <PackageReference Include="runtime.osx-$(TargetArchitecture).runtime.native.System.Net.MsQuic.Transport"
-                        Version="$(SystemNetMsQuicTransportVersion)"
-                        PrivateAssets="all"
-                        GeneratePathProperty="true" />
-      <NativeBinPlaceItem Include="$(PkgRuntime_osx-x64_runtime_native_System_Net_MsQuic_Transport)\runtimes\osx-$(TargetArchitecture)\native\*" />
-  </ItemGroup>
   <ItemGroup Condition="'@(NativeBinplaceItem)' != ''">
     <BinPlaceDir Include="$(MicrosoftNetCoreAppRuntimePackNativeDir)" ItemName="NativeBinPlaceItem" />
     <BinPlaceDir Include="$(NetCoreAppCurrentTestHostSharedFrameworkPath)" ItemName="NativeBinPlaceItem" />

src/libraries/tests.proj

@@ -40,6 +40,11 @@
     <HighAOTResourceRequiringProject Include="$(MSBuildThisFileDirectory)System.Text.Json\tests\System.Text.Json.Tests\System.Text.Json.Tests.csproj" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(TargetOS)' == 'osx' and ('$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'x64')">
+    <!-- https://github.com/dotnet/dnceng/issues/3008 - osx Helix queue doesn't have OpenSSL -->
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Security.Cryptography.OpenSsl\tests\System.Security.Cryptography.OpenSsl.Tests.csproj" />
+  </ItemGroup>
+
   <ItemGroup Condition="'$(TargetOS)' == 'browser'">
     <!-- Samples which are too complex for CI -->
     <ProjectExclusions Include="$(MonoProjectRoot)sample\wasm\console-node-ts\Wasm.Console.Node.TS.Sample.csproj" />