Jettison vxsort support in the GC when optimizing for size (#85036)

Vxsort comes with precomputed tables that are more than 100 kB in size.

Jettison them when optimizing for size. Hello world with optimize for size + invariant globalization is now 1.08 MB.

Fixes #84749.

src/coreclr/gc/vxsort/dummy.cpp

@@ -0,0 +1,30 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#include "common.h"
+
+#include "do_vxsort.h"
+
+//
+// Dummy replacement VXSORT support that always says the CPU doesn't
+// support the required instruction set.
+//
+
+bool IsSupportedInstructionSet (InstructionSet instructionSet)
+{
+    return false;
+}
+
+void InitSupportedInstructionSet (int32_t configSetting)
+{
+}
+
+void do_vxsort_avx2 (uint8_t** low, uint8_t** high, uint8_t *range_low, uint8_t *range_high)
+{
+    assert(false);
+}
+
+void do_vxsort_avx512 (uint8_t** low, uint8_t** high, uint8_t* range_low, uint8_t* range_high)
+{
+    assert(false);
+}

src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Windows.targets

@@ -22,6 +22,8 @@ The .NET Foundation licenses this file to you under the MIT license.
     <FullRuntimeName Condition="'$(ServerGarbageCollection)' == 'true' or '$(ControlFlowGuard)' == 'Guard'">Runtime.ServerGC</FullRuntimeName>
     <BootstrapperName>bootstrapper</BootstrapperName>
     <BootstrapperName Condition="'$(NativeLib)' != '' or '$(CustomNativeMain)' == 'true'">bootstrapperdll</BootstrapperName>
+    <VxSortSupportName>Runtime.VxsortEnabled</VxSortSupportName>
+    <VxSortSupportName Condition="'$(ControlFlowGuard)' != 'Guard' and ('$(IlcOptimizationPreference)' == 'Size' or '$(IlcDisableVxsort)' == 'true')">Runtime.VxsortDisabled</VxSortSupportName>
     <EntryPointSymbol Condition="'$(EntryPointSymbol)' == ''">wmainCRTStartup</EntryPointSymbol>
     <LinkerSubsystem Condition="'$(OutputType)' == 'WinExe' and '$(LinkerSubsystem)' == ''">WINDOWS</LinkerSubsystem>
     <LinkerSubsystem Condition="'$(OutputType)' == 'Exe' and '$(LinkerSubsystem)' == ''">CONSOLE</LinkerSubsystem>
@@ -36,6 +38,7 @@ The .NET Foundation licenses this file to you under the MIT license.
       <NativeLibrary Include="$(IlcSdkPath)$(BootstrapperName)$(LibrarySuffix)" />
       <NativeLibrary Include="$(IlcSdkPath)$(FullRuntimeName)$(LibrarySuffix)" />
       <NativeLibrary Include="$(IlcSdkPath)$(EventPipeName)$(LibrarySuffix)" />
+      <NativeLibrary Condition="'$(_targetArchitecture)' == 'x64'" Include="$(IlcSdkPath)$(VxSortSupportName)$(LibrarySuffix)" />
       <NativeLibrary Condition="'$(IlcMultiModule)' == 'true'" Include="$(SharedLibrary)" />
     </ItemGroup>
 

src/coreclr/nativeaot/Runtime/CMakeLists.txt

@@ -177,7 +177,7 @@ if (CLR_CMAKE_TARGET_APPLE)
 endif (CLR_CMAKE_TARGET_APPLE)
 
 if (CLR_CMAKE_TARGET_ARCH_AMD64 AND CLR_CMAKE_TARGET_WIN32)
-  list(APPEND COMMON_RUNTIME_SOURCES
+  set(VXSORT_SOURCES
     ${GC_DIR}/vxsort/isa_detection.cpp
     ${GC_DIR}/vxsort/do_vxsort_avx2.cpp
     ${GC_DIR}/vxsort/do_vxsort_avx512.cpp
@@ -187,7 +187,11 @@ if (CLR_CMAKE_TARGET_ARCH_AMD64 AND CLR_CMAKE_TARGET_WIN32)
     ${GC_DIR}/vxsort/smallsort/bitonic_sort.AVX512.int64_t.generated.cpp
     ${GC_DIR}/vxsort/smallsort/bitonic_sort.AVX512.int32_t.generated.cpp
     ${GC_DIR}/vxsort/smallsort/avx2_load_mask_tables.cpp
-)
+  )
+
+  set(DUMMY_VXSORT_SOURCES
+    ${GC_DIR}/vxsort/dummy.cpp
+  )
 endif (CLR_CMAKE_TARGET_ARCH_AMD64 AND CLR_CMAKE_TARGET_WIN32)
 
 list(APPEND RUNTIME_SOURCES_ARCH_ASM
@@ -267,6 +271,8 @@ convert_to_absolute_path(COMMON_RUNTIME_SOURCES ${COMMON_RUNTIME_SOURCES})
 convert_to_absolute_path(FULL_RUNTIME_SOURCES ${FULL_RUNTIME_SOURCES})
 convert_to_absolute_path(SERVER_GC_SOURCES ${SERVER_GC_SOURCES})
 convert_to_absolute_path(RUNTIME_SOURCES_ARCH_ASM ${RUNTIME_SOURCES_ARCH_ASM})
+convert_to_absolute_path(VXSORT_SOURCES ${VXSORT_SOURCES})
+convert_to_absolute_path(DUMMY_VXSORT_SOURCES ${DUMMY_VXSORT_SOURCES})
 
 if(NOT CLR_CMAKE_TARGET_ARCH_WASM)
   add_subdirectory(Full)

src/coreclr/nativeaot/Runtime/Full/CMakeLists.txt

@@ -30,12 +30,22 @@ add_library(Runtime.WorkstationGC STATIC ${COMMON_RUNTIME_SOURCES} ${FULL_RUNTIM
 
 add_library(Runtime.ServerGC STATIC ${COMMON_RUNTIME_SOURCES} ${FULL_RUNTIME_SOURCES} ${RUNTIME_SOURCES_ARCH_ASM} ${SERVER_GC_SOURCES} ${RUNTIME_ARCH_ASM_OBJECTS})
 
+if (CLR_CMAKE_TARGET_WIN32 AND CLR_CMAKE_TARGET_ARCH_AMD64)
+  add_library(Runtime.VxsortEnabled STATIC ${VXSORT_SOURCES})
+  add_library(Runtime.VxsortDisabled STATIC ${DUMMY_VXSORT_SOURCES})
+endif (CLR_CMAKE_TARGET_WIN32 AND CLR_CMAKE_TARGET_ARCH_AMD64)
+
 target_compile_definitions(Runtime.ServerGC PRIVATE -DFEATURE_SVR_GC)
 
 if (CLR_CMAKE_TARGET_WIN32)
   add_library(Runtime.ServerGC.GuardCF STATIC ${COMMON_RUNTIME_SOURCES} ${FULL_RUNTIME_SOURCES} ${RUNTIME_SOURCES_ARCH_ASM} ${SERVER_GC_SOURCES} ${RUNTIME_ARCH_ASM_OBJECTS})
   target_compile_definitions(Runtime.ServerGC.GuardCF PRIVATE -DFEATURE_SVR_GC)
   target_compile_options(Runtime.ServerGC.GuardCF PRIVATE $<$<OR:$<COMPILE_LANGUAGE:C>,$<COMPILE_LANGUAGE:CXX>>:/guard:cf>)
+
+  if (CLR_CMAKE_TARGET_ARCH_AMD64)
+    add_library(Runtime.VxsortEnabled.GuardCF STATIC ${VXSORT_SOURCES})
+    target_compile_options(Runtime.VxsortEnabled.GuardCF PRIVATE $<$<OR:$<COMPILE_LANGUAGE:C>,$<COMPILE_LANGUAGE:CXX>>:/guard:cf>)
+  endif (CLR_CMAKE_TARGET_ARCH_AMD64)
 endif (CLR_CMAKE_TARGET_WIN32)
 
 # Get the current list of definitions
@@ -85,5 +95,10 @@ endif (CLR_CMAKE_TARGET_WIN32)
 install_static_library(Runtime.WorkstationGC aotsdk nativeaot)
 install_static_library(Runtime.ServerGC aotsdk nativeaot)
 if (CLR_CMAKE_TARGET_WIN32)
+  if (CLR_CMAKE_TARGET_ARCH_AMD64)
+    install_static_library(Runtime.VxsortEnabled aotsdk nativeaot)
+    install_static_library(Runtime.VxsortDisabled aotsdk nativeaot)
+    install_static_library(Runtime.VxsortEnabled.GuardCF aotsdk nativeaot)
+  endif (CLR_CMAKE_TARGET_ARCH_AMD64)
   install_static_library(Runtime.ServerGC.GuardCF aotsdk nativeaot)
 endif (CLR_CMAKE_TARGET_WIN32)