Add reference assembly for System.Private.CoreLib.dll (#72143)

We now compile against the reference assembly in all places where we were compiling against the mono/coreclr System.Private.CoreLib.dll implementation assembly before.

The new reference assembly consumes sources from the existing contracts to avoid checking in a generated version of SPC.dll (this would add ~20k lines of .cs which is mostly duplicated with System.Runtime.cs)

Since a few contracts have only partially moved types to SPC we wrap contract types with `#if !BUILDING_CORELIB_REFERENCE` so we can hide them when compiling the SPC reference assembly. This needed a few GenAPI changes which are implemented here: dotnet/arcade#10003.

Note that this means that the types which live in CoreLib are moved to the end of the ref .cs file which causes a GitHub diff to show up, but it is actually a no-op.

Regenerating the ref .cs files works the same way as before, by running the `GenerateReferenceAssemblySource` target in the contract's src\ folder.

Fixes #67660

Co-authored-by: Viktor Hofer <viktor.hofer@microsoft.com>

Directory.Build.props

@@ -255,9 +255,11 @@
 
   <PropertyGroup>
     <CoreLibSharedDir>$([MSBuild]::NormalizeDirectory('$(LibrariesProjectRoot)', 'System.Private.CoreLib', 'src'))</CoreLibSharedDir>
+    <CoreLibRefDir>$([MSBuild]::NormalizeDirectory('$(LibrariesProjectRoot)', 'System.Private.CoreLib', 'ref'))</CoreLibRefDir>
     <CoreLibProject Condition="'$(RuntimeFlavor)' == 'CoreCLR'">$([MSBuild]::NormalizePath('$(CoreClrProjectRoot)', 'System.Private.CoreLib', 'System.Private.CoreLib.csproj'))</CoreLibProject>
     <CoreLibProject Condition="'$(RuntimeFlavor)' == 'Mono'">$([MSBuild]::NormalizePath('$(MonoProjectRoot)', 'System.Private.CoreLib', 'System.Private.CoreLib.csproj'))</CoreLibProject>
     <CoreLibProject Condition="'$(UseNativeAotCoreLib)' == 'true'">$([MSBuild]::NormalizePath('$(CoreClrProjectRoot)', 'nativeaot', 'System.Private.CoreLib', 'src', 'System.Private.CoreLib.csproj'))</CoreLibProject>
+    <UriProject>$([MSBuild]::NormalizePath('$(LibrariesProjectRoot)', 'System.Private.Uri', 'src', 'System.Private.Uri.csproj'))</UriProject>
 
     <!-- this property is used by the SDK to pull in mono-based runtime packs -->
     <UseMonoRuntime Condition="'$(UseMonoRuntime)' == '' and '$(RuntimeFlavor)' == 'Mono'">true</UseMonoRuntime>

docs/coding-guidelines/updating-ref-source.md

@@ -13,7 +13,7 @@ This document provides the steps you need to take to update the reference assemb
 
 These steps can also be applied to some unique assemblies which depend on changes in System.Private.Corelib. (partial facades like System.Memory, for example).
 
-1) Run `dotnet build --no-incremental /t:GenerateReferenceSource` from the System.Runtime/src directory.
+1) Run `dotnet build --no-incremental /t:GenerateReferenceAssemblySource` from the System.Runtime/src directory.
 2) Filter out all unrelated changes and extract the changes you care about (ignore certain attributes being removed). Generally, this step is not required for other reference assemblies.
 
 ## For Full Facade Assemblies implementation assemblies

eng/Version.Details.xml

@@ -66,9 +66,9 @@
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>11672d906390046e77a34b6406d9e02229fd7e45</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.GenAPI" Version="7.0.0-beta.22372.1">
+    <Dependency Name="Microsoft.DotNet.GenAPI" Version="7.0.0-beta.22401.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>11672d906390046e77a34b6406d9e02229fd7e45</Sha>
+      <Sha>aa90e21c63248b4d6d61e8de14a0d8e7a9275130</Sha>
     </Dependency>
     <Dependency Name="Microsoft.DotNet.GenFacades" Version="7.0.0-beta.22372.1">
       <Uri>https://github.com/dotnet/arcade</Uri>

eng/Versions.props

@@ -57,7 +57,7 @@
     <MicrosoftDotNetApiCompatVersion>7.0.0-beta.22372.1</MicrosoftDotNetApiCompatVersion>
     <MicrosoftDotNetBuildTasksFeedVersion>7.0.0-beta.22372.1</MicrosoftDotNetBuildTasksFeedVersion>
     <MicrosoftDotNetCodeAnalysisVersion>7.0.0-beta.22372.1</MicrosoftDotNetCodeAnalysisVersion>
-    <MicrosoftDotNetGenAPIVersion>7.0.0-beta.22372.1</MicrosoftDotNetGenAPIVersion>
+    <MicrosoftDotNetGenAPIVersion>7.0.0-beta.22401.3</MicrosoftDotNetGenAPIVersion>
     <MicrosoftDotNetGenFacadesVersion>7.0.0-beta.22372.1</MicrosoftDotNetGenFacadesVersion>
     <MicrosoftDotNetXUnitExtensionsVersion>7.0.0-beta.22372.1</MicrosoftDotNetXUnitExtensionsVersion>
     <MicrosoftDotNetXUnitConsoleRunnerVersion>2.5.1-beta.22372.1</MicrosoftDotNetXUnitConsoleRunnerVersion>

eng/references.targets

@@ -29,15 +29,6 @@
                       Condition="'$(IsSourceProject)' == 'true' and '@(_coreLibProjectReference)' != ''" />
   </ItemGroup>
 
-  <!-- Disable TargetArchitectureMismatch warning when we reference CoreLib as it is platform specific. -->
-  <Target Name="DisableProjectReferenceArchitectureMismatchWarningForCoreLib"
-          Condition="'@(_coreLibProjectReference)' != ''"
-          BeforeTargets="ResolveAssemblyReferences">
-    <PropertyGroup>
-      <ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch>None</ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch>
-    </PropertyGroup>
-  </Target>
-
   <!-- Filter out transitive P2Ps which should be excluded. -->
   <Target Name="FilterTransitiveProjectReferences"
           AfterTargets="IncludeTransitiveProjectReferences"
@@ -94,6 +85,16 @@
     </ItemGroup>
   </Target>
 
+  <Target Name="ReplaceCoreLibSrcWithRefAssemblyForCompilation"
+          AfterTargets="FindReferenceAssembliesForReferences"
+          Condition="'$(CompileUsingReferenceAssemblies)' != 'true' and '@(_coreLibProjectReference)' != ''">
+    <ItemGroup>
+      <_resolvedCoreLibProjectReference Include="@(_ResolvedProjectReferencePaths->WithMetadataValue('MSBuildSourceProjectFile','$(CoreLibProject)'))" />
+      <ReferencePathWithRefAssemblies Remove="@(_resolvedCoreLibProjectReference)" />
+      <ReferencePathWithRefAssemblies Include="@(_resolvedCoreLibProjectReference->Metadata('ReferenceAssembly'))" />
+    </ItemGroup>
+  </Target>
+
   <!-- For experimental ref assemblies (which typically have the same name as a regular ref
        assembly), bump their minor file version by 100 to make it distinguishable from the regular
        ref assembly. -->

eng/resolveContract.props

@@ -0,0 +1,13 @@
+<Project>
+  <!-- ApiCompat setting -->
+  <PropertyGroup>
+    <RunApiCompatForSrc>$([MSBuild]::ValueOrDefault('$(IsSourceProject)', 'false'))</RunApiCompatForSrc>
+    <RunMatchingRefApiCompat>$([MSBuild]::ValueOrDefault('$(IsSourceProject)', 'false'))</RunMatchingRefApiCompat>
+    <ApiCompatEnforceOptionalRules>true</ApiCompatEnforceOptionalRules>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ApiCompatExcludeAttributesFile Include="$(RepositoryEngineeringDir)DefaultGenApiDocIds.txt" />
+    <ApiCompatExcludeAttributesFile Include="$(RepositoryEngineeringDir)ApiCompatExcludeAttributes.txt" />
+  </ItemGroup>
+</Project>

eng/resolveContract.targets

@@ -12,6 +12,7 @@
     <GenAPIHeaderFile>$(RepositoryEngineeringDir)LicenseHeader.txt</GenAPIHeaderFile>
     <GenAPITargetPath>$([MSBuild]::NormalizePath('$(MSBuildProjectDirectory)', '..', 'ref', '$(AssemblyName).cs'))</GenAPITargetPath>
     <GenAPILangVersion Condition="'$(LangVersion)' != ''">$(LangVersion)</GenAPILangVersion>
+    <ProjectForGenAPIDocIdGeneration Condition="'$(IsSourceProject)' == 'true' and '$(ProjectForGenAPIDocIdGeneration)' == ''">$(CoreLibProject)</ProjectForGenAPIDocIdGeneration>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(IsSourceProject)' == 'true'">
@@ -21,6 +22,10 @@
     <RunApiCompat Condition="'$(HasMatchingContract)' != 'true'">false</RunApiCompat>
   </PropertyGroup>
 
+  <ItemGroup Condition="'$(IsSourceProject)' == 'true' or '$(IsReferenceAssemblyProject)' == 'true' or '$(IsPartialFacadeAssembly)' == 'true'">
+    <PackageReference Include="Microsoft.DotNet.ApiCompat" Condition="'$(DotNetBuildFromSource)' != 'true'" Version="$(MicrosoftDotNetApiCompatVersion)" PrivateAssets="all" IsImplicitlyDefined="true" />
+  </ItemGroup>
+
   <ItemGroup Condition="'$(HasMatchingContract)' == 'true' and '$(ContractProject)' != '' and '@(ResolvedMatchingContract)' == ''">
     <ProjectReference Include="$(ContractProject)" ReferenceOutputAssembly="false" OutputItemType="ResolvedMatchingContract" />
     <!-- We aren't referencing the contract, but make sure it's considered as an input to Compile so that if it changes we rebuild and rerun API compat -->
@@ -60,11 +65,40 @@
                                       ReferenceAssembly="" />
     </ItemGroup>
   </Target>
-
+
+  <!-- Generate a .txt file with all public types of the project referenced by ProjectForGenAPIDocIdGeneration (e.g. System.Private.CoreLib or System.Private.Uri implementation assembly).
+       This file is then later on passed to GenAPI as the list of types which should be "split out" with the conditional compilation logic. -->
+  <Target Name="GenerateConditionalTypeLists"
+          Condition="'@(ProjectReference)' != '' and @(ProjectReference->AnyHaveMetadataValue('Identity', '$(ProjectForGenAPIDocIdGeneration)')) and ('$(ContractTypesPartiallyMoved)' == 'true' or '$(ContractTypesPartiallyMoved)' == 'true')">
+    <PropertyGroup>
+      <!-- We either wrap all contract types in a #if condition (when SymbolForGenAPIConditionalTypeLists is empty),
+           or just the types mentioned in GenAPIConditionalTypeLists with the symbol specified in SymbolForGenAPIConditionalTypeLists -->
+      <GenAPIDefaultCondition Condition="'$(SymbolForGenAPIConditionalTypeLists)' == ''">!BUILDING_CORELIB_REFERENCE</GenAPIDefaultCondition>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <_genAPILibPathForGenAPIDocIdGeneration Include="@(ReferencePath->'%(RootDir)%(Directory)'->Distinct())" />
+      <_genAPIResolvedProjectForGenAPIDocIdGeneration Include="@(_ResolvedProjectReferencePaths->WithMetadataValue('MSBuildSourceProjectFile','$(ProjectForGenAPIDocIdGeneration)'))" />
+
+      <GenAPIConditionalTypeLists Include="$(IntermediateOutputPath)GenAPIConditionalTypeList.txt">
+        <Symbol Condition="'$(SymbolForGenAPIConditionalTypeLists)' != ''">$(SymbolForGenAPIConditionalTypeLists)</Symbol>
+      </GenAPIConditionalTypeLists>
+    </ItemGroup>
+
+    <!-- Generate conditional compilation symbols to hide types implemented in contracts so we can include the .cs in the System.Private.CoreLib ref -->
+    <Microsoft.DotNet.GenAPI.GenAPITask
+        Assembly="@(_genAPIResolvedProjectForGenAPIDocIdGeneration)"
+        LibPath="@(_genAPILibPathForGenAPIDocIdGeneration)"
+        WriterType="DocIds"
+        DocIdKinds="Namespace, Type"
+        OutputPath="@(GenAPIConditionalTypeLists)" />
+  </Target>
+
   <Target Name="SetGenAPIProperties"
+          DependsOnTargets="GenerateConditionalTypeLists"
           BeforeTargets="GenerateReferenceAssemblySource">
     <PropertyGroup>
-      <GenAPIFollowTypeForwards Condition="'%(ProjectReference.Identity)' == '$(CoreLibProject)'">true</GenAPIFollowTypeForwards>
+      <GenAPIFollowTypeForwards Condition="'%(ProjectReference.Identity)' == '$(CoreLibProject)' or '%(ProjectReference.Identity)' == '$(UriProject)'">true</GenAPIFollowTypeForwards>
     </PropertyGroup>
   </Target>
 

src/coreclr/System.Private.CoreLib/MatchingRefApiCompatBaseline.txt

@@ -0,0 +1,35 @@
+Compat issues with assembly System.Private.CoreLib:
+TypesMustExist : Type 'Internal.Console' does not exist in the reference but it does exist in the implementation.
+TypesMustExist : Type 'Internal.Runtime.InteropServices.ComActivationContextInternal' does not exist in the reference but it does exist in the implementation.
+TypesMustExist : Type 'Internal.Runtime.InteropServices.ComActivator' does not exist in the reference but it does exist in the implementation.
+TypesMustExist : Type 'Internal.Runtime.InteropServices.ComponentActivator' does not exist in the reference but it does exist in the implementation.
+TypesMustExist : Type 'Internal.Runtime.InteropServices.InMemoryAssemblyLoader' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'public void System.AppContext.add_FirstChanceException(System.EventHandler<System.Runtime.ExceptionServices.FirstChanceExceptionEventArgs>)' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'public void System.AppContext.add_ProcessExit(System.EventHandler)' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'public void System.AppContext.add_UnhandledException(System.UnhandledExceptionEventHandler)' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'public void System.AppContext.remove_FirstChanceException(System.EventHandler<System.Runtime.ExceptionServices.FirstChanceExceptionEventArgs>)' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'public void System.AppContext.remove_ProcessExit(System.EventHandler)' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'public void System.AppContext.remove_UnhandledException(System.UnhandledExceptionEventHandler)' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'public void System.Environment.FailFast(System.String, System.Exception, System.String)' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'protected void System.WeakReference..ctor()' does not exist in the reference but it does exist in the implementation.
+CannotRemoveAttribute : Attribute 'System.Runtime.CompilerServices.TypeDependencyAttribute' exists on 'System.Collections.Generic.Comparer<T>' in the implementation but not the reference.
+CannotRemoveAttribute : Attribute 'System.Runtime.CompilerServices.TypeDependencyAttribute' exists on 'System.Collections.Generic.EqualityComparer<T>' in the implementation but not the reference.
+MembersMustExist : Member 'protected void System.Diagnostics.Tracing.EventCounter.Flush()' does not exist in the reference but it does exist in the implementation.
+CannotMakeTypeAbstract : Type 'System.Diagnostics.Tracing.EventListener' is abstract in the reference but is not abstract in the implementation.
+CannotMakeMoreVisible : Visibility of member 'System.Diagnostics.Tracing.EventListener..ctor()' is 'protected' in the reference but 'public' in the implementation.
+CannotMakeMoreVisible : Visibility of member 'System.Diagnostics.Tracing.EventListener.EventSourceIndex(System.Diagnostics.Tracing.EventSource)' is 'protected' in the reference but 'public' in the implementation.
+TypesMustExist : Type 'System.Diagnostics.Tracing.TraceLoggingEventTypes' does not exist in the reference but it does exist in the implementation.
+CannotMakeMemberAbstract : Member 'public System.Boolean System.IO.FileSystemInfo.Exists' is abstract in the reference but is not abstract in the implementation.
+CannotMakeMemberAbstract : Member 'public System.String System.IO.FileSystemInfo.Name' is abstract in the reference but is not abstract in the implementation.
+CannotMakeMemberAbstract : Member 'public System.Boolean System.IO.FileSystemInfo.Exists.get()' is abstract in the reference but is not abstract in the implementation.
+CannotMakeMemberAbstract : Member 'public System.String System.IO.FileSystemInfo.Name.get()' is abstract in the reference but is not abstract in the implementation.
+MembersMustExist : Member 'protected System.ModuleHandle System.Reflection.Module.GetModuleHandleImpl()' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'protected System.ModuleHandle System.Reflection.Emit.ModuleBuilder.GetModuleHandleImpl()' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'protected System.String System.String System.Resources.ResourceManager.BaseNameField' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'protected System.Resources.IResourceReader System.Resources.IResourceReader System.Resources.ResourceSet.Reader' does not exist in the reference but it does exist in the implementation.
+TypesMustExist : Type 'System.Runtime.CompilerServices.ICastable' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'public void System.Runtime.InteropServices.UnmanagedFunctionPointerAttribute..ctor()' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'public System.Boolean System.Runtime.Serialization.SerializationInfo.DeserializationInProgress.get()' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'public void System.Runtime.Serialization.SerializationInfo.ThrowIfDeserializationInProgress()' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'public void System.Runtime.Serialization.SerializationInfo.ThrowIfDeserializationInProgress(System.String, System.Int32)' does not exist in the reference but it does exist in the implementation.
+MembersMustExist : Member 'public void System.Runtime.Serialization.SerializationInfo.UpdateValue(System.String, System.Object, System.Type)' does not exist in the reference but it does exist in the implementation.

src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj

@@ -346,4 +346,12 @@
       <FileWrites Include="@(EventingSourceFile)" />
     </ItemGroup>
   </Target>
+
+  <!-- Import refererence assembly and ApiCompat logic -->
+  <PropertyGroup>
+    <IsSourceProject>true</IsSourceProject>
+    <ApiCompatValidateBaseline>true</ApiCompatValidateBaseline>
+  </PropertyGroup>
+  <Import Project="$(RepositoryEngineeringDir)resolveContract.props" />
+  <Import Project="$(RepositoryEngineeringDir)resolveContract.targets" />
 </Project>