Add support for source-build

.azure/pipelines/ci.yml

@@ -471,3 +471,52 @@ jobs:
     - name: Linux_Test_Results
       path: artifacts/TestResults/
       publishOnError: true
+
+# Source build
+- job: Source_Build
+  displayName: 'Test: Linux Source Build'
+  container: centos:7
+  pool:
+    vmImage: 'ubuntu-16.04'
+  variables:
+    DotNetCoreSdkDir: $(Agent.ToolsDirectory)/dotnet
+    DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: true
+  steps:
+  - script: |
+      source eng/common/native/common-library.sh
+      mkdir -p $HOME/bin
+      GetFile https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 $HOME/bin/jq
+      chmod +x $HOME/bin/jq
+      echo "##vso[task.prependpath]$HOME/bin"
+    displayName: Install jq
+  - task: UseDotNet@2
+    displayName: 'Use .NET Core sdk'
+    inputs:
+      packageType: sdk
+      # The SDK version selected here is intentionally supposed to use the latest release
+      # For the purpose of building Linux distros, we can't depend on features of the SDK
+      # which may not exist in pre-built versions of the SDK
+      version: 3.0.x
+      installationPath: $(DotNetCoreSdkDir)
+      includePreviewVersions: true
+  - script: ./eng/scripts/ci-source-build.sh --ci --configuration Release /p:BuildManaged=true
+    displayName: Run ci-source-build.sh
+  - task: PublishBuildArtifacts@1
+    displayName: Upload logs
+    condition: always()
+    continueOnError: true
+    inputs:
+      pathtoPublish: artifacts/log/
+      artifactName: Source_Build_Logs
+      artifactType: Container
+      parallel: true
+  - task: PublishBuildArtifacts@1
+    displayName: Upload package artifacts
+    # Only capture source build artifacts in PRs for the sake of inspecting
+    # changes that impact source-build. The artifacts from this build pipeline are never actually used.
+    condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
+    inputs:
+      pathtoPublish: artifacts/packages/
+      artifactName: Source_Build_Packages
+      artifactType: Container
+      parallel: true

Directory.Build.targets

@@ -14,6 +14,9 @@
       By default, assemblies which are only in the Microsoft.AspNetCore.App shared framework are not available as NuGet packages.
     -->
     <IsPackable Condition="'$(IsAspNetCoreApp)' == 'true' AND '$(IsShippingPackage)' != 'true'">false</IsPackable>
+
+    <!-- Only build assemblies in Microsoft.AspNetCore.App in source build -->
+    <ExcludeFromSourceBuild Condition="'$(ExcludeFromSourceBuild)' == '' and '$(DotNetBuildFromSource)' == 'true' and '$(IsAspNetCoreApp)' != 'true'">true</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />

eng/AfterSolutionBuild.targets

@@ -17,7 +17,7 @@
       SharedFrameworkTargetFramework="netcoreapp$(AspNetCoreMajorVersion).$(AspNetCoreMinorVersion)" />
   </Target>
 
-  <Target Name="GenerateBuildAssetManifest" AfterTargets="Pack" Condition="'$(ContinuousIntegrationBuild)' == 'true'">
+  <Target Name="GenerateBuildAssetManifest" AfterTargets="Pack" Condition="'$(ContinuousIntegrationBuild)' == 'true' and '$(DotNetBuildFromSource)' != 'true'">
     <!-- Generate build manifests. These manifests are used by Maestro and the Build Asset Registry to flow dependencies to other repos. -->
     <MSBuild Projects="$(MSBuildThisFileDirectory)tools\Maestro\Maestro.csproj"
           Targets="Restore"

eng/Workarounds.targets

@@ -21,10 +21,15 @@
   </ItemGroup>
 
   <!-- Workaround https://github.com/aspnet/AspNetCore/issues/7503. This chains GenerateSourceLinkFile before razor component targets run. -->
-  <Target Name="_EnsureSourceLinkHappensBeforeRazorComponentGeneration"
+  <!-- Workaround https://github.com/dotnet/source-build/issues/1112. Source link is currently disabled in source build so do not apply this worksaround. -->
+  <Target Condition="'$(DotNetBuildFromSource)' != 'true'"
+          Name="_EnsureSourceLinkHappensBeforeRazorComponentGeneration"
           BeforeTargets="PrepareForRazorComponentGenerate"
           DependsOnTargets="GenerateSourceLinkFile" />
 
+  <!-- Workaround https://github.com/dotnet/source-build/issues/1112. Source link is currently disabled in source build so define this dummy target which is required for pack. -->
+  <Import Condition="'$(DotNetBuildFromSource)' == 'true'" Project="WorkaroundsImported.targets" />
+
   <!-- Workaround https://github.com/aspnet/websdk/pull/646. If merged, once we update to a websdk with this fix, we can move the setting below to Directory.Build.props. -->
   <PropertyGroup>
     <!-- Ignore warning about calling the Pack target on Web SDK projects. Our build scripts call /t:pack on everything in this repo. -->

eng/WorkaroundsImported.targets

@@ -0,0 +1,5 @@
+<!-- Use this file to workaround issues. List the issue tracking the item to fix so we can remove the workaround when the issue is resolved. -->
+<Project>
+  <!-- Workaround https://github.com/dotnet/source-build/issues/1112. Source link is currently disabled in source build so define this dummy target which is required for pack. -->
+  <Target Name="InitializeSourceControlInformation"/>
+</Project>

eng/scripts/ci-source-build.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+ #
+# This script is meant for testing source build by imitating some of the input parameters and conditions.
+#
+
+set -euo pipefail
+
+scriptroot="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+reporoot="$(dirname "$(dirname "$scriptroot")")"
+
+ # For local development, make a backup copy of this file first
+if [ ! -f "$reporoot/global.bak.json" ]; then
+    mv "$reporoot/global.json" "$reporoot/global.bak.json"
+fi
+
+ # Detect the current version of .NET Core installed
+export SDK_VERSION=$(dotnet --version)
+echo "The ambient version of .NET Core SDK version = $SDK_VERSION"
+
+ # Update the global.json file to match the current .NET environment
+cat "$reporoot/global.bak.json" | \
+    jq '.sdk.version=env.SDK_VERSION' | \
+    jq '.tools.dotnet=env.SDK_VERSION' | \
+    jq 'del(.tools.runtimes)' \
+    > "$reporoot/global.json"
+
+ # Restore the original global.json file
+trap "{
+    mv "$reporoot/global.bak.json" "$reporoot/global.json"
+}" EXIT
+
+export DotNetBuildFromSource='true'
+
+ # Build repo tasks
+"$reporoot/eng/common/build.sh" --restore --build --ci --configuration Release /p:ProjectToBuild=$reporoot/eng/tools/RepoTasks/RepoTasks.csproj
+
+ # Build projects
+"$reporoot/eng/common/build.sh" --restore --build --pack "$@"

eng/targets/CSharp.Common.props

@@ -11,7 +11,7 @@
     <!-- Required to exist in the NuGet package cache to enable code-signing. -->
     <PackageReference Include="MicroBuild.Core" Version="0.3.0" PrivateAssets="All" AllowExplicitReference="true" ExcludeAssets="All" />
 
-    <PackageReference Include="Microsoft.DotNet.GenAPI" PrivateAssets="All" Version="$(MicrosoftDotNetGenApiPackageVersion)" IsImplicitlyDefined="true" />
+    <PackageReference Condition="'$(DotNetBuildFromSource)' != 'true'" Include="Microsoft.DotNet.GenAPI" PrivateAssets="All" Version="$(MicrosoftDotNetGenApiPackageVersion)" IsImplicitlyDefined="true" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(UsingMicrosoftNETSdkWeb)' == 'true' OR '$(RazorSdkCurrentVersionProps)' != ''">

eng/targets/ResolveReferences.targets

@@ -19,6 +19,8 @@
 <Project>
 
   <PropertyGroup>
+    <EnableCustomReferenceResolution Condition="'$(EnableCustomReferenceResolution)' == '' AND ('$(DotNetBuildFromSource)' != 'true' OR '$(ExcludeFromSourceBuild)' != 'true')">true</EnableCustomReferenceResolution>
+
     <ResolveReferencesDependsOn>
       ResolveCustomReferences;
       $(ResolveReferencesDependsOn);
@@ -60,7 +62,7 @@
     </Reference>
   </ItemDefinitionGroup>
 
-  <ItemGroup>
+  <ItemGroup Condition="'$(EnableCustomReferenceResolution)' == 'true'">
     <Reference Update="@(Reference)">
       <IsSharedSource Condition="'%(IsSharedSource)' == '' AND $([System.String]::new('%(Identity)').EndsWith('.Sources'))">true</IsSharedSource>
     </Reference>
@@ -129,7 +131,7 @@
     This target resolves remaining Referene items to Packages, if possible. If not, they are left as Reference items fo the SDK to resolve.
     This executes on NuGet restore and during DesignTimeBuild. It should not run in the outer, cross-targeting build.
    -->
-  <Target Name="ResolveCustomReferences" BeforeTargets="CollectPackageReferences;ResolveAssemblyReferencesDesignTime;ResolveAssemblyReferences" Condition=" '$(TargetFramework)' != '' ">
+  <Target Name="ResolveCustomReferences" BeforeTargets="CollectPackageReferences;ResolveAssemblyReferencesDesignTime;ResolveAssemblyReferences" Condition=" '$(TargetFramework)' != '' AND '$(EnableCustomReferenceResolution)' == 'true' ">
     <ItemGroup>
       <!-- Ensure only content asset are consumed from .Sources packages -->
       <Reference>

eng/tools/RepoTasks/RepoTasks.csproj

@@ -7,6 +7,8 @@
     <Optimize>false</Optimize>
     <DebugType>embedded</DebugType>
     <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
+    <!-- Need to build this project in source build -->
+    <ExcludeFromSourceBuild>false</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <ItemGroup>

src/Framework/ref/Microsoft.AspNetCore.App.Ref.csproj

@@ -76,6 +76,9 @@ This package is an internal implementation of the .NET Core SDK and is not meant
       _InstallTargetingPackIntoLocalDotNet;
       _CreateTargetingPackArchive;
     </BuildDependsOn>
+
+    <!-- No-op when in source build -->
+    <BuildDependsOn Condition="'$(IsTargetingPackBuilding)' != 'false' and '$(DotNetBuildFromSource)' == 'true'"/>
   </PropertyGroup>
 
   <!-- Override the default MSBuild targets so that nothing is returned from the project since it represents a collection of assemblies. -->

src/Framework/src/Microsoft.AspNetCore.App.Runtime.csproj

@@ -8,6 +8,8 @@
     <SelfContained>false</SelfContained>
     <PackageId>$(MSBuildProjectName).$(RuntimeIdentifier)</PackageId>
     <IsShippingPackage>true</IsShippingPackage>
+    <!-- Do not exclude this project from source build -->
+    <ExcludeFromSourceBuild>false</ExcludeFromSourceBuild>
     <HasReferenceAssembly>false</HasReferenceAssembly>
     <PackageDescription>Provides a default set of APIs for building an ASP.NET Core application. Contains assets used for self-contained deployments.
 
@@ -361,8 +363,8 @@ This package is an internal implementation of the .NET Core SDK and is not meant
   Targets related to creating .zip/.tar.gz
   #########################################
   -->
-  <Target Name="_DownloadAndExtractDotNetRuntime">
-   <DownloadFile Condition=" ! Exists('$(DotNetRuntimeArchive)')"
+  <Target Name="_DownloadAndExtractDotNetRuntime" Condition="'$(DotNetBuildFromSource)' != 'true'">
+    <DownloadFile Condition=" ! Exists('$(DotNetRuntimeArchive)')"
       SourceUrl="$(DotNetRuntimeDownloadUrl)$(DotNetAssetRootAccessTokenSuffix)"
       DestinationFolder="$(BaseIntermediateOutputPath)"
       DestinationFileName="$(DotNetRuntimeArchiveFileName)" />
@@ -412,7 +414,9 @@ This package is an internal implementation of the .NET Core SDK and is not meant
   </Target>
 
   <!-- Required to workaround https://github.com/dotnet/core-setup/issues/4809. This copies the shared framework into the $reporoot/.dotnet folder so tests can run against the shared framework. -->
-  <Target Name="_InstallFrameworkIntoLocalDotNet"
+  <!-- This workaround is not needed in source build -->
+  <Target Condition="'$(DotNetBuildFromSource)' != 'true'"
+          Name="_InstallFrameworkIntoLocalDotNet"
           Inputs="@(SharedFxContent)"
           Outputs="@(SharedFxContent->'$(LocalInstallationOutputPath)%(FileName)%(Extension)')">
 
@@ -437,7 +441,9 @@ This package is an internal implementation of the .NET Core SDK and is not meant
       Condition="'$(ArchiveExtension)' == '.tar.gz'" />
   </Target>
 
-  <Target Name="_CreateRedistSharedFxArchive"
+  <!-- Redist tarball including the dotnet-runtime is not needed in source build -->
+  <Target Condition="'$(DotNetBuildFromSource)' != 'true'"
+          Name="_CreateRedistSharedFxArchive"
           Inputs="@(SharedFxContent)"
           Outputs="$(RedistArchiveOutputPath)">
     <Message Importance="High" Text="$(MSbuildProjectFile) -> $(RedistArchiveOutputPath)" />

src/Identity/UI/src/Microsoft.AspNetCore.Identity.UI.csproj

@@ -40,7 +40,8 @@
     <UIFrameworkVersionMoniker Include="V4" />
   </ItemGroup>
 
-  <Target Name="BuildRazorViews" DependsOnTargets="Compile" BeforeTargets="Build">
+  <!-- Source build doesn't build this package -->
+  <Target Name="BuildRazorViews" DependsOnTargets="Compile" BeforeTargets="Build" Condition="'$(DotNetBuildFromSource)' != 'true'">
     <Message Text="Building razor views assemblies" Importance="High" />
     <MSbuild Projects="$(MSBuildThisFile)"
              Targets="BuildForUI"
@@ -128,7 +129,8 @@
     </ItemGroup>
   </Target>
 
-  <Target Name="VerifyBuildOutputs" AfterTargets="BuildRazorViews">
+  <!-- Source build doesn't build this package -->
+  <Target Name="VerifyBuildOutputs" AfterTargets="BuildRazorViews" Condition="'$(DotNetBuildFromSource)' != 'true'">
     <ItemGroup>
       <ExpectedOutputFile Include="$(TargetDir)Microsoft.AspNetCore.Identity.UI.Views.V3.dll" />
       <ExpectedOutputFile Include="$(TargetDir)Microsoft.AspNetCore.Identity.UI.Views.V4.dll" />

src/Tools/FirstRunCertGenerator/src/Microsoft.AspNetCore.DeveloperCertificates.XPlat.csproj

@@ -9,6 +9,7 @@
     <!-- This package contains API for the .NET CLI to generate the aspnet HTTPs dev cert during CLI first-run initialization. -->
     <IsShippingPackage>false</IsShippingPackage>
     <HasReferenceAssembly>false</HasReferenceAssembly>
+    <ExcludeFromSourceBuild>false</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <ItemGroup>

src/Tools/dotnet-watch/test/dotnet-watch.Tests.csproj

@@ -28,13 +28,15 @@
     <RemoveDir Directories="$(TargetDir)TestProjects" Condition="Exists('$(TargetDir)TestProjects')" />
   </Target>
 
-  <Target Name="PublishDotNetWatchOnBuild" BeforeTargets="Build">
+  <!-- Do not publish in source build -->
+  <Target Name="PublishDotNetWatchOnBuild" BeforeTargets="Build" Condition="'$(DotNetBuildFromSource)' != 'true'">
     <MSBuild Projects="..\src\dotnet-watch.csproj"
       Targets="Publish"
       Properties="PublishDir=$(OutputPath)\tool\;Configuration=$(Configuration)" />
   </Target>
 
-  <Target Name="PublishDotNetWatchOnPublish" BeforeTargets="Publish">
+  <!-- Do not publish in source build -->
+  <Target Name="PublishDotNetWatchOnPublish" BeforeTargets="Publish" Condition="'$(DotNetBuildFromSource)' != 'true'">
     <MSBuild Projects="..\src\dotnet-watch.csproj"
       Targets="Publish"
       Properties="PublishDir=$(PublishDir)\tool\;Configuration=$(Configuration)" />