Always build portable runtime in addition to non-portable.

.vsts.pipelines/jobs/ci-linux.yml

@@ -14,7 +14,7 @@ jobs:
   strategy:
     matrix: ${{ parameters.matrix }}
   pool: ${{ parameters.pool }}
-  timeoutInMinutes: 270
+  timeoutInMinutes: 300
   variables:
     # Prefix to distinguish artifacts from different legs.
     artifactName: ${{ format('{0} $(type)', parameters.job) }}
@@ -72,7 +72,7 @@ jobs:
         /p:AzDoPat=$(System.AccessToken)
       du -h $(rootDirectory) | sort -h | tail -n 50
     displayName: Build source-build
-    timeoutInMinutes: 150
+    timeoutInMinutes: 180
 
   # Generate prebuilt burndown data
   - script: |
@@ -208,7 +208,7 @@ jobs:
         $poisonArg
       du -h $(rootDirectory) | sort -h | tail -n 50
     displayName: Build tarball
-    timeoutInMinutes: 120
+    timeoutInMinutes: 150
     condition: and(succeeded(), eq(variables['sb.tarball'], true))
 
   # Run smoke tests.

.vsts.pipelines/jobs/ci-linux_bootstrap.yml

@@ -35,6 +35,7 @@ jobs:
     type: Production
 
   steps:
+  - template: ../steps/cleanup-unneeded-files.yml
   - template: ../steps/docker-cleanup-linux.yml
 
   - task: DownloadBuildArtifacts@0
@@ -49,7 +50,7 @@ jobs:
       set -ex
       df -h
       $(docker.run) $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
-      git clone https://github.com/dotnet/source-build-reference-packages'
+      git clone --depth 1 https://github.com/dotnet/source-build-reference-packages'
       du -h $(rootDirectory) | sort -h | tail -n 50
     displayName: clone source-build-reference-packages
     condition: always()

.vsts.pipelines/steps/run-bootstrap.yml

@@ -1,15 +1,14 @@
-# Delete some files from the default image since we're tight on disk space.
-# List is from https://github.com/microsoft/azure-pipelines-image-generation/pull/1231#issuecomment-537174621
-
 # Extract the tarball fetched from the previous Offline stage 
 steps:
 - script: |
       set -ex 
+      mount
       df -h 
       $(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
       mkdir -p /bst/sb/tarball
-      tar -xvf "/bst/Tarball centos71 Offline/$(tarballName).tar.gz" -C /bst/sb/tarball --no-same-owner --strip-components=1
+      tar -xf "/bst/Tarball centos71 Offline/$(tarballName).tar.gz" -C /bst/sb/tarball --no-same-owner --strip-components=1
       rm -rf "/bst/Tarball centos71 Offline"'
+      df -h
   displayName: Unpack Tarball
   condition: always()
 
@@ -19,6 +18,7 @@ steps:
       df -h 
       $(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
       /bst/sb/tarball/tool-bootstrapping/bootstrap.sh /bst/bootstrap_dir /bst/sb/tarball /bst/source-build-reference-packages 1'
+      df -h
   displayName: Build stage1
   condition: succeeded()
 
@@ -105,7 +105,8 @@ steps:
       set -ex 
       df -h 
       $(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
-      rm -rf /bst/source-build-reference-packages/* 
+      rm -rf /bst/source-build-reference-packages/*
+      rm -rf /bst/source-build-reference-packages/.git
       pushd /bst/bootstrap_dir/reference-packages
       echo "Removing everything in reference-packages except artifacts"
       find . -type f -not -name '*SourceBuild.ReferencePackages.bootstrap.tar.gz' -delete
@@ -226,4 +227,4 @@ steps:
   inputs:
     PathtoPublish: $(Build.ArtifactStagingDirectory)/drop/tarball
     ArtifactName: Tarball Artifacts $(artifactName)
-    ArtifactType: Container
+    ArtifactType: Container

eng/Version.Details.xml

@@ -32,6 +32,11 @@
       <Sha>1ed7226e39b7214807318d2cc8ff37492527ffa1</Sha>
       <RepoName>coreclr</RepoName>
     </Dependency>
+    <Dependency Name="Microsoft.NETCore.Runtime.CoreCLR" Version="3.1.0-rtm.19564.2" CoherentParentDependency="Microsoft.Private.CoreFx.NETCoreApp">
+      <Uri>https://github.com/dotnet/coreclr</Uri>
+      <Sha>1ed7226e39b7214807318d2cc8ff37492527ffa1</Sha>
+      <RepoName>coreclr-portable</RepoName>
+    </Dependency>
     <Dependency Name="NETStandard.Library" Version="2.1.0" CoherentParentDependency="Microsoft.Dotnet.Sdk.Internal">
       <Uri>https://github.com/dotnet/standard</Uri>
       <Sha>a5b5f2e1e369972c8ff1e2183979fab6099f52ef</Sha>
@@ -41,11 +46,21 @@
       <Sha>0f7f38c4fd323b26da10cce95f857f77f0f09b48</Sha>
       <RepoName>corefx</RepoName>
     </Dependency>
+    <Dependency Name="Microsoft.Private.CoreFx.NETCoreApp" Version="4.7.0-rtm.19564.4" CoherentParentDependency="Microsoft.NETCore.App">
+      <Uri>https://github.com/dotnet/corefx</Uri>
+      <Sha>0f7f38c4fd323b26da10cce95f857f77f0f09b48</Sha>
+      <RepoName>corefx-portable</RepoName>
+    </Dependency>
     <Dependency Name="Microsoft.NETCore.App" Version="3.1.0-rtm.19565.2" CoherentParentDependency="Microsoft.Dotnet.Sdk.Internal">
       <Uri>https://github.com/dotnet/core-setup</Uri>
       <Sha>65f04fb6db7a5e198d05dbebd5c4ad21eb018f89</Sha>
       <RepoName>core-setup</RepoName>
     </Dependency>
+    <Dependency Name="Microsoft.NETCore.App" Version="3.1.0-rtm.19565.2" CoherentParentDependency="Microsoft.Dotnet.Sdk.Internal">
+      <Uri>https://github.com/dotnet/core-setup</Uri>
+      <Sha>65f04fb6db7a5e198d05dbebd5c4ad21eb018f89</Sha>
+      <RepoName>core-setup-portable</RepoName>
+    </Dependency>
     <Dependency Name="Microsoft.AspNetCore.Mvc.Razor.Extensions" Version="3.1.0" CoherentParentDependency="Microsoft.AspNetCore.App.Ref">
       <Uri>https://github.com/aspnet/AspNetCore-Tooling</Uri>
       <Sha>29036777e7cd2172a81c45c69e0dc257e8b8184e</Sha>

repos/core-setup-portable.proj

@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <SourceDirectory>core-setup</SourceDirectory>
+    <PortableBuild>true</PortableBuild>
+    <BuildingPortableRuntime>true</BuildingPortableRuntime>
+  </PropertyGroup>
+
+  <Import Project="core-setup.common.props" />
+
+  <ItemGroup>
+    <RepositoryReference Include="coreclr" />
+    <RepositoryReference Include="corefx" />
+    <RepositoryReference Include="newtonsoft-json" />
+    <RepositoryReference Include="newtonsoft-json901" />
+    <RepositoryReference Include="roslyn" />
+    <RepositoryReference Include="standard" />
+  </ItemGroup>
+
+  <Target Name="CleanupRepoForNonPortableBuild" AfterTargets="RemoveBuiltPackagesFromCache">
+    <ItemGroup>
+      <DirsToDelete Include="$(ProjectDirectory)artifacts" Condition="Exists('$(ProjectDirectory)artifacts')" />
+      <DirsToDelete Include="$(ProjectDirectory)bin" Condition="Exists('$(ProjectDirectory)bin')" />
+    </ItemGroup>
+
+    <RemoveDir Directories="@(DirsToDelete)" />
+  </Target>
+
+  <Import Project="core-setup.common.targets" />
+
+  <Target Name="GatherBuiltPackages" BeforeTargets="CopyPackage">
+    <ItemGroup>
+      <_BuiltPackages Include="$(ShippingPackagesOutput)/*linux*nupkg" Exclude="$(ShippingPackagesOutput)/*.symbols.nupkg" />
+      <_BuiltPackages Include="$(NonShippingPackagesOutput)/*linux*nupkg" Exclude="$(NonShippingPackagesOutput)/*.symbols.nupkg" />
+    </ItemGroup>
+  </Target>
+</Project>

repos/core-setup.common.props

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))/dir.props" />
+
+  <PropertyGroup>
+    <!-- core-sdk always wants a portable runtime on OSX and FreeBSD-->
+    <OverrideTargetRid>$(TargetRid)</OverrideTargetRid>
+    <OverrideTargetRid Condition="'$(TargetOS)' == 'OSX'">osx-x64</OverrideTargetRid>
+    <OverrideTargetRid Condition="'$(TargetOS)' == 'FreeBSD'">freebsd-x64</OverrideTargetRid>
+    <OverrideTargetRid Condition="'$(TargetOS)' == 'Linux' and ('$(PortableBuild)' == 'true' or '$(BuildingPortableRuntime)' == 'true')">linux-$(Platform)</OverrideTargetRid>
+    <OverridePortable>$(PortableBuild)</OverridePortable>
+    <OverridePortable Condition="'$(TargetOS)' == 'OSX' or '$(TargetOS)' == 'FreeBSD'">true</OverridePortable>
+    <OverridePortable Condition="'$(BuildingPortableRuntime)' == 'true'">true</OverridePortable>
+
+    <BuildArguments>$(FlagParameterPrefix)restore $(FlagParameterPrefix)build</BuildArguments>
+    <BuildArguments>$(BuildArguments) $(FlagParameterPrefix)configuration $(Configuration)</BuildArguments>
+    <BuildArguments>$(BuildArguments) $(FlagParameterPrefix)ci</BuildArguments>
+    <BuildArguments>$(BuildArguments) /p:PortableBuild=$(OverridePortable)</BuildArguments>
+    <BuildArguments Condition="$(Platform.Contains('arm')) AND '$(BuildArchitecture)' != 'arm64'">$(BuildArguments) /p:TargetArchitecture=$(Platform) /p:DisableCrossgen=true /p:CrossBuild=true</BuildArguments>
+    <BuildArguments>$(BuildArguments) /p:BuildDebPackage=false</BuildArguments>
+    <BuildArguments>$(BuildArguments) /p:BuildAllPackages=true</BuildArguments>
+    <BuildArguments>$(BuildArguments) /p:RestoreAllBuildRids=false</BuildArguments>
+    <BuildArguments>$(BuildArguments) /p:OutputRid=$(OverrideTargetRid)</BuildArguments>
+    <BuildArguments>$(BuildArguments) /p:DotNetOutputBlobFeedDir=$(SourceBuiltBlobFeedDir)</BuildArguments>
+    <BuildArguments>$(BuildArguments) /m:1</BuildArguments>
+
+    <LogVerbosityOptOut>true</LogVerbosityOptOut>
+    <BuildArguments>$(BuildArguments) $(FlagParameterPrefix)verbosity $(LogVerbosity)</BuildArguments>
+    <BuildArguments>$(BuildArguments) $(FlagParameterPrefix)binaryLog</BuildArguments>
+    <BuildArguments>$(BuildArguments) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg)</BuildArguments>
+    <BuildArguments>$(BuildArguments) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg)</BuildArguments>
+
+    <BuildCommand>$(ProjectDirectory)/build$(ShellExtension) $(BuildArguments)</BuildCommand>
+    <BuildCommand Condition="$(Platform.Contains('arm')) AND '$(BuildArchitecture)' != 'arm64'">$(ArmEnvironmentVariables) $(BuildCommand)</BuildCommand>
+
+    <OutputPlacementRepoApiImplemented>false</OutputPlacementRepoApiImplemented>
+    <ShippingPackagesOutput>$(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/</ShippingPackagesOutput>
+    <NonShippingPackagesOutput>$(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/</NonShippingPackagesOutput>
+
+    <NuGetConfigFile>$(ProjectDirectory)/NuGet.config</NuGetConfigFile>
+    <DependencyVersionInputRepoApiImplemented>true</DependencyVersionInputRepoApiImplemented>
+
+    <EnvironmentRestoreSources>$(SourceBuiltPackagesPath)</EnvironmentRestoreSources>
+    <EnvironmentRestoreSources>$(EnvironmentRestoreSources)%3B$(ShippingPackagesOutput)</EnvironmentRestoreSources>
+    <EnvironmentRestoreSources>$(EnvironmentRestoreSources)%3B$(NonShippingPackagesOutput)</EnvironmentRestoreSources>
+    <EnvironmentRestoreSources Condition="'$(OfflineBuild)' == 'true'">$(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath)</EnvironmentRestoreSources>
+    <EnvironmentRestoreSources Condition="'$(OfflineBuild)' != 'true'">$(EnvironmentRestoreSources)%3Bhttps://api.nuget.org/v3/index.json</EnvironmentRestoreSources>
+    <EnvironmentRestoreSources Condition="'$(OfflineBuild)' != 'true'">$(EnvironmentRestoreSources)%3Bhttps://dotnetfeed.blob.core.windows.net/dotnet-core/index.json</EnvironmentRestoreSources>
+    <EnvironmentRestoreSources Condition="'$(OfflineBuild)' != 'true'">$(EnvironmentRestoreSources)%3Bhttps://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json</EnvironmentRestoreSources>
+
+    <!-- core-setup does not destabilize for intermediate servicing builds -->
+    <IsStable>true</IsStable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <UseSourceBuiltSdkOverride Include="@(ArcadeSdkOverride)" />
+    <UseSourceBuiltSdkOverride Include="@(ArcadePackagingOverride)" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <EnvironmentVariables Include="RestoreSources=$(EnvironmentRestoreSources)" />
+    <EnvironmentVariables Include="BuildInParallel=false" />
+  </ItemGroup>
+
+</Project>

repos/core-setup.common.targets

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))/dir.targets" />
+
+  <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="PublishCoreSetupBinaries" />
+
+  <!-- Need to "publish" the tarballs for cli, until cli respects the assets blob -->
+  <Target Name="PublishBinaries" AfterTargets="Package">
+    <ItemGroup>
+      <CoreSetupTarballs Include="$(SourceBuiltAssetsDir)dotnet-runtime-*$(TarBallExtension)" />
+      <CoreSetupTarballs Include="$(SourceBuiltAssetsDir)dotnet-hostfxr-*$(TarBallExtension)" />
+    </ItemGroup>
+    <PublishCoreSetupBinaries Binaries="@(CoreSetupTarballs)"
+                              DestinationFolderTemplate="$(SourceBuiltRuntimeDir){version}/" >
+      <Output TaskParameter="PublishedVersion" PropertyName="PublishedVersion" />
+    </PublishCoreSetupBinaries>
+    <Message Text="Published core-setup binaries version $(PublishedVersion)" Importance="High" />
+  </Target>
+
+  <Target Name="CopyBinariesToBinFolder" AfterTargets="PublishBinaries">
+    <ItemGroup>
+      <_builtRuntimePackages Include="$(SourceBuiltAssetsDir)*.symbols.nupkg" />
+      <_builtRuntimePackages>
+        <TransformedFileName>$([System.String]::Copy('%(FileName)').Replace('symbols', 'nupkg'))</TransformedFileName>
+      </_builtRuntimePackages>
+      <BinariesToCopy Include="$(SourceBuiltAssetsDir)*.*" Exclude="$(SourceBuiltAssetsDir)*.nupkg;$(SourceBuiltAssetsDir)*.requires_nupkg_signing" />
+      <BinariesToCopy Include="@(_builtRuntimePackages->'$(SourceBuiltPackagesPath)%(TransformedFileName)')" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(BinariesToCopy)"
+          DestinationFolder="$(OutputPath)runtime"
+          Condition="'@(BinariesToCopy)'!=''" />
+  </Target>
+
+  <Target Name="SetOutputList" AfterTargets="RepoBuild">
+    <ItemGroup>
+      <PackagesOutputList Include="$(ShippingPackagesOutput)" />
+      <PackagesOutputList Include="$(NonShippingPackagesOutput)" />
+    </ItemGroup>
+  </Target>
+
+</Project>