Update Arcade SDK publishing

NuGet.config

@@ -5,5 +5,6 @@
     <add key="arcade" value="https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json" />
     <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+    <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
   </packageSources>
 </configuration>

azure-pipelines/builds/ci.yml

@@ -1,12 +1,29 @@
-jobs:
-- template: ../jobs/ci-linux.yml
-  parameters:
-    job: fedora29
-    imageName: microsoft/dotnet-buildtools-prereqs:fedora-29-2f0798a-20181105183801
+variables:
+- name: _DotNetArtifactsCategory
+  value: .NETCore
 
+stages:
+- stage: build
+  displayName: Build
+  jobs:
+  - template: ../jobs/ci-linux.yml
+    parameters:
+      job: fedora29
+      imageName: microsoft/dotnet-buildtools-prereqs:fedora-29-2f0798a-20181105183801
+
+  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - template: ../../eng/common/templates/job/publish-build-assets.yml
+      parameters:
+        dependsOn: fedora29
+        publishUsingPipelines: true
+        pool:
+          vmImage: vs2017-win2016
+
+# Based on - https://github.com/dotnet/arcade/blob/9b3f304c7bc9fd4d11be9ca0b466b83e98d2a191/Documentation/CorePackages/Publishing.md#moving-away-from-the-legacy-pushtoblobfeed-task
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-  - template: ../../eng/common/templates/job/publish-build-assets.yml
+  - template: ../../eng/common/templates/post-build/post-build.yml
     parameters:
-      dependsOn: fedora29
-      pool:
-        vmImage: vs2017-win2016
+      enableSourceLinkValidation: true
+      #Nuget and Signing Validation are not needed as we use an empty placeholder nupkg which is never utilized
+      enableSigningValidation: false
+      enableNugetValidation: false

azure-pipelines/jobs/ci-linux.yml

@@ -20,6 +20,7 @@ jobs:
     imageName: ${{ parameters.imageName }}
     rootDirectory: $(Build.SourcesDirectory)/..
     stagingDirectory: $(rootDirectory)/source-build-reference-packages/staging
+    _PublishUsingPipelines: true
 
   steps:
   - template: ../steps/docker-cleanup-linux.yml
@@ -56,16 +57,28 @@ jobs:
             in(variables['Build.Reason'], 'PullRequest')
         )"
     continueOnError: true
+  # Below step's sourced from : https://github.com/dotnet/runtime/blob/4e72737a8ec246297bce916ad747e4a2fb1265ac/eng/pipelines/installer/jobs/base-job.yml#L455-L468
+  # Files may be owned by root because builds don't set user ID. Later build steps run 'find' in
+  # the source tree, which fails due to permissions in the 'NetCore*-Int-Pool' queues. This step
+  # prevents the failure by using chown to clean up our source tree.
+  - script: |
+      set -x
+      docker run --rm \
+        -v "$(Agent.BuildDirectory):/root/build" \
+        -w /root/build \
+        bash -c "chown -R $(id -u):$(id -g) *"
+    displayName: Update file ownership from root to build agent account
+    continueOnError: true
+    condition: succeededOrFailed()
 
   # Create reference packages tarball and upload to azure storage
   - script: |
       set -x
       df -h
-      $(docker.run) $(docker.src.map) $(docker.src.work) $(imageName) ./publish.sh $(BUILD.BUILDNUMBER) --ci \
-        /p:AzureAccountName=$(publish.blobStorage.account) \
-        /p:ContainerName=$(publish.blobStorage.container) \
-        /p:AzureAccessToken=$(publish.blobStorage.accessToken) \
-        /p:ManifestName=$(BUILD.REPOSITORY.NAME) \
+      cd "$(rootDirectory)/source-build-reference-packages/src"
+      ./publish.sh $(BUILD.BUILDNUMBER) --ci \
+        /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) \
+        /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory) \
         /p:ManifestBuildId=$(BUILD.BUILDID) \
         /p:ManifestBranch=$(BUILD.SOURCEBRANCHNAME) \
         /p:ManifestCommit=$(BUILD.SOURCEVERSION)
@@ -114,22 +127,7 @@ jobs:
     continueOnError: true
     inputs:
       PathtoPublish: $(Build.ArtifactStagingDirectory)
-      ArtifactName: Artifacts
-      ArtifactType: Container
-
-  # Publish manifest.
-  - task: PublishBuildArtifacts@1
-    displayName: Publish Manifest
-    condition: "
-        and(
-            succeeded(),
-            ne(variables['System.TeamProject'], 'public'),
-            notin(variables['Build.Reason'], 'PullRequest')
-        )"
-    continueOnError: true
-    inputs:
-      PathtoPublish: $(Build.ArtifactStagingDirectory)/asset-manifests/
-      ArtifactName: AssetManifests
+      ArtifactName: All_Artifacts
       ArtifactType: Container
 
   # Clean up working directory.

src/emptyPackageProject/Private.SourceBuild.ReferencePackages.csproj

@@ -11,10 +11,14 @@
     <RepositoryCommit>0000</RepositoryCommit>
     <RepositoryCommit Condition="'$(ManifestCommit)' != ''">$(ManifestCommit)</RepositoryCommit>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
+    <PackageIcon>icon.png</PackageIcon>
     <TargetFramework>netcoreapp2.1</TargetFramework>
     <IsPackable>true</IsPackable>
     <IsShipping>false</IsShipping>
   </PropertyGroup>
 
-</Project>
+  <ItemGroup>
+    <None Include="icon.png" Pack="true" PackagePath="" />
+  </ItemGroup>
 
+</Project>

src/publishPackages/publish.proj

@@ -2,11 +2,13 @@
 <Project ToolsVersion="15.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$(RepoRoot)eng/Directory.Build.props" />
   <Import Project="$(RepoRoot)src/dir.props" />
+  <Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
 
   <PropertyGroup>
     <ArtifactsPkgDir>$(RepoRoot)artifacts/packages/</ArtifactsPkgDir>
     <ManifestDir>$(RepoRoot)artifacts/asset-manifests/</ManifestDir>
     <TarballDir>$(RepoRoot)artifacts/tarball/</TarballDir>
+    <ManifestName>source-build-reference-packages</ManifestName>
     <RestoreSources>
       $(RestoreSources);
       https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json;
@@ -18,11 +20,11 @@
     <BuildDependsOn>
       $(CreateTargets)
     </BuildDependsOn>
-    <BuildDependsOn Condition="'$(AzureAccessToken)' != ''">
+    <BuildDependsOn>
       $(BuildDependsOn)
       PreparePublish;
-      PublishPackagesToBlobFeed;
-      PublishFilesToBlobFeed;
+      PreparePublishPackages;
+      PreparePublishFiles;
     </BuildDependsOn>
   </PropertyGroup>
 
@@ -32,16 +34,17 @@
 
   <Target Name="GetTarballFileName">
     <PropertyGroup>
-      <TarballFileName>$(TarballDir)Private.SourceBuild.ReferencePackages.$(VersionPrefix)-$(VersionSuffix).tar.gz</TarballFileName>
+      <TarballFileName>Private.SourceBuild.ReferencePackages.$(VersionPrefix)-$(VersionSuffix).tar.gz</TarballFileName>
+      <TarballFilePath>$(TarballDir)$(TarballFileName)</TarballFilePath>
     </PropertyGroup>
   </Target>
 
   <Target Name="CreateTarball" DependsOnTargets="GetTarballFileName" >
     <MakeDir Directories="$(TarballDir)" />
 
-    <Exec Command="tar --numeric-owner -czf $(TarballFileName) *.nupkg" WorkingDirectory="$(ArtifactsRefPkgDir)" />
+    <Exec Command="tar --numeric-owner -czf $(TarballFilePath) *.nupkg" WorkingDirectory="$(ArtifactsRefPkgDir)" />
 
-    <Message Importance="High" Text="Created $(TarballFileName)" />
+    <Message Importance="High" Text="Created $(TarballFilePath)" />
   </Target>
 
   <Target Name="CreateEmptyPackage">
@@ -53,33 +56,102 @@
 
   <Target Name="PreparePublish" DependsOnTargets="$(CreateDependsOn)">
     <MakeDir Directories="$(ManifestDir)" />
+    <MakeDir Directories="$(ArtifactsTempWorkDir)" />
+    <!-- Include other build data attributes : (Src) https://github.com/dotnet/arcade/blob/master/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj#L167 -->
     <PropertyGroup>
-      <ExpectedFeedUrl>https://$(AzureAccountName).blob.core.windows.net/$(ContainerName)/index.json</ExpectedFeedUrl>
-      <AccountKey>$(AzureAccessToken)</AccountKey>
+      <CollectionUri>$(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI)</CollectionUri>
+
+			<!-- When we have dev.azure.com/<account>/ -->
+      <AzureDevOpsAccount Condition="$(CollectionUri.IndexOf('dev.azure.com')) >= 0">$(CollectionUri.Split('/')[3])</AzureDevOpsAccount>
+
     </PropertyGroup>
+    <ItemGroup>
+      <NewManifestBuildData Include="InitialAssetsLocation=$(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI)$(SYSTEM_TEAMPROJECT)/_apis/build/builds/$(BUILD_BUILDID)/artifacts" />
+      <NewManifestBuildData Include="AzureDevOpsBuildId=$(BUILD_BUILDID)" />
+      <NewManifestBuildData Include="AzureDevOpsBuildDefinitionId=$(SYSTEM_DEFINITIONID)" />
+      <NewManifestBuildData Include="AzureDevOpsAccount=$(AzureDevOpsAccount)" />
+      <NewManifestBuildData Include="AzureDevOpsProject=$(SYSTEM_TEAMPROJECT)" />
+      <NewManifestBuildData Include="AzureDevOpsBuildNumber=$(BUILD_BUILDNUMBER)" />
+      <NewManifestBuildData Include="AzureDevOpsRepository=$(BUILD_REPOSITORY_URI)" />
+      <NewManifestBuildData Include="AzureDevOpsBranch=$(ManifestBranch)" />
+      <!-- 
+        Adding this just to not fail the validation when creating the manifest. 
+        TODO: Should be removed once PublishingUsingPipelines is the default.
+      -->
+      <NewManifestBuildData Include="Location=$(AzureFeedUrl)" />
+    </ItemGroup>
   </Target>
 
-  <Target Name="PreparePublishPackages" DependsOnTargets="PreparePublish" BeforeTargets="PublishPackagesToBlobFeed">
+  <Target Name="PreparePublishPackages" DependsOnTargets="PreparePublish">
     <PropertyGroup>
-      <AssetManifestFilePath>$(ManifestDir)reference-packages-nupkg-manifest.xml</AssetManifestFilePath>
+      <AssetManifestFilename>reference-packages-nupkg-manifest.xml</AssetManifestFilename>
+      <AssetManifestFilePath>$(ManifestDir)$(AssetManifestFileName)</AssetManifestFilePath>
+      <!-- Create a temporary directory to store the generated asset manifest by the task -->
+      <TempWorkingDirectory>$(ArtifactsDir)AssetsTmpDir\$([System.Guid]::NewGuid())/</TempWorkingDirectory>
     </PropertyGroup>
     <ItemGroup>
       <PackagesToPublish Include="$(ArtifactsPkgDir)**/*.nupkg" />
     </ItemGroup>
+    <MakeDir Directories="$(TempWorkingDirectory)"/>
+    <PushToAzureDevOpsArtifacts
+      ItemsToPush="@(PackagesToPublish)"
+      ManifestBuildData="@(NewManifestBuildData)"
+      ManifestRepoUri="$(ManifestName)"
+      ManifestBranch="$(ManifestBranch)"
+      ManifestBuildId="$(ManifestBuildId)"
+      ManifestCommit="$(ManifestCommit)"
+      IsStableBuild="false"
+      AssetManifestPath="$(AssetManifestFilePath)"
+      AssetsTemporaryDirectory="$(TempWorkingDirectory)" />
+
+    <!-- Copy the generated manifest to the build's artifacts -->
+    <Copy
+      SourceFiles="$(AssetManifestFilePath)"
+      DestinationFolder="$(TempWorkingDirectory)$(AssetManifestFileName)" />
+
+    <Message
+      Text="##vso[artifact.upload containerfolder=AssetManifests;artifactname=AssetManifests]$(TempWorkingDirectory)$(AssetManifestFileName)"
+      Importance="high" />
+
   </Target>
 
-  <Target Name="PreparePublishFiles" DependsOnTargets="PreparePublish;GetTarballFileName" BeforeTargets="PublishFilesToBlobFeed">
+  <Target Name="PreparePublishFiles" DependsOnTargets="PreparePublish;GetTarballFileName">
     <PropertyGroup>
-      <AssetManifestFilePath>$(ManifestDir)reference-packages-tar-manifest.xml</AssetManifestFilePath>
+      <AssetManifestFilename>reference-packages-tar-manifest.xml</AssetManifestFilename>
+      <AssetManifestFilePath>$(ManifestDir)$(AssetManifestFileName)</AssetManifestFilePath>
+      <!-- Create a temporary directory to store the generated asset manifest by the task -->
+      <TempWorkingDirectory>$(ArtifactsDir)AssetsTmpDir/$([System.Guid]::NewGuid())/</TempWorkingDirectory>
     </PropertyGroup>
     <ItemGroup>
-      <FilesToPublish Include="$(TarballFileName)" />
+      <FilesToPublish Include="$(TarballFilePath)" RelativeBlobPath="assets/$(TarballFileName)" />
     </ItemGroup>
+    <MakeDir Directories="$(TempWorkingDirectory)"/>
+    <PushToAzureDevOpsArtifacts
+      ItemsToPush="@(FilesToPublish)"
+      ManifestBuildData="@(NewManifestBuildData)"
+      ManifestRepoUri="$(ManifestName)"
+      ManifestBranch="$(ManifestBranch)"
+      ManifestBuildId="$(ManifestBuildId)"
+      ManifestCommit="$(ManifestCommit)"
+      PublishFlatContainer="true"
+      IsStableBuild="false"
+      AssetManifestPath="$(AssetManifestFilePath)"
+      AssetsTemporaryDirectory="$(TempWorkingDirectory)" />
+
+    <!-- Copy the generated manifest to the build's artifacts -->
+    <Copy
+      SourceFiles="$(AssetManifestFilePath)"
+      DestinationFolder="$(TempWorkingDirectory)$(AssetManifestFileName)" />
+
+    <Message
+      Text="##vso[artifact.upload containerfolder=AssetManifests;artifactname=AssetManifests]$(TempWorkingDirectory)$(AssetManifestFileName)"
+        Importance="high" />
+
   </Target>
 
 
 
-  <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />
+  <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed/$(MicrosoftDotNetBuildTasksFeedVersion)/build/Microsoft.DotNet.Build.Tasks.Feed.targets" />
   <Import Project="$(RepoRoot)eng/Directory.Build.targets" />
 
 </Project>