Skip to content

Commit a8fc576

Browse files
jkoritzinskyViktorHofer
jkoritzinsky
and
ViktorHofer
authored
Use asset manifests exclusively to produce/consume previously source built artifacts (#40745)
Co-authored-by: Viktor Hofer <viktor.hofer@microsoft.com>
1 parent 31f0b65 commit a8fc576

File tree

6 files changed

+85
-67
lines changed

6 files changed

+85
-67
lines changed

‎src/SourceBuild/content/eng/bootstrap/buildBootstrapPreviouslySB.csproj

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,16 @@
2626
<UnixRid Include="osx-arm64" />
2727
</ItemGroup>
2828

29-
<!-- These packages will be replaced with ms-built packages downloaded from official package feeds-->
29+
<!--
30+
These packages will be replaced with ms-built packages downloaded from official package feeds.
31+
The runtime packs do not have a RIDless package, so we'll use the version of their corresponding ref pack
32+
to determine the version.
33+
-->
3034
<ItemGroup>
31-
<RuntimePack Include="Microsoft.Aspnetcore.App.Runtime" Version="[$(MicrosoftAspNetCoreAppRuntimeVersion)]" />
32-
<RuntimePack Include="Microsoft.NETCore.App.Crossgen2" Version="[$(MicrosoftNETCoreAppCrossgen2Version)]" />
33-
<RuntimePack Include="Microsoft.NETCore.App.Host" Version="[$(MicrosoftNETCoreAppHostPackageVersion)]" />
34-
<RuntimePack Include="Microsoft.NETCore.App.Runtime" Version="[$(MicrosoftNETCoreAppRuntimeVersion)]" />
35+
<RuntimePack Include="Microsoft.AspNetCore.App.Runtime" Version="[$(MicrosoftAspNetCoreAppRefPackageVersion)]" />
36+
<RuntimePack Include="Microsoft.NETCore.App.Crossgen2" Version="[$(MicrosoftNETCoreAppRefPackageVersion)]" />
37+
<RuntimePack Include="Microsoft.NETCore.App.Host" Version="[$(MicrosoftNETCoreAppRefPackageVersion)]" />
38+
<RuntimePack Include="Microsoft.NETCore.App.Runtime" Version="[$(MicrosoftNETCoreAppRefPackageVersion)]" />
3539

3640
<PortablePackage Include="Microsoft.DotNet.ILCompiler" Version="[$(MicrosoftDotNetILCompilerVersion)]" />
3741
<PortablePackage Include="Microsoft.NETCore.DotNetAppHost" Version="[$(MicrosoftNETCoreDotNetAppHostVersion)]" />
@@ -63,7 +67,7 @@
6367
<ItemGroup>
6468
<!--
6569
Generate package names for runtime packs by concatenating the base name with the Unix RID
66-
(e.g. Microsoft.Aspnetcore.App.Runtime.linux-x64)
70+
(e.g. Microsoft.AspNetCore.App.Runtime.linux-x64)
6771
-->
6872
<PackageWithName Include="@(RuntimePackWithUnixRid)">
6973
<PackageName>%(RuntimePackWithUnixRid.Identity).%(RuntimePackWithUnixRid.UnixRid)</PackageName>
@@ -153,7 +157,7 @@
153157

154158
<!-- Repack tarball with new bootstrap name -->
155159
<Message Text=" Repacking tarball to $(NewTarballName)" Importance="High" />
156-
<Exec Command="tar --numeric-owner -czf $(NewTarballName) *.nupkg *.props SourceBuildReferencePackages/" WorkingDirectory="$(UnpackedTarPath)" />
160+
<Exec Command="tar --numeric-owner -czf $(NewTarballName) *.nupkg *.props *.xml SourceBuildReferencePackages/" WorkingDirectory="$(UnpackedTarPath)" />
157161

158162
</Target>
159163

‎src/SourceBuild/content/eng/finish-source-only.proj

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -196,11 +196,19 @@
196196
</PropertyGroup>
197197
</Target>
198198

199+
<!-- Discover the produced packages from the merged asset manifest -->
200+
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.GetKnownPackagesFromAssetManifests" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
201+
<Target Name="GetProducedPackages" Returns="@(ProducedPackage)">
202+
<GetKnownPackagesFromAssetManifests AssetManifests="$(MergedAssetManifestOutputPath)">
203+
<Output TaskParameter="KnownPackages" ItemName="ProducedPackage" />
204+
</GetKnownPackagesFromAssetManifests>
205+
</Target>
206+
199207
<!-- Create the SourceBuilt.Private.Artifacts archive when building source-only. -->
200208
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.WritePackageVersionsProps" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
201209
<Target Name="CreatePrivateSourceBuiltArtifactsArchive"
202210
AfterTargets="Build"
203-
DependsOnTargets="GetInputsOutputForCreatePrivateSourceBuiltArtifactsArchive"
211+
DependsOnTargets="GetInputsOutputForCreatePrivateSourceBuiltArtifactsArchive;GetProducedPackages"
204212
Inputs="@(ArtifactsPackageToBundle);@(ReferencePackageToBundle);@(MergedAssetManifest)"
205213
Outputs="$(SourceBuiltTarballName);$(SourceBuiltVersionName);$(AllPackageVersionsPropsName);$(SourceBuiltMergedAssetManifestName)">
206214
<!-- Copy packages to layout directory. Since there are a large number of files,
@@ -222,20 +230,12 @@
222230
Overwrite="true" />
223231

224232
<!-- Copy the merged asset manifest into the tarball -->
225-
<Copy SourceFiles="$(MergedAssetManifestOutputPath)"
233+
<Copy SourceFiles="@(MergedAssetManifest)"
226234
DestinationFolder="$(SourceBuiltLayoutDir)"
227235
UseSymbolicLinksIfPossible="true" />
228236

229-
<!-- non-rid-specific versions of RID-specific version variables to use for bootstrapping -->
230-
<ItemGroup>
231-
<ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppRuntimeVersion" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />
232-
<ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppHostPackageVersion" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />
233-
<ExtraPackageVersionPropsPackageInfo Include="MicrosoftAspNetCoreAppRuntimeVersion" Version="%24(MicrosoftAspNetCoreAppRefPackageVersion)" />
234-
<ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppCrossgen2Version" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />
235-
</ItemGroup>
236-
237237
<!-- Create a PackageVersions.props file that includes entries for all packages. -->
238-
<WritePackageVersionsProps NuGetPackages="@(ArtifactsPackageToBundle)"
238+
<WritePackageVersionsProps KnownPackages="@(ProducedPackage)"
239239
ExtraProperties="@(ExtraPackageVersionPropsPackageInfo)"
240240
VersionPropsFlowType="AllPackages"
241241
OutputPath="$(AllPackageVersionsPropsName)" />

‎src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/GetKnownPackagesFromAssetManifests.cs

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System.Collections.Generic;
6+
using System.Linq;
7+
using System.Xml.Linq;
8+
using Microsoft.Build.Framework;
9+
using Microsoft.Build.Utilities;
10+
11+
namespace Microsoft.DotNet.UnifiedBuild.Tasks
12+
{
13+
/// <summary>
14+
/// Get a list of MSBuild Items that represent the packages described in the asset manifests.
15+
/// </summary>
16+
public sealed class GetKnownPackagesFromAssetManifests : Task
17+
{
18+
[Required]
19+
public ITaskItem[] AssetManifests { get; set; }
20+
21+
[Output]
22+
public ITaskItem[] KnownPackages { get; set; }
23+
24+
public override bool Execute()
25+
{
26+
var knownPackages = from assetManifest in AssetManifests
27+
let doc = XDocument.Load(assetManifest.ItemSpec)
28+
from package in doc.Root.Descendants("Package")
29+
select new TaskItem(package.Attribute("Id").Value, new Dictionary<string, string>{ { "Version", package.Attribute("Version").Value } });
30+
KnownPackages = knownPackages.ToArray();
31+
return true;
32+
}
33+
}
34+
}

‎src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/WritePackageVersionsProps.cs

Lines changed: 2 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@
44

55
using Microsoft.Build.Framework;
66
using Microsoft.Build.Utilities;
7-
using NuGet.Packaging;
8-
using NuGet.Packaging.Core;
97
using NuGet.Versioning;
108
using System;
119
using System.Collections.Generic;
@@ -49,17 +47,13 @@ public class WritePackageVersionsProps : Microsoft.Build.Utilities.Task
4947
private const string DependenciesOnlyVersionPropsFlowType = "DependenciesOnly";
5048
private const string DefaultVersionPropsFlowType = AllPackagesVersionPropsFlowType;
5149

52-
/// <summary>
53-
/// Set of input nuget package files to generate version properties for.
54-
/// </summary>
55-
public ITaskItem[] NuGetPackages { get; set; }
56-
5750
/// <summary>
5851
/// Set of packages built by dependencies of this repo during this build.
5952
///
6053
/// %(Identity): Package identity.
6154
/// %(Version): Package version.
6255
/// </summary>
56+
[Required]
6357
public ITaskItem[] KnownPackages { get; set; }
6458

6559
/// <summary>
@@ -176,25 +170,10 @@ public override bool Execute()
176170
return !Log.HasLoggedErrors;
177171
}
178172

179-
NuGetPackages ??= Array.Empty<ITaskItem>();
180173
KnownPackages ??= Array.Empty<ITaskItem>();
181174

182175
// First, obtain version information from the packages and additional assets that
183176
// are provided.
184-
var latestPackages = NuGetPackages
185-
.Select(item =>
186-
{
187-
using (var reader = new PackageArchiveReader(item.GetMetadata("FullPath")))
188-
{
189-
return reader.GetIdentity();
190-
}
191-
})
192-
.Select(identity => new VersionEntry()
193-
{
194-
Name = identity.Id,
195-
Version = identity.Version
196-
});
197-
198177
var knownPackages = KnownPackages
199178
.Select(item => new VersionEntry()
200179
{
@@ -205,7 +184,7 @@ public override bool Execute()
205184
// We may have multiple versions of the same package. We'll keep the latest one.
206185
// This can even happen in the KnownPackages list, as a repo (such as source-build-reference-packages)
207186
// may have multiple versions of the same package.
208-
IEnumerable<VersionEntry> packageElementsToWrite = latestPackages.Concat(knownPackages)
187+
IEnumerable<VersionEntry> packageElementsToWrite = knownPackages
209188
.GroupBy(identity => identity.Name)
210189
.Select(g => g.OrderByDescending(id => id.Version).First())
211190
.OrderBy(id => id.Name);

‎src/SourceBuild/content/prep-source-build.sh

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -231,10 +231,16 @@ if [ "$removeBinaries" == true ]; then
231231
echo " Unpacking Private.SourceBuilt.Artifacts.*.tar.gz into $packagesDir"
232232
mkdir -p "$packagesDir"
233233
tar -xzf "$sourceBuiltArchive" -C "$packagesDir"
234-
elif [ ! -f "$packagesDir/PackageVersions.props" ] && [ -f "$sourceBuiltArchive" ]; then
235-
echo " Creating $packagesDir/PackageVersions.props..."
236-
tar -xzf "$sourceBuiltArchive" -C "$packagesDir" PackageVersions.props
237-
elif [ ! -f "$sourceBuiltArchive" ]; then
234+
elif [ -f "$sourceBuiltArchive" ]; then
235+
if [ ! -f "$packagesDir/PackageVersions.props" ]; then
236+
echo " Creating $packagesDir/PackageVersions.props..."
237+
tar -xzf "$sourceBuiltArchive" -C "$packagesDir" PackageVersions.props
238+
fi
239+
if [ ! -f "$packagesDir/VerticalManifest.xml" ]; then
240+
echo " Unpacking Asset manifests into $packagesDir..."
241+
tar -xzf "$sourceBuiltArchive" -C "$packagesDir" VerticalManifest.xml
242+
fi
243+
else
238244
echo " ERROR: Private.SourceBuilt.Artifacts.*.tar.gz does not exist..."\
239245
"Cannot remove non-SB allowed binaries. Either pass --with-packages or download the artifacts."
240246
exit 1

‎src/SourceBuild/content/repo-projects/Directory.Build.targets

Lines changed: 17 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@
105105
<ExtraSourcesNuGetSourceName>ExtraSources</ExtraSourcesNuGetSourceName>
106106
<SbrpRepoSrcPath>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'source-build-reference-packages', 'src'))</SbrpRepoSrcPath>
107107
</PropertyGroup>
108-
108+
109109
<PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
110110
<PrebuiltNuGetSourceName>prebuilt</PrebuiltNuGetSourceName>
111111
<PreviouslySourceBuiltNuGetSourceName>previously-source-built</PreviouslySourceBuiltNuGetSourceName>
@@ -116,7 +116,7 @@
116116
<_CommonBuildSources Include="@(DependentRepoSourceName)" />
117117
<_CommonBuildSources Include="$(ExtraSourcesNuGetSourceName)" Condition="'$(ExtraRestoreSourcePath)' != ''" />
118118
</ItemGroup>
119-
119+
120120
<PropertyGroup Condition="'$(DotNetBuildSourceOnly)' != 'true'">
121121
<!-- When not building source-only, repositories lagging behind on the .NET SDK (e.g. tooling repos)
122122
that need to utilize a VS-aligned version for development purposes may not have the latest product
@@ -250,14 +250,21 @@
250250
<!-- Before a repository builds, set up the version property files that override the repo's defaults.
251251
There are 3 files generated -->
252252
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.WritePackageVersionsProps" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
253+
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.GetKnownPackagesFromAssetManifests" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
253254
<Target Name="CreateBuildInputProps"
254255
DependsOnTargets="GetProducedPackagesFromTransitiveReferences"
255256
Inputs="$(MSBuildProjectFullPath)"
256257
Outputs="$(BaseIntermediateOutputPath)CreateBuildInputProps.complete">
258+
259+
<!-- Get the previously-built-source-built package information from the manifest from that build. -->
257260
<ItemGroup>
258-
<_PreviouslyBuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" />
261+
<_PrebuiltSourceBuiltAssetManifests Include="$(PrebuiltSourceBuiltPackagesPath)VerticalManifest.xml" />
259262
</ItemGroup>
260263

264+
<GetKnownPackagesFromAssetManifests AssetManifests="@(_PrebuiltSourceBuiltAssetManifests)" Condition="Exists(@(_PrebuiltSourceBuiltAssetManifests))">
265+
<Output TaskParameter="KnownPackages" ItemName="_PreviouslyBuiltSourceBuiltPackages" />
266+
</GetKnownPackagesFromAssetManifests>
267+
261268
<Error Condition="'$(PackageVersionPropsFlowType)' != 'AllPackages' and '$(PackageVersionPropsFlowType)' != 'DependenciesOnly'"
262269
Text="Invalid PackageVersionPropsFlowType '$(PackageVersionPropsFlowType)'. Must be 'AllPackages' or 'DependenciesOnly'." />
263270

@@ -273,7 +280,7 @@
273280
OutputPath="$(CurrentSourceBuiltPackageVersionPropsPath)" />
274281

275282
<!-- Create previously source-built inputs info -->
276-
<WritePackageVersionsProps NuGetPackages="@(_PreviouslyBuiltSourceBuiltPackages)"
283+
<WritePackageVersionsProps KnownPackages="@(_PreviouslyBuiltSourceBuiltPackages)"
277284
VersionPropsFlowType="$(PackageVersionPropsFlowType)"
278285
VersionDetails="$(_VersionDetailsXml)"
279286
OutputPath="$(PreviouslySourceBuiltPackageVersionPropsPath)" />
@@ -465,28 +472,16 @@
465472
<Message Importance="High" Text=" -> %(RepoManifestArtifact.Identity)" />
466473
</Target>
467474

468-
<!-- Discover the produced packages from all repo asset manifests for this repository. -->
469-
<Target Name="DiscoverProducedPackages" DependsOnTargets="GetRepoAssetManifests" Inputs="@(RepoAssetManifest)" Outputs="%(Identity).ForBatching">
470-
<XmlPeek XmlInputPath="%(RepoAssetManifest.Identity)" Query="/Build/*[self::Package]">
471-
<Output TaskParameter="Result" ItemName="ProducedPackageEntry" />
472-
</XmlPeek>
473-
</Target>
474-
475-
<Target Name="ProcessPackageEntries" DependsOnTargets="DiscoverProducedPackages" Inputs="@(ProducedPackageEntry)" Outputs="%(Identity).ForBatching">
476-
<XmlPeek XmlContent="%(ProducedPackageEntry.Identity)" Query="/Package/@Id">
477-
<Output TaskParameter="Result" PropertyName="ProducedPackageId" />
478-
</XmlPeek>
479-
<XmlPeek XmlContent="%(ProducedPackageEntry.Identity)" Query="/Package/@Version">
480-
<Output TaskParameter="Result" PropertyName="ProducedPackageVersion" />
481-
</XmlPeek>
482-
475+
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.GetKnownPackagesFromAssetManifests" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
476+
<Target Name="GetProducedPackages" DependsOnTargets="GetRepoAssetManifests" Returns="@(ProducedPackage)">
477+
<GetKnownPackagesFromAssetManifests AssetManifests="@(RepoAssetManifest)">
478+
<Output TaskParameter="KnownPackages" ItemName="ProducedPackage" />
479+
</GetKnownPackagesFromAssetManifests>
483480
<ItemGroup>
484-
<ProducedPackage Include="$(ProducedPackageId)" Version="$(ProducedPackageVersion)" ReferenceOnly="$([MSBuild]::ValueOrDefault('$(ReferenceOnlyRepoArtifacts)', 'false'))" />
481+
<ProducedPackage ReferenceOnly="$([MSBuild]::ValueOrDefault('$(ReferenceOnlyRepoArtifacts)', 'false'))" />
485482
</ItemGroup>
486483
</Target>
487484

488-
<Target Name="GetProducedPackages" DependsOnTargets="GetRepoAssetManifests;DiscoverProducedPackages;ProcessPackageEntries" Returns="@(ProducedPackage)" />
489-
490485
<Target Name="GetProducedPackagesFromTransitiveReferences" DependsOnTargets="GetTransitiveRepositoryReferences" Returns="@(_DependencyProducedPackage)">
491486
<MSBuild Projects="@(TransitiveRepositoryReference->'%(Identity).proj')"
492487
Targets="GetProducedPackages"

0 commit comments

Comments
 (0)