Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
802d827
PGO artifacts are vertical visibility by default except for dotnet-sd…
jkoritzinsky Jan 16, 2025
f1fcb12
Remove PGO exclusion
jkoritzinsky Jan 16, 2025
59cca10
Add backport as arcade->sdk flow is blocked
jkoritzinsky Jan 21, 2025
3e9416f
Fix toolset asset visibility
jkoritzinsky Jan 21, 2025
c20182d
Add patch for NuGet.Client publishing
jkoritzinsky Jan 22, 2025
a09be6a
Fix folder path
jkoritzinsky Jan 22, 2025
5ab898c
Update patch
jkoritzinsky Jan 22, 2025
0c7dfe7
Update patch from PR
jkoritzinsky Jan 22, 2025
0beba5b
Merge branch 'main' into pgo-visibility-default
ViktorHofer Jan 23, 2025
454b4b3
Remove DevVersions exclusion from join-verticals.proj
ViktorHofer Jan 23, 2025
e4f57a9
Use Artifact extension point for the FSharp packages
jkoritzinsky Jan 23, 2025
5843805
Merge branch 'main' of https://github.com/dotnet/sdk into pgo-visibil…
jkoritzinsky Jan 23, 2025
dfa98dd
Remove outdated patch
jkoritzinsky Jan 23, 2025
98a908a
Pass ArtifactVisibilitiesToPublish into PushToBuildStorage
jkoritzinsky Jan 23, 2025
45874a9
Pass the visibility as manifest artifact data, otherwise it doesn't g…
jkoritzinsky Jan 24, 2025
dbb743e
Merge branch 'main' of https://github.com/dotnet/sdk into pgo-visibil…
jkoritzinsky Jan 24, 2025
fe24e02
Change how the SDK looks things up as it publishes blobs in non-stand…
jkoritzinsky Jan 24, 2025
9f9ecf3
SDK currently classifies the PGO assets as shipping, so follow that.
jkoritzinsky Jan 27, 2025
fa14530
PR feedback
jkoritzinsky Jan 27, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 57 additions & 40 deletions eng/Publishing.props
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,13 @@
<ManifestBuildData Include="AzureDevOpsBranch=$(BUILD_SOURCEBRANCH)" />
</ItemGroup>

<!-- The PGO sdk should always have External visibility, even if someone changes the default artifact visibility -->
<ItemGroup>
<ToolsetAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)dotnet-toolset*.zip" />
<Artifact Update="$(ArtifactsShippingPackagesDir)/dotnet-sdk-pgo-*" Visibility="External" />
</ItemGroup>

<ItemGroup>
<ToolsetAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)dotnet-toolset*.zip" Visibility="$(DefaultArtifactVisibility)" />
</ItemGroup>

<!-- Include RID specific packages when globbing for default artifacts is disabled. -->
Expand All @@ -72,19 +77,15 @@
To achieve this, we find the FSharp compiler package, then the stable or non-stable FSharp.Core and Compiler service
package contained within, depending on the stability switch of the SDK. The SDK then treats these packages as its own outputs,
which means they get automatically pushed on release day. -->
<PropertyGroup>
<PublishDependsOnTargets>$(PublishDependsOnTargets);_ResolvePublishFSharpNuGetPackages</PublishDependsOnTargets>
</PropertyGroup>

<Target Name="_ResolvePublishFSharpNuGetPackages" Condition="'$(EnableDefaultArtifacts)' == 'true'">
<Target Name="_ResolvePublishFSharpNuGetPackages" Condition="'$(EnableDefaultArtifacts)' == 'true'" BeforeTargets="BeforePublish">
<PropertyGroup>
<FSharpCorePath Condition="'$(DotNetFinalVersionKind)' != 'release'">Shipping</FSharpCorePath>
<FSharpCorePath Condition="'$(DotNetFinalVersionKind)' == 'release'">Release</FSharpCorePath>
</PropertyGroup>
<ItemGroup>
<FSharpPackagesToPush Include="$(NuGetPackageRoot)\Microsoft.FSharp.Compiler\$(MicrosoftFSharpCompilerPackageVersion)\contentFiles\$(FSharpCorePath)\FSharp.Core.*.nupkg" />
<FSharpPackagesToPush Include="$(NuGetPackageRoot)\Microsoft.FSharp.Compiler\$(MicrosoftFSharpCompilerPackageVersion)\contentFiles\$(FSharpCorePath)\FSharp.Compiler.Service.*.nupkg" />
<ItemsToPushToBlobFeed Include="@(FSharpPackagesToPush)" IsShipping="true" />
<Artifact Include="@(FSharpPackagesToPush)" PublishFlatContainer="false" />
</ItemGroup>
</Target>

Expand All @@ -94,6 +95,7 @@
<ToolsetAssetsToPushToBlobFeed Include="@(ToolsetAssetsToPublish)"
RelativeBlobPath="$(BlobStoragePartialRelativePath)/$(ToolsetVersionValue)/$([System.String]::Copy('%(Filename)%(Extension)').Replace('\' ,'/'))"
IsShipping="false" />
<ToolsetAssetsToPushToBlobFeed ManifestArtifactData="Visibility=%(Visibility)" />
</ItemGroup>

<ItemGroup Condition="'$(PostBuildSign)' == 'true'">
Expand Down Expand Up @@ -126,7 +128,8 @@
AssetsLocalStorageDir="$(SourceBuiltAssetsDir)"
ShippingPackagesLocalStorageDir="$(SourceBuiltShippingPackagesDir)"
NonShippingPackagesLocalStorageDir="$(SourceBuiltNonShippingPackagesDir)"
AssetManifestsLocalStorageDir="$(SourceBuiltAssetManifestsDir)" />
AssetManifestsLocalStorageDir="$(SourceBuiltAssetManifestsDir)"
ArtifactVisibilitiesToPublish="@(ArtifactVisibilityToPublish)" />
</Target>

<PropertyGroup>
Expand All @@ -144,37 +147,50 @@
<InstallersAssetManifestFilePath>$(ArtifactsLogDir)AssetManifest\$(InstallersAssetManifestFileName).xml</InstallersAssetManifestFilePath>
</PropertyGroup>

<ItemGroup>
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.zip" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.tar.gz" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.pkg" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.exe" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.deb" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.rpm" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.msi" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.cab" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.svg" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
<!-- Only publish this file from windows x64 so that we don't end up with duplicates -->
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)productVersion.txt"
Condition=" '$(PublishBinariesAndBadge)' == 'true' and '$(OS)' == 'Windows_NT' and '$(Architecture)' == 'x64' and '$(PgoInstrument)' != 'true'" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)sdk-productVersion.txt"
Condition=" '$(PublishBinariesAndBadge)' == 'true' and '$(OS)' == 'Windows_NT' and '$(Architecture)' == 'x64' and '$(PgoInstrument)' != 'true'" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)productCommit-*.json" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)productCommit-*.txt" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
<SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.swr" />
<SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.msi" />
<SdkNonShippingAssetsToPublish Condition="'$(PublishBinariesAndBadge)' != 'false'" Include="$(ArtifactsNonShippingPackagesDir)*.tar.gz" />
<!-- dotnet-toolset zips are published in the PublishToolsetAssets target. -->
<SdkNonShippingAssetsToPublish Condition="'$(PublishBinariesAndBadge)' != 'false'" Include="$(ArtifactsNonShippingPackagesDir)*.zip" Exclude="$(ArtifactsNonShippingPackagesDir)dotnet-toolset*.zip" />
<SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.pkg" />
<CheckSumsToPublish Include="$(ArtifactsShippingPackagesDir)*.sha512" />
<CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)productCommit-*.json.sha512" Condition=" '$(PublishBinariesAndBadge)' == 'false' " />
<CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)productCommit-*.txt.sha512" Condition=" '$(PublishBinariesAndBadge)' == 'false' " />
<CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)productVersion.txt.sha512" Condition=" '$(OS)' != 'Windows_NT' or '$(Architecture)' != 'x64' " />
<CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)sdk-productVersion.txt.sha512" Condition=" '$(OS)' != 'Windows_NT' or '$(Architecture)' != 'x64' " />
<CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)*.zip.sha512" Condition=" '$(PublishBinariesAndBadge)' == 'false' " />
<CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)*.tar.gz.sha512" Condition=" '$(PublishBinariesAndBadge)' == 'false' " />
</ItemGroup>
<!-- When doing a PGO build in the VMR, only ship the SDK's PGO archives -->
<Choose>
<When Condition="'$(PgoInstrument)' == 'true' and '$(DotNetBuildOrchestrator)' == 'true'">
<ItemGroup>
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)/dotnet-sdk-pgo-*.zip" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)/dotnet-sdk-pgo-*.tar.gz" />
<CheckSumsToPublish Include="$(ArtifactsShippingPackagesDir)/dotnet-sdk-pgo-*.zip.sha512" />
<CheckSumsToPublish Include="$(ArtifactsShippingPackagesDir)/dotnet-sdk-pgo-*.tar.gz.sha512" />
</ItemGroup>
</When>
<Otherwise>
<ItemGroup>
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.zip" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.tar.gz" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.pkg" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.exe" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.deb" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.rpm" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.msi" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.cab" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.svg" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
<!-- Only publish this file from windows x64 so that we don't end up with duplicates -->
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)productVersion.txt"
Condition=" '$(PublishBinariesAndBadge)' == 'true' and '$(OS)' == 'Windows_NT' and '$(Architecture)' == 'x64' and '$(PgoInstrument)' != 'true'" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)sdk-productVersion.txt"
Condition=" '$(PublishBinariesAndBadge)' == 'true' and '$(OS)' == 'Windows_NT' and '$(Architecture)' == 'x64' and '$(PgoInstrument)' != 'true'" />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)productCommit-*.json" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
<SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)productCommit-*.txt" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
<SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.swr" />
<SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.msi" />
<SdkNonShippingAssetsToPublish Condition="'$(PublishBinariesAndBadge)' != 'false'" Include="$(ArtifactsNonShippingPackagesDir)*.tar.gz" />
<!-- dotnet-toolset zips are published in the PublishToolsetAssets target. -->
<SdkNonShippingAssetsToPublish Condition="'$(PublishBinariesAndBadge)' != 'false'" Include="$(ArtifactsNonShippingPackagesDir)*.zip" Exclude="$(ArtifactsNonShippingPackagesDir)dotnet-toolset*.zip" />
<SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.pkg" />
<CheckSumsToPublish Include="$(ArtifactsShippingPackagesDir)*.sha512" />
<CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)productCommit-*.json.sha512" Condition=" '$(PublishBinariesAndBadge)' == 'false' " />
<CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)productCommit-*.txt.sha512" Condition=" '$(PublishBinariesAndBadge)' == 'false' " />
<CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)productVersion.txt.sha512" Condition=" '$(OS)' != 'Windows_NT' or '$(Architecture)' != 'x64' " />
<CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)sdk-productVersion.txt.sha512" Condition=" '$(OS)' != 'Windows_NT' or '$(Architecture)' != 'x64' " />
<CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)*.zip.sha512" Condition=" '$(PublishBinariesAndBadge)' == 'false' " />
<CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)*.tar.gz.sha512" Condition=" '$(PublishBinariesAndBadge)' == 'false' " />
</ItemGroup>
</Otherwise>
</Choose>

<!-- Publish the sdk and the checksums only in the initial build pass. -->
<Target Name="PublishSdkAssetsAndChecksums"
Expand Down Expand Up @@ -238,7 +254,8 @@
AssetsLocalStorageDir="$(SourceBuiltAssetsDir)"
ShippingPackagesLocalStorageDir="$(SourceBuiltShippingPackagesDir)"
NonShippingPackagesLocalStorageDir="$(SourceBuiltNonShippingPackagesDir)"
AssetManifestsLocalStorageDir="$(SourceBuiltAssetManifestsDir)" />
AssetManifestsLocalStorageDir="$(SourceBuiltAssetManifestsDir)"
ArtifactVisibilitiesToPublish="@(ArtifactVisibilityToPublish)" />
</Target>

</Project>
2 changes: 1 addition & 1 deletion src/SourceBuild/content/eng/join-verticals.proj
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
<ItemGroup>
<VerticalManifest
Include="$(VerticalManifestsPath)\*.xml"
Exclude="$(VerticalManifestsPath)\*Pgo*.xml;$(VerticalManifestsPath)\*Shortstack*.xml;$(VerticalManifestsPath)\*DevVersions*.xml" />
Exclude="$(VerticalManifestsPath)\*Shortstack*.xml" />
</ItemGroup>

<!-- AzureDevOpsToken shouldn't be set when running in dnceng-public -->
Expand Down
2 changes: 2 additions & 0 deletions src/SourceBuild/content/repo-projects/Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@
<BuildArgs>$(BuildArgs) /p:SourceBuiltAssetManifestsDir=$(RepoAssetManifestsDir)</BuildArgs>
<BuildArgs Condition="'$(OfficialBuildId)' != ''">$(BuildArgs) /p:OfficialBuildId=$(OfficialBuildId)</BuildArgs>
<BuildArgs Condition="'$(ForceDryRunSigning)' != ''">$(BuildArgs) /p:ForceDryRunSigning=$(ForceDryRunSigning)</BuildArgs>
<!-- PGO assets by default are "Vertical" visibilty. Each repo will enable the specific artifacts it must publish externally -->
<BuildArgs Condition="'$(PgoInstrument)' == 'true'">$(BuildArgs) /p:DefaultArtifactVisibility=Vertical</BuildArgs>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ViktorHofer @jkoritzinsky Is your thinking that default artifact visibility is something that the VMR will be responsible for passing, rather than runtime setting this under certain conditions?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the default visibility should be a property of the VMR for PGO as we need to tell every repo (not just runtime) that its assets are Vertical-only.

For other scenarios (like filtering out RID-agnostic packages if we wanted to get rid of the concept of a main vertical entirely) I could see runtime controlling the default rules itself.

</PropertyGroup>

<PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
From 62c4b5f114850d39efdca3df3b9ee83cc0f424ff Mon Sep 17 00:00:00 2001
From: Jeremy Koritzinsky <jekoritz@microsoft.com>
Date: Tue, 21 Jan 2025 16:16:49 -0800
Subject: [PATCH 1/2] Use the documented Artifact extension point to add
artifacts

The Artifact extension point handles symbol packages automatically for source-build, and it automatically gets support for new features like Artifact Visibility (required for https://github.com/dotnet/sdk/pull/46063)
Backport: https://github.com/NuGet/NuGet.Client/pull/6233
---
eng/Publishing.props | 11 +++--------
1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/eng/Publishing.props b/eng/Publishing.props
index 38810c73428..1175200dd59 100644
--- a/eng/Publishing.props
+++ b/eng/Publishing.props
@@ -1,11 +1,6 @@
<Project>
-
<ItemGroup>
- <ItemsToPushToBlobFeed Include="$(ArtifactsDir)nupkgs/*.nupkg"
- IsShipping="true"
- UploadPathSegment="nuget-client" />
- <_SymbolsPackages Include="$(ArtifactsDir)nupkgs/*.symbols.nupkg" />
- <ItemsToPushToBlobFeed Remove="@(_SymbolsPackages)" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+ <Artifact Include="$(ArtifactsDir)nupkgs/*.nupkg"
+ IsShipping="true" />
</ItemGroup>
-
-</Project>
\ No newline at end of file
+</Project>

From 9072cb03cb5c2958df318f59389547caa5de2c8a Mon Sep 17 00:00:00 2001
From: Jeremy Koritzinsky <jkoritzinsky@gmail.com>
Date: Wed, 22 Jan 2025 10:07:47 -0800
Subject: [PATCH 2/2] Formatting and PR feedback

---
eng/Publishing.props | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/eng/Publishing.props b/eng/Publishing.props
index 1175200dd59..41f498d4d4a 100644
--- a/eng/Publishing.props
+++ b/eng/Publishing.props
@@ -1,6 +1,8 @@
-<Project>
- <ItemGroup>
- <Artifact Include="$(ArtifactsDir)nupkgs/*.nupkg"
- IsShipping="true" />
- </ItemGroup>
-</Project>
+<Project>
+
+ <ItemGroup>
+ <Artifact Include="$(ArtifactsDir)nupkgs/*.nupkg"
+ PublishFlatContainer="false" />
+ </ItemGroup>
+
+</Project>
Loading