Skip to content

Commit

Permalink
Make the calculation of the build graph a task so we can pass metadat…
Browse files Browse the repository at this point in the history
…a around
  • Loading branch information
pranavkm committed Jul 12, 2017
1 parent d8a897b commit aa25401
Show file tree
Hide file tree
Showing 18 changed files with 163 additions and 304 deletions.
1 change: 1 addition & 0 deletions NuGet.Config
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<configuration>
<packageSources>
<clear />
<add key="AspNetCore" value="https://dotnet.myget.org/F/aspnetcore-ci-release/api/v3/index.json" />
<add key="NuGet" value="https://api.nuget.org/v3/index.json" />
</packageSources>
</configuration>
6 changes: 0 additions & 6 deletions Universe.sln
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.26228.0
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BuildGraph", "tools\BuildGraph\BuildGraph.csproj", "{B0621D49-4770-4552-9425-D6BD2CF0FB50}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PinVersions", "tools\PinVersions\PinVersions.csproj", "{DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{085280EC-7055-426A-BF9C-1B692B9599AB}"
Expand All @@ -18,10 +16,6 @@ Global
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{B0621D49-4770-4552-9425-D6BD2CF0FB50}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B0621D49-4770-4552-9425-D6BD2CF0FB50}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B0621D49-4770-4552-9425-D6BD2CF0FB50}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B0621D49-4770-4552-9425-D6BD2CF0FB50}.Release|Any CPU.Build.0 = Release|Any CPU
{DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{DACA9DFB-508E-45EA-A5CF-C0F5C2BA181B}.Release|Any CPU.ActiveCfg = Release|Any CPU
Expand Down
41 changes: 12 additions & 29 deletions build/RepositoryBuild.targets
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
<Project>
<Import Project="$(_BuildGraphFile)" />

<Target Name="BuildRepositories">
<Target Name="_BuildRepositories">
<ItemGroup>
<BatchedRepository Include="$(MSBuildProjectFullPath)">
<BuildGroup>%(RepositoryToBuildInOrder.Order)</BuildGroup>
<Repository>%(RepositoryToBuildInOrder.Identity)</Repository>
<AdditionalProperties>
RepositoryToBuild=%(RepositoryToBuildInOrder.Identity);
BuildRepositoryRoot=$(_CloneRepositoryRoot)%(RepositoryToBuildInOrder.Identity)\
BuildRepositoryRoot=%(RepositoryToBuildInOrder.RepositoryPath)\;
CommitHash=%(RepositoryToBuildInOrder.Commit)
</AdditionalProperties>
</BatchedRepository>
</ItemGroup>
Expand All @@ -18,26 +17,6 @@
<BatchBuilds Condition="'$(BatchBuilds)'==''">false</BatchBuilds>
</PropertyGroup>

<Message
Text="Building repositories in the following batches:"
Importance="High"
Condition="'$(BatchBuilds)'=='true'" />

<Message
Text="%(BatchedRepository.BuildGroup): @(BatchedRepository -> '%(Repository)', ', ')"
Importance="High"
Condition="'$(BatchBuilds)'=='true'" />

<Message
Text="Building repositories in the following order:"
Importance="High"
Condition="'$(BatchBuilds)'!='true'" />

<Message
Text="%(BatchedRepository.Repository)"
Importance="High"
Condition="'$(BatchBuilds)'!='true'" />

<MSBuild
Projects="@(BatchedRepository)"
BuildInParallel="$(BatchBuilds)"
Expand All @@ -48,6 +27,9 @@

<Target Name="_BuildRepository" DependsOnTargets="_PinVersions">
<PropertyGroup>
<!-- If there are duplicate properties, the properties which are defined later in the order would override the earlier ones -->
<RepositoryBuildArguments>$(RepositoryBuildArguments) /p:BuildNumber=$(BuildNumber) /p:Configuration=$(Configuration) /p:CommitHash=$(CommitHash)</RepositoryBuildArguments>

<BuildArguments>$(_RepositoryBuildTargets) $(RepositoryBuildArguments)</BuildArguments>
<RepositoryArtifactsRoot>$(BuildRepositoryRoot)artifacts</RepositoryArtifactsRoot>
<RepositoryArtifactsBuildDirectory>$(RepositoryArtifactsRoot)\build\</RepositoryArtifactsBuildDirectory>
Expand Down Expand Up @@ -83,26 +65,27 @@

<Copy
SourceFiles="@(RepositoryArtifacts)"
DestinationFolder="$(UniverseBuildDir)" />
DestinationFolder="$(BuildDir)" />

<Move
SourceFiles="@(RepositoryMSBuildArtifacts)"
DestinationFolder="$(UniverseMSBuildDir)\$(RepositoryToBuild)\%(RecursiveDir)" />
DestinationFolder="$(ArtifactsDir)msbuild\$(RepositoryToBuild)\%(RecursiveDir)" />

<Message Text="Publishing the following packages to the volatile feed: @(RepositoryNupkgs -> '%(Filename)%(Extension)', ', ')"
Condition="'$(PublishPackages)'=='true' AND '@(RepositoryNupkgs)' != ''" />

<Exec
Command="$(DotNetPath) $(PackagePublisherPath) -d $(RepositoryArtifactsBuildDirectory) -f $(NuGetPublishVolatileFeed)"
Command="$(DotNetPath) $(PackagePublisherNetCoreApp) -d $(RepositoryArtifactsBuildDirectory) -f $(NuGetPublishVolatileFeed)"
Condition="'$(PublishPackages)'=='true' AND '@(RepositoryNupkgs)' != ''" />

<Message Text="============ Done building $(RepositoryToBuild) ============" Importance="High" />
</Target>

<Target Name="_PinVersions">
<Target Name="_PinVersions" DependsOnTargets="_FindDotNetPath">

<PropertyGroup>
<PinToolBinary>$(RepositoryRoot)tools\PinVersions\bin\$(Configuration)\netcoreapp1.1\PinVersions.dll</PinToolBinary>
<PinVersionArgs>$(DotNetPath) $(PinToolBinary) --graph-specs-root &quot;$(_RestoreGraphSpecsDirectory) &quot; -s &quot;$(UniverseBuildDir) &quot; &quot;$(BuildRepositoryRoot) &quot;</PinVersionArgs>
<PinVersionArgs>$(DotNetPath) $(PinToolBinary) --graph-specs-root &quot;$(_RestoreGraphSpecsDirectory) &quot; -s &quot;$(BuildDir) &quot; &quot;$(BuildRepositoryRoot) &quot;</PinVersionArgs>
<PinVersionArgs Condition="Exists('$(_DependencyPackagesDirectory)')">$(PinVersionArgs) -s &quot;$(_DependencyPackagesDirectory) &quot;</PinVersionArgs>
</PropertyGroup>

Expand Down
80 changes: 25 additions & 55 deletions build/repo.targets
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
<Project>
<Import Project="RepositoryBuild.targets" />

<PropertyGroup>
<NuGetPublishVolatileFeed>https://dotnet.myget.org/F/aspnetcore-volatile-dev/api/v2/package</NuGetPublishVolatileFeed>

<_BuildGraphFile>$(BuildDir)BuildGraph.proj</_BuildGraphFile>
<_CloneRepositoryRoot>$(RepositoryRoot).r\</_CloneRepositoryRoot>
<_DependencyBuildDirectory>$(RepositoryRoot).deps\build\</_DependencyBuildDirectory>
<_DependencyPackagesDirectory>$(_DependencyBuildDirectory)</_DependencyPackagesDirectory>
Expand Down Expand Up @@ -33,7 +34,7 @@
<RemoveDir Directories="$(_CloneRepositoryRoot)" Condition="Exists('$(_CloneRepositoryRoot)')" />
</Target>

<Target Name="_FilterRepositories">
<Target Name="_PrepareRepositories">
<ItemGroup Condition="'$(KOREBUILD_REPOSITORY_INCLUDE)'!=''">
<_RepositoriesToInclude Include="$(KOREBUILD_REPOSITORY_INCLUDE)" />
<Repository
Expand All @@ -49,10 +50,12 @@
<Error Text="KOREBUILD_REPOSITORY_EXCLUDE AND KOREBUILD_REPOSITORY_INCLUDE are specified."
Condition="'$(KOREBUILD_REPOSITORY_INCLUDE)' != '' AND '$(KOREBUILD_REPOSITORY_EXCLUDE)' != ''" />

<Message Text="%(Repository.CloneUrl)" />
<ItemGroup>
<Repository Update="%(Identity)" RepositoryPath="$(_CloneRepositoryRoot)%(Identity)" />
</ItemGroup>
</Target>

<Target Name="CloneRepositories" DependsOnTargets="_FilterRepositories">
<Target Name="CloneRepositories" DependsOnTargets="_PrepareRepositories">
<ItemGroup>
<_CloneRepository Include="$(MSBuildProjectFullPath)">
<AdditionalProperties>
Expand All @@ -72,6 +75,7 @@
<MSBuild Projects="@(_CloneRepository)"
Targets="_CloneRepository"
BuildInParallel="$(BuildInParallel)" />

</Target>

<Target Name="_CloneRepository">
Expand Down Expand Up @@ -107,45 +111,9 @@
</Target>

<Target Name="BuildRepositories"
DependsOnTargets="_FilterRepositories;_FindDotNetPath;_GenerateRestoreGraphSpecs;_GenerateBuildGraph;_UpdateNuGetConfig;_CreateRepositoriesListWithCommits">

<PropertyGroup>
<!-- If there are duplicate properties, the properties which are defined later in the order would override the earlier ones -->
<RepositoryBuildArguments>$(RepositoryBuildArguments) /p:BuildNumber=$(BuildNumber) /p:Configuration=$(Configuration)</RepositoryBuildArguments>

<_BuildRepositoryProperties>
UniverseBuildDir=$(BuildDir);
UniverseMSBuildDir=$(ArtifactsDir)msbuild;
BuildInParallel=$(BuildInParallel);
Configuration=$(Configuration);
DotNetPath=$(DotNetPath);
KoreBuildDirectory=$(MSBuildProjectDirectory)\;
KoreBuildProject=$(MSBuildProjectFile);
RepositoryRoot=$(RepositoryRoot);
_BuildGraphFile=$(_BuildGraphFile);
_CloneRepositoryRoot=$(_CloneRepositoryRoot);
_DependencyPackagesDirectory=$(_DependencyPackagesDirectory);
_RepositoryBuildTargets=$(_RepositoryBuildTargets);
RepositoryBuildArguments=$(RepositoryBuildArguments);
_RestoreGraphSpecsDirectory=$(_RestoreGraphSpecsDirectory);
PackagePublisherPath=$(PackagePublisherNetCoreApp)
</_BuildRepositoryProperties>

<_BuildRepositoryProperties Condition="'$(PublishPackages)'=='true'">
$(_BuildRepositoryProperties);
APIKey=$(APIKey);
NuGetPublishVolatileFeed=$(NuGetPublishVolatileFeed);
PublishPackages=$(PublishPackages)
</_BuildRepositoryProperties>
</PropertyGroup>

<MSBuild
Projects="$(MSBuildThisFileDirectory)RepositoryBuild.targets"
Targets="BuildRepositories"
Properties="$(_BuildRepositoryProperties)" />
</Target>
DependsOnTargets="_PrepareRepositories;_FindDotNetPath;_CreateRepositoriesListWithCommits;_UpdateNuGetConfig;_GenerateBuildGraph;_BuildRepositories" />

<Target Name="_GenerateRestoreGraphSpecs" DependsOnTargets="_FindDotNetPath">
<Target Name="_PrepareRestoreGraphSpecs" DependsOnTargets="_PrepareRepositories">
<ItemGroup>
<Solution Include="$(_CloneRepositoryRoot)%(Repository.Identity)\*.sln">
<Repository>%(Repository.Identity)</Repository>
Expand All @@ -154,22 +122,26 @@
<Solution>
<AdditionalProperties>RestoreGraphOutputPath=$(_RestoreGraphSpecsDirectory)%(Solution.Repository)\%(Solution.FileName)%(Solution.Extension).json</AdditionalProperties>
</Solution>

<GraphSpecInputs Include="
@(Solution);
$(_CloneRepositoryRoot)**\*.csproj;
$(_CloneRepositoryRoot)**\dependencies.props" />
<GraphSpecOutputs Include="$(_RestoreGraphSpecsDirectory)%(Solution.Repository)\%(Solution.FileName)%(Solution.Extension).json" />
</ItemGroup>
</Target>

<Target Name="_GenerateRestoreGraphSpecs" DependsOnTargets="_PrepareRestoreGraphSpecs" Inputs="@(GraphSpecInputs)" Outputs="@(GraphSpecOutputs)">
<MSBuild
Projects="@(Solution)"
Targets="GenerateRestoreGraphFile"
BuildInParallel="$(BuildInParallel)" />
</Target>

<Target Name="_GenerateBuildGraph" DependsOnTargets="_FindDotNetPath">
<PropertyGroup>
<BuildGrapArgs>$(DotNetPath) run -r &quot;$(_CloneRepositoryRoot) &quot; --graph-specs-root &quot;$(_RestoreGraphSpecsDirectory) &quot; &quot;$(_BuildGraphFile)&quot;</BuildGrapArgs>
<BuildGrapArgs Condition="'$(BuildGraphOf)'!=''">$(BuildGrapArgs) --start-at $(BuildGraphOf)</BuildGrapArgs>
</PropertyGroup>
<Exec
Command="$(BuildGrapArgs)"
WorkingDirectory="$(RepositoryRoot)tools\BuildGraph\" />
<Target Name="_GenerateBuildGraph" DependsOnTargets="_GenerateRestoreGraphSpecs">
<RepoTasks.CalculateBuildGraph Repositories="@(Repository)" StartGraphAt="$(BuildGraphOf)" PackageSpecsDirectory="$(_RestoreGraphSpecsDirectory)">
<Output TaskParameter="RepositoriesToBuildInOrder" ItemName="RepositoryToBuildInOrder" />
</RepoTasks.CalculateBuildGraph>
</Target>

<Target Name="_UpdateNuGetConfig">
Expand Down Expand Up @@ -216,16 +188,14 @@
-->
<_CloneUrl>$([System.Environment]::GetEnvironmentVariable("vcsroot.%(Repository.Identity).url"))</_CloneUrl>
<_CommitHash>$([System.Environment]::GetEnvironmentVariable("build.vcs.number.%(Repository.Identity)"))</_CommitHash>

<RepositoryCloneDirectory>$(_CloneRepositoryRoot)%(Repository.Identity)</RepositoryCloneDirectory>
</PropertyGroup>

<Warning Text="%(Repository.Identity) has not been cloned."
Condition="!Exists('$(RepositoryCloneDirectory)')" />
Condition="!Exists('%(Repository.RepositoryPath)')" />

<GetGitCommitInfo
WorkingDirectory="$(RepositoryCloneDirectory)"
Condition="'$(_CommitHash)'=='' AND Exists('$(RepositoryCloneDirectory)')">
WorkingDirectory="%(Repository.RepositoryPath)"
Condition="'$(_CommitHash)'=='' AND Exists('%(Repository.RepositoryPath)')">

<Output TaskParameter="CommitHash" PropertyName="_CommitHash" />
</GetGitCommitInfo>
Expand Down
65 changes: 65 additions & 0 deletions build/tasks/BuildGraph/CalculateBuildGraph.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Linq;
using RepoTools.BuildGraph;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;

namespace RepoTasks
{
public class CalculateBuildGraph : Task
{
[Required]
public ITaskItem[] Repositories { get; set; }

[Output]
public ITaskItem[] RepositoriesToBuildInOrder { get; set; }

/// <summary>
/// The repository at which to root the graph at
/// </summary>
public string StartGraphAt { get; set; }

/// <summary>
/// Directory that contains the package spec files.
/// </summary>
[Required]
public string PackageSpecsDirectory { get; set; }

public override bool Execute()
{
var graphSpecProvider = new DependencyGraphSpecProvider(PackageSpecsDirectory.Trim());

var repositoryPaths = Repositories.Select(r => r.GetMetadata("RepositoryPath")).ToList();
var repositories = Repository.ReadAllRepositories(repositoryPaths, graphSpecProvider);

var graph = GraphBuilder.Generate(repositories, StartGraphAt);
var repositoriesWithOrder = new List<(ITaskItem repository, int order)>();
foreach (var repositoryTaskItem in Repositories)
{
var repositoryName = repositoryTaskItem.ItemSpec;
var graphNodeRepository = graph.First(g => g.Repository.Name == repositoryName);
var order = TopologicalSort.GetOrder(graphNodeRepository);
repositoryTaskItem.SetMetadata("Order", order.ToString());
repositoriesWithOrder.Add((repositoryTaskItem, order));
}

Log.LogMessage(MessageImportance.High, "Repository build order:");
foreach (var buildGroup in repositoriesWithOrder.GroupBy(r => r.order).OrderBy(g => g.Key))
{
var buildGroupRepos = buildGroup.Select(b => b.repository.ItemSpec);
Log.LogMessage(MessageImportance.High, $"{buildGroup.Key.ToString().PadLeft(2, ' ')}: {string.Join(", ", buildGroupRepos)}");
}

RepositoriesToBuildInOrder = repositoriesWithOrder
.OrderBy(r => r.order)
.Select(r => r.repository)
.ToArray();

return true;
}
}
}
21 changes: 21 additions & 0 deletions build/tasks/BuildGraph/DependencyGraphSpecProvider.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
using System.IO;
using NuGet.ProjectModel;

namespace RepoTools.BuildGraph
{
public class DependencyGraphSpecProvider
{
readonly string _packageSpecDirectory;

public DependencyGraphSpecProvider(string packageSpecDirectory)
{
_packageSpecDirectory = packageSpecDirectory;
}

public DependencyGraphSpec GetDependencyGraphSpec(string repositoryName, string solutionPath)
{
var outputFile = Path.Combine(_packageSpecDirectory, repositoryName, Path.GetFileName(solutionPath) + ".json");
return DependencyGraphSpec.Load(outputFile);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
using System.Collections.Generic;
using System.Linq;

namespace BuildGraph
namespace RepoTools.BuildGraph
{
public static class GraphBuilder
{
Expand All @@ -19,7 +19,7 @@ public static IList<GraphNode> Generate(IList<Repository> repositories, string r
foreach (var project in repositories.SelectMany(r => r.AllProjects))
{
var thisProjectRepositoryNode = graphNodes[project.Repository];
if (root != null && string.Equals(root, project.Repository.Name, StringComparison.OrdinalIgnoreCase))
if (!string.IsNullOrEmpty(root) && string.Equals(root, project.Repository.Name, StringComparison.OrdinalIgnoreCase))
{
searchRoot = thisProjectRepositoryNode;
}
Expand Down Expand Up @@ -58,4 +58,4 @@ private static void Visit(HashSet<GraphNode> results, GraphNode searchRoot)
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
using System.Collections.Generic;
using System.Diagnostics;

namespace BuildGraph
namespace RepoTools.BuildGraph
{
[DebuggerDisplay("{Repository.Name}")]
public class GraphNode
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
using System.Collections.Generic;
using System.Diagnostics;

namespace BuildGraph
namespace RepoTools.BuildGraph
{
[DebuggerDisplay("{Name}")]
public class Project
Expand All @@ -20,4 +20,4 @@ public Project(string name)

public ISet<string> PackageReferences { get; } = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
}
}
}
Loading

0 comments on commit aa25401

Please sign in to comment.