Skip to content

Commit b764409

Browse files
ellahathawayellahathaway
authored
Binary tooling (#18726)
1 parent 15f6667 commit b764409

File tree

12 files changed

+753
-41
lines changed

12 files changed

+753
-41
lines changed

‎eng/pipelines/templates/jobs/vmr-build.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,7 @@ jobs:
386386
find artifacts/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \;
387387
if [[ "${{ parameters.buildSourceOnly }}" == "True" ]]; then
388388
find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \;
389+
find artifacts/log/binary-report/ -exec rsync -R {} -t ${targetFolder} \;
389390
fi
390391
find src/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
391392
find src/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;

‎src/SourceBuild/content/eng/prep-source-build.sh

Lines changed: 143 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22

33
### Usage: $0
44
###
5-
### Prepares the environment for a source build by downloading Private.SourceBuilt.Artifacts.*.tar.gz and
6-
### installing the version of dotnet referenced in global.json
5+
### Prepares the environment for a source build by downloading Private.SourceBuilt.Artifacts.*.tar.gz,
6+
### installing the version of dotnet referenced in global.json,
7+
### and detecting binaries and removing any non-SB allowed binaries.
78
###
89
### Options:
910
### --no-artifacts Exclude the download of the previously source-built artifacts archive
@@ -15,6 +16,21 @@
1516
### --runtime-source-feed URL of a remote server or a local directory, from which SDKs and
1617
### runtimes can be downloaded
1718
### --runtime-source-feed-key Key for accessing the above server, if necessary
19+
###
20+
### Binary-Tooling options:
21+
### --no-binary-tooling Don't run the binary tooling
22+
### --allowed-binaries Path to the file containing the list of known binaries that are allowed
23+
### in the VMR and can be kept for source-building.
24+
### Default is src/installer/src/VirtualMonoRepo/allowed-binaries.txt
25+
### --disallowed-sb-binaries Path to the file containing the list of known binaries that are allowed
26+
### in the VMR but cannot be kept for source-building.
27+
### Default is null.
28+
### --with-sdk Use the SDK in the specified directory
29+
### Default is the .NET SDK
30+
### --with-packages URL or specified directory to use as the source feed for packages
31+
### Default is the previously source-built artifacts archive
32+
### --no-validate Do not run validation. Only remove the binaries.
33+
### --no-clean Do not remove the binaries. Only run the validation.
1834

1935
set -euo pipefail
2036
IFS=$'\n\t'
@@ -26,15 +42,32 @@ function print_help () {
2642
sed -n '/^### /,/^$/p' "$source" | cut -b 5-
2743
}
2844

45+
# SB prep default arguments
2946
defaultArtifactsRid='centos.8-x64'
3047

48+
# Binary Tooling default arguments
49+
defaultAllowedBinaries="$REPO_ROOT/src/installer/src/VirtualMonoRepo/allowed-binaries.txt"
50+
defaultDotnetSdk="$REPO_ROOT/.dotnet"
51+
defaultPackagesDir="$REPO_ROOT/prereqs/packages"
52+
defaultMode="All"
53+
54+
# SB prep arguments
3155
buildBootstrap=true
3256
downloadArtifacts=true
3357
downloadPrebuilts=true
3458
installDotnet=true
3559
artifactsRid=$defaultArtifactsRid
3660
runtime_source_feed='' # IBM requested these to support s390x scenarios
3761
runtime_source_feed_key='' # IBM requested these to support s390x scenarios
62+
63+
# Binary Tooling arguments
64+
runBinaryTool=true
65+
allowedBinaries=$defaultAllowedBinaries
66+
disallowedSbBinaries=''
67+
dotnetSdk=$defaultDotnetSdk
68+
packagesSourceFeed=$defaultPackagesDir
69+
mode=$defaultMode
70+
3871
positional_args=()
3972
while :; do
4073
if [ $# -le 0 ]; then
@@ -69,6 +102,47 @@ while :; do
69102
runtime_source_feed_key=$2
70103
shift
71104
;;
105+
--no-binary-tooling)
106+
runBinaryTool=false
107+
;;
108+
--allowed-binaries)
109+
allowedBinaries=$2
110+
if [ ! -f "$allowedBinaries" ]; then
111+
echo "Allowed binaries file '$allowedBinaries' does not exist"
112+
exit 1
113+
fi
114+
shift
115+
;;
116+
--disallowed-sb-binaries)
117+
disallowedSbBinaries=$2
118+
if [ ! -f "$disallowedSbBinaries" ]; then
119+
echo "Disallowed source build binaries file '$disallowedSbBinaries' does not exist"
120+
exit 1
121+
fi
122+
shift
123+
;;
124+
--with-sdk)
125+
dotnetSdk=$2
126+
if [ ! -d "$dotnetSdk" ]; then
127+
echo "Custom SDK directory '$dotnetSdk' does not exist"
128+
exit 1
129+
fi
130+
if [ ! -x "$dotnetSdk/dotnet" ]; then
131+
echo "Custom SDK '$dotnetSdk/dotnet' does not exist or is not executable"
132+
exit 1
133+
fi
134+
shift
135+
;;
136+
--with-packages)
137+
packagesSourceFeed=$2
138+
shift
139+
;;
140+
--no-clean)
141+
mode="Validate"
142+
;;
143+
--no-validate)
144+
mode="Clean"
145+
;;
72146
*)
73147
positional_args+=("$1")
74148
;;
@@ -112,6 +186,56 @@ if [ "$installDotnet" == true ] && [ -d "$REPO_ROOT/.dotnet" ]; then
112186
installDotnet=false;
113187
fi
114188

189+
function ParseBinaryArgs {
190+
# Attempting to run the binary tooling without an SDK will fail. So either the --with-sdk flag must be passed
191+
# or a pre-existing .dotnet SDK directory must exist.
192+
if [ "$dotnetSdk" == "$defaultDotnetSdk" ] && [ ! -d "$dotnetSdk" ]; then
193+
echo " ERROR: A pre-existing .dotnet SDK directory is needed if --with-sdk is not provided. \
194+
Please either supply an SDK using --with-sdk or execute ./eng/prep-source-build.sh before proceeding. Exiting..."
195+
exit 1
196+
fi
197+
198+
## Attemping to run the binary tooling without a packages directory or source-feed will fail. So either the
199+
## --with-packages flag must be passed with a valid directory or a pre-existing packages directory must exist.
200+
if [ "$packagesSourceFeed" == "$defaultPackagesDir" ] && [ ! -d "$packagesSourceFeed" ]; then
201+
echo " ERROR: A pre-existing packages directory is needed if --with-packages is not provided. \
202+
Please either supply a packages directory using --with-packages or \
203+
execute ./eng/prep-source-build.sh with download artifacts enabled before proceeding. Exiting..."
204+
exit 1
205+
fi
206+
207+
# Attempting to run the binary tooling with a custom packages feed that does not
208+
# have PackageVersions.props in the packages directory or source-feed will fail.
209+
if [ "$packagesSourceFeed" != "$defaultPackagesDir" ] && [ ! -f "$packagesSourceFeed/PackageVersions.props" ]; then
210+
echo " ERROR: PackageVersions.props is needed in the packages directory or source-feed. Exiting..."
211+
exit 1
212+
fi
213+
214+
# Set up the packages source feed if we're using the default artifacts
215+
previouslyBuiltPackagesDir="$defaultPackagesDir/previously-source-built"
216+
packageArtifacts="$defaultPackagesDir/archive/Private.SourceBuilt.Artifacts.*.tar.gz"
217+
if [ "$packagesSourceFeed" == "$defaultPackagesDir" ]; then
218+
if [ -d "$previouslyBuiltPackagesDir" ]; then
219+
echo " Previously source built packages directory exists..."
220+
echo " Using $previouslyBuiltPackagesDir as the source-feed for the binary tooling..."
221+
packagesSourceFeed="$previouslyBuiltPackagesDir"
222+
elif [ -f ${packageArtifacts} ]; then
223+
echo " Unpacking Private.SourceBuilt.Artifacts.*.tar.gz to $previouslyBuiltPackagesDir..."
224+
mkdir -p "$previouslyBuiltPackagesDir"
225+
tar -xzf ${packageArtifacts} -C "$previouslyBuiltPackagesDir"
226+
tar -xzf ${packageArtifacts} -C "$previouslyBuiltPackagesDir" PackageVersions.props
227+
228+
echo " Using $previouslyBuiltPackagesDir as the source-feed for the binary tooling..."
229+
packagesSourceFeed="$previouslyBuiltPackagesDir"
230+
else
231+
echo " ERROR: A pre-existing package archive is needed if --with-packages is not provided. \
232+
Please either supply a source-feed using --with-packages or execute ./eng/prep-source-build.sh \
233+
with download artifacts enabled before proceeding. Exiting..."
234+
exit 1
235+
fi
236+
fi
237+
}
238+
115239
function DownloadArchive {
116240
archiveType="$1"
117241
isRequired="$2"
@@ -171,6 +295,18 @@ function BootstrapArtifacts {
171295
rm -rf "$workingDir"
172296
}
173297

298+
function RunBinaryTool {
299+
BinaryTool="$REPO_ROOT/eng/tools/BinaryToolKit"
300+
TargetDir="$REPO_ROOT"
301+
OutputDir="$REPO_ROOT/artifacts/log/binary-report"
302+
303+
# Set the environment variable for the packages source feed
304+
export ARTIFACTS_PATH="$packagesSourceFeed"
305+
306+
# Run the BinaryDetection tool
307+
"$dotnetSdk/dotnet" run --project "$BinaryTool" -c Release -p PackagesPropsDirectory="$packagesSourceFeed" "$TargetDir" "$OutputDir" -ab "$allowedBinaries" -db "$disallowedSbBinaries" -m $mode -l Debug
308+
}
309+
174310
# Check for the version of dotnet to install
175311
if [ "$installDotnet" == true ]; then
176312
echo " Installing dotnet..."
@@ -189,3 +325,8 @@ fi
189325
if [ "$downloadPrebuilts" == true ]; then
190326
DownloadArchive Prebuilts false $artifactsRid
191327
fi
328+
329+
if [ "$runBinaryTool" == true ]; then
330+
ParseBinaryArgs
331+
RunBinaryTool
332+
fi

‎src/SourceBuild/content/eng/tools/BinaryToolKit/BinaryTool.cs

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
namespace BinaryToolKit;
5+
6+
public class BinaryTool
7+
{
8+
public async Task ExecuteAsync(
9+
string targetDirectory,
10+
string outputReportDirectory,
11+
string? allowedBinariesFile,
12+
string? disallowedSbBinariesFile,
13+
Modes mode)
14+
{
15+
DateTime startTime = DateTime.Now;
16+
17+
Log.LogInformation($"Starting binary tool at {startTime} in {mode} mode");
18+
19+
// Parse args
20+
targetDirectory = GetAndValidateFullPath(
21+
"TargetDirectory",
22+
targetDirectory,
23+
isDirectory: true,
24+
createIfNotExist: false,
25+
isRequired: true)!;
26+
outputReportDirectory = GetAndValidateFullPath(
27+
"OutputReportDirectory",
28+
outputReportDirectory,
29+
isDirectory: true,
30+
createIfNotExist: true,
31+
isRequired: true)!;
32+
allowedBinariesFile = GetAndValidateFullPath(
33+
"AllowedBinariesFile",
34+
allowedBinariesFile,
35+
isDirectory: false,
36+
createIfNotExist: false,
37+
isRequired: false);
38+
disallowedSbBinariesFile = GetAndValidateFullPath(
39+
"DisallowedSbBinariesFile",
40+
disallowedSbBinariesFile,
41+
isDirectory: false,
42+
createIfNotExist: false,
43+
isRequired: false);
44+
45+
// Run the tooling
46+
var detectedBinaries = await DetectBinaries.ExecuteAsync(targetDirectory);
47+
48+
var comparedBinaries = CompareBinariesAgainstBaselines
49+
.Execute(
50+
detectedBinaries,
51+
allowedBinariesFile,
52+
disallowedSbBinariesFile,
53+
outputReportDirectory,
54+
targetDirectory,
55+
mode);
56+
57+
if (mode.HasFlag(Modes.Clean))
58+
{
59+
RemoveBinaries.Execute(comparedBinaries, targetDirectory);
60+
}
61+
62+
Log.LogInformation("Finished all binary tasks. Took " + (DateTime.Now - startTime).TotalSeconds + " seconds.");
63+
}
64+
65+
private string? GetAndValidateFullPath(
66+
string parameterName,
67+
string? path,
68+
bool isDirectory,
69+
bool createIfNotExist,
70+
bool isRequired)
71+
{
72+
if (string.IsNullOrWhiteSpace(path))
73+
{
74+
if (isRequired)
75+
{
76+
Log.LogError($"Required path for '{parameterName}' is empty or contains whitespace.");
77+
Environment.Exit(1);
78+
}
79+
return null;
80+
}
81+
82+
string fullPath = Path.GetFullPath(path);
83+
bool exists = isDirectory ? Directory.Exists(fullPath) : File.Exists(fullPath);
84+
85+
if (!exists)
86+
{
87+
if (createIfNotExist && isDirectory)
88+
{
89+
Log.LogInformation($"Creating directory '{fullPath}' for '{parameterName}'.");
90+
Directory.CreateDirectory(fullPath);
91+
}
92+
else
93+
{
94+
Log.LogError($"{(isDirectory ? "Directory" : "File")} '{fullPath}' for '{parameterName}' does not exist.");
95+
Environment.Exit(1);
96+
}
97+
}
98+
return fullPath;
99+
}
100+
}

‎src/SourceBuild/content/eng/tools/BinaryToolKit/BinaryToolKit.csproj

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<Project Sdk="Microsoft.NET.Sdk" InitialTargets="CheckPackagesPropsDirectory">
2+
3+
<PropertyGroup>
4+
<TargetFramework>$(NetCurrent)</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
<PackagesPropsDirectory></PackagesPropsDirectory>
8+
<OutputType>Exe</OutputType>
9+
</PropertyGroup>
10+
11+
<Target Name="CheckPackagesPropsDirectory">
12+
<Error Condition="'$(PackagesPropsDirectory)' == ''" Text="PackagesPropsDirectory is not set. Please pass as an MSBuild Property" />
13+
<Error Condition="!Exists('$(PackagesPropsDirectory)/PackageVersions.props')" Text="PackageVersions.props does not exist in $(PackagesPropsDirectory)." />
14+
</Target>
15+
16+
<!-- Need to condition this import because msbuild will complain about the project not being valid otherwise. -->
17+
<!-- With the condition, the CheckPackagesPropsDirectory will run as expected and show the respective errors. -->
18+
<Import Project="$(PackagesPropsDirectory)/PackageVersions.props" Condition="'$(PackagesPropsDirectory)' != ''" />
19+
20+
<ItemGroup>
21+
<PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="$(MicrosoftExtensionsFileSystemGlobbingVersion)" />
22+
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleVersion)" />
23+
<PackageReference Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsLoggingVersion)" />
24+
<PackageReference Include="System.CommandLine" Version="$(SystemCommandLineVersion)" />
25+
</ItemGroup>
26+
27+
</Project>

0 commit comments

Comments
 (0)