Skip to content

[mono][tasks] reduce large allocations in build tasks #112295

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Feb 13, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/tasks/Common/FileCache.cs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ public FileCache(string? cacheFilePath, TaskLoggingHelper log)
Enabled = true;
if (File.Exists(cacheFilePath))
{
_oldCache = JsonSerializer.Deserialize<CompilerCache>(File.ReadAllText(cacheFilePath), s_jsonOptions);
using FileStream fs = File.OpenRead(cacheFilePath);
_oldCache = JsonSerializer.Deserialize<CompilerCache>(fs, s_jsonOptions);
}

_oldCache ??= new();
Expand Down
100 changes: 67 additions & 33 deletions src/tasks/Common/Utils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -214,15 +214,66 @@ public static (int, string) TryRunProcess(
return (process.ExitCode, outputBuilder.ToString().Trim('\r', '\n'));
}

private static bool ContentEqual(string filePathA, string filePathB)
{
const int bufferSize = 8192;
using FileStream streamA = new(filePathA, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: bufferSize, FileOptions.SequentialScan);
using FileStream streamB = new(filePathB, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: bufferSize, FileOptions.SequentialScan);

if (streamA.Length != streamB.Length)
return false;

byte[] bufferA = new byte[bufferSize];
byte[] bufferB = new byte[bufferSize];

int readA = 0;
int readB = 0;
int consumedA = 0;
int consumedB = 0;

/*
Read both streams in parallel into rolling buffers, comparing overlapping bytes.
Advance the consumed amount by the overlap. Refill a buffer when the previous read is exhausted.
This keeps the comparison position in sync, even if the read amounts differ.
*/

while (true)
{
if (consumedA == readA)
{
readA = streamA.Read(bufferA, 0, bufferSize);
consumedA = 0;
}

if (consumedB == readB)
{
readB = streamB.Read(bufferB, 0, bufferSize);
consumedB = 0;
}

if (readA == 0 && readB == 0)
return true;

if (readA == 0 || readB == 0)
return false;

int overlap = Math.Min(readA - consumedA, readB - consumedB);
if (!bufferA.AsSpan(consumedA, overlap).SequenceEqual(bufferB.AsSpan(consumedB, overlap)))
return false;

consumedA += overlap;
consumedB += overlap;
}
}

#pragma warning disable IDE0060 // Remove unused parameter
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm certain there is a difference between the text comparison and bytewise comparison but I think any value is likely an artifact of history that no longer applies. I'm happy to remove the setting at the callers and will unless someone objects.

public static bool CopyIfDifferent(string src, string dst, bool useHash)
#pragma warning restore IDE0060 // Remove unused parameter
{
if (!File.Exists(src))
throw new ArgumentException($"Cannot find {src} file to copy", nameof(src));

bool areDifferent = !File.Exists(dst) ||
(useHash && ComputeHash(src) != ComputeHash(dst)) ||
(File.ReadAllText(src) != File.ReadAllText(dst));

bool areDifferent = !File.Exists(dst) || !ContentEqual(src, dst);
if (areDifferent)
File.Copy(src, dst, true);

Expand Down Expand Up @@ -252,41 +303,24 @@ private static string ToBase64SafeString(byte[] data)

private static byte[] ComputeHashFromStream(Stream stream, HashAlgorithmType algorithm)
{
if (algorithm == HashAlgorithmType.SHA512)
{
using HashAlgorithm hashAlgorithm = SHA512.Create();
return hashAlgorithm.ComputeHash(stream);
}
else if (algorithm == HashAlgorithmType.SHA384)
{
using HashAlgorithm hashAlgorithm = SHA384.Create();
return hashAlgorithm.ComputeHash(stream);
}
else if (algorithm == HashAlgorithmType.SHA256)
{
using HashAlgorithm hashAlgorithm = SHA256.Create();
return hashAlgorithm.ComputeHash(stream);
}
else
using HashAlgorithm hash = algorithm switch
{
throw new ArgumentException($"Unsupported hash algorithm: {algorithm}");
}
HashAlgorithmType.SHA512 => SHA512.Create(),
HashAlgorithmType.SHA384 => SHA384.Create(),
HashAlgorithmType.SHA256 => SHA256.Create(),
_ => throw new ArgumentException($"Unsupported hash algorithm: {algorithm}")
};
return hash.ComputeHash(stream);
}

private static string EncodeHash(byte[] data, HashEncodingType encoding)
{
if (encoding == HashEncodingType.Base64)
{
return Convert.ToBase64String(data);
}
else if (encoding == HashEncodingType.Base64Safe)
return encoding switch
{
return ToBase64SafeString(data);
}
else
{
throw new ArgumentException($"Unsupported hash encoding: {encoding}");
}
HashEncodingType.Base64 => Convert.ToBase64String(data),
HashEncodingType.Base64Safe => ToBase64SafeString(data),
_ => throw new ArgumentException($"Unsupported hash encoding: {encoding}")
};
}

public static string ComputeHash(string filepath)
Expand Down
4 changes: 2 additions & 2 deletions src/tasks/WasmAppBuilder/WasmAppBuilderBaseTask.cs
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,8 @@ protected virtual void UpdateRuntimeConfigJson()
if (matchingAssemblies.Length > 1)
throw new LogAsErrorException($"Found more than one assembly matching the main assembly name {MainAssemblyName}: {string.Join(",", matchingAssemblies)}");

var rootNode = JsonNode.Parse(File.ReadAllText(RuntimeConfigJsonPath),
new JsonNodeOptions { PropertyNameCaseInsensitive = true });
using FileStream rcs = File.OpenRead(RuntimeConfigJsonPath);
var rootNode = JsonNode.Parse(rcs, new JsonNodeOptions { PropertyNameCaseInsensitive = true });
if (rootNode == null)
throw new LogAsErrorException($"Failed to parse {RuntimeConfigJsonPath}");

Expand Down
Loading