[flutter_tools] chunk the hashing of large files (flutter#57506)

 For files larger than 250 KB (roughly the size of framework.dart), chunk the conversion. This may be important for large assets like images.

Author: jonahwilliams

packages/flutter_tools/lib/src/build_system/file_store.dart

@@ -16,6 +16,9 @@ import '../base/utils.dart';
 import '../convert.dart';
 import 'build_system.dart';
 
+/// The default threshold for file chunking is 250 KB, or about the size of `framework.dart`.
+const int kDefaultFileChunkThresholdBytes = 250000;
+
 /// An encoded representation of all file hashes.
 class FileStorage {
   FileStorage(this.version, this.files);
@@ -91,13 +94,16 @@ class FileStore {
     @required File cacheFile,
     @required Logger logger,
     FileStoreStrategy strategy = FileStoreStrategy.hash,
+    int fileChunkThreshold = kDefaultFileChunkThresholdBytes,
   }) : _logger = logger,
        _strategy = strategy,
-       _cacheFile = cacheFile;
+       _cacheFile = cacheFile,
+       _fileChunkThreshold = fileChunkThreshold;
 
   final File _cacheFile;
   final Logger _logger;
   final FileStoreStrategy _strategy;
+  final int _fileChunkThreshold;
 
   final HashMap<String, String> previousAssetKeys = HashMap<String, String>();
   final HashMap<String, String> currentAssetKeys = HashMap<String, String>();
@@ -229,7 +235,18 @@ class FileStore {
         dirty.add(file);
         return;
       }
-      final Digest digest = md5.convert(await file.readAsBytes());
+      Digest digest;
+      final int fileBytes = file.lengthSync();
+      // For files larger than a given threshold, chunk the conversion.
+      if (fileBytes > _fileChunkThreshold) {
+        final StreamController<Digest> digests = StreamController<Digest>();
+        final ByteConversionSink inputSink = md5.startChunkedConversion(digests);
+        await file.openRead().forEach(inputSink.add);
+        inputSink.close();
+        digest = await digests.stream.last;
+      } else {
+        digest = md5.convert(await file.readAsBytes());
+      }
       final String currentHash = digest.toString();
       if (currentHash != previousHash) {
         dirty.add(file);

packages/flutter_tools/test/general.shard/build_system/file_store_test.dart

@@ -4,6 +4,7 @@
 
 import 'dart:typed_data';
 
+import 'package:crypto/crypto.dart';
 import 'package:file/memory.dart';
 import 'package:file_testing/file_testing.dart';
 import 'package:flutter_tools/src/artifacts.dart';
@@ -173,6 +174,31 @@ void main() {
 
     expect(logger.errorText, contains('Out of space!'));
   });
+
+  testWithoutContext('FileStore handles chunked conversion of a file', () async {
+    final FileSystem fileSystem = MemoryFileSystem.test();
+    final File cacheFile = fileSystem
+      .directory('example')
+      .childFile(FileStore.kFileCache)
+      ..createSync(recursive: true);
+    final FileStore fileCache = FileStore(
+      cacheFile: cacheFile,
+      logger: BufferLogger.test(),
+      fileChunkThreshold: 1, // Chunk files larger than 1 byte.
+    );
+    final File file = fileSystem.file('foo.dart')
+      ..createSync()
+      ..writeAsStringSync('hello');
+    fileCache.initialize();
+
+    cacheFile.parent.deleteSync(recursive: true);
+
+    await fileCache.diffFileList(<File>[file]);
+
+    // Validate that chunked hash is the same as non-chunked.
+    expect(fileCache.currentAssetKeys['foo.dart'],
+      md5.convert(file.readAsBytesSync()).toString());
+  });
 }
 
 class MockFile extends Mock implements File {}