[CFE] Add incremental serializer

Add an incremental serializer that can be used to avoid re-serializing
the same thing again and again.
It does this by grouping libraries into Components / bundles,
serializing them individually and using the concatenated dill feature to
output the wanted data --- just potentially faster if we had the data
in cache.

Note that the serialized output might contain *more* than the input given
if we cached the wanted data into a bigger bundle.
The output will always be "closed" though, i.e. if the stuff that is
included that is too much added a new dependency, that dependency will be
included as well.
This should generally make it safe, although one can imagine situations
where it could pull in lots of dependencies that it wouldn't otherwise
have.

It is being driven by the incremental compiler which makes sure to
invalidate the cache when the data changes.

Except for situations where the libraries are changed externally after
being serialized, but before being serialized again, where one then wants
the updated library serialized, the feature should be safe.

Change-Id: I2a504abe6dbb68434c3b04abff13480ef72a6a6e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/120786
Reviewed-by: Johnni Winther <johnniwinther@google.com>
Commit-Queue: Jens Johansen <jensj@google.com>

Author: jensjoha

pkg/front_end/lib/src/api_prototype/incremental_kernel_generator.dart

@@ -17,18 +17,25 @@ import '../fasta/compiler_context.dart' show CompilerContext;
 
 import '../fasta/incremental_compiler.dart' show IncrementalCompiler;
 
+import '../fasta/incremental_serializer.dart' show IncrementalSerializer;
+
 import '../fasta/scanner/string_scanner.dart' show StringScanner;
 
 import 'compiler_options.dart' show CompilerOptions;
 
+export '../fasta/incremental_serializer.dart' show IncrementalSerializer;
+
 abstract class IncrementalKernelGenerator {
   factory IncrementalKernelGenerator(CompilerOptions options, Uri entryPoint,
-      [Uri initializeFromDillUri, bool outlineOnly]) {
+      [Uri initializeFromDillUri,
+      bool outlineOnly,
+      IncrementalSerializer incrementalSerializer]) {
     return new IncrementalCompiler(
         new CompilerContext(
             new ProcessedOptions(options: options, inputs: [entryPoint])),
         initializeFromDillUri,
-        outlineOnly);
+        outlineOnly,
+        incrementalSerializer);
   }
 
   /// Initialize the incremental compiler from a component.

pkg/front_end/lib/src/api_unstable/vm.dart

@@ -17,7 +17,7 @@ export '../api_prototype/file_system.dart'
 export '../api_prototype/front_end.dart' show CompilerResult;
 
 export '../api_prototype/incremental_kernel_generator.dart'
-    show IncrementalKernelGenerator, isLegalIdentifier;
+    show IncrementalKernelGenerator, IncrementalSerializer, isLegalIdentifier;
 
 export '../api_prototype/kernel_generator.dart'
     show kernelForModule, kernelForProgram;

pkg/front_end/lib/src/fasta/incremental_compiler.dart

@@ -14,7 +14,8 @@ import 'package:kernel/binary/ast_from_binary.dart'
         BinaryBuilderWithMetadata,
         CanonicalNameError,
         CanonicalNameSdkError,
-        InvalidKernelVersionError;
+        InvalidKernelVersionError,
+        SubComponentView;
 
 import 'package:kernel/class_hierarchy.dart'
     show ClassHierarchy, ClosedWorldClassHierarchy;
@@ -63,6 +64,8 @@ import 'dill/dill_library_builder.dart' show DillLibraryBuilder;
 
 import 'dill/dill_target.dart' show DillTarget;
 
+import 'incremental_serializer.dart' show IncrementalSerializer;
+
 import 'util/error_reporter_file_copier.dart' show saveAsGzip;
 
 import 'fasta_codes.dart'
@@ -108,13 +111,15 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
   final Uri initializeFromDillUri;
   final Component componentToInitializeFrom;
   bool initializedFromDill = false;
+  bool initializedIncrementalSerializer = false;
   Uri previousPackagesUri;
   Map<String, Uri> previousPackagesMap;
   Map<String, Uri> currentPackagesMap;
   bool hasToCheckPackageUris = false;
   Map<Uri, List<DiagnosticMessageFromJson>> remainingComponentProblems =
       new Map<Uri, List<DiagnosticMessageFromJson>>();
   List<Component> modulesToLoad;
+  IncrementalSerializer incrementalSerializer;
 
   static final Uri debugExprUri =
       new Uri(scheme: "org-dartlang-debug", path: "synthetic_debug_expression");
@@ -123,16 +128,20 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
 
   IncrementalCompiler.fromComponent(
       this.context, Component this.componentToInitializeFrom,
-      [bool outlineOnly])
+      [bool outlineOnly, IncrementalSerializer incrementalSerializer])
       : ticker = context.options.ticker,
         initializeFromDillUri = null,
-        this.outlineOnly = outlineOnly ?? false;
+        this.outlineOnly = outlineOnly ?? false,
+        this.incrementalSerializer = incrementalSerializer;
 
   IncrementalCompiler(this.context,
-      [this.initializeFromDillUri, bool outlineOnly])
+      [this.initializeFromDillUri,
+      bool outlineOnly,
+      IncrementalSerializer incrementalSerializer])
       : ticker = context.options.ticker,
         componentToInitializeFrom = null,
-        this.outlineOnly = outlineOnly ?? false;
+        this.outlineOnly = outlineOnly ?? false,
+        this.incrementalSerializer = incrementalSerializer;
 
   @override
   Future<Component> computeDelta(
@@ -240,7 +249,9 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
 
       bool removedDillBuilders = false;
       for (LibraryBuilder builder in notReusedLibraries) {
+        // TODO(jensj): What about parts in uriToSource?
         CompilerContext.current.uriToSource.remove(builder.fileUri);
+        incrementalSerializer?.invalidate(builder.fileUri);
 
         LibraryBuilder dillBuilder =
             dillLoadedData.loader.builders.remove(builder.uri);
@@ -703,6 +714,9 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
         uriToSource.remove(uri);
         userBuilders?.remove(uri);
         removeLibraryFromRemainingComponentProblems(lib, uriTranslator);
+
+        // Technically this isn't necessary as the uri is not a package-uri.
+        incrementalSerializer?.invalidate(builder.fileUri);
       }
     }
     hierarchy?.applyTreeChanges(removedLibraries, const []);
@@ -755,9 +769,14 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
 
         // We're going to output all we read here so lazy loading it
         // doesn't make sense.
-        new BinaryBuilderWithMetadata(initializationBytes,
+        List<SubComponentView> views = new BinaryBuilderWithMetadata(
+                initializationBytes,
                 disableLazyReading: true)
-            .readComponent(data.component, checkCanonicalNames: true);
+            .readComponent(data.component,
+                checkCanonicalNames: true, createView: true);
+        initializedIncrementalSerializer =
+            incrementalSerializer?.initialize(initializationBytes, views) ??
+                false;
 
         // Check the any package-urls still point to the same file
         // (e.g. the package still exists and hasn't been updated).