Get DDC and DDK test builds working with the forked NNBD SDK.

With these changes, I can run:

  ./tools/build.py -m release --nnbd dartdevc_test

In a checkout that also includes a migrated core library that uses
NNBD features.

I haven't verified if the resulting JS is *correct*, but the build
doesn't error out.

Change-Id: I7d89efe5da8c46e2a9805743e4e61858da8097dd
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/122280
Reviewed-by: Nicholas Shahan <nshahan@google.com>
Commit-Queue: Bob Nystrom <rnystrom@google.com>

Author: munificent

pkg/async_helper/lib/async_helper.dart

@@ -90,6 +90,8 @@ Future<void> asyncTest(f()) {
   return f().then(asyncSuccess);
 }
 
+bool _pass(dynamic object) => true;
+
 /// Calls [f] and verifies that it throws a `T`.
 ///
 /// The optional [check] function can provide additional validation that the
@@ -103,7 +105,7 @@ Future<void> asyncTest(f()) {
 /// exception is not caught by [asyncExpectThrows]. The test is still considered
 /// failing.
 void asyncExpectThrows<T>(Future<void> f(),
-    [bool check(T error), String reason]) {
+    [bool check(T error) = _pass, String reason = ""]) {
   var type = "";
   if (T != dynamic && T != Object) type = "<$T>";
   var header = "asyncExpectThrows$type(${reason ?? ''}):";

pkg/dev_compiler/tool/build_pkgs.dart

@@ -23,6 +23,13 @@ String kernelSummary;
 /// packages will be placed in a "pkg" subdirectory of this.
 String outputDirectory;
 
+/// List of language experiments to enable when building.
+List<String> experiments;
+
+/// Whether to force the analyzer backend to generate code even if there are
+/// errors.
+bool unsafeForceCompile;
+
 /// Compiles the packages that the DDC tests use to JS into the given output
 /// directory.
 ///
@@ -46,6 +53,10 @@ Future main(List<String> arguments) async {
       abbr: "o", help: "Directory to write output to.");
   argParser.addFlag("travis",
       help: "Build the additional packages tested on Travis.");
+  argParser.addMultiOption("enable-experiment",
+      help: "Enable experimental language features.");
+  argParser.addFlag("unsafe-force-compile",
+      help: "Generate output even if compile errors are reported.");
 
   ArgResults argResults;
   try {
@@ -63,6 +74,8 @@ Future main(List<String> arguments) async {
   analyzerSummary = argResults["analyzer-sdk"] as String;
   kernelSummary = argResults["kernel-sdk"] as String;
   outputDirectory = argResults["output"] as String;
+  experiments = argResults["enable-experiment"] as List<String>;
+  unsafeForceCompile = argResults["unsafe-force-compile"] as bool;
 
   // Build leaf packages. These have no other package dependencies.
 
@@ -126,21 +139,18 @@ Future compileModule(String module,
   makeArgs({bool kernel = false}) {
     var pkgDirectory = p.join(outputDirectory, kernel ? 'pkg_kernel' : 'pkg');
     Directory(pkgDirectory).createSync(recursive: true);
-    var args = <String>[];
-    if (kernel) args.add('-k');
 
-    args.addAll([
+    return [
+      if (kernel) '-k',
+      if (experiments.isNotEmpty)
+        '--enable-experiment=${experiments.join(",")}',
+      if (unsafeForceCompile && !kernel) '--unsafe-force-compile',
       '--dart-sdk-summary=${kernel ? kernelSummary : analyzerSummary}',
       '-o${pkgDirectory}/$module.js',
-      'package:$module/$module.dart'
-    ]);
-    for (var lib in libs) {
-      args.add('package:$module/$lib.dart');
-    }
-    for (var dep in deps) {
-      args.add('-s${pkgDirectory}/$dep.${kernel ? "dill" : "sum"}');
-    }
-    return args;
+      'package:$module/$module.dart',
+      for (var lib in libs) 'package:$module/$lib.dart',
+      for (var dep in deps) '-s${pkgDirectory}/$dep.${kernel ? "dill" : "sum"}',
+    ];
   }
 
   if (analyzerSummary != null) {

pkg/dev_compiler/tool/patch_sdk.dart

@@ -9,6 +9,8 @@
 import 'dart:io';
 import 'dart:math' as math;
 
+import 'package:analyzer/dart/analysis/features.dart';
+import 'package:analyzer/dart/analysis/results.dart';
 import 'package:analyzer/dart/analysis/utilities.dart';
 import 'package:analyzer/dart/ast/ast.dart';
 import 'package:analyzer/dart/ast/token.dart';
@@ -35,8 +37,16 @@ void main(List<String> argv) {
   }
 
   var sdk = 'sdk';
+  var useNnbd = false;
   if (argv.length > 3) {
     sdk = argv[3];
+
+    // TODO(38701): While the core libraries have been forked for NNBD, use the
+    // SDK directory name to determine whether to enable the NNBD experiment
+    // when parsing the lib sources. Once the libraries have been unforked, we
+    // should unconditionally enable the experiment flag since then the
+    // canonical SDK libs will use NNBD syntax.
+    useNnbd = sdk.contains("nnbd");
   }
 
   var selfModifyTime = File(self).lastModifiedSync().millisecondsSinceEpoch;
@@ -63,7 +73,7 @@ void main(List<String> argv) {
       File(p.join(repoDir, 'tools', 'VERSION')).readAsStringSync());
 
   // Parse libraries.dart
-  var sdkLibraries = _getSdkLibraries(libContents);
+  var sdkLibraries = _getSdkLibraries(libContents, useNnbd: useNnbd);
 
   // Enumerate core libraries and apply patches
   for (SdkLibrary library in sdkLibraries) {
@@ -91,7 +101,8 @@ void main(List<String> argv) {
       int inputModifyTime = math.max(selfModifyTime,
           libraryFile.lastModifiedSync().millisecondsSinceEpoch);
       var partFiles = <File>[];
-      for (var part in parseString(content: libraryContents).unit.directives) {
+      for (var part
+          in _parseString(libraryContents, useNnbd: false).unit.directives) {
         if (part is PartDirective) {
           var partPath = part.uri.stringValue;
           outPaths.add(p.join(p.dirname(libraryOut), partPath));
@@ -136,7 +147,7 @@ void main(List<String> argv) {
         contents.addAll(partFiles.map((f) => f.readAsStringSync()));
         if (patchExists) {
           var patchContents = patchFile.readAsStringSync();
-          contents = _patchLibrary(contents, patchContents);
+          contents = _patchLibrary(contents, patchContents, useNnbd: useNnbd);
         }
 
         if (contents != null) {
@@ -178,18 +189,19 @@ void _writeSync(String filePath, String contents) {
 /// in the Dart language. Since this feature is only for the convenience of
 /// writing the dart:* libraries, and not a tool given to Dart developers, it
 /// seems like a non-ideal situation. Instead we keep the preprocessing simple.
-List<String> _patchLibrary(List<String> partsContents, String patchContents) {
+List<String> _patchLibrary(List<String> partsContents, String patchContents,
+    {bool useNnbd = false}) {
   var results = <StringEditBuffer>[];
 
   // Parse the patch first. We'll need to extract bits of this as we go through
   // the other files.
-  var patchFinder = PatchFinder.parseAndVisit(patchContents);
+  var patchFinder = PatchFinder.parseAndVisit(patchContents, useNnbd: useNnbd);
 
   // Merge `external` declarations with the corresponding `@patch` code.
   bool failed = false;
   for (var partContent in partsContents) {
     var partEdits = StringEditBuffer(partContent);
-    var partUnit = parseString(content: partContent).unit;
+    var partUnit = _parseString(partContent, useNnbd: useNnbd).unit;
     var patcher = PatchApplier(partEdits, patchFinder);
     partUnit.accept(patcher);
     if (!failed) failed = patcher.patchWasMissing;
@@ -314,9 +326,9 @@ class PatchFinder extends GeneralizingAstVisitor {
   final mergeMembers = <String, List<ClassMember>>{};
   final mergeDeclarations = <CompilationUnitMember>[];
 
-  PatchFinder.parseAndVisit(String contents)
+  PatchFinder.parseAndVisit(String contents, {bool useNnbd})
       : contents = contents,
-        unit = parseString(content: contents).unit {
+        unit = _parseString(contents, useNnbd: false).unit {
     visitCompilationUnit(unit);
   }
 
@@ -478,11 +490,16 @@ class _StringEdit implements Comparable<_StringEdit> {
   }
 }
 
-List<SdkLibrary> _getSdkLibraries(String contents) {
+List<SdkLibrary> _getSdkLibraries(String contents, {bool useNnbd}) {
   // TODO(jmesserly): fix SdkLibrariesReader_LibraryBuilder in Analyzer.
   // It doesn't understand optional new/const in Dart 2. For now, we keep
   // redundant `const` in tool/input_sdk/libraries.dart as a workaround.
   var libraryBuilder = SdkLibrariesReader_LibraryBuilder();
-  parseString(content: contents).unit.accept(libraryBuilder);
+  _parseString(contents, useNnbd: false).unit.accept(libraryBuilder);
   return libraryBuilder.librariesMap.sdkLibraries;
 }
+
+ParseStringResult _parseString(String source, {bool useNnbd}) {
+  var features = FeatureSet.fromEnableFlags([if (useNnbd) "non-nullable"]);
+  return parseString(content: source, featureSet: features);
+}

utils/dartdevc/BUILD.gn

@@ -211,6 +211,16 @@ prebuilt_dart_action("dartdevc_sdk") {
     "-o",
     rebase_path("$target_gen_dir/js/legacy/dart_sdk.js"),
   ]
+
+  # TODO(38701): Cleanup after merging the forked SDK into mainline.
+  if (use_nnbd) {
+    args += [
+      "--enable-experiment=non-nullable",
+
+      # TODO(38813): Ignore incorrect analyzer errors.
+      "--unsafe-force-compile",
+    ]
+  }
 }
 
 # Builds everything needed to run dartdevc tests using test.dart.
@@ -321,6 +331,19 @@ prebuilt_dart_action("dartdevc_test_pkg") {
     "--output",
     rebase_path("$target_gen_dir"),
   ]
+
+  # TODO(38701): Cleanup after merging the forked SDK into mainline.
+  if (use_nnbd) {
+    args += [
+      "--enable-experiment=non-nullable",
+
+      # TODO(rnystrom): Most of the packages used by tests can be cleanly
+      # compiled as opted-in libraries, but js.dart has an optional parameter
+      # of type String with no default value. Changing that would be a breaking
+      # API change. For now, ignore the error.
+      "--unsafe-force-compile",
+    ]
+  }
 }
 
 prebuilt_dart_action("dartdevc_kernel_sdk_outline") {