src: compile native modules and their code cache in C++

This patch refactors out a part of NativeModule.prototype.compile
(in JS land) into a C++ NativeModule class, this enables a
couple of possibilities:

1. By moving the code to the C++ land, we have more opportunity
  to specialize the compilation process of the native modules
  (e.g. compilation options, code cache) that is orthogonal to
  how user land modules are compiled
2. We can reuse the code to compile bootstrappers and context
  fixers and enable them to be compiled with the code cache later,
  since they are not loaded by NativeModule in the JS land their
  caching must be done in C++.
3. Since there is no need to pass the static data to JS for
  compilation anymore, this enables us to use
  (std::map<std::string, const char*>) in the generated
  node_code_cache.cc and node_javascript.cc later, and scope
  every actual access to the source of native modules to a
  std::map lookup instead of a lookup on a v8::Object in
  dictionary mode.

This patch also refactor the code cache generator and tests
a bit and trace the `withCodeCache` and `withoutCodeCache`
in a Set instead of an Array, and makes sure that all the cachable
builtins are tested.

PR-URL: #24221
Reviewed-By: Refael Ackermann <refack@gmail.com>
Reviewed-By: Anna Henningsen <anna@addaleax.net>

Author: joyeecheung

lib/internal/bootstrap/cache.js

@@ -8,20 +8,13 @@
 const {
   NativeModule
 } = require('internal/bootstrap/loaders');
+const {
+  source,
+  compileCodeCache
+} = internalBinding('native_module');
 const { hasTracing } = process.binding('config');
 
-function getCodeCache(id) {
-  const cached = NativeModule.getCached(id);
-  if (cached && (cached.loaded || cached.loading)) {
-    return cached.script.createCachedData();
-  }
-
-  // The script has not been compiled and run
-  NativeModule.require(id);
-  return getCodeCache(id);
-}
-
-const depsModule = Object.keys(NativeModule._source).filter(
+const depsModule = Object.keys(source).filter(
   (key) => NativeModule.isDepsModule(key) || key.startsWith('internal/deps')
 );
 
@@ -75,17 +68,10 @@ if (!process.versions.openssl) {
 }
 
 module.exports = {
-  cachableBuiltins: Object.keys(NativeModule._source).filter(
+  cachableBuiltins: Object.keys(source).filter(
     (key) => !cannotUseCache.includes(key)
   ),
-  builtinSource: Object.assign({}, NativeModule._source),
-  getCodeCache,
-  getSource: NativeModule.getSource,
-  codeCache: internalBinding('code_cache'),
-  compiledWithoutCache: NativeModule.compiledWithoutCache,
-  compiledWithCache: NativeModule.compiledWithCache,
-  nativeModuleWrap(script) {
-    return NativeModule.wrap(script);
-  },
+  getSource(id) { return source[id]; },
+  getCodeCache: compileCodeCache,
   cannotUseCache
 };

lib/internal/bootstrap/loaders.js

@@ -149,7 +149,6 @@
 
   // Create this WeakMap in js-land because V8 has no C++ API for WeakMap
   internalBinding('module_wrap').callbackMap = new WeakMap();
-  const { ContextifyScript } = internalBinding('contextify');
 
   // Set up NativeModule
   function NativeModule(id) {
@@ -160,20 +159,13 @@
     this.exportKeys = undefined;
     this.loaded = false;
     this.loading = false;
-    this.script = null;  // The ContextifyScript of the module
   }
 
   NativeModule._source = getInternalBinding('natives');
   NativeModule._cache = {};
 
   const config = getBinding('config');
 
-  const codeCache = getInternalBinding('code_cache');
-  const codeCacheHash = getInternalBinding('code_cache_hash');
-  const sourceHash = getInternalBinding('natives_hash');
-  const compiledWithoutCache = NativeModule.compiledWithoutCache = [];
-  const compiledWithCache = NativeModule.compiledWithCache = [];
-
   // Think of this as module.exports in this file even though it is not
   // written in CommonJS style.
   const loaderExports = { internalBinding, NativeModule };
@@ -332,67 +324,18 @@
     this.exports = new Proxy(this.exports, handler);
   };
 
+  const { compileFunction } = getInternalBinding('native_module');
   NativeModule.prototype.compile = function() {
     const id = this.id;
-    let source = NativeModule.getSource(id);
-    source = NativeModule.wrap(source);
 
     this.loading = true;
 
     try {
-      // Currently V8 only checks that the length of the source code is the
-      // same as the code used to generate the hash, so we add an additional
-      // check here:
-      // 1. During compile time, when generating node_javascript.cc and
-      //    node_code_cache.cc, we compute and include the hash of the
-      //   (unwrapped) JavaScript source in both.
-      // 2. At runtime, we check that the hash of the code being compiled
-      //   and the hash of the code used to generate the cache
-      //   (inside the wrapper) is the same.
-      // This is based on the assumptions:
-      // 1. `internalBinding('code_cache_hash')` must be in sync with
-      //    `internalBinding('code_cache')` (same C++ file)
-      // 2. `internalBinding('natives_hash')` must be in sync with
-      //    `internalBinding('natives')` (same C++ file)
-      // 3. If `internalBinding('natives_hash')` is in sync with
-      //    `internalBinding('natives_hash')`, then the (unwrapped)
-      //    code used to generate `internalBinding('code_cache')`
-      //    should be in sync with the (unwrapped) code in
-      //    `internalBinding('natives')`
-      // There will be, however, false positives if the wrapper used
-      // to generate the cache is different from the one used at run time,
-      // and the length of the wrapper somehow stays the same.
-      // But that should be rare and can be eased once we make the
-      // two bootstrappers cached and checked as well.
-      const cache = codeCacheHash[id] &&
-        (codeCacheHash[id] === sourceHash[id]) ? codeCache[id] : undefined;
-
-      // (code, filename, lineOffset, columnOffset
-      // cachedData, produceCachedData, parsingContext)
-      const script = new ContextifyScript(
-        source, this.filename, 0, 0,
-        cache, false, undefined
-      );
-
-      // This will be used to create code cache in tools/generate_code_cache.js
-      this.script = script;
-
-      // One of these conditions may be false when any of the inputs
-      // of the `node_js2c` target in node.gyp is modified.
-      // FIXME(joyeecheung): Figure out how to resolve the dependency issue.
-      // When the code cache was introduced we were at a point where refactoring
-      // node.gyp may not be worth the effort.
-      if (!cache || script.cachedDataRejected) {
-        compiledWithoutCache.push(this.id);
-      } else {
-        compiledWithCache.push(this.id);
-      }
-
-      // Arguments: timeout, displayErrors, breakOnSigint
-      const fn = script.runInThisContext(-1, true, false);
       const requireFn = this.id.startsWith('internal/deps/') ?
         NativeModule.requireForDeps :
         NativeModule.require;
+
+      const fn = compileFunction(id);
       fn(this.exports, requireFn, this, process, internalBinding);
 
       if (config.experimentalModules && !NativeModule.isInternal(this.id)) {

node.gyp

@@ -416,6 +416,8 @@
         'src/node_javascript.h',
         'src/node_messaging.h',
         'src/node_mutex.h',
+        'src/node_native_module.h',
+        'src/node_native_module.cc',
         'src/node_options.h',
         'src/node_options-inl.h',
         'src/node_perf.h',