From 48e6750183fdea31852c03429a9cd008d5038f4d Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 7 Mar 2023 17:46:04 +0100 Subject: [PATCH] build: replace js2c.py with js2c.cc PR-URL: https://github.com/nodejs/node/pull/46997 Reviewed-By: Yagiz Nizipli --- deps/simdutf/simdutf.gyp | 1 + deps/uv/uv.gyp | 1 + node.gyp | 19 ++- test/tools/test_js2c.py | 14 -- tools/js2c.py | 268 --------------------------------------- 5 files changed, 11 insertions(+), 292 deletions(-) delete mode 100644 test/tools/test_js2c.py delete mode 100755 tools/js2c.py diff --git a/deps/simdutf/simdutf.gyp b/deps/simdutf/simdutf.gyp index a86e92eb608d7e..ca8cd2fa7bc079 100644 --- a/deps/simdutf/simdutf.gyp +++ b/deps/simdutf/simdutf.gyp @@ -7,6 +7,7 @@ 'targets': [ { 'target_name': 'simdutf', + 'toolsets': ['host', 'target'], 'type': 'static_library', 'include_dirs': ['.'], 'direct_dependent_settings': { diff --git a/deps/uv/uv.gyp b/deps/uv/uv.gyp index ba6d715867fb90..ddc30fafca8294 100644 --- a/deps/uv/uv.gyp +++ b/deps/uv/uv.gyp @@ -155,6 +155,7 @@ 'targets': [ { 'target_name': 'libuv', + 'toolsets': ['host', 'target'], 'type': '<(uv_library)', 'include_dirs': [ 'include', diff --git a/node.gyp b/node.gyp index b7febacfbc3783..d57ab5f21c575e 100644 --- a/node.gyp +++ b/node.gyp @@ -27,7 +27,7 @@ 'node_lib_target_name%': 'libnode', 'node_intermediate_lib_type%': 'static_library', 'node_builtin_modules_path%': '', - # We list the deps/ files out instead of globbing them in js2c.py since we + # We list the deps/ files out instead of globbing them in js2c.cc since we # only include a subset of all the files under these directories. # The lengths of their file names combined should not exceed the # Windows command length limit or there would be an error. @@ -362,6 +362,7 @@ 'src/quic/transportparams.h', ], 'node_mksnapshot_exec': '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)node_mksnapshot<(EXECUTABLE_SUFFIX)', + 'node_js2c_exec': '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)node_js2c<(EXECUTABLE_SUFFIX)', 'conditions': [ ['GENERATOR == "ninja"', { 'node_text_start_object_path': 'src/large_pages/node_text_start.node_text_start.o' @@ -770,6 +771,7 @@ 'deps/uvwasi/uvwasi.gyp:uvwasi', 'deps/simdutf/simdutf.gyp:simdutf', 'deps/ada/ada.gyp:ada', + 'node_js2c#host', ], 'sources': [ @@ -925,8 +927,7 @@ 'action_name': 'node_js2c', 'process_outputs_as_sources': 1, 'inputs': [ - # Put the code first so it's a dependency and can be used for invocation. - 'tools/js2c.py', + '<(node_js2c_exec)', '<@(library_files)', '<@(deps_files)', 'config.gypi' @@ -935,12 +936,9 @@ '<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc', ], 'action': [ - '<(python)', - 'tools/js2c.py', - '--directory', - 'lib', - '--target', + '<(node_js2c_exec)', '<@(_outputs)', + 'lib', 'config.gypi', '<@(deps_files)', ], @@ -1179,8 +1177,9 @@ { 'target_name': 'node_js2c', 'type': 'executable', + 'toolsets': ['host'], 'dependencies': [ - 'deps/simdutf/simdutf.gyp:simdutf', + 'deps/simdutf/simdutf.gyp:simdutf#host', ], 'include_dirs': [ 'tools' @@ -1191,7 +1190,7 @@ ], 'conditions': [ [ 'node_shared_libuv=="false"', { - 'dependencies': [ 'deps/uv/uv.gyp:libuv' ], + 'dependencies': [ 'deps/uv/uv.gyp:libuv#host' ], }], [ 'debug_node=="true"', { 'cflags!': [ '-O3' ], diff --git a/test/tools/test_js2c.py b/test/tools/test_js2c.py deleted file mode 100644 index 204562117086e4..00000000000000 --- a/test/tools/test_js2c.py +++ /dev/null @@ -1,14 +0,0 @@ -import unittest -import sys, os -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), - '..', '..', 'tools'))) -from js2c import NormalizeFileName - -class Js2ctest(unittest.TestCase): - def testNormalizeFileName(self): - self.assertEqual(NormalizeFileName('dir/mod.js'), 'mod') - self.assertEqual(NormalizeFileName('deps/mod.js'), 'internal/deps/mod') - self.assertEqual(NormalizeFileName('mod.js'), 'mod') - -if __name__ == '__main__': - unittest.main() diff --git a/tools/js2c.py b/tools/js2c.py deleted file mode 100755 index f93312c10b32fe..00000000000000 --- a/tools/js2c.py +++ /dev/null @@ -1,268 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2006-2008 the V8 project authors. All rights reserved. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -""" -This is a utility for converting JavaScript source code into uint16_t[], -that are used for embedding JavaScript code into the Node.js binary. -""" -import argparse -import os -import re -import functools -import codecs -import utils - -def ReadFile(filename): - if is_verbose: - print(filename) - with codecs.open(filename, "r", "utf-8") as f: - lines = f.read() - return lines - - -TEMPLATE = """ -#include "env-inl.h" -#include "node_builtins.h" -#include "node_external_reference.h" -#include "node_internals.h" - -namespace node {{ - -namespace builtins {{ -{0} -namespace {{ -const ThreadsafeCopyOnWrite global_source_map {{ - BuiltinSourceMap {{ -{1} - }} // BuiltinSourceMap -}}; // ThreadsafeCopyOnWrite -}} // anonymous namespace - -void BuiltinLoader::LoadJavaScriptSource() {{ - source_ = global_source_map; -}} - -void RegisterExternalReferencesForInternalizedBuiltinCode( - ExternalReferenceRegistry* registry) {{ -{2} -}} - -UnionBytes BuiltinLoader::GetConfig() {{ - return UnionBytes(&{3}); -}} - -}} // namespace builtins - -}} // namespace node -""" - -ONE_BYTE_STRING = """ -static const uint8_t {0}[] = {{ -{1} -}}; - -static StaticExternalOneByteResource {2}({0}, {3}, nullptr); -""" - -TWO_BYTE_STRING = """ -static const uint16_t {0}[] = {{ -{1} -}}; - -static StaticExternalTwoByteResource {2}({0}, {3}, nullptr); -""" - -INITIALIZER = ' {{"{0}", UnionBytes(&{1}) }},' - -REGISTRATION = ' registry->Register(&{0});' - -CONFIG_GYPI_ID = 'config_raw' - -CONFIG_GYPI_RESOURCE_ID = 'config_resource' - -SLUGGER_RE = re.compile(r'[.\-/]') - -is_verbose = False - -def GetDefinition(var, source, resource_var): - template = ONE_BYTE_STRING - code_points = [ord(c) for c in source] - if any(c > 127 for c in code_points): - template = TWO_BYTE_STRING - # Treat non-ASCII as UTF-8 and encode as UTF-16 Little Endian. - encoded_source = bytearray(source, 'utf-16le') - code_points = [ - encoded_source[i] + (encoded_source[i + 1] * 256) - for i in range(0, len(encoded_source), 2) - ] - - elements_s = ['%s' % x for x in code_points] - array_content = ','.join(elements_s) + ',' - length = len(code_points) - definition = template.format(var, array_content, resource_var, length) - - return definition - - -def AddModule(filename, definitions, initializers, registrations): - code = ReadFile(filename) - name = NormalizeFileName(filename) - slug = SLUGGER_RE.sub('_', name) - var = slug + '_raw' - resource_var = slug + '_resource' - definition = GetDefinition(var, code, resource_var) - initializer = INITIALIZER.format(name, resource_var) - registration = REGISTRATION.format(resource_var) - definitions.append(definition) - initializers.append(initializer) - registrations.append(registration) - -def NormalizeFileName(filename): - split = filename.split('/') - if split[0] == 'deps': - split = ['internal'] + split - else: # `lib/**/*.js` so drop the 'lib' part - split = split[1:] - if len(split): - filename = '/'.join(split) - return os.path.splitext(filename)[0] - - -def JS2C(source_files, target): - # Build source code lines - definitions = [] - initializers = [] - registrations = [] - - for filename in source_files['.js']: - AddModule(filename, definitions, initializers, registrations) - for filename in source_files['.mjs']: - AddModule(filename, definitions, initializers, registrations) - - config_def = handle_config_gypi(source_files['config.gypi']) - definitions.append(config_def) - - # Emit result - definitions = ''.join(definitions) - initializers = '\n'.join(initializers) - registrations = '\n'.join(registrations) - out = TEMPLATE.format(definitions, initializers, - registrations, CONFIG_GYPI_RESOURCE_ID) - write_if_chaged(out, target) - - -def handle_config_gypi(config_filename): - # if its a gypi file we're going to want it as json - # later on anyway, so get it out of the way now - config = ReadFile(config_filename) - config = jsonify(config) - config_def = GetDefinition(CONFIG_GYPI_ID, config, CONFIG_GYPI_RESOURCE_ID) - return config_def - - -def jsonify(config): - # 1. string comments - config = re.sub(r'#.*?\n', '', config) - # 2. join multiline strings - config = re.sub(r"'$\s+'", '', config, flags=re.M) - # 3. normalize string literals from ' into " - config = re.sub('\'', '"', config) - # 4. turn pseudo-booleans strings into Booleans - config = re.sub('"true"', 'true', config) - config = re.sub('"false"', 'false', config) - return config - - -def write_if_chaged(content, target): - if os.path.exists(target): - with open(target, 'rt') as existing: - old_content = existing.read() - else: - old_content = '' - if old_content == content: - os.utime(target, None) - return - with open(target, "wt") as output: - output.write(content) - - -def SourceFileByExt(files_by_ext, filename): - """ - :type files_by_ext: dict - :type filename: str - :rtype: dict - """ - ext = os.path.splitext(filename)[-1] - files_by_ext.setdefault(ext, []).append(filename) - return files_by_ext - -def main(): - parser = argparse.ArgumentParser( - description='Convert code files into `uint16_t[]`s', - fromfile_prefix_chars='@' - ) - parser.add_argument('--target', help='output file') - parser.add_argument( - '--directory', - default=None, - help='input file directory') - parser.add_argument( - '--root', - default=None, - help='root directory containing the sources') - parser.add_argument('--verbose', action='store_true', help='output file') - parser.add_argument('sources', nargs='*', help='input files') - options = parser.parse_args() - global is_verbose - is_verbose = options.verbose - sources = options.sources - - if options.root is not None: - os.chdir(options.root) - - if options.directory is not None: - js_files = utils.SearchFiles(options.directory, 'js') - mjs_files = utils.SearchFiles(options.directory, 'mjs') - sources = js_files + mjs_files + options.sources - - source_files = functools.reduce(SourceFileByExt, sources, {}) - - # Should have exactly 3 types: `.js`, `.mjs` and `.gypi` - assert len(source_files) == 3 - # Currently config.gypi is the only `.gypi` file allowed - assert len(source_files['.gypi']) == 1 - assert os.path.basename(source_files['.gypi'][0]) == 'config.gypi' - source_files['config.gypi'] = source_files.pop('.gypi')[0] - source_files['.js'] = sorted(source_files['.js']) - source_files['.mjs'] = sorted(source_files['.mjs']) - - JS2C(source_files, options.target) - -if __name__ == "__main__": - main()