[GR-69706] Add support to build ESPRESSO_NI_NATIVE_STANDALONE with musl based toolchains

lewurm · lewurm · commit 56eed3f5d39c · 2025-10-02T02:19:48.000Z
PullRequest: graal/22116
diff --git a/espresso-compiler-stub/mx.espresso-compiler-stub/mx_espresso_compiler_stub.py b/espresso-compiler-stub/mx.espresso-compiler-stub/mx_espresso_compiler_stub.py
@@ -79,10 +79,13 @@ def create_ni_standalone(base_standalone_name, register_distribution):
         if _find_native_image_command(espresso_java_home.java_home):
             # ESPRESSO_JAVA_HOME has native-image, keep that
             pass
-        elif has_component('ni') and espresso_java_home.java_home == mx_sdk_vm.base_jdk().home:
+        elif has_component('ni'):
             if graalvm_skip_archive():
                 mx.abort("Cannot build NI standalones with GRAALVM_SKIP_ARCHIVE enabled")
 
+            if espresso_java_home.java_home != mx_sdk_vm.base_jdk(stage1=False).home:
+                mx.abort(f"ESPRESSO_JAVA_HOME(={espresso_java_home.java_home}) must match JAVA_HOME (={mx_sdk_vm.base_jdk(stage1=True).home}) (or FINAL_STAGE_JAVA_HOME (={mx_sdk_vm.base_jdk(stage1=False).home}) if set)")
+
             # substratevm is available and ESPRESSO_JAVA_HOME is JAVA_HOME, use GraalVM
             layout['./'][0]['source_type'] = 'extracted-dependency'
             layout['./'][0]['dependency'] = get_final_graalvm_distribution().qualifiedName()
diff --git a/sdk/mx.sdk/mx_sdk_vm.py b/sdk/mx.sdk/mx_sdk_vm.py
@@ -78,7 +78,8 @@
     ('jvm-3-compiler-threads', [], ['--jvm', '--engine.CompilerThreads=3'], 50),
     ('native-3-compiler-threads', [], ['--native', '--engine.CompilerThreads=3'], 100)
 ]
-_base_jdk = None
+_base_jdk_stage1 = None
+_base_jdk_final = None
 
 
 class AbstractNativeImageConfig(object, metaclass=ABCMeta):
@@ -262,6 +263,7 @@ def __init__(self,
                  has_relative_home=True,
                  jvm_configs=None,
                  extra_native_targets=None,
+                 stage1_only=False,
                  final_stage_only=False):
         """
         :param suite mx.Suite: the suite this component belongs to
@@ -280,6 +282,7 @@ def __init__(self,
                 'priority': -1,  # 0 is invalid; < 0 prepends to the default configs; > 0 appends
             }
         :param extra_native_targets: list of str, enables extra targets in multi-target projects.
+        :param stage1_only: bool, this component should be only included in the stage1 GraalVM, not in final GraalVM
         :param final_stage_only: bool, this component should be only included in the final GraalVM, not in stage1
         :type license_files: list[str]
         :type third_party_license_files: list[str]
@@ -305,6 +308,7 @@ def __init__(self,
         :type extra_installable_qualifiers: list[str] | None
         :type has_relative_home: bool
         :type jvm_configs: list[dict] or None
+        :type stage1_only: bool
         :type final_stage_only: bool
         """
         if dependencies is None:
@@ -340,8 +344,12 @@ def __init__(self,
         self.has_relative_home = has_relative_home
         self.jvm_configs = jvm_configs or []
         self.extra_native_targets = extra_native_targets
+        self.stage1_only = stage1_only
         self.final_stage_only = final_stage_only
 
+        if stage1_only and final_stage_only:
+            mx.abort("{}: Cannot set both `stage1_only` and `final_stage_only`".format(name))
+
         if supported is not None or early_adopter:
             if stability is not None:
                 raise mx.abort("{}: Cannot use `stability` attribute in combination with deprecated `supported` and `early_adopter` attributes".format(name))
@@ -545,16 +553,42 @@ def register_vm_config(config_name, components, suite, dist_name=None, env_file=
 def get_graalvm_hostvm_configs():
     return _graalvm_hostvm_configs
 
+def base_jdk(stage1=True):
+    global _base_jdk_stage1, _base_jdk_final
+
+    if _base_jdk_stage1 is None:
+        _base_jdk_stage1 = mx.get_jdk(tag='default')
+
+    if stage1:
+        return _base_jdk_stage1
+
+    if _base_jdk_final is None:
+        final_stage_java_home = mx.get_env('FINAL_STAGE_JAVA_HOME')
+        if final_stage_java_home:
+            _base_jdk_final = mx.JDKConfig(final_stage_java_home)
 
-def base_jdk():
-    global _base_jdk
-    if _base_jdk is None:
-        _base_jdk = mx.get_jdk(tag='default')
-    return _base_jdk
+            assert _base_jdk_stage1.version == _base_jdk_final.version, f"version of JAVA_HOME ({_base_jdk_stage1.version}) and FINAL_STAGE_JAVA_HOME ({_base_jdk_final.version}) must match"
+            assert _base_jdk_stage1.release_dict['MODULES'] == _base_jdk_final.release_dict['MODULES'], "JAVA_HOME and FINAL_STAGE_JAVA_HOME do not include the same set of modules"
+
+            try:
+                # attempt to initialize JVMCI related attributes
+                jdk_enables_jvmci_by_default(_base_jdk_final)
+            except AssertionError as e:
+                assert "Could not execute" in e.args[0]
+
+                # ._probe_jvmci_info() and .get_modules() need a working java launcher, which might not be the case for FINAL_STAGE_JAVA_HOME.
+                # Copy infos from stage1 JAVA_HOME instead.
+                setattr(_base_jdk_final, '.enables_jvmci_by_default', jdk_enables_jvmci_by_default(_base_jdk_stage1))
+                setattr(_base_jdk_final, '.jvmciThreadsPerNativeLibraryRuntime', get_JVMCIThreadsPerNativeLibraryRuntime(_base_jdk_stage1))
+
+                setattr(_base_jdk_final, '.modules', _base_jdk_stage1.get_modules())
+        else:
+            _base_jdk_final = _base_jdk_stage1
+    return _base_jdk_final
 
 
-def base_jdk_version():
-    return base_jdk().javaCompliance.value
+def base_jdk_version(stage1=True):
+    return base_jdk(stage1=stage1).javaCompliance.value
 
 def get_jdk_version_for_profiles():
     jdk_version = mx.get_jdk().javaCompliance.value
@@ -564,7 +598,9 @@ def _probe_jvmci_info(jdk, attribute_name):
     if not hasattr(jdk, '.enables_jvmci_by_default'):
         out = mx.LinesOutputCapture()
         sink = lambda x: x
-        mx.run([jdk.java, '-XX:+UnlockExperimentalVMOptions', '-XX:+PrintFlagsFinal', '-version'], out=out, err=sink)
+        rc = mx.run([jdk.java, '-XX:+UnlockExperimentalVMOptions', '-XX:+PrintFlagsFinal', '-version'], out=out, err=sink, nonZeroIsFatal=False)
+        if rc != 0:
+            raise AssertionError(f"Could not execute {jdk.java}")
         enableJVMCI = False
         jvmciThreadsPerNativeLibraryRuntime = None
         for line in out.lines:
@@ -599,7 +635,7 @@ def _read_java_base_hashes(jdk):
     """
     hashes = {}
     out = mx.LinesOutputCapture()
-    mx.run([jdk.exe_path('jmod'), 'describe', join(jdk.home, 'jmods', 'java.base.jmod')], out=out)
+    mx.run([base_jdk(stage1=True).exe_path('jmod'), 'describe', join(jdk.home, 'jmods', 'java.base.jmod')], out=out)
     lines = out.lines
     for line in lines:
         if line.startswith('hashes'):
@@ -948,7 +984,7 @@ def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, ignore_dists,
 
                     with open(module_info_java, 'w') as fp:
                         fp.write(module_info)
-                    mx.run([jdk.javac, '-d', module_build_dir,
+                    mx.run([base_jdk(stage1=True).javac, '-d', module_build_dir,
                             '--limit-modules=java.base,' + ','.join(jmd.requires.keys()),
                             '--module-path=' + os.pathsep.join((m.jarpath for m in modules)),
                             module_info_java])
@@ -958,14 +994,14 @@ def jlink_new_jdk(jdk, dst_jdk_dir, module_dists, ignore_dists,
                     if exists(jmd.get_jmod_path()):
                         os.remove(jmd.get_jmod_path())
                     if not use_upgrade_module_path:
-                        mx.run([jdk.javac.replace('javac', 'jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])
+                        mx.run([base_jdk(stage1=True).exe_path('jmod'), 'create', '--class-path=' + module_build_dir, jmd.get_jmod_path()])
 
                 modules.extend(synthetic_modules.keys())
                 module_names = frozenset((m.name for m in modules))
                 all_module_names = frozenset(list(jdk_modules.keys())) | module_names
 
         # Now build the new JDK image with jlink
-        jlink = [jdk.javac.replace('javac', 'jlink')]
+        jlink = [base_jdk(stage1=True).exe_path('jlink')]
         jlink_persist = []
 
         if jdk_enables_jvmci_by_default(jdk):
@@ -1100,6 +1136,8 @@ def get_jmod_path_with_specified_module_info(self, m):
             if "Shared spaces are not supported in this VM" in out.data:
                 # GR-37047: CDS support in darwin-aarch64 jdk11 is missing.
                 assert mx.get_os() == 'darwin' and mx.get_arch() == 'aarch64' and jdk.javaCompliance == '11'
+            elif base_jdk(stage1=True) != jdk:
+                mx.log("Cross build, skip -Xshare.  FINAL_STAGE_JAVA_HOME might not run on the build host")
             else:
                 mx.log(out.data)
                 mx.abort('Error generating CDS shared archive')
@@ -1182,7 +1220,7 @@ def verify_graalvm_configs(suites=None, start_from=None, check_all=False):
                 base_name=mx_sdk_vm_impl._graalvm_base_name,
                 delimiter='_' if dist_name else '',
                 dist_name=dist_name,
-                jdk_version=mx_sdk_vm_impl._src_jdk_version
+                jdk_version=base_jdk_version(stage1=False)
             ).upper().replace('-', '_')
             mx.log("{}Checking that the env file '{}' in suite '{}' produces a GraalVM distribution named '{}'".format('' if started else '[SKIPPED] ', _env_file, suite.name, graalvm_dist_name))
 
diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py
@@ -138,6 +138,8 @@ def add_dependencies(dependencies, excludes=True):
             components = dependencies[:]
             while components:
                 component = components.pop(0)
+                if component.stage1_only and not stage1:
+                    continue
                 if component.final_stage_only and stage1:
                     continue
                 if component not in components_to_build and not (excludes and is_excluded(component)):
@@ -240,18 +242,6 @@ def _get_main_component(components):
     return main_component
 
 
-_src_jdk = mx_sdk_vm.base_jdk()
-_src_jdk_version = mx_sdk_vm.base_jdk_version()
-
-# Example:
-#   macOS:
-#     _src_jdk_dir  = /Library/Java/JavaVirtualMachines/oraclejdk1.8.0_221-jvmci-19.2-b02
-#     _src_jdk_base = Contents/Home
-#   Others:
-#     _src_jdk_dir  = $JAVA_HOME (e.g. /usr/lib/jvm/oraclejdk1.8.0_212-jvmci-19.2-b01)
-#     _src_jdk_base = .
-_src_jdk_dir, _src_jdk_base = _get_jdk_base(_src_jdk)
-
 """:type: dict[str, (str, str)]"""
 _parent_info_cache = {}
 
@@ -282,6 +272,15 @@ def __init__(self, suite, name, deps, components, is_graalvm, exclLibs, platform
         self.stage1 = stage1
         self.skip_archive = stage1 or graalvm_skip_archive()  # *.tar archives for stage1 distributions are never built
 
+        # Example:
+        #   macOS:
+        #     _src_jdk_dir  = /Library/Java/JavaVirtualMachines/oraclejdk1.8.0_221-jvmci-19.2-b02
+        #     _src_jdk_base = Contents/Home
+        #   Others:
+        #     _src_jdk_dir  = $JAVA_HOME (e.g. /usr/lib/jvm/oraclejdk1.8.0_212-jvmci-19.2-b01)
+        #     _src_jdk_base = .
+        _src_jdk_dir, _src_jdk_base = _get_jdk_base(mx_sdk_vm.base_jdk(stage1=stage1))
+
         layout = {}
         src_jdk_base = _src_jdk_base if add_jdk_base else '.'
         assert src_jdk_base
@@ -442,7 +441,7 @@ def _add_link(_dest, _target, _component=None, _dest_base_name=None):
             if stage1:
                 # 1. we do not want a GraalVM to be used as base-JDK
                 # 2. we don't need to check if the base JDK is JVMCI-enabled, since JVMCIVersionCheck takes care of that when the GraalVM compiler is a registered component
-                check_versions(_src_jdk, expect_graalvm=False, check_jvmci=False)
+                check_versions(mx_sdk_vm.base_jdk(stage1=stage1), expect_graalvm=False, check_jvmci=False)
 
             # Add base JDK
             if mx.get_os() == 'darwin':
@@ -679,7 +678,7 @@ def _add_link(_dest, _target, _component=None, _dest_base_name=None):
         _add(layout, '<jre_base>/lib/graalvm/', ['dependency:' + d for d in sorted(graalvm_dists)], with_sources=True)
 
         for _base, _suites in component_suites.items():
-            _metadata = self._get_metadata(_suites)
+            _metadata = self._get_metadata(_suites, stage1=stage1)
             _add(layout, _base + 'release', "string:{}".format(_metadata))
 
         if "archive_factory" not in kw_args and self.skip_archive:
@@ -712,7 +711,7 @@ def needsUpdate(self, newestInput):
             return super(BaseGraalVmLayoutDistribution, self).needsUpdate(newestInput)
 
     @staticmethod
-    def _get_metadata(suites, parent_release_file=None, java_version=None):
+    def _get_metadata(suites, parent_release_file=None, java_version=None, stage1=True):
         """
         :type suites: list[mx.Suite]
         :type parent_release_file: str | None
@@ -735,7 +734,7 @@ def _get_metadata(suites, parent_release_file=None, java_version=None):
         else:
             _metadata_dict = OrderedDict()
 
-        _metadata_dict.setdefault('JAVA_VERSION', java_version or _src_jdk.version)
+        _metadata_dict.setdefault('JAVA_VERSION', java_version or mx_sdk_vm.base_jdk(stage1=stage1).version)
         _metadata_dict.setdefault('OS_NAME', get_graalvm_os())
         _metadata_dict.setdefault('OS_ARCH', mx.get_arch())
         if mx.get_os_variant():
@@ -919,7 +918,7 @@ def _get_graalvm_configuration(base_name, components=None, stage1=False):
             name_prefix = '{base_name}{vm_dist_name}_java{jdk_version}'.format(
                 base_name=base_name,
                 vm_dist_name=('_' + vm_dist_name) if vm_dist_name else '',
-                jdk_version=_src_jdk_version
+                jdk_version=mx_sdk_vm.base_jdk_version(stage1=stage1)
             )
             name = '{name_prefix}{stage_suffix}'.format(name_prefix=name_prefix, stage_suffix='_stage1' if stage1 else '')
         else:
@@ -934,7 +933,7 @@ def _get_graalvm_configuration(base_name, components=None, stage1=False):
             else:
                 m.update("not-jlinked".encode())
             short_sha1_digest = m.hexdigest()[:10]  # to keep paths short
-            base_dir = '{base_name}_{hash}_java{jdk_version}'.format(base_name=base_name, hash=short_sha1_digest, jdk_version=_src_jdk_version)
+            base_dir = '{base_name}_{hash}_java{jdk_version}'.format(base_name=base_name, hash=short_sha1_digest, jdk_version=mx_sdk_vm.base_jdk_version(stage1=stage1))
             name = '{base_dir}{stage_suffix}'.format(base_dir=base_dir, stage_suffix='_stage1' if stage1 else '')
             base_dir += '_' + _suite.release_version()
         name = name.upper()
@@ -1590,11 +1589,12 @@ def with_source(dep):
             return not isinstance(dep, mx.Dependency) or (_include_sources(dep.qualifiedName()) and dep.isJARDistribution() and not dep.is_stripped())
         vendor_info = {'vendor-version': graalvm_vendor_version()}
         out_dir = self.subject.output_directory()
+        stage1 = 'stage1' in self.subject.name
 
         if _jlink_libraries():
             use_upgrade_module_path = mx.get_env('MX_BUILD_EXPLODED') == 'true'
 
-            built = mx_sdk.jlink_new_jdk(_src_jdk,
+            built = mx_sdk.jlink_new_jdk(mx_sdk_vm.base_jdk(stage1=stage1),
                                  out_dir,
                                  self.subject.deps,
                                  self.subject.jimage_ignore_jars,
@@ -1607,12 +1607,12 @@ def with_source(dep):
             mx.warn("--no-jlinking flag used. The resulting VM will be HotSpot, not GraalVM")
             if exists(out_dir):
                 mx.rmtree(out_dir)
-            shutil.copytree(_src_jdk.home, out_dir, symlinks=True)
+            shutil.copytree(mx_sdk_vm.base_jdk(stage1=stage1).home, out_dir, symlinks=True)
             built = True
 
         release_file = join(out_dir, 'release')
         _sorted_suites = sorted(mx.suites(), key=lambda s: s.name)
-        _metadata = BaseGraalVmLayoutDistribution._get_metadata(_sorted_suites, release_file)
+        _metadata = BaseGraalVmLayoutDistribution._get_metadata(_sorted_suites, release_file, stage1=stage1)
         with open(release_file, 'w') as f:
             f.write(_metadata)
 
@@ -1656,7 +1656,7 @@ def _config(self):
         # Save the path and timestamp of the JDK image so that graalvm-jimage
         # is rebuilt if the JDK at JAVA_HOME is rebuilt. The JDK image file is
         # always updated when the JDK is rebuilt.
-        src_jimage = mx.TimeStampFile(join(_src_jdk.home, 'lib', 'modules'))
+        src_jimage = mx.TimeStampFile(join(mx_sdk_vm.base_jdk().home, 'lib', 'modules'))
         return [
             f'include sources: {_include_sources_str()}',
             f'strip jars: {mx.get_opts().strip_jars}',
@@ -2109,7 +2109,7 @@ def graalvm_home_relative_classpath(dependencies, start=None, with_boot_jars=Fal
         if jimage_deps and _jlink_libraries() and _cp_entry in jimage_deps:
             continue
         if _cp_entry.isJdkLibrary() or _cp_entry.isJreLibrary():
-            jdk = _src_jdk
+            jdk = mx_sdk_vm.base_jdk(stage1=graal_vm.stage1)
             if hasattr(_cp_entry, 'jdkStandardizedSince') and jdk.javaCompliance >= _cp_entry.jdkStandardizedSince:
                 continue
             jdk_location = relpath(_cp_entry.classpath_repr(jdk), jdk.home)
@@ -2294,12 +2294,6 @@ def __str__(self):
         return 'Building {}'.format(self.subject.name)
 
 
-def _get_component_stability(component):
-    if _src_jdk_version not in (17, 20):
-        return "experimental"
-    return component.stability
-
-
 def default_jvm_components():
     """
     Components that, for now, must be included in the JVM.
@@ -2312,7 +2306,7 @@ def _get_jvm_cfg():
     candidates = (['lib', 'jvm.cfg'], ['jre', 'lib', 'jvm.cfg'], ['jre', 'lib', mx.get_arch(), 'jvm.cfg'])
     probed = []
     for candidate in candidates:
-        jvm_cfg = join(_src_jdk.home, *candidate)
+        jvm_cfg = join(mx_sdk_vm.base_jdk().home, *candidate)
         if exists(jvm_cfg):
             return jvm_cfg
         probed.append(jvm_cfg)
@@ -2761,6 +2755,7 @@ def register_main_dist(dist, label):
             defaultBuild=False,
             missing_export_target_action='warn',
             default_to_jvmci='lib' if has_lib_graal else False,
+            stage1=False,
         )
         register_project(java_standalone_jimage)
 
@@ -2831,6 +2826,11 @@ def register_main_dist(dist, label):
 
 def _needs_stage1_jimage(stage1_dist, final_dist):
     assert isinstance(stage1_dist.jimage_jars, set) and isinstance(final_dist.jimage_jars, set)
+
+    if mx_sdk_vm.base_jdk(stage1=True) != mx_sdk_vm.base_jdk(stage1=False):
+        # cross-build, always need a stage1_jimage
+        return True
+
     return stage1_dist.jimage_jars != final_dist.jimage_jars
 
 
@@ -3239,7 +3239,7 @@ def graalvm_show(args, forced_graalvm_dist=None):
     print("Config name: {}".format(graalvm_dist.vm_config_name))
     print("Components:")
     for component in sorted(graalvm_dist.components, key=lambda c: c.name):
-        print(" - {} ('{}', /{}, {})".format(component.name, component.short_name, component.dir_name, _get_component_stability(component)))
+        print(" - {} ('{}', /{})".format(component.name, component.short_name, component.dir_name))
 
     if forced_graalvm_dist is None:
         # Custom GraalVM distributions with a forced component list do not yet support launchers and libraries.
@@ -3388,7 +3388,7 @@ def print_graalvm_vm_name(args):
     """Print the VM name of GraalVM"""
     parser = ArgumentParser(prog='mx graalvm-vm-name', description='Print the VM name of GraalVM')
     _ = parser.parse_args(args)
-    print(graalvm_vm_name(get_final_graalvm_distribution(), _src_jdk))
+    print(graalvm_vm_name(get_final_graalvm_distribution(), mx_sdk_vm.base_jdk(stage1=False)))
 
 def graalvm_vm_name(graalvm_dist, jdk):
     """
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterCHelper.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporterCHelper.java
