Skip to content

[GR-59050] Switched Truffle gates using native-image to downloaded GraalVM-21. #10786

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 28, 2025
Merged

[GR-59050] Switched Truffle gates using native-image to downloaded GraalVM-21. #10786

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ci/ci_common/common.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ common + common.frequencies + {
labsjdk21:: self["labsjdk-" + repo_config.graalvm_edition + "-21"],
labsjdk21Debug:: self["labsjdk-" + repo_config.graalvm_edition + "-21Debug"],
labsjdk21LLVM:: self["labsjdk-" + repo_config.graalvm_edition + "-21-llvm"],
graalvmee21:: self["graalvm-ee-21"],

labsjdkLatest:: self["labsjdk-" + repo_config.graalvm_edition + "-latest"],
labsjdkLatestDebug:: self["labsjdk-" + repo_config.graalvm_edition + "-latestDebug"],
Expand Down
61 changes: 51 additions & 10 deletions truffle/ci/ci.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,18 +51,25 @@
],
},

local graalVMCELatest = common.labsjdkLatest + common.deps.svm + {
mx_build_graalvm_cmd: ["mx", "-p", "../vm", "--env", "ce", "--native-images=lib:jvmcicompiler"],
run+: [
self.mx_build_graalvm_cmd + ["build"],
["set-export", "JAVA_HOME", self.mx_build_graalvm_cmd + ["--quiet", "--no-warning", "graalvm-home"]]
]
},

local simple_tool_maven_project_gate = truffle_common + {
name: 'gate-external-mvn-simpletool-' + self.jdk_name,
packages+: {
maven: "==3.3.9"
},
mx_cmd: ["mx", "-p", "../vm", "--env", "ce", "--dynamicimports", "/graal-js", "--native-images=none"],
mx_cmd: ["mx", "-p", "../vm", "--dynamicimports", "/graal-js"],
run+: [
["set-export", "ROOT_DIR", ["pwd"]],
self.mx_cmd + ["build"],
["mkdir", "mxbuild/tmp_mvn_repo"],
self.mx_cmd + ["maven-deploy", "--tags=public", "--all-suites", "--all-distribution-types", "--validate=full", "--licenses=EPL-2.0,PSF-License,GPLv2-CPE,ICU,GPLv2,BSD-simplified,BSD-new,UPL,MIT", "--version-string", self.mx_cmd + ["graalvm-version"], "--suppress-javadoc", "local", "file://$ROOT_DIR/mxbuild/tmp_mvn_repo"],
["set-export", "JAVA_HOME", self.mx_cmd + ["--quiet", "--no-warning", "graalvm-home"]],
["cd", "external_repos/"],
["python", "populate.py"],
["cd", "simpletool"],
Expand All @@ -77,13 +84,12 @@
maven: "==3.3.9",
ruby: ">=2.1.0",
},
mx_cmd: ["mx", "-p", "../vm", "--env", "ce", "--native-images=none"],
mx_cmd: ["mx"],
run+: [
["set-export", "ROOT_DIR", ["pwd"]],
self.mx_cmd + ["build"],
["mkdir", "mxbuild/tmp_mvn_repo"],
self.mx_cmd + ["maven-deploy", "--tags=public", "--all-suites", "--all-distribution-types", "--validate=full", "--licenses=EPL-2.0,PSF-License,GPLv2-CPE,ICU,GPLv2,BSD-simplified,BSD-new,UPL,MIT", "--version-string", self.mx_cmd + ["graalvm-version"], "--suppress-javadoc", "local", "file://$ROOT_DIR/mxbuild/tmp_mvn_repo"],
["set-export", "JAVA_HOME", self.mx_cmd + ["--quiet", "--no-warning", "graalvm-home"]],
["cd", "external_repos"],
["python", "populate.py"],
["cd", "simplelanguage"],
Expand All @@ -96,9 +102,31 @@
],
},

local truffle_jvm_gate = truffle_common + {
run+: [
['mx', '--no-jlinking', 'gate', '--no-warning-as-error', '--tags', 'build,truffle-jvm'],
],
notify_groups: ["truffle"],
components+: ["truffle"],
timelimit: '1:15:00',
name: 'gate-truffle-ce-jvm-' + self.jdk_name + '-linux-amd64',
},

local truffle_native_gate = truffle_common + {
gate_tag_suffix: '',
run+: [
['mx', '--no-jlinking', 'gate', '--no-warning-as-error', '--tags', 'build,truffle-native' + self.gate_tag_suffix],
],
notify_groups: ["truffle"],
components+: ["truffle"],
timelimit: '1:00:00',
name: 'gate-truffle-ce-native' + self.gate_tag_suffix + '-jvm-' + self.jdk_name + '-linux-amd64',
},

local truffle_gate = truffle_common + common.deps.eclipse + common.deps.jdt + common.deps.spotbugs {
name: 'gate-truffle-oraclejdk-' + self.jdk_name,
run: [["mx", "--strict-compliance", "gate", "--strict-mode"]],
# The `fulltest` tag includes all Truffle test gate tasks except those that require GraalVM.
run: [["mx", "--strict-compliance", "gate", "--strict-mode", "--tag", "style,fullbuild,fulltest"]],
},

local truffle_weekly = common.weekly + {notify_groups:: ["truffle"]},
Expand All @@ -112,11 +140,24 @@
]) +
[
# The simple_language_maven_project_gate uses native-image, so we must run on labsjdk rather than oraclejdk
linux_amd64 + common.labsjdk21 + simple_language_maven_project_gate,
linux_amd64 + common.labsjdkLatest + simple_language_maven_project_gate,
linux_amd64 + common.graalvmee21 + simple_language_maven_project_gate,
linux_amd64 + graalVMCELatest + simple_language_maven_project_gate,
# The simple_tool_maven_project_gate builds compiler, so we must run on labsjdk rather than oraclejdk because of compiler module rename
linux_amd64 + common.labsjdk21 + simple_tool_maven_project_gate,
linux_amd64 + common.labsjdkLatest + simple_tool_maven_project_gate,
linux_amd64 + common.graalvmee21 + simple_tool_maven_project_gate,
linux_amd64 + graalVMCELatest + simple_tool_maven_project_gate,
# Truffle JVM gate
linux_amd64 + common.graalvmee21 + truffle_jvm_gate,
linux_amd64 + common.oraclejdk23 + truffle_jvm_gate,
linux_amd64 + graalVMCELatest + truffle_jvm_gate,
# Truffle Native gate
linux_amd64 + common.graalvmee21 + truffle_native_gate,
linux_amd64 + common.graalvmee21 + truffle_native_gate + {
gate_tag_suffix: '-quickbuild'
},
linux_amd64 + graalVMCELatest + truffle_native_gate,
linux_amd64 + graalVMCELatest + truffle_native_gate + {
gate_tag_suffix: '-quickbuild'
},

linux_amd64 + common.oraclejdk21 + truffle_gate + guard + {timelimit: "45:00"},
linux_amd64 + common.oraclejdkLatest + truffle_gate + guard + {environment+: {DISABLE_DSL_STATE_BITS_TESTS: "true"}},
Expand Down Expand Up @@ -147,7 +188,7 @@
truffle_common + linux_amd64 + common.oraclejdk21 + common.deps.eclipse + common.deps.jdt + guard + {
name: "weekly-truffle-coverage-21-linux-amd64",
run: [
["mx", "--strict-compliance", "gate", "--strict-mode", "--jacoco-relativize-paths", "--jacoco-omit-src-gen", "--jacocout", "coverage", "--jacoco-format", "lcov"],
["mx", "--strict-compliance", "gate", "--strict-mode", "--jacoco-relativize-paths", "--jacoco-omit-src-gen", "--jacocout", "coverage", "--jacoco-format", "lcov", "build,fulltest"],
],
teardown+: [
["mx", "sversions", "--print-repositories", "--json", "|", "coverage-uploader.py", "--associated-repos", "-"],
Expand Down
163 changes: 93 additions & 70 deletions truffle/mx.truffle/mx_truffle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -398,9 +398,9 @@ def _sl_command(jdk, vm_args, sl_args, use_optimized_runtime=True, use_enterpris
main_class = ["--module", "org.graalvm.sl_launcher/com.oracle.truffle.sl.launcher.SLMain"]

if force_cp:
vm_args += ["--enable-native-access=ALL-UNNAMED"]
vm_args = vm_args + ["--enable-native-access=ALL-UNNAMED"]
else:
vm_args += ["--enable-native-access=org.graalvm.truffle"]
vm_args = vm_args + ["--enable-native-access=org.graalvm.truffle"]
enable_sun_misc_unsafe(vm_args)

return [jdk.java] + jdk.processArgs(vm_args + mx.get_runtime_jvm_args(names=dist_names, force_cp=force_cp) + main_class + sl_args)
Expand Down Expand Up @@ -460,6 +460,9 @@ class TruffleGateTags:
string_test = ['string-test', 'test', 'fulltest']
dsl_max_state_bit_test = ['dsl-max-state-bit-test', 'fulltest']
parser_test = ['parser-test', 'test', 'fulltest']
truffle_jvm = ['truffle-jvm']
truffle_native = ['truffle-native']
truffle_native_quickbuild = ['truffle-native-quickbuild']

def _truffle_gate_runner(args, tasks):
jdk = mx.get_jdk(tag=mx.DEFAULT_JDK_TAG)
Expand All @@ -484,46 +487,92 @@ def _truffle_gate_runner(args, tasks):
_truffle_gate_state_bitwidth_tests()
with Task('Validate parsers', tasks, tags=TruffleGateTags.parser_test) as t:
if t: validate_parsers()
gate_truffle_jvm(tasks)
gate_truffle_native(tasks)
gate_truffle_native(tasks, quickbuild=True)


# Run in vm suite with:
# mx --env ce --native-images=. build
# mx --env ce --native-images=. gate -o -t "Truffle ModulePath Unit Tests Optimized"
def truffle_jvm_module_path_optimized_unit_tests_gate():
_truffle_jvm_module_path_unit_tests_gate()


# Run in vm suite with:
# mx --env ce --native-images=. build
# mx --env ce --native-images=. gate -o -t "Truffle ModulePath Unit Tests Fallback"
def truffle_jvm_module_path_fallback_unit_tests_gate():
_truffle_jvm_module_path_unit_tests_gate(['--disable-truffle-optimized-runtime'])
def gate_truffle_jvm(tasks):
if mx_sdk.GraalVMJDKConfig.is_libgraal_jdk(mx.get_jdk(tag='default').home):
additional_jvm_args = ['-XX:+UnlockExperimentalVMOptions', '-XX:+EnableJVMCI', '-XX:+UseJVMCINativeLibrary', '-XX:-UnlockExperimentalVMOptions']
else:
additional_jvm_args = []
with Task('Truffle ModulePath Unit Tests Optimized', tasks, tags=TruffleGateTags.truffle_jvm) as t:
if t:
truffle_jvm_module_path_optimized_unit_tests_gate(additional_jvm_args)
with Task('Truffle ModulePath Unit Tests Fallback', tasks, tags=TruffleGateTags.truffle_jvm) as t:
if t:
truffle_jvm_module_path_fallback_unit_tests_gate(additional_jvm_args)
with Task('Truffle ClassPath Unit Tests Optimized', tasks, tags=TruffleGateTags.truffle_jvm) as t:
if t:
truffle_jvm_class_path_optimized_unit_tests_gate(additional_jvm_args)
with Task('Truffle ClassPath Unit Tests Fallback', tasks, tags=TruffleGateTags.truffle_jvm) as t:
if t:
truffle_jvm_class_path_fallback_unit_tests_gate(additional_jvm_args)
with Task('Truffle SL JVM', tasks, tags=TruffleGateTags.truffle_jvm) as t:
if t:
sl_jvm_gate_tests(additional_jvm_args)

def gate_truffle_native(tasks, quickbuild=False):
tag = TruffleGateTags.truffle_native_quickbuild if quickbuild else TruffleGateTags.truffle_native
name_suffix = ' with quickbuild' if quickbuild else ''
with Task('Truffle SL Native Fallback' + name_suffix, tasks, tags=tag) as t:
if t:
sl_native_fallback_gate_tests(quickbuild)
with Task('Truffle SL Native Optimized' + name_suffix, tasks, tags=tag) as t:
if t:
sl_native_optimized_gate_tests(quickbuild)
with Task('Truffle API Native Tests' + name_suffix, tasks, tags=tag) as t:
if t:
truffle_native_unit_tests_gate(True, quickbuild)
truffle_native_unit_tests_gate(False, quickbuild)

# Run with:
# mx -p ../vm --env ce build
# export JAVA_HOME=`mx -p ../vm --env ce --quiet --no-warning graalvm-home`
# mx build
# mx gate -o -t "Truffle ModulePath Unit Tests Optimized"
def truffle_jvm_module_path_optimized_unit_tests_gate(additional_jvm_args):
_truffle_jvm_module_path_unit_tests_gate(['--'] + additional_jvm_args)


# Run with:
# mx -p ../vm --env ce build
# export JAVA_HOME=`mx -p ../vm --env ce --quiet --no-warning graalvm-home`
# mx build
# mx gate -o -t "Truffle ModulePath Unit Tests Fallback"
def truffle_jvm_module_path_fallback_unit_tests_gate(additional_jvm_args):
_truffle_jvm_module_path_unit_tests_gate(['--disable-truffle-optimized-runtime'] + ['--'] + additional_jvm_args)


def _truffle_jvm_module_path_unit_tests_gate(additional_unittest_options=None):
if additional_unittest_options is None:
additional_unittest_options = []
unittest(list(['--suite', 'truffle', '--use-graalvm', '--enable-timing', '--verbose', '--max-class-failures=25'] + additional_unittest_options))
unittest(list(['--suite', 'truffle', '--enable-timing', '--verbose', '--max-class-failures=25'] + additional_unittest_options))


# Run in VM suite with:
# mx --env ce --native-images=. build
# mx --env ce --native-images=. gate -o -t "Truffle ClassPath Unit Tests Optimized"
def truffle_jvm_class_path_optimized_unit_tests_gate():
_truffle_jvm_class_path_unit_tests_gate()
# Run with:
# mx -p ../vm --env ce build
# export JAVA_HOME=`mx -p ../vm --env ce --quiet --no-warning graalvm-home`
# mx build
# mx gate -o -t "Truffle ClassPath Unit Tests Optimized"
def truffle_jvm_class_path_optimized_unit_tests_gate(additional_jvm_args):
_truffle_jvm_class_path_unit_tests_gate(['--'] + additional_jvm_args)


# Run in VM suite with:
# mx --env ce --native-images=. build
# mx --env ce --native-images=. gate -o -t "Truffle ClassPath Unit Tests Fallback"
def truffle_jvm_class_path_fallback_unit_tests_gate():
_truffle_jvm_class_path_unit_tests_gate(['--disable-truffle-optimized-runtime'])
# Run with:
# mx -p ../vm --env ce build
# export JAVA_HOME=`mx -p ../vm --env ce --quiet --no-warning graalvm-home`
# mx build
# mx gate -o -t "Truffle ClassPath Unit Tests Fallback"
def truffle_jvm_class_path_fallback_unit_tests_gate(additional_jvm_args):
_truffle_jvm_class_path_unit_tests_gate(['--disable-truffle-optimized-runtime'] + ['--'] + additional_jvm_args)


def _truffle_jvm_class_path_unit_tests_gate(additional_unittest_options=None):
if additional_unittest_options is None:
additional_unittest_options = []
unittest(list(['--suite', 'truffle', '--use-graalvm', '--enable-timing', '--force-classpath', '--verbose',
unittest(list(['--suite', 'truffle', '--enable-timing', '--force-classpath', '--verbose',
'--max-class-failures=25'] + additional_unittest_options))

@mx.command(_suite.name, 'native-truffle-unittest')
Expand Down Expand Up @@ -670,32 +719,31 @@ def _allow_runtime_reflection(for_types):
mx.warn(f'The native Truffle unit test has failed, preserving the working directory at {tmp} for further investigation.')


# Run in VM suite with:
# mx --env ce --native-images=. build
# mx --env ce --native-images=. gate -o -t "Truffle SL JVM"
def sl_jvm_gate_tests():
_sl_jvm_gate_tests(mx.get_jdk(tag='graalvm'), force_cp=False, supports_optimization=True)
_sl_jvm_gate_tests(mx.get_jdk(tag='graalvm'), force_cp=True, supports_optimization=True)

_sl_jvm_gate_tests(mx.get_jdk(tag='default'), force_cp=False, supports_optimization=False)
_sl_jvm_gate_tests(mx.get_jdk(tag='default'), force_cp=True, supports_optimization=False)

_sl_jvm_compiler_on_upgrade_module_path_gate_tests(mx.get_jdk(tag='default'))
# Run with:
# mx -p ../vm --env ce build
# export JAVA_HOME=`mx -p ../vm --env ce --quiet --no-warning graalvm-home`
# mx build
# mx gate -o -t "Truffle SL JVM"
def sl_jvm_gate_tests(additional_vm_args):
jdk = mx.get_jdk(tag='default')
supports_optimization = mx_sdk.GraalVMJDKConfig.is_graalvm(jdk.home) or mx_sdk.GraalVMJDKConfig.is_libgraal_jdk(jdk.home)
_sl_jvm_gate_tests(jdk, additional_vm_args, force_cp=False, supports_optimization=supports_optimization)
_sl_jvm_gate_tests(jdk, additional_vm_args, force_cp=True, supports_optimization=supports_optimization)


def _sl_jvm_gate_tests(jdk, force_cp=False, supports_optimization=True):
def _sl_jvm_gate_tests(jdk, vm_args, force_cp=False, supports_optimization=True):
default_args = []
if not supports_optimization:
default_args += ['--engine.WarnInterpreterOnly=false']

def run_jvm_fallback(test_file):
return _sl_command(jdk, [], [test_file, '--disable-launcher-output', '--engine.WarnInterpreterOnly=false'] + default_args, use_optimized_runtime=False, force_cp=force_cp)
return _sl_command(jdk, vm_args, [test_file, '--disable-launcher-output', '--engine.WarnInterpreterOnly=false'] + default_args, use_optimized_runtime=False, force_cp=force_cp)
def run_jvm_optimized(test_file):
return _sl_command(jdk, [], [test_file, '--disable-launcher-output'] + default_args, use_optimized_runtime=True, force_cp=force_cp)
return _sl_command(jdk, vm_args, [test_file, '--disable-launcher-output'] + default_args, use_optimized_runtime=True, force_cp=force_cp)
def run_jvm_optimized_immediately(test_file):
return _sl_command(jdk, [], [test_file, '--disable-launcher-output', '--engine.CompileImmediately', '--engine.BackgroundCompilation=false'] + default_args, use_optimized_runtime=True, force_cp=force_cp)
return _sl_command(jdk, vm_args, [test_file, '--disable-launcher-output', '--engine.CompileImmediately', '--engine.BackgroundCompilation=false'] + default_args, use_optimized_runtime=True, force_cp=force_cp)
def run_jvmci_disabled(test_file):
return _sl_command(jdk, [], [test_file, '--disable-launcher-output', '--engine.WarnInterpreterOnly=false', '-XX:-EnableJVMCI'] + default_args, use_optimized_runtime=True, force_cp=force_cp)
return _sl_command(jdk, vm_args, [test_file, '--disable-launcher-output', '--engine.WarnInterpreterOnly=false', '-XX:-EnableJVMCI'] + default_args, use_optimized_runtime=True, force_cp=force_cp)

mx.log(f'Run SL JVM Fallback Test on {jdk.home} force_cp={force_cp}')
_run_sl_tests(run_jvm_fallback)
Expand All @@ -713,11 +761,11 @@ def run_jvmci_disabled(test_file):
enterprise = _get_enterprise_truffle()
if enterprise:
def run_jvm_no_enterprise_optimized(test_file):
return _sl_command(jdk, [], [test_file, '--disable-launcher-output'] + default_args, use_optimized_runtime=True, use_enterprise=False, force_cp=force_cp)
return _sl_command(jdk, vm_args, [test_file, '--disable-launcher-output'] + default_args, use_optimized_runtime=True, use_enterprise=False, force_cp=force_cp)
def run_jvm_no_enterprise_optimized_immediately(test_file):
return _sl_command(jdk, [], [test_file, '--disable-launcher-output', '--engine.CompileImmediately', '--engine.BackgroundCompilation=false'] + default_args, use_optimized_runtime=True, use_enterprise=False, force_cp=force_cp)
return _sl_command(jdk, vm_args, [test_file, '--disable-launcher-output', '--engine.CompileImmediately', '--engine.BackgroundCompilation=false'] + default_args, use_optimized_runtime=True, use_enterprise=False, force_cp=force_cp)
def run_jvm_no_enterprise_jvmci_disabled(test_file):
return _sl_command(jdk, [], [test_file, '--disable-launcher-output', '--engine.WarnInterpreterOnly=false', '-XX:-EnableJVMCI'] + default_args, use_optimized_runtime=True, use_enterprise=False, force_cp=force_cp)
return _sl_command(jdk, vm_args, [test_file, '--disable-launcher-output', '--engine.WarnInterpreterOnly=false', '-XX:-EnableJVMCI'] + default_args, use_optimized_runtime=True, use_enterprise=False, force_cp=force_cp)

mx.log(f'Run SL JVM Optimized Test No Truffle Enterprise on {jdk.home} force_cp={force_cp}')
_run_sl_tests(run_jvm_no_enterprise_optimized)
Expand All @@ -730,31 +778,6 @@ def run_jvm_no_enterprise_jvmci_disabled(test_file):
_run_sl_tests(run_jvm_no_enterprise_jvmci_disabled)


def _sl_jvm_compiler_on_upgrade_module_path_gate_tests(jdk):
if mx_sdk.GraalVMJDKConfig.is_graalvm(jdk.home) or mx_sdk.GraalVMJDKConfig.is_libgraal_jdk(jdk.home):
# Ignore tests for Truffle LTS gate using GraalVM as a base JDK
mx.log(f'Ignoring SL JVM Optimized with Compiler on Upgrade Module Path on {jdk.home} because JDK is GraalVM')
return
compiler = mx.distribution('compiler:GRAAL')
vm_args = [
'-XX:+UnlockExperimentalVMOptions',
'-XX:+EnableJVMCI',
'-Djdk.graal.CompilationFailureAction=ExitVM',
f'--upgrade-module-path={compiler.classpath_repr()}',
]

def run_jvm_optimized(test_file):
return _sl_command(jdk, vm_args, [test_file, '--disable-launcher-output'], use_optimized_runtime=True)

def run_jvm_optimized_immediately(test_file):
return _sl_command(jdk, vm_args, [test_file, '--disable-launcher-output', '--engine.CompileImmediately', '--engine.BackgroundCompilation=false'], use_optimized_runtime=True)

mx.log(f'Run SL JVM Optimized Test on {jdk.home} with Compiler on Upgrade Module Path')
_run_sl_tests(run_jvm_optimized)
mx.log(f'Run SL JVM Optimized Immediately Test on {jdk.home} with Compiler on Upgrade Module Path')
_run_sl_tests(run_jvm_optimized_immediately)


# Run in VM suite with:
# mx --env ce --native-images=. build
# mx --env ce --native-images=. gate -o -t "Truffle SL Native Optimized"
Expand Down
Loading
Loading