Skip to content

Commit 42c8b8e

Browse files
MarcusSorealheisMarcusSorealheis
committed
Initial implementation of remote persistent workers
1 parent f380d7d commit 42c8b8e

File tree

18 files changed

+3294
-27
lines changed

18 files changed

+3294
-27
lines changed

‎.bazelrc‎

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ build --aspects=@rules_rust//rust:defs.bzl%rust_clippy_aspect
5656
# Kept in sync with top-level Cargo.toml via generate-bazel-rc
5757
## BEGIN CARGO LINTS ##
5858

59-
build --@rules_rust//:extra_rustc_flag=-Wunused
59+
build --@rules_rust//:extra_rustc_flag=-Dunused
6060
build --@rules_rust//:extra_rustc_flag=-Dambiguous_glob_reexports
6161
build --@rules_rust//:extra_rustc_flag=-Dclashing_extern_declarations
6262
build --@rules_rust//:extra_rustc_flag=-Dconst_item_mutation
@@ -72,49 +72,49 @@ build --@rules_rust//:extra_rustc_flag=-Dinvalid_macro_export_arguments
7272
build --@rules_rust//:extra_rustc_flag=-Dinvalid_nan_comparisons
7373
build --@rules_rust//:extra_rustc_flag=-Dinvalid_reference_casting
7474
build --@rules_rust//:extra_rustc_flag=-Dinvalid_value
75-
build --@rules_rust//:extra_rustc_flag=-Wkeyword_idents
76-
build --@rules_rust//:extra_rustc_flag=-Wlet_underscore
77-
build --@rules_rust//:extra_rustc_flag=-Wmacro_use_extern_crate
78-
build --@rules_rust//:extra_rustc_flag=-Wmeta_variable_misuse
79-
build --@rules_rust//:extra_rustc_flag=-Wmissing_abi
80-
build --@rules_rust//:extra_rustc_flag=-Wmissing_copy_implementations
81-
build --@rules_rust//:extra_rustc_flag=-Wmissing_debug_implementations
82-
build --@rules_rust//:extra_rustc_flag=-Wnoop_method_call
75+
build --@rules_rust//:extra_rustc_flag=-Dkeyword_idents
76+
build --@rules_rust//:extra_rustc_flag=-Dlet_underscore
77+
build --@rules_rust//:extra_rustc_flag=-Dmacro_use_extern_crate
78+
build --@rules_rust//:extra_rustc_flag=-Dmeta_variable_misuse
79+
build --@rules_rust//:extra_rustc_flag=-Dmissing_abi
80+
build --@rules_rust//:extra_rustc_flag=-Dmissing_copy_implementations
81+
build --@rules_rust//:extra_rustc_flag=-Dmissing_debug_implementations
82+
build --@rules_rust//:extra_rustc_flag=-Dnoop_method_call
8383
build --@rules_rust//:extra_rustc_flag=-Dopaque_hidden_inferred_bound
8484
build --@rules_rust//:extra_rustc_flag=-Doverlapping_range_endpoints
85-
build --@rules_rust//:extra_rustc_flag=-Wsingle_use_lifetimes
85+
build --@rules_rust//:extra_rustc_flag=-Dsingle_use_lifetimes
8686
build --@rules_rust//:extra_rustc_flag=-Dsuspicious_double_ref_op
87-
build --@rules_rust//:extra_rustc_flag=-Wtrivial_casts
88-
build --@rules_rust//:extra_rustc_flag=-Wtrivial_numeric_casts
87+
build --@rules_rust//:extra_rustc_flag=-Dtrivial_casts
88+
build --@rules_rust//:extra_rustc_flag=-Dtrivial_numeric_casts
8989
build --@rules_rust//:extra_rustc_flag=-Dunconditional_recursion
9090
build --@rules_rust//:extra_rustc_flag=-Dunexpected_cfgs
9191
build --@rules_rust//:extra_rustc_flag=-Dunnameable_test_items
92-
build --@rules_rust//:extra_rustc_flag=-Wunreachable_pub
92+
build --@rules_rust//:extra_rustc_flag=-Dunreachable_pub
9393
build --@rules_rust//:extra_rustc_flag=-Dunsafe_op_in_unsafe_fn
9494
build --@rules_rust//:extra_rustc_flag=-Dunstable_syntax_pre_expansion
95-
build --@rules_rust//:extra_rustc_flag=-Wunused_import_braces
96-
build --@rules_rust//:extra_rustc_flag=-Wunused_lifetimes
97-
build --@rules_rust//:extra_rustc_flag=-Wunused_qualifications
98-
build --@rules_rust//:extra_rustc_flag=-Wvariant_size_differences
99-
100-
build --@rules_rust//:clippy_flag=-Wclippy::all
101-
build --@rules_rust//:clippy_flag=-Wclippy::nursery
102-
build --@rules_rust//:clippy_flag=-Wclippy::pedantic
95+
build --@rules_rust//:extra_rustc_flag=-Dunused_import_braces
96+
build --@rules_rust//:extra_rustc_flag=-Dunused_lifetimes
97+
build --@rules_rust//:extra_rustc_flag=-Dunused_qualifications
98+
build --@rules_rust//:extra_rustc_flag=-Dvariant_size_differences
99+
100+
build --@rules_rust//:clippy_flag=-Dclippy::all
101+
build --@rules_rust//:clippy_flag=-Dclippy::nursery
102+
build --@rules_rust//:clippy_flag=-Dclippy::pedantic
103103
build --@rules_rust//:clippy_flag=-Dclippy::alloc_instead_of_core
104104
build --@rules_rust//:clippy_flag=-Dclippy::as_underscore
105-
build --@rules_rust//:clippy_flag=-Wclippy::dbg_macro
106-
build --@rules_rust//:clippy_flag=-Wclippy::decimal_literal_representation
105+
build --@rules_rust//:clippy_flag=-Dclippy::dbg_macro
106+
build --@rules_rust//:clippy_flag=-Dclippy::decimal_literal_representation
107107
build --@rules_rust//:clippy_flag=-Aclippy::get_unwrap
108108
build --@rules_rust//:clippy_flag=-Dclippy::missing_const_for_fn
109109
build --@rules_rust//:clippy_flag=-Aclippy::missing_docs_in_private_items
110-
build --@rules_rust//:clippy_flag=-Wclippy::print_stdout
110+
build --@rules_rust//:clippy_flag=-Dclippy::print_stdout
111111
build --@rules_rust//:clippy_flag=-Dclippy::std_instead_of_core
112-
build --@rules_rust//:clippy_flag=-Wclippy::todo
112+
build --@rules_rust//:clippy_flag=-Dclippy::todo
113113
build --@rules_rust//:clippy_flag=-Aclippy::too_long_first_doc_paragraph
114-
build --@rules_rust//:clippy_flag=-Wclippy::unimplemented
114+
build --@rules_rust//:clippy_flag=-Dclippy::unimplemented
115115
build --@rules_rust//:clippy_flag=-Aclippy::unwrap_in_result
116116
build --@rules_rust//:clippy_flag=-Aclippy::unwrap_used
117-
build --@rules_rust//:clippy_flag=-Wclippy::use_debug
117+
build --@rules_rust//:clippy_flag=-Dclippy::use_debug
118118
build --@rules_rust//:clippy_flag=-Aclippy::cast_possible_truncation
119119
build --@rules_rust//:clippy_flag=-Aclippy::cast_possible_wrap
120120
build --@rules_rust//:clippy_flag=-Aclippy::cast_precision_loss

‎deployment-examples/persistent-workers/.bazelrc‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# Bazel configuration for using NativeLink with Remote Persistent Workers
2+
# This configuration is optimized for JVM-based projects (Java, Scala, Kotlin)
3+
# and large TypeScript/JavaScript projects
4+
5+
# Remote execution configuration
6+
build --remote_executor=grpc://localhost:50051
7+
build --remote_cache=grpc://localhost:50051
8+
9+
# Enable remote persistent workers
10+
build --experimental_remote_mark_tool_inputs
11+
build --experimental_remote_persistent_workers
12+
13+
# Instance name for the remote execution service
14+
build --remote_instance_name=main
15+
16+
# Default remote execution properties
17+
build --remote_default_exec_properties=OSFamily=linux
18+
build --remote_default_exec_properties=cpu_count=4
19+
20+
# Java/Scala persistent worker configuration
21+
build:java --strategy=Javac=remote
22+
build:java --strategy=JavaIjar=remote
23+
build:java --strategy=JavaDeployJar=remote
24+
build:java --strategy=JavaSourceJar=remote
25+
build:java --remote_default_exec_properties=persistentWorkerTool=javac
26+
build:java --worker_extra_flag=Javac=--persistent_worker
27+
build:java --worker_max_instances=Javac=5
28+
build:java --worker_quit_after_build=false
29+
30+
# Scala persistent worker configuration
31+
build:scala --strategy=Scalac=remote
32+
build:scala --remote_default_exec_properties=persistentWorkerTool=scalac
33+
build:scala --worker_extra_flag=Scalac=--persistent_worker
34+
build:scala --worker_max_instances=Scalac=3
35+
build:scala --worker_quit_after_build=false
36+
37+
# Kotlin persistent worker configuration
38+
build:kotlin --strategy=KotlinCompile=remote
39+
build:kotlin --remote_default_exec_properties=persistentWorkerTool=kotlinc
40+
build:kotlin --worker_extra_flag=KotlinCompile=--persistent_worker
41+
build:kotlin --worker_max_instances=KotlinCompile=3
42+
build:kotlin --worker_quit_after_build=false
43+
44+
# TypeScript persistent worker configuration
45+
build:typescript --strategy=TypeScriptCompile=remote
46+
build:typescript --remote_default_exec_properties=persistentWorkerTool=tsc
47+
build:typescript --worker_extra_flag=TypeScriptCompile=--persistent_worker
48+
build:typescript --worker_max_instances=TypeScriptCompile=5
49+
build:typescript --worker_quit_after_build=false
50+
51+
# Webpack bundling with persistent workers
52+
build:webpack --strategy=WebpackBundle=remote
53+
build:webpack --remote_default_exec_properties=persistentWorkerTool=webpack
54+
build:webpack --worker_extra_flag=WebpackBundle=--persistent_worker
55+
build:webpack --worker_max_instances=WebpackBundle=2
56+
build:webpack --worker_quit_after_build=false
57+
58+
# General persistent worker settings
59+
build --worker_verbose
60+
build --worker_sandboxing=false # Persistent workers don't support sandboxing
61+
build --worker_multiplex_sandboxing=false
62+
build --experimental_worker_strict_flagfiles
63+
build --experimental_worker_for_repo_fetching=off
64+
65+
# Memory management for JVM workers
66+
build --worker_extra_flag=Javac=-J-Xmx2g
67+
build --worker_extra_flag=Scalac=-J-Xmx4g
68+
build --worker_extra_flag=KotlinCompile=-J-Xmx2g
69+
70+
# Performance optimizations
71+
build --jobs=100
72+
build --remote_download_minimal
73+
build --experimental_remote_cache_compression
74+
build --experimental_remote_cache_async
75+
build --remote_timeout=3600
76+
77+
# Debugging persistent workers (uncomment when needed)
78+
# build --worker_verbose
79+
# build --sandbox_debug
80+
# build --verbose_failures
81+
# build --subcommands=pretty_print
82+
83+
# Platform-specific configurations
84+
build:linux --remote_default_exec_properties=OSFamily=linux
85+
build:linux --remote_default_exec_properties=container-image=docker://gcr.io/nativelink/ubuntu-22.04-java17
86+
87+
build:macos --remote_default_exec_properties=OSFamily=darwin
88+
build:macos --remote_default_exec_properties=arch=x86_64
89+
90+
build:windows --remote_default_exec_properties=OSFamily=windows
91+
build:windows --remote_default_exec_properties=arch=x64
92+
93+
# CI configuration with persistent workers
94+
build:ci --config=java
95+
build:ci --config=remote
96+
build:ci --remote_upload_local_results=true
97+
build:ci --remote_timeout=3600
98+
build:ci --worker_quit_after_build=true # Clean up workers after CI build
99+
100+
# Development configuration
101+
build:dev --config=java
102+
build:dev --worker_quit_after_build=false
103+
build:dev --worker_max_instances=10
104+
build:dev --remote_download_outputs=toplevel

‎deployment-examples/persistent-workers/nativelink-config.json‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
{
2+
"stores": {
3+
"AC_FAST_SLOW_STORE": {
4+
"fast_slow": {
5+
"fast": {
6+
"filesystem": {
7+
"content_path": "/tmp/nativelink/content_path-ac",
8+
"temp_path": "/tmp/nativelink/tmp_path-ac",
9+
"eviction_policy": {
10+
"max_bytes": 5000000000
11+
}
12+
}
13+
},
14+
"slow": {
15+
"noop": {}
16+
}
17+
}
18+
},
19+
"CAS_FAST_SLOW_STORE": {
20+
"fast_slow": {
21+
"fast": {
22+
"filesystem": {
23+
"content_path": "/tmp/nativelink/content_path-cas",
24+
"temp_path": "/tmp/nativelink/tmp_path-cas",
25+
"eviction_policy": {
26+
"max_bytes": 5000000000
27+
}
28+
}
29+
},
30+
"slow": {
31+
"noop": {}
32+
}
33+
}
34+
}
35+
},
36+
"schedulers": {
37+
"MAIN_SCHEDULER": {
38+
"simple": {
39+
"supported_platform_properties": {
40+
"cpu_count": "minimum",
41+
"memory_kb": "minimum",
42+
"network_kbps": "minimum",
43+
"disk_read_iops": "minimum",
44+
"disk_read_bps": "minimum",
45+
"disk_write_iops": "minimum",
46+
"disk_write_bps": "minimum",
47+
"persistentWorkerKey": "exact",
48+
"persistentWorkerTool": "exact",
49+
"OSFamily": "exact",
50+
"container-image": "exact"
51+
},
52+
"worker_timeout_s": 60,
53+
"max_job_retries": 3,
54+
"allocation_strategy": "least_recently_used",
55+
"work_queue_timeout_s": 60,
56+
"worker_pool_config": {
57+
"max_persistent_workers": 20,
58+
"persistent_worker_idle_timeout_s": 300,
59+
"persistent_worker_max_instances_per_key": 5
60+
}
61+
}
62+
}
63+
},
64+
"workers": [
65+
{
66+
"local": {
67+
"worker_api_endpoint": {
68+
"uri": "grpc://127.0.0.1:50061"
69+
},
70+
"entrypoint_cmd": "/nativelink/bin/nativelink_worker_wrapper.sh",
71+
"work_dir": "/tmp/nativelink/work",
72+
"cas_fast_slow_store": "CAS_FAST_SLOW_STORE",
73+
"upload_action_result": {
74+
"ac_store": "AC_FAST_SLOW_STORE"
75+
},
76+
"platform_properties": {
77+
"cpu_count": {
78+
"values": ["16"]
79+
},
80+
"memory_kb": {
81+
"values": ["64000000"]
82+
},
83+
"network_kbps": {
84+
"values": ["100000"]
85+
},
86+
"OSFamily": {
87+
"values": ["linux"]
88+
}
89+
},
90+
"persistent_worker_config": {
91+
"enabled": true,
92+
"runner_mode": true,
93+
"max_worker_instances": 10,
94+
"worker_idle_timeout_s": 300,
95+
"fork_strategy": "on_demand",
96+
"supported_tools": [
97+
{
98+
"tool": "javac",
99+
"max_instances": 5,
100+
"memory_limit_mb": 2048,
101+
"cpu_limit": 2
102+
},
103+
{
104+
"tool": "scalac",
105+
"max_instances": 3,
106+
"memory_limit_mb": 4096,
107+
"cpu_limit": 4
108+
},
109+
{
110+
"tool": "kotlinc",
111+
"max_instances": 3,
112+
"memory_limit_mb": 2048,
113+
"cpu_limit": 2
114+
},
115+
{
116+
"tool": "tsc",
117+
"max_instances": 5,
118+
"memory_limit_mb": 1024,
119+
"cpu_limit": 2
120+
},
121+
{
122+
"tool": "webpack",
123+
"max_instances": 2,
124+
"memory_limit_mb": 4096,
125+
"cpu_limit": 4
126+
}
127+
]
128+
}
129+
}
130+
}
131+
],
132+
"servers": [
133+
{
134+
"name": "public",
135+
"listener": {
136+
"http": {
137+
"socket_address": "0.0.0.0:50051"
138+
}
139+
},
140+
"services": {
141+
"cas": {
142+
"main": {
143+
"cas_store": "CAS_FAST_SLOW_STORE"
144+
}
145+
},
146+
"ac": {
147+
"main": {
148+
"ac_store": "AC_FAST_SLOW_STORE"
149+
}
150+
},
151+
"execution": {
152+
"main": {
153+
"cas_store": "CAS_FAST_SLOW_STORE",
154+
"scheduler": "MAIN_SCHEDULER"
155+
}
156+
},
157+
"capabilities": {
158+
"main": {
159+
"remote_execution": {
160+
"scheduler": "MAIN_SCHEDULER"
161+
}
162+
}
163+
},
164+
"worker_api": {
165+
"scheduler": "MAIN_SCHEDULER"
166+
}
167+
}
168+
},
169+
{
170+
"name": "worker_api",
171+
"listener": {
172+
"http": {
173+
"socket_address": "0.0.0.0:50061"
174+
}
175+
},
176+
"services": {
177+
"worker_api": {
178+
"scheduler": "MAIN_SCHEDULER"
179+
},
180+
"admin": {}
181+
}
182+
}
183+
]
184+
}

0 commit comments

Comments
 (0)