Bump bb-storage

tomgr · EdSchouten · commit 0ef36fa4ed17 · 2025-08-15T10:21:54.000+02:00
This also switches to pre-built protoc toolchains to improve compilation
time.
diff --git a/.bazelrc b/.bazelrc
@@ -1,2 +1,7 @@
-build --host_cxxopt=-std=c++17
 run --workspace_status_command="bash tools/workspace-status.sh"
+common --enable_platform_specific_config
+common --incompatible_enable_proto_toolchain_resolution
+common --protocopt=--fatal_warnings
+
+# Required to make protobuf compile on Windows
+common:windows --host_cxxopt=/std:c++17 --define=protobuf_allow_msvc=true
diff --git a/.github/workflows/master.yaml b/.github/workflows/master.yaml
diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml
@@ -1,89 +1,136 @@
 {
    "jobs": {
       "build_and_test": {
-         "runs-on": "ubuntu-latest",
+         "name": "build_and_test ${{ matrix.host.os }}",
+         "runs-on": "${{ matrix.host.os }}",
          "steps": [
             {
                "name": "Check out source code",
                "uses": "actions/checkout@v1"
             },
             {
                "name": "Installing Bazel",
-               "run": "v=$(cat .bazelversion) && curl -L https://github.com/bazelbuild/bazel/releases/download/${v}/bazel-${v}-linux-x86_64 > ~/bazel && chmod +x ~/bazel && echo ~ >> ${GITHUB_PATH}"
+               "run": "v=$(cat .bazelversion) && curl -L https://github.com/bazelbuild/bazel/releases/download/${v}/bazel-${v}-${{matrix.host.bazel_os}}-x86_64 > ~/bazel && chmod +x ~/bazel && echo ~ >> ${GITHUB_PATH}",
+               "shell": "bash"
             },
             {
+               "if": "matrix.host.platform_name == 'windows_amd64'",
+               "name": "Override .bazelrc",
+               "run": "echo \"startup --output_base=D:/bazel_output\" >> .bazelrc"
+            },
+            {
+               "if": "matrix.host.lint",
                "name": "Bazel mod tidy",
                "run": "bazel mod tidy"
             },
             {
+               "if": "matrix.host.lint",
                "name": "Gazelle",
                "run": "rm -f $(find . -name '*.pb.go' | sed -e 's/[^/]*$/BUILD.bazel/') && bazel run //:gazelle"
             },
             {
+               "if": "matrix.host.lint",
                "name": "Buildifier",
                "run": "bazel run @com_github_bazelbuild_buildtools//:buildifier"
             },
             {
+               "if": "matrix.host.lint",
                "name": "Gofmt",
                "run": "bazel run @cc_mvdan_gofumpt//:gofumpt -- -w -extra $(pwd)"
             },
             {
+               "if": "matrix.host.lint",
                "name": "Clang format",
                "run": "find . -name '*.proto' -exec bazel run @llvm_toolchain_llvm//:bin/clang-format -- -i {} +"
             },
             {
+               "if": "matrix.host.lint",
                "name": "GitHub workflows",
                "run": "bazel build //tools/github_workflows && cp bazel-bin/tools/github_workflows/*.yaml .github/workflows"
             },
             {
+               "if": "matrix.host.lint",
                "name": "Protobuf generation",
                "run": "if [ -d pkg/proto ]; then\n  find . bazel-bin/pkg/proto -name '*.pb.go' -delete || true\n  bazel build $(bazel query --output=label 'kind(\"go_proto_library\", //...)')\n  find bazel-bin/pkg/proto -name '*.pb.go' | while read f; do\n    cat $f > $(echo $f | sed -e 's|.*/pkg/proto/|pkg/proto/|')\n  done\nfi\n"
             },
             {
+               "if": "matrix.host.lint",
                "name": "Embedded asset generation",
                "run": "bazel build $(git grep '^[[:space:]]*//go:embed ' | sed -e 's|\\(.*\\)/.*//go:embed |//\\1:|; s|\"||g; s| .*||' | sort -u)\ngit grep '^[[:space:]]*//go:embed ' | sed -e 's|\\(.*\\)/.*//go:embed |\\1/|' | while read o; do\n  if [ -e \"bazel-bin/$o\" ]; then\n    rm -rf \"$o\"\n    cp -r \"bazel-bin/$o\" \"$o\"\n    find \"$o\" -type f -exec chmod -x {} +\n  fi\ndone\n"
             },
             {
+               "if": "matrix.host.lint",
                "name": "Test style conformance",
                "run": "git add . && git diff --exit-code HEAD --"
             },
             {
+               "if": "matrix.host.lint",
                "name": "Golint",
                "run": "bazel run @org_golang_x_lint//golint -- -set_exit_status $(pwd)/..."
             },
             {
-               "name": "linux_amd64: build and test",
-               "run": "bazel test --test_output=errors --platforms=@rules_go//go/toolchain:linux_amd64 //..."
+               "if": "matrix.host.cross_compile || matrix.host.platform_name == 'linux_amd64'",
+               "name": "linux_amd64: build${{ matrix.host.platform_name == 'linux_amd64' && ' and test' || '' }}",
+               "run": "bazel ${{ matrix.host.platform_name == 'linux_amd64' && 'test --test_output=errors' || 'build' }} --platforms=@rules_go//go/toolchain:linux_amd64 //..."
             },
             {
-               "name": "linux_386: build and test",
-               "run": "bazel test --test_output=errors --platforms=@rules_go//go/toolchain:linux_386 //..."
+               "if": "matrix.host.cross_compile || matrix.host.platform_name == 'linux_386'",
+               "name": "linux_386: build${{ matrix.host.platform_name == 'linux_amd64' && ' and test' || '' }}",
+               "run": "bazel ${{ matrix.host.platform_name == 'linux_amd64' && 'test --test_output=errors' || 'build' }} --platforms=@rules_go//go/toolchain:linux_386 //..."
             },
             {
-               "name": "linux_arm: build and test",
-               "run": "bazel build --platforms=@rules_go//go/toolchain:linux_arm //..."
+               "if": "matrix.host.cross_compile || matrix.host.platform_name == 'linux_arm'",
+               "name": "linux_arm: build${{ matrix.host.platform_name == 'linux_arm' && ' and test' || '' }}",
+               "run": "bazel ${{ matrix.host.platform_name == 'linux_arm' && 'test --test_output=errors' || 'build' }} --platforms=@rules_go//go/toolchain:linux_arm //..."
             },
             {
-               "name": "linux_arm64: build and test",
-               "run": "bazel build --platforms=@rules_go//go/toolchain:linux_arm64 //..."
+               "if": "matrix.host.cross_compile || matrix.host.platform_name == 'linux_arm64'",
+               "name": "linux_arm64: build${{ matrix.host.platform_name == 'linux_arm64' && ' and test' || '' }}",
+               "run": "bazel ${{ matrix.host.platform_name == 'linux_arm64' && 'test --test_output=errors' || 'build' }} --platforms=@rules_go//go/toolchain:linux_arm64 //..."
             },
             {
-               "name": "darwin_amd64: build and test",
-               "run": "bazel build --platforms=@rules_go//go/toolchain:darwin_amd64 //..."
+               "if": "matrix.host.cross_compile || matrix.host.platform_name == 'darwin_amd64'",
+               "name": "darwin_amd64: build${{ matrix.host.platform_name == 'darwin_amd64' && ' and test' || '' }}",
+               "run": "bazel ${{ matrix.host.platform_name == 'darwin_amd64' && 'test --test_output=errors' || 'build' }} --platforms=@rules_go//go/toolchain:darwin_amd64 //..."
             },
             {
-               "name": "darwin_arm64: build and test",
-               "run": "bazel build --platforms=@rules_go//go/toolchain:darwin_arm64 //..."
+               "if": "matrix.host.cross_compile || matrix.host.platform_name == 'darwin_arm64'",
+               "name": "darwin_arm64: build${{ matrix.host.platform_name == 'darwin_arm64' && ' and test' || '' }}",
+               "run": "bazel ${{ matrix.host.platform_name == 'darwin_arm64' && 'test --test_output=errors' || 'build' }} --platforms=@rules_go//go/toolchain:darwin_arm64 //..."
             },
             {
-               "name": "freebsd_amd64: build and test",
-               "run": "bazel build --platforms=@rules_go//go/toolchain:freebsd_amd64 //cmd/bb_noop_worker //cmd/bb_runner //cmd/bb_scheduler //cmd/bb_virtual_tmp //cmd/bb_worker //cmd/fake_python //cmd/fake_xcrun"
+               "if": "matrix.host.cross_compile || matrix.host.platform_name == 'freebsd_amd64'",
+               "name": "freebsd_amd64: build${{ matrix.host.platform_name == 'freebsd_amd64' && ' and test' || '' }}",
+               "run": "bazel ${{ matrix.host.platform_name == 'freebsd_amd64' && 'test --test_output=errors' || 'build' }} --platforms=@rules_go//go/toolchain:freebsd_amd64 //..."
             },
             {
-               "name": "windows_amd64: build and test",
-               "run": "bazel build --platforms=@rules_go//go/toolchain:windows_amd64 //cmd/bb_noop_worker //cmd/bb_runner //cmd/bb_scheduler //cmd/bb_virtual_tmp //cmd/bb_worker //cmd/fake_python //cmd/fake_xcrun"
+               "if": "matrix.host.cross_compile || matrix.host.platform_name == 'windows_amd64'",
+               "name": "windows_amd64: build${{ matrix.host.platform_name == 'windows_amd64' && ' and test' || '' }}",
+               "run": "bazel ${{ matrix.host.platform_name == 'windows_amd64' && 'test --test_output=errors' || 'build' }} --platforms=@rules_go//go/toolchain:windows_amd64 //..."
             }
-         ]
+         ],
+         "strategy": {
+            "matrix": {
+               "host": [
+                  {
+                     "bazel_os": "linux",
+                     "cross_compile": true,
+                     "lint": true,
+                     "os": "ubuntu-latest",
+                     "platform_name": "linux_amd64",
+                     "upload": true
+                  },
+                  {
+                     "bazel_os": "windows",
+                     "cross_compile": false,
+                     "lint": false,
+                     "os": "windows-latest",
+                     "platform_name": "windows_amd64",
+                     "upload": false
+                  }
+               ]
+            }
+         }
       }
    },
    "name": "pull-requests",
diff --git a/MODULE.bazel b/MODULE.bazel
@@ -7,6 +7,8 @@ bazel_dep(name = "com_github_buildbarn_bb_storage")
 bazel_dep(name = "com_github_buildbarn_go_xdr")
 bazel_dep(name = "gazelle", version = "0.45.0")
 bazel_dep(name = "googleapis", version = "0.0.0-20250703-f9d6fe4a")
+bazel_dep(name = "platforms", version = "1.0.0")
+bazel_dep(name = "toolchains_protoc", version = "0.4.3")  # must come BEFORE protobuf so the toolchain registration wins
 bazel_dep(name = "protobuf", version = "31.1")
 bazel_dep(name = "rules_go", version = "0.56.1")
 bazel_dep(name = "rules_jsonnet", version = "0.7.2")
@@ -23,7 +25,7 @@ git_override(
 
 git_override(
     module_name = "com_github_buildbarn_bb_storage",
-    commit = "48ed13203b8710ffdde5cc4441d7e58033a76957",
+    commit = "139cf51268e0870b937633dd8b7fb255225ad9c8",
     remote = "https://github.com/buildbarn/bb-storage.git",
 )
 
@@ -48,6 +50,10 @@ single_version_override(
     patches = ["//:patches/gazelle/dont-flatten-srcs.diff"],
 )
 
+# Match protobuf above, until it ships a pre-built binary toolchain itself
+protoc = use_extension("@toolchains_protoc//protoc:extensions.bzl", "protoc")
+protoc.toolchain(version = "v31.1")
+
 go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
 go_deps.from_file(go_mod = "//:go.mod")
 use_repo(
diff --git a/cmd/bb_runner/BUILD.bazel b/cmd/bb_runner/BUILD.bazel
@@ -90,6 +90,7 @@ oci_image(
     platform_transition_filegroup(
         name = "bb_runner_installer_image_" + arch,
         srcs = ["bb_runner_installer_image"],
+        tags = ["manual"],
         target_platform = "@rules_go//go/toolchain:linux_" + arch,
     )
     for arch in [
@@ -107,6 +108,7 @@ oci_image_index(
             "arm64",
         ]
     ],
+    tags = ["manual"],
     visibility = ["//visibility:public"],
 )
 
diff --git a/cmd/bb_scheduler/BUILD.bazel b/cmd/bb_scheduler/BUILD.bazel
@@ -1,3 +1,5 @@
+load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup")
+load("@aspect_rules_js//js:defs.bzl", "js_run_binary")
 load("@com_github_buildbarn_bb_remote_execution_npm//:purgecss/package_json.bzl", purgecss_bin = "bin")
 load("@com_github_buildbarn_bb_storage//tools:container.bzl", "container_push_official", "multiarch_go_image")
 load("@rules_go//go:def.bzl", "go_binary", "go_library")
@@ -8,8 +10,9 @@ go_library(
         "build_queue_state_service.go",
         "main.go",
     ],
+    # keep
     embedsrcs = [
-        "stylesheet.css",
+        ":stylesheet",
         "templates/error.html",
         "templates/footer.html",
         "templates/get_build_queue_state.html",
@@ -59,9 +62,11 @@ go_library(
     ],
 )
 
+templates = glob(["templates/*.html"])
+
 filegroup(
     name = "templates",
-    srcs = glob(["templates/*.html"]),
+    srcs = templates,
 )
 
 purgecss_bin.purgecss_binary(
@@ -70,16 +75,37 @@ purgecss_bin.purgecss_binary(
 )
 
 # Create a copy of Bootstrap that only contains the style attributes
-# used by the HTML templates.
-genrule(
-    name = "stylesheet",
+# used by the HTML template.
+js_run_binary(
+    name = "stylesheet_platform",
     srcs = [
         "@com_github_twbs_bootstrap//:css/bootstrap.min.css",
-        ":templates",
-    ],
+    ] + templates,
     outs = ["stylesheet.css"],
-    cmd = "BAZEL_BINDIR=$(BINDIR) $(location :purgecss) --css $${PWD}/$(location @com_github_twbs_bootstrap//:css/bootstrap.min.css) --content $$(for i in $(locations templates); do echo $${PWD}/$${i}; done) --output $${PWD}/$@",
-    tools = [":purgecss"],
+    args = [
+        # js_run_binary runs in the output dir;
+        # see https://github.com/aspect-build/rules_js/blob/main/docs/migrate.md#account-for-change-to-working-directory
+        "--css",
+        "../../../$(location @com_github_twbs_bootstrap//:css/bootstrap.min.css)",
+        "--output",
+        "../../../$(location stylesheet.css)",
+        "--content",
+    ] + [
+        "../../../$(location %s)" % f
+        for f in templates
+    ],
+    copy_srcs_to_bin = False,
+    tags = ["manual"],
+    tool = ":purgecss",
+)
+
+# Work around https://github.com/aspect-build/rules_js/issues/2121 by always
+# building the stylesheet for the host platform, even if we are cross
+# compiling.
+platform_transition_filegroup(
+    name = "stylesheet",
+    srcs = [":stylesheet_platform"],
+    target_platform = "@platforms//host",
 )
 
 go_binary(
diff --git a/go.mod b/go.mod
@@ -12,7 +12,7 @@ require (
 	cloud.google.com/go/longrunning v0.6.7
 	github.com/bazelbuild/buildtools v0.0.0-20250715102656-62b9413b08bb
 	github.com/bazelbuild/remote-apis v0.0.0-20250728120203-e94a7ece2a1e
-	github.com/buildbarn/bb-storage v0.0.0-20250811173226-48ed13203b87
+	github.com/buildbarn/bb-storage v0.0.0-20250814113139-139cf51268e0
 	github.com/buildbarn/go-xdr v0.0.0-20240702182809-236788cf9e89
 	github.com/golang/protobuf v1.5.4
 	github.com/google/uuid v1.6.0
diff --git a/go.sum b/go.sum
@@ -75,8 +75,8 @@ github.com/bazelbuild/remote-apis v0.0.0-20250728120203-e94a7ece2a1e/go.mod h1:/
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/buildbarn/bb-storage v0.0.0-20250811173226-48ed13203b87 h1:A/kPfCDwlhNpPEJ6qnhPdPmMdeTzlschpM97c76dEik=
-github.com/buildbarn/bb-storage v0.0.0-20250811173226-48ed13203b87/go.mod h1:ro83180H9FprjjO6nTjBfhvke3edvWtsLuZkfSSMgpo=
+github.com/buildbarn/bb-storage v0.0.0-20250814113139-139cf51268e0 h1:S6iMu1p4Wv7B8nqnowOLeiY8CTW0v9D1zxpH4L2bJmk=
+github.com/buildbarn/bb-storage v0.0.0-20250814113139-139cf51268e0/go.mod h1:ro83180H9FprjjO6nTjBfhvke3edvWtsLuZkfSSMgpo=
 github.com/buildbarn/go-sha256tree v0.0.0-20250310211320-0f70f20e855b h1:IKUxixGBm9UxobU7c248z0BF0ojG19uoSLz8MFZM/KA=
 github.com/buildbarn/go-sha256tree v0.0.0-20250310211320-0f70f20e855b/go.mod h1:e7g3/yWApcg+PpDqd4eQEEV8pexQmfCgK3frP+1Wuvk=
 github.com/buildbarn/go-xdr v0.0.0-20240702182809-236788cf9e89 h1:Wtpgk4CIkoEJ7Qx3BwjaMp3TOVv834heqyCC9jMKStM=
diff --git a/pkg/runner/local_runner_test.go b/pkg/runner/local_runner_test.go
@@ -49,7 +49,7 @@ func TestLocalRunnerCheckReadiness(t *testing.T) {
 		_, err := runner.CheckReadiness(ctx, &runner_pb.CheckReadinessRequest{
 			Path: "does_not_exist",
 		})
-		testutil.RequirePrefixedStatus(t, status.Error(codes.Internal, "Failed to check existence of path \"does_not_exist\" in build directory: no such file or directory"), err)
+		testutil.RequirePrefixedStatus(t, status.Error(codes.Internal, "Failed to check existence of path \"does_not_exist\" in build directory: "), err)
 	})
 
 	t.Run("NonExistentDirectory", func(t *testing.T) {
@@ -59,7 +59,7 @@ func TestLocalRunnerCheckReadiness(t *testing.T) {
 		_, err := runner.CheckReadiness(ctx, &runner_pb.CheckReadinessRequest{
 			Path: "does/not/exist",
 		})
-		testutil.RequirePrefixedStatus(t, status.Error(codes.Internal, "Failed to resolve path \"does/not/exist\" in build directory: no such file or directory"), err)
+		testutil.RequirePrefixedStatus(t, status.Error(codes.Internal, "Failed to resolve path \"does/not/exist\" in build directory: "), err)
 	})
 
 	t.Run("Success", func(t *testing.T) {
@@ -90,7 +90,7 @@ func TestLocalRunnerRun(t *testing.T) {
 	defer buildDirectory.Close()
 
 	buildDirectoryPathBuilder, scopeWalker := path.EmptyBuilder.Join(path.VoidScopeWalker)
-	require.NoError(t, path.Resolve(path.UNIXFormat.NewParser(buildDirectoryPath), scopeWalker))
+	require.NoError(t, path.Resolve(path.LocalFormat.NewParser(buildDirectoryPath), scopeWalker))
 
 	var cmdPath string
 	var getEnvCommand []string

Original file line number	Diff line number	Diff line change
`@@ -90,6 +90,7 @@ oci_image(`
`90`	`90`	`platform_transition_filegroup(`
`91`	`91`	`name = "bb_runner_installer_image_" + arch,`
`92`	`92`	`srcs = ["bb_runner_installer_image"],`
	`93`	`+ tags = ["manual"],`
`93`	`94`	`target_platform = "@rules_go//go/toolchain:linux_" + arch,`
`94`	`95`	`)`
`95`	`96`	`for arch in [`
`@@ -107,6 +108,7 @@ oci_image_index(`
`107`	`108`	`"arm64",`
`108`	`109`	`]`
`109`	`110`	`],`
	`111`	`+ tags = ["manual"],`
`110`	`112`	`visibility = ["//visibility:public"],`
`111`	`113`	`)`
`112`	`114`