Merge pull request MaterializeInc#24069 from def-/pr-buildkite-trim-b…

…uild

ci: Trim unnecessary builds

ci/mkpipeline.py

@@ -35,6 +35,7 @@
 from materialize.ci_util.trim_pipeline import permit_rerunning_successful_steps
 from materialize.mzcompose.composition import Composition
 from materialize.ui import UIError
+from materialize.xcompile import Arch
 
 from .deploy.deploy_util import rust_version
 
@@ -137,6 +138,8 @@ def visit(step: dict[str, Any]) -> None:
 
     check_depends_on(pipeline, args.pipeline)
 
+    trim_builds(pipeline, args.coverage)
+
     # Remove the Materialize-specific keys from the configuration that are
     # only used to inform how to trim the pipeline and for coverage runs.
     def visit(step: dict[str, Any]) -> None:
@@ -390,6 +393,34 @@ def visit(step: PipelineStep) -> None:
     ]
 
 
+def trim_builds(pipeline: Any, coverage: bool) -> None:
+    """Trim unnecessary x86-64/aarch64 builds if all artifacts already exist."""
+
+    def builds_published(arch: Arch) -> bool:
+        repo = mzbuild.Repository(Path("."), arch=arch, coverage=coverage)
+        deps_publish = repo.resolve_dependencies(
+            image for image in repo if image.publish
+        )
+        return deps_publish.check()
+
+    def visit(step: dict[str, Any]) -> None:
+        if step.get("id") == "build-x86_64":
+            if builds_published(Arch.X86_64):
+                step["skip"] = True
+        if step.get("id") == "build-aarch64":
+            branch = os.environ["BUILDKITE_BRANCH"]
+            if (
+                branch == "main" or branch.startswith("v") and "." in branch
+            ) and builds_published(Arch.AARCH64):
+                step["skip"] = True
+
+    for step in pipeline["steps"]:
+        visit(step)
+        if "group" in step:
+            for inner_step in step.get("steps", []):
+                visit(inner_step)
+
+
 def have_paths_changed(globs: Iterable[str]) -> bool:
     """Reports whether the specified globs have diverged from origin/main."""
     diff = subprocess.run(

misc/python/materialize/mzbuild.py

@@ -780,6 +780,11 @@ def ensure(self, post_build: Callable[[ResolvedImage], None] | None = None):
             if returncode:
                 raise subprocess.CalledProcessError(returncode, push.args)
 
+    def check(self) -> bool:
+        """Check all publishable images in this dependency set exist on Docker
+        Hub. Don't try to download or build them."""
+        return all(dep.is_published_if_necessary() for dep in self)
+
     def __iter__(self) -> Iterator[ResolvedImage]:
         return iter(self._dependencies.values())