ops: merge go op-stack build into dockerfile targets, rely on docker …

…for ccross-build caching

.circleci/config.yml

@@ -1081,12 +1081,10 @@ jobs:
           command: |
             IMAGE_BASE_PREFIX="us-docker.pkg.dev/oplabs-tools-artifacts/images"
             # Load from previous docker-build job
-            docker load < "/tmp/workspace/op-stack-go.tar"
             docker load < "/tmp/workspace/op-node.tar"
             docker load < "/tmp/workspace/op-proposer.tar"
             docker load < "/tmp/workspace/op-batcher.tar"
             # rename to the tags that the docker-compose of the devnet expects
-            docker tag "$IMAGE_BASE_PREFIX/op-stack-go:<<pipeline.git.revision>>" "$IMAGE_BASE_PREFIX/op-stack-go:devnet"
             docker tag "$IMAGE_BASE_PREFIX/op-node:<<pipeline.git.revision>>" "$IMAGE_BASE_PREFIX/op-node:devnet"
             docker tag "$IMAGE_BASE_PREFIX/op-proposer:<<pipeline.git.revision>>" "$IMAGE_BASE_PREFIX/op-proposer:devnet"
             docker tag "$IMAGE_BASE_PREFIX/op-batcher:<<pipeline.git.revision>>" "$IMAGE_BASE_PREFIX/op-batcher:devnet"
@@ -1479,46 +1477,35 @@ workflows:
             - op-e2e-WS-tests
             - op-e2e-HTTP-tests
             - op-e2e-ext-geth-tests
-      - docker-build: # just to warm up the cache (other jobs run in parallel)
-          name: op-stack-go-docker-build
-          docker_name: op-stack-go
-          docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          save_image_tag: <<pipeline.git.revision>> # for devnet later
       - docker-build:
           name: op-node-docker-build
           docker_name: op-node
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          requires: ['op-stack-go-docker-build']
           save_image_tag: <<pipeline.git.revision>> # for devnet later
       - docker-build:
           name: op-batcher-docker-build
           docker_name: op-batcher
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          requires: ['op-stack-go-docker-build']
           save_image_tag: <<pipeline.git.revision>> # for devnet later
       - docker-build:
           name: op-program-docker-build
           docker_name: op-program
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          requires: ['op-stack-go-docker-build']
           save_image_tag: <<pipeline.git.revision>> # for devnet later
       - docker-build:
           name: op-proposer-docker-build
           docker_name: op-proposer
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          requires: ['op-stack-go-docker-build']
           save_image_tag: <<pipeline.git.revision>> # for devnet later
       - docker-build:
           name: op-challenger-docker-build
           docker_name: op-challenger
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          requires: ['op-stack-go-docker-build']
           save_image_tag: <<pipeline.git.revision>> # for devnet later
       - docker-build:
           name: op-heartbeat-docker-build
           docker_name: op-heartbeat
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          requires: ['op-stack-go-docker-build']
           save_image_tag: <<pipeline.git.revision>> # for devnet later
       - cannon-prestate:
           requires: ["op-stack-go-lint"]
@@ -1554,18 +1541,6 @@ workflows:
               only: /^(proxyd|chain-mon|indexer|ci-builder|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
             branches:
               ignore: /.*/
-      - docker-build: # just to warm up the cache (other jobs run in parallel)
-          name: op-stack-go-docker-build-release
-          filters:
-            tags:
-              only: /^(proxyd|chain-mon|indexer|ci-builder|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
-            branches:
-              ignore: /.*/
-          docker_name: op-stack-go
-          docker_tags: <<pipeline.git.revision>>
-          platforms: "linux/amd64,linux/arm64"
-          requires:
-            - hold
       - docker-build:
           name: op-heartbeat-release
           filters:
@@ -1575,7 +1550,7 @@ workflows:
               ignore: /.*/
           docker_name: op-heartbeat
           docker_tags: <<pipeline.git.revision>>
-          requires: ['op-stack-go-docker-build-release']
+          requires: ['hold']
           platforms: "linux/amd64,linux/arm64"
           publish: true
           release: true
@@ -1590,7 +1565,7 @@ workflows:
               ignore: /.*/
           docker_name: op-node
           docker_tags: <<pipeline.git.revision>>
-          requires: ['op-stack-go-docker-build-release']
+          requires: ['hold']
           platforms: "linux/amd64,linux/arm64"
           publish: true
           release: true
@@ -1605,7 +1580,7 @@ workflows:
               ignore: /.*/
           docker_name: op-batcher
           docker_tags: <<pipeline.git.revision>>
-          requires: ['op-stack-go-docker-build-release']
+          requires: ['hold']
           platforms: "linux/amd64,linux/arm64"
           publish: true
           release: true
@@ -1620,7 +1595,7 @@ workflows:
               ignore: /.*/
           docker_name: op-proposer
           docker_tags: <<pipeline.git.revision>>
-          requires: ['op-stack-go-docker-build-release']
+          requires: ['hold']
           platforms: "linux/amd64,linux/arm64"
           publish: true
           release: true
@@ -1635,7 +1610,7 @@ workflows:
               ignore: /.*/
           docker_name: op-challenger
           docker_tags: <<pipeline.git.revision>>
-          requires: ['op-stack-go-docker-build-release']
+          requires: ['hold']
           platforms: "linux/amd64,linux/arm64"
           publish: true
           release: true
@@ -1763,19 +1738,10 @@ workflows:
     when:
       equal: [ build_hourly, <<pipeline.schedule.name>> ]
     jobs:
-      - docker-build: # just to warm up the cache (other jobs run in parallel)
-          name: op-stack-go-docker-build-publish
-          docker_name: op-stack-go
-          docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          platforms: "linux/amd64,linux/arm64"
-          context:
-            - oplabs-gcr
-            - slack
       - docker-build:
           name: op-node-docker-publish
           docker_name: op-node
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          requires: [ 'op-stack-go-docker-build-publish' ]
           platforms: "linux/amd64,linux/arm64"
           publish: true
           context:
@@ -1785,7 +1751,6 @@ workflows:
           name: op-batcher-docker-publish
           docker_name: op-batcher
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          requires: [ 'op-stack-go-docker-build-publish' ]
           platforms: "linux/amd64,linux/arm64"
           publish: true
           context:
@@ -1795,7 +1760,6 @@ workflows:
           name: op-program-docker-publish
           docker_name: op-program
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          requires: [ 'op-stack-go-docker-build-publish' ]
           platforms: "linux/amd64,linux/arm64"
           publish: true
           context:
@@ -1805,7 +1769,6 @@ workflows:
           name: op-proposer-docker-publish
           docker_name: op-proposer
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          requires: [ 'op-stack-go-docker-build-publish' ]
           platforms: "linux/amd64,linux/arm64"
           publish: true
           context:
@@ -1815,7 +1778,6 @@ workflows:
           name: op-challenger-docker-publish
           docker_name: op-challenger
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          requires: [ 'op-stack-go-docker-build-publish' ]
           platforms: "linux/amd64,linux/arm64"
           publish: true
           context:
@@ -1825,7 +1787,6 @@ workflows:
           name: op-heartbeat-docker-publish
           docker_name: op-heartbeat
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
-          requires: [ 'op-stack-go-docker-build-publish' ]
           platforms: "linux/amd64,linux/arm64"
           publish: true
           context:

docker-bake.hcl

@@ -14,8 +14,10 @@ variable "GIT_DATE" {
   default = "0"
 }
 
+// The default version to embed in the built images.
+// During CI release builds this is set to <<pipeline.git.tag>>
 variable "GIT_VERSION" {
-  default = "docker"  // original default as set in proxyd file, not used by full go stack, yet
+  default = "v0.0.0"
 }
 
 variable "IMAGE_TAGS" {
@@ -30,95 +32,126 @@ variable "PLATFORMS" {
   default = "linux/amd64"
 }
 
-target "op-stack-go" {
+// Each of the services can have a customized version, but defaults to the global specified version.
+variable "OP_NODE_VERSION" {
+  default = "${GIT_VERSION}"
+}
+
+variable "OP_BATCHER_VERSION" {
+  default = "${GIT_VERSION}"
+}
+
+variable "OP_PROPOSER_VERSION" {
+  default = "${GIT_VERSION}"
+}
+
+variable "OP_CHALLENGER_VERSION" {
+  default = "${GIT_VERSION}"
+}
+
+variable OP_HEARTBEAT_VERSION {
+  default = "${GIT_VERSION}"
+}
+
+variable OP_PROGRAM_VERSION {
+  default = "${GIT_VERSION}"
+}
+
+variable CANNON_VERSION {
+  default = "${GIT_VERSION}"
+}
+
+target "op-node" {
   dockerfile = "ops/docker/op-stack-go/Dockerfile"
   context = "."
   args = {
     GIT_COMMIT = "${GIT_COMMIT}"
     GIT_DATE = "${GIT_DATE}"
+    OP_NODE_VERSION = "${OP_NODE_VERSION}"
   }
-  platforms = split(",", PLATFORMS)
-  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-stack-go:${tag}"]
-}
-
-target "op-node" {
-  dockerfile = "Dockerfile"
-  context = "./op-node"
-  args = {
-    OP_STACK_GO_BUILDER = "op-stack-go"
-  }
-  contexts = {
-    op-stack-go: "target:op-stack-go"
-  }
+  target = "op-node-target"
   platforms = split(",", PLATFORMS)
   tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-node:${tag}"]
 }
 
 target "op-batcher" {
-  dockerfile = "Dockerfile"
-  context = "./op-batcher"
+  dockerfile = "ops/docker/op-stack-go/Dockerfile"
+  context = "."
   args = {
-    OP_STACK_GO_BUILDER = "op-stack-go"
-  }
-  contexts = {
-    op-stack-go: "target:op-stack-go"
+    GIT_COMMIT = "${GIT_COMMIT}"
+    GIT_DATE = "${GIT_DATE}"
+    OP_BATCHER_VERSION = "${OP_BATCHER_VERSION}"
   }
+  target = "op-batcher-target"
   platforms = split(",", PLATFORMS)
   tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-batcher:${tag}"]
 }
 
 target "op-proposer" {
-  dockerfile = "Dockerfile"
-  context = "./op-proposer"
+  dockerfile = "ops/docker/op-stack-go/Dockerfile"
+  context = "."
   args = {
-    OP_STACK_GO_BUILDER = "op-stack-go"
-  }
-  contexts = {
-    op-stack-go: "target:op-stack-go"
+    GIT_COMMIT = "${GIT_COMMIT}"
+    GIT_DATE = "${GIT_DATE}"
+    OP_PROPOSER_VERSION = "${OP_PROPOSER_VERSION}"
   }
+  target = "op-proposer-target"
   platforms = split(",", PLATFORMS)
   tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-proposer:${tag}"]
 }
 
 target "op-challenger" {
-  dockerfile = "Dockerfile"
-  context = "./op-challenger"
+  dockerfile = "ops/docker/op-stack-go/Dockerfile"
+  context = "."
   args = {
-    OP_STACK_GO_BUILDER = "op-stack-go"
-  }
-  contexts = {
-    op-stack-go: "target:op-stack-go"
+    GIT_COMMIT = "${GIT_COMMIT}"
+    GIT_DATE = "${GIT_DATE}"
+    OP_CHALLENGER_VERSION = "${OP_CHALLENGER_VERSION}"
   }
+  target = "op-challenger-target"
   platforms = split(",", PLATFORMS)
   tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-challenger:${tag}"]
 }
 
 target "op-heartbeat" {
-  dockerfile = "Dockerfile"
-  context = "./op-heartbeat"
+  dockerfile = "ops/docker/op-stack-go/Dockerfile"
+  context = "."
   args = {
-    OP_STACK_GO_BUILDER = "op-stack-go"
-  }
-  contexts = {
-    op-stack-go: "target:op-stack-go"
+    GIT_COMMIT = "${GIT_COMMIT}"
+    GIT_DATE = "${GIT_DATE}"
+    OP_HEARTBEAT_VERSION = "${OP_HEARTBEAT_VERSION}"
   }
+  target = "op-heartbeat-target"
   platforms = split(",", PLATFORMS)
   tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-heartbeat:${tag}"]
 }
 
 target "op-program" {
-  dockerfile = "Dockerfile"
-  context = "./op-program"
+  dockerfile = "ops/docker/op-stack-go/Dockerfile"
+  context = "."
   args = {
-    OP_STACK_GO_BUILDER = "op-stack-go"
-  }
-  contexts = {
-    op-stack-go: "target:op-stack-go"
+    GIT_COMMIT = "${GIT_COMMIT}"
+    GIT_DATE = "${GIT_DATE}"
+    OP_PROGRAM_VERSION = "${OP_PROGRAM_VERSION}"
   }
+  target = "op-program-target"
   platforms = split(",", PLATFORMS)
   tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/op-program:${tag}"]
 }
 
+target "cannon" {
+  dockerfile = "ops/docker/op-stack-go/Dockerfile"
+  context = "."
+  args = {
+    GIT_COMMIT = "${GIT_COMMIT}"
+    GIT_DATE = "${GIT_DATE}"
+    CANNON_VERSION = "${CANNON_VERSION}"
+  }
+  target = "cannon-target"
+  platforms = split(",", PLATFORMS)
+  tags = [for tag in split(",", IMAGE_TAGS) : "${REGISTRY}/${REPOSITORY}/cannon:${tag}"]
+}
+
 target "proxyd" {
   dockerfile = "Dockerfile"
   context = "./proxyd"