From 344b6d790f24eec21816ec4a038eb78b53e4b31a Mon Sep 17 00:00:00 2001 From: Andrea Frittoli Date: Tue, 17 Dec 2024 12:38:16 +0000 Subject: [PATCH] Update the release to use ghcr.io Start using ghcr.io (as all other Tekton projects) for nightly and full releases. Update the release tasks and pipeline to use remote resolution as well, to align to what other projects do and simplify running releases. Update the docs accordingly. Signed-off-by: Andrea Frittoli --- tekton/build-publish-images-manifests.yaml | 63 ++++++++--- tekton/operator-release-pipeline.yaml | 115 ++++++++++++++++++--- tekton/release-cheat-sheet.md | 53 ++++------ tekton/setup-release-from-scratch.md | 37 ++----- 4 files changed, 181 insertions(+), 87 deletions(-) diff --git a/tekton/build-publish-images-manifests.yaml b/tekton/build-publish-images-manifests.yaml index 5913f18104..8cc820a853 100644 --- a/tekton/build-publish-images-manifests.yaml +++ b/tekton/build-publish-images-manifests.yaml @@ -10,6 +10,9 @@ spec: - name: images description: List of cmd//* paths to be published as images default: "operator webhook proxy-webhook" + - name: koExtraArgs + description: Extra args to be passed to ko + default: "--preserve-import-paths" - name: versionTag description: The vX.Y.Z version that the artifacts should be tagged with (including `v`) - name: imageRegistry @@ -20,6 +23,9 @@ spec: - name: imageRegistryRegions description: The target image registry regions default: "us eu asia" + - name: imageRegistryUser + description: Username to be used to login to the container registry + default: "_json_key" - name: releaseAsLatest description: Whether to tag and publish this release as Pipelines' latest default: "true" @@ -54,6 +60,8 @@ spec: value: "$(workspaces.release-secret.path)/$(params.serviceAccountPath)" - name: CONTAINER_REGISTRY value: "$(params.imageRegistry)/$(params.imageRegistryPath)" + - name: CONTAINER_REGISTRY_USER + value: "$(params.imageRegistryUser)" - name: REGIONS value: "$(params.imageRegistryRegions)" - name: OUTPUT_RELEASE_DIR @@ -62,17 +70,19 @@ spec: value: "$(params.kubeDistro)" - name: FILENAME_PREFIX value: "$(params.releaseFilePrefix)" + - name: KO_EXTRA_ARGS + value: "$(params.koExtraArgs)" steps: - name: container-registy-auth - image: gcr.io/go-containerregistry/crane:debug@sha256:ff0e08eeae8097d28b2381c7f7123bf542757abc68d11bff58fb882b72843785 + image: cgr.dev/chainguard/crane:latest-dev@sha256:6fc6fcdeb173c7951f038e6a7b230f586c1be05a011d9e6f9db6c614ec412c2f script: | #!/busybox/sh set -ex # Login to the container registry - DOCKER_CONFIG=$(cat ${CONTAINER_REGISTY_CREDENTIALS} | \ - crane auth login -u _json_key --password-stdin $(params.imageRegistry) 2>&1 | \ + DOCKER_CONFIG=$(cat ${CONTAINER_REGISTRY_CREDENTIALS} | \ + crane auth login -u ${CONTAINER_REGISTRY_USER} --password-stdin $(params.imageRegistry) 2>&1 | \ sed 's,^.*logged in via \(.*\)$,\1,g') # Auth with account credentials for all regions. @@ -127,31 +137,62 @@ spec: # Publish images and create release.yaml mkdir -p $OUTPUT_RELEASE_DIR - kustomize build ${PROJECT_ROOT}/config/${KUBE_DISTRO}/overlays/default | ko resolve --platform=$(params.platforms) --preserve-import-paths -t $(params.versionTag) -f - > $OUTPUT_RELEASE_DIR/${FILENAME_PREFIX}release.yaml + # Make a local git tag to make git status happy :) + # The real "tagging" will happen with the "create-release" pipeline. + git tag $(params.versionTag) + + kustomize build ${PROJECT_ROOT}/config/${KUBE_DISTRO}/overlays/default | \ + ko resolve \ + --image-label=org.opencontainers.image.source=https://$(params.package) \ + --platform=$(params.platforms) ${KO_EXTRA_ARGS} \ + -t $(params.versionTag) \ + -f - > $OUTPUT_RELEASE_DIR/${FILENAME_PREFIX}release.yaml + # Publish images and create release.notags.yaml # This is useful if your container runtime doesn't support the `image-reference:tag@digest` notation # This is currently the case for `cri-o` (and most likely others) - kustomize build ${PROJECT_ROOT}/config/${KUBE_DISTRO}/overlays/default | ko resolve --platform=$(params.platforms) --preserve-import-paths -f - > $OUTPUT_RELEASE_DIR/${FILENAME_PREFIX}release.notags.yaml + kustomize build ${PROJECT_ROOT}/config/${KUBE_DISTRO}/overlays/default | \ + ko resolve \ + --image-label=org.opencontainers.image.source=https://$(params.package) \ + --platform=$(params.platforms) ${KO_EXTRA_ARGS} \ + -f - > $OUTPUT_RELEASE_DIR/${FILENAME_PREFIX}release.notags.yaml - name: koparse - image: gcr.io/tekton-releases/dogfooding/koparse:v20240910-ec3cf3c749@sha256:5e8a522fc1e587fc00b69a6d73e0bfdf7a29ca143537a5542eb224680d2dbf2f + image: gcr.io/tekton-releases/dogfooding/koparse@sha256:194c2ab9dce5f778ed757af13c626d6b85f15452e2c2902c79b0d0f5a0adf4d1 script: | set -ex - IMAGES_PATH=${CONTAINER_REGISTRY}/$(params.package) + # Find "--preserve-import-paths" in a list of args + function find_preserve_import_path() { + for arg in $@; do + if [[ "$arg" == "--preserve-import-paths" ]]; then + return 0 + fi + done + return 1 + } + + # If "--preserve-import-paths" is used, include "package" in the expected path + find_preserve_import_path \ + $(echo $KO_EXTRA_ARGS) && \ + PRESERVE_IMPORT_PATH="--preserve-path" || \ + PRESERVE_IMPORT_PATH="--no-preserve-path" for cmd in $(params.images) do - IMAGES="${IMAGES} ${IMAGES_PATH}/cmd/${KUBE_DISTRO}/${cmd}:$(params.versionTag)" + IMAGES="${IMAGES} $(params.package)/cmd/${KUBE_DISTRO}/${cmd}:$(params.versionTag)" done # Parse the built images from the release.yaml generated by ko koparse \ --path $OUTPUT_RELEASE_DIR/${FILENAME_PREFIX}release.yaml \ - --base ${IMAGES_PATH} --images ${IMAGES} > /workspace/built_images + --base $(params.package) \ + --container-registry ${CONTAINER_REGISTRY} \ + --images ${IMAGES} \ + ${PRESERVE_IMPORT_PATH} > /workspace/built_images - name: tag-images - image: gcr.io/go-containerregistry/crane:debug@sha256:ff0e08eeae8097d28b2381c7f7123bf542757abc68d11bff58fb882b72843785 + image: cgr.dev/chainguard/crane:latest-dev@sha256:6fc6fcdeb173c7951f038e6a7b230f586c1be05a011d9e6f9db6c614ec412c2f script: | #!/busybox/sh set -ex @@ -161,8 +202,6 @@ spec: mkdir -p ${DOCKER_CONFIG} cp /workspace/docker-config.json ${DOCKER_CONFIG}/config.json - REGIONS="us eu asia" - # Tag the images and put them in all the regions for IMAGE in $(cat /workspace/built_images) do diff --git a/tekton/operator-release-pipeline.yaml b/tekton/operator-release-pipeline.yaml index f594a0c276..d7cad3ee43 100644 --- a/tekton/operator-release-pipeline.yaml +++ b/tekton/operator-release-pipeline.yaml @@ -16,6 +16,12 @@ spec: - name: imageRegistryPath description: The path (project) in the image registry default: tekton-releases + - name: imageRegistryRegions + description: The target image registry regions + default: "us eu asia" + - name: imageRegistryUser + description: The user for the image registry credentials + default: _json_key - name: versionTag description: The X.Y.Z version that the artifacts should be tagged with - name: releaseBucket @@ -27,8 +33,16 @@ spec: - name: platforms description: Platforms to publish for the images (e.g. linux/amd64,linux/arm64) default: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le + - name: koExtraArgs + description: Extra args to be passed to ko + default: "--preserve-import-paths" - name: serviceAccountPath description: The path to the service account file within the release-secret workspace + - name: serviceAccountImagesPath + description: The path to the service account file or credentials within the release-images-secret workspace + - name: runTests + description: If set to something other than "true", skip the build and test tasks + default: "true" - name: kubeDistros description: The kubernetes platform (e.g. kubernetes or openshift ... ) targeted by a pipeline run default: "kubernetes openshift" @@ -40,6 +54,8 @@ spec: description: The workspace where the repo will be cloned. - name: release-secret description: The secret that contains a service account authorized to push to the imageRegistry and to the output bucket + - name: release-images-secret + description: The secret that contains a service account authorized to push to the imageRegistry results: - name: commit-sha description: the sha of the commit that was released @@ -59,11 +75,16 @@ spec: tasks: - name: git-clone taskRef: - name: git-clone + resolver: hub + params: + - name: name + value: git-clone + - name: version + value: "0.7" workspaces: - - name: output - workspace: workarea - subpath: git + - name: output + workspace: workarea + subpath: git params: - name: url value: https://$(params.package) @@ -73,7 +94,16 @@ spec: runAfter: - git-clone taskRef: - name: prerelease-checks + resolver: git + params: + - name: repo + value: plumbing + - name: org + value: tektoncd + - name: revision + value: aeed19e5a36f335ebfdc4b96fa78d1ce5bb4f7b8 + - name: pathInRepo + value: tekton/resources/release/base/prerelease_checks.yaml params: - name: package value: $(params.package) @@ -88,8 +118,17 @@ spec: - name: unit-tests runAfter: - precheck + when: + - cel: "'$(params.runTests)' == 'true'" taskRef: - name: golang-test + resolver: bundles + params: + - name: bundle + value: ghcr.io/tektoncd/catalog/upstream/tasks/golang-test:0.2 + - name: name + value: golang-test + - name: kind + value: task params: - name: package value: $(params.package) @@ -100,10 +139,19 @@ spec: workspace: workarea subpath: git - name: fetch-component-releases - taskRef: - name: operator-fetch-component-releases runAfter: - unit-tests + taskRef: + resolver: git + params: + - name: repo + value: operator + - name: org + value: tektoncd + - name: revision + value: $(params.gitRevision) + - name: pathInRepo + value: tekton/task-fetch-components.yaml workspaces: - name: source workspace: workarea @@ -116,8 +164,17 @@ spec: - name: build-test runAfter: - fetch-component-releases + when: + - cel: "'$(params.runTests)' == 'true'" taskRef: - name: golang-build + resolver: bundles + params: + - name: bundle + value: ghcr.io/tektoncd/catalog/upstream/tasks/golang-build:0.3 + - name: name + value: golang-build + - name: kind + value: task params: - name: package value: $(params.package) @@ -131,7 +188,16 @@ spec: runAfter: - build-test taskRef: - name: publish-operator-release + resolver: git + params: + - name: repo + value: operator + - name: org + value: tektoncd + - name: revision + value: $(params.gitRevision) + - name: pathInRepo + value: tekton/build-publish-images-manifests.yaml params: - name: package value: $(params.package) @@ -164,7 +230,16 @@ spec: runAfter: - build-test taskRef: - name: publish-operator-release + resolver: git + params: + - name: repo + value: operator + - name: org + value: tektoncd + - name: revision + value: $(params.gitRevision) + - name: pathInRepo + value: tekton/build-publish-images-manifests.yaml params: - name: package value: $(params.package) @@ -198,7 +273,14 @@ spec: - publish-images-platform-kubernetes - publish-images-platform-openshift taskRef: - name: gcs-upload + resolver: bundles + params: + - name: bundle + value: ghcr.io/tektoncd/catalog/upstream/tasks/gcs-upload:0.3 + - name: name + value: gcs-upload + - name: kind + value: task workspaces: - name: credentials workspace: release-secret @@ -222,7 +304,14 @@ spec: values: - "true" taskRef: - name: gcs-upload + resolver: bundles + params: + - name: bundle + value: ghcr.io/tektoncd/catalog/upstream/tasks/gcs-upload:0.3 + - name: name + value: gcs-upload + - name: kind + value: task workspaces: - name: credentials workspace: release-secret diff --git a/tekton/release-cheat-sheet.md b/tekton/release-cheat-sheet.md index 6ef22f7249..535d0cd73b 100644 --- a/tekton/release-cheat-sheet.md +++ b/tekton/release-cheat-sheet.md @@ -56,32 +56,13 @@ need a checkout of the operator repo, a terminal window and a text editor. 2`cd` to root of Operator git checkout. -3. Make sure the release `Task` and `Pipeline` are up-to-date on the - cluster. To do that, you can use `kustomize`: - - ```bash - kustomize build tekton | kubectl replace -f - - ``` - - - [publish-operator-release](https://github.com/tektoncd/operator/blob/main/tekton/build-publish-images-manifests.yaml) - - This task uses [ko](https://github.com/google/ko) to build all container images we release and generate - the `release.yaml` - ```shell script - kubectl apply -f tekton/bases/build-publish-images-manifests.yaml - ``` - - [operator-release](https://github.com/tektoncd/operator/blob/main/tekton/operator-release-pipeline.yaml) - ```shell script - kubectl apply -f tekton/overlays/versioned-releases/operator-release-pipeline.yaml - ``` - -4. Confirm commit SHA matches what you want to release. +3. Confirm commit SHA matches what you want to release. ```bash git show $TEKTON_RELEASE_GIT_SHA ``` -6. Create a workspace template file: +4. Create a workspace template file: ```bash cat < workspace-template.yaml @@ -94,23 +75,29 @@ need a checkout of the operator repo, a terminal window and a text editor. EOF ``` -7. Execute the release pipeline. +5. Execute the release pipeline. ```bash tkn --context dogfooding pipeline start operator-release \ + --filename=tekton/operator-release-pipeline.yaml \ --serviceaccount=release-right-meow \ - --param=components=components.yaml \ - --param=gitRevision="${TEKTON_RELEASE_GIT_SHA}" \ - --param=versionTag="${TEKTON_RELEASE_VERSION}" \ - --param=serviceAccountPath=release.json \ - --param=releaseBucket=gs://tekton-releases/operator \ - --param=imageRegistry=gcr.io \ - --param=imageRegistryPath=tekton-releases \ - --param=releaseAsLatest=true \ - --param=platforms=linux/amd64,linux/arm64,linux/s390x,linux/ppc64le \ - --param=kubeDistros="kubernetes openshift" \ - --param=package=github.com/tektoncd/operator \ + --param package=github.com/tektoncd/operator \ + --param components=components.yaml \ + --param gitRevision="${TEKTON_RELEASE_GIT_SHA}" \ + --param imageRegistry=ghcr.io \ + --param imageRegistryPath=tektoncd/operator \ + --param imageRegistryRegions="" \ + --param imageRegistryUser=tekton-robot \ + --param serviceAccountPath=release.json \ + --param serviceAccountImagesPath=credentials \ + --param versionTag="${TEKTON_RELEASE_VERSION}" \ + --param releaseBucket=gs://tekton-releases/operator \ + --param koExtraArgs="" \ + --param releaseAsLatest=true \ + --param platforms=linux/amd64,linux/arm64,linux/s390x,linux/ppc64le \ + --param kubeDistros="kubernetes openshift" \ --workspace name=release-secret,secret=release-secret \ + --workspace name=release-images-secret,secret=ghcr-creds \ --workspace name=workarea,volumeClaimTemplateFile=workspace-template.yaml \ --pipeline-timeout 2h0m0s ``` diff --git a/tekton/setup-release-from-scratch.md b/tekton/setup-release-from-scratch.md index 96e4534ac8..a7ce22f4ed 100644 --- a/tekton/setup-release-from-scratch.md +++ b/tekton/setup-release-from-scratch.md @@ -20,35 +20,14 @@ kubectl apply --filename https://storage.googleapis.com/tekton-releases/pipelin ### Install tasks and pipelines -Add all the `Tasks` and `Pipelines` needed for creating to the cluster:, - -#### Tasks from Tekton Catalog - -- [`golang-test`](https://hub-preview.tekton.dev/detail/45) - ```shell script - tkn hub install task golang-test - ``` -- [`golang-build`](https://hub-preview.tekton.dev/detail/44) - ```shell script - tkn hub install task golang-build - ``` -- [`gcs-upload`](https://hub-preview.tekton.dev/detail/30) - ```shell script - tkn hub install task gcs-upload - ``` - -#### Tasks and Pipelines from this repository - -- [publish-operator-release](https://github.com/tektoncd/operator/blob/main/tekton/build-publish-images-manifests.yaml) - - This task uses [ko](https://github.com/google/ko) to build all container images we release and generate the `release.yaml` - ```shell script - kubectl apply -f tekton/bases/build-publish-images-manifests.yaml - ``` -- [operator-release](https://github.com/tektoncd/operator/blob/main/tekton/operator-release-pipeline.yaml) - ```shell script - kubectl apply -f tekton/overlays/versioned-releases/operator-release-pipeline.yaml - ``` +All the `Tasks` required are fetched via remote resolution. +The `Pipeline` can be installed on the cluster: + +```shell script +kubectl apply -f tekton/operator-release-pipeline.yaml +``` + +or it can be executed from the local file using `tkn pipeline start --filename`. ### Service account and secrets