From befb1d33b202bb4be2568d9b43aa64647fc5e4bc Mon Sep 17 00:00:00 2001 From: Jiaqi Luo <6218999+jiaqiluo@users.noreply.github.com> Date: Thu, 20 Jun 2024 11:41:10 -0700 Subject: [PATCH 1/2] Migrate from Drone to GitHub Action --- .drone.yml | 324 ---------------------- .github/workflows/ci-on-pull-requset.yaml | 35 +++ .github/workflows/fossa.yaml | 30 ++ .github/workflows/release.yaml | 266 ++++++++++++++++++ .golangci.json | 8 +- Dockerfile.dapper | 4 +- main.go | 2 +- package/Dockerfile | 1 + package/Dockerfile.suc | 1 + pkg/k8splan/watcher.go | 2 +- pkg/localplan/watcher.go | 2 +- scripts/gha-ci | 9 + scripts/version | 4 + 13 files changed, 352 insertions(+), 336 deletions(-) delete mode 100644 .drone.yml create mode 100644 .github/workflows/ci-on-pull-requset.yaml create mode 100644 .github/workflows/fossa.yaml create mode 100644 .github/workflows/release.yaml create mode 100755 scripts/gha-ci diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 50b2d1f4..00000000 --- a/.drone.yml +++ /dev/null @@ -1,324 +0,0 @@ ---- -kind: pipeline -name: amd64 - -platform: - os: linux - arch: amd64 - -steps: -- name: build - image: rancher/dapper:v0.6.0 - commands: - - dapper ci - volumes: - - name: docker - path: /var/run/docker.sock - -- name: github_binary_release - image: plugins/github-release - settings: - api_key: - from_secret: github_token - prerelease: true - checksum: - - sha256 - checksum_file: CHECKSUMsum-amd64.txt - checksum_flatten: true - files: - - "dist/artifacts/*" - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -- name: docker-publish - image: plugins/docker - settings: - dockerfile: package/Dockerfile - password: - from_secret: docker_password - repo: "rancher/system-agent" - tag: "${DRONE_TAG}-amd64" - username: - from_secret: docker_username - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -- name: docker-publish-suc - image: plugins/docker - settings: - dockerfile: package/Dockerfile.suc - password: - from_secret: docker_password - repo: "rancher/system-agent" - tag: "${DRONE_TAG}-suc-amd64" - username: - from_secret: docker_username - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -volumes: -- name: docker - host: - path: /var/run/docker.sock - ---- -kind: pipeline -name: arm64 - -platform: - os: linux - arch: arm64 - -steps: -- name: build - image: rancher/dapper:v0.6.0 - commands: - - dapper ci - volumes: - - name: docker - path: /var/run/docker.sock - -- name: github_binary_release - image: plugins/github-release - settings: - api_key: - from_secret: github_token - prerelease: true - checksum: - - sha256 - checksum_file: CHECKSUMsum-arm64.txt - checksum_flatten: true - files: - - "dist/artifacts/*" - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -- name: docker-publish - image: plugins/docker - settings: - dockerfile: package/Dockerfile - password: - from_secret: docker_password - repo: "rancher/system-agent" - tag: "${DRONE_TAG}-arm64" - username: - from_secret: docker_username - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -- name: docker-publish-suc - image: plugins/docker - settings: - dockerfile: package/Dockerfile.suc - password: - from_secret: docker_password - repo: "rancher/system-agent" - tag: "${DRONE_TAG}-suc-arm64" - username: - from_secret: docker_username - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -volumes: -- name: docker - host: - path: /var/run/docker.sock - -#--- -#kind: pipeline -#name: s390x -# -#platform: -# os: linux -# arch: amd64 -# -## Hack needed for s390x: https://gist.github.com/colstrom/c2f359f72658aaabb44150ac20b16d7c#gistcomment-3858388 -#node: -# arch: s390x -# -#steps: -#- name: build -# image: rancher/dapper:v0.6.0 -# commands: -# - dapper ci -# volumes: -# - name: docker -# path: /var/run/docker.sock -# -#- name: github_binary_release -# image: rancher/drone-images:github-release-s390x -# settings: -# api_key: -# from_secret: github_token -# prerelease: true -# checksum: -# - sha256 -# checksum_file: CHECKSUMsum-s390x.txt -# checksum_flatten: true -# files: -# - "dist/artifacts/*" -# when: -# instance: -# - drone-publish.rancher.io -# ref: -# - refs/head/master -# - refs/tags/* -# event: -# - tag -# -#- name: docker-publish -# image: rancher/drone-images:docker-s390x -# volumes: -# - name: docker -# path: /var/run/docker.sock -# settings: -# dockerfile: package/Dockerfile -# password: -# from_secret: docker_password -# repo: "rancher/system-agent" -# tag: "${DRONE_TAG}-s390x" -# username: -# from_secret: docker_username -# when: -# instance: -# - drone-publish.rancher.io -# ref: -# - refs/head/master -# - refs/tags/* -# event: -# - tag -# -#- name: docker-publish-suc -# image: rancher/drone-images:docker-s390x -# volumes: -# - name: docker -# path: /var/run/docker.sock -# settings: -# dockerfile: package/Dockerfile.suc -# password: -# from_secret: docker_password -# repo: "rancher/system-agent" -# tag: "${DRONE_TAG}-suc-s390x" -# username: -# from_secret: docker_username -# when: -# instance: -# - drone-publish.rancher.io -# ref: -# - refs/head/master -# - refs/tags/* -# event: -# - tag -# -#volumes: -#- name: docker -# host: -# path: /var/run/docker.sock - ---- -kind: pipeline -name: manifest - -platform: - os: linux - arch: amd64 - -steps: -- name: manifest - image: plugins/manifest:1.4.0 - settings: - username: - from_secret: docker_username - password: - from_secret: docker_password - platforms: - - linux/amd64 - - linux/arm64 -# - linux/s390x - target: "rancher/system-agent:${DRONE_TAG}" - template: "rancher/system-agent:${DRONE_TAG}-ARCH" - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -depends_on: -- amd64 -- arm64 -#- s390x - ---- -kind: pipeline -name: manifest-suc - -platform: - os: linux - arch: amd64 - -steps: -- name: manifest - image: plugins/manifest:1.4.0 - settings: - username: - from_secret: docker_username - password: - from_secret: docker_password - platforms: - - linux/amd64 - - linux/arm64 -# - linux/s390x - target: "rancher/system-agent:${DRONE_TAG}-suc" - template: "rancher/system-agent:${DRONE_TAG}-suc-ARCH" - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -depends_on: -- amd64 -- arm64 -#- s390x diff --git a/.github/workflows/ci-on-pull-requset.yaml b/.github/workflows/ci-on-pull-requset.yaml new file mode 100644 index 00000000..8e4bc461 --- /dev/null +++ b/.github/workflows/ci-on-pull-requset.yaml @@ -0,0 +1,35 @@ +name: CI on Pull Request + +on: + pull_request: + +jobs: + test-build-linux: + runs-on: ubuntu-latest + container: + image: rancher/dapper:v0.6.0 + permissions: + contents: read + strategy: + matrix: + os: [ linux ] + arch: [ amd64, arm64 ] + steps: + - name: Fix the not-a-git-repository issue + run: | + apk -U add git + git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set environment variables + run: | + echo "DAPPER_HOST_ARCH=${{ matrix.arch }}" >> "$GITHUB_ENV" + echo "GH_VERSION=${{ github.ref_name }}" >> "$GITHUB_ENV" + echo "GOARCH=${{ matrix.arch }}" >> "$GITHUB_ENV" + echo "GOOS=${{ matrix.os }}" >> "$GITHUB_ENV" + echo "CROSS=false" >> "$GITHUB_ENV" + + - name: build with Dapper + run: dapper gha-ci diff --git a/.github/workflows/fossa.yaml b/.github/workflows/fossa.yaml new file mode 100644 index 00000000..9f417e62 --- /dev/null +++ b/.github/workflows/fossa.yaml @@ -0,0 +1,30 @@ +name: Run Fossa Scan + +on: + push: + branches: + - "main" + # For manual scans. + workflow_dispatch: + +jobs: + fossa: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # needed for the Vault authentication + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Read FOSSA token + uses: rancher-eio/read-vault-secrets@main + with: + secrets: | + secret/data/github/org/rancher/fossa/push token | FOSSA_API_KEY_PUSH_ONLY + + - name: FOSSA scan + uses: fossas/fossa-action@main + with: + api-key: ${{ env.FOSSA_API_KEY_PUSH_ONLY }} + run-tests: false diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000..346b13c1 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,266 @@ +name: Release + +on: + push: + tags: + - '*' + +env: + IMAGE: rancher/system-agent + TAG: ${{ github.ref_name }} + TAG_SUC: ${{ github.ref_name }}-suc + +jobs: + build-binary: + runs-on: ubuntu-latest + container: + image: rancher/dapper:v0.6.0 + permissions: + contents: read + strategy: + matrix: + os: [ linux ] + arch: [ amd64, arm64 ] + steps: + - name: Fix the not-a-git-repository issue + run: | + apk -U add git + git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set environment variables + run: | + echo "DAPPER_HOST_ARCH=${{ matrix.arch }}" >> "$GITHUB_ENV" + echo "GH_VERSION=${{ github.ref_name }}" >> "$GITHUB_ENV" + echo "GOARCH=${{ matrix.arch }}" >> "$GITHUB_ENV" + echo "GOOS=${{ matrix.os }}" >> "$GITHUB_ENV" + echo "CROSS=false" >> "$GITHUB_ENV" + + - name: build with Dapper + run: dapper gha-ci + + - name: upload the binaries + uses: actions/upload-artifact@v4 + with: + name: binary-${{ matrix.os }}-${{ matrix.arch }} + path: bin/rancher-system-agent + if-no-files-found: error + overwrite: true + + build-push-image: + runs-on: ubuntu-latest + needs: + - build-binary + permissions: + contents: read + id-token: write # needed for the Vault authentication + strategy: + matrix: + os: [ linux ] + arch: [ amd64, arm64 ] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download binary + uses: actions/download-artifact@v4 + with: + name: binary-${{ matrix.os }}-${{ matrix.arch }} + path: bin + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Load secrets from Vault + uses: rancher-eio/read-vault-secrets@main + with: + secrets: | + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials username | DOCKER_USERNAME ; + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials password | DOCKER_PASSWORD + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + + - name: Build and push tag + id: build + uses: docker/build-push-action@v5 + with: + context: . + file: package/Dockerfile + push: true + tags: ${{ env.IMAGE }}:${{ env.TAG }} + platforms: ${{ matrix.os }}/${{ matrix.arch }} + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: "digests-${{ matrix.os }}-${{ matrix.arch }}" + path: /tmp/digests/* + if-no-files-found: error + overwrite: true + + merge: + runs-on: ubuntu-latest + needs: + - build-push-image + permissions: + contents: read + id-token: write # needed for the Vault authentication + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Load secrets from Vault + uses: rancher-eio/read-vault-secrets@main + with: + secrets: | + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials username | DOCKER_USERNAME ; + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials password | DOCKER_PASSWORD + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create --tag ${{ env.IMAGE }}:${{ env.TAG }} $(printf '${{ env.IMAGE }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.IMAGE }}:${{ env.TAG }} + + build-push-image-suc: + runs-on: ubuntu-latest + needs: + - build-binary + permissions: + contents: read + id-token: write # needed for the Vault authentication + strategy: + matrix: + os: [ linux ] + arch: [ amd64, arm64 ] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download binary + uses: actions/download-artifact@v4 + with: + name: binary-${{ matrix.os }}-${{ matrix.arch }} + path: bin + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Load secrets from Vault + uses: rancher-eio/read-vault-secrets@main + with: + secrets: | + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials username | DOCKER_USERNAME ; + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials password | DOCKER_PASSWORD + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + + - name: Build and push Docker tag + id: build + uses: docker/build-push-action@v5 + with: + context: . + file: package/Dockerfile.suc + push: true + tags: ${{ env.IMAGE }}:${{ env.TAG_SUC }} + platforms: ${{ matrix.os }}/${{ matrix.arch }} + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: "suc-digests-${{ matrix.os }}-${{ matrix.arch }}" + path: /tmp/digests/* + if-no-files-found: error + overwrite: true + + + merge-suc: + runs-on: ubuntu-latest + needs: + - build-push-image-suc + permissions: + contents: read + id-token: write # needed for the Vault authentication + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Load secrets from Vault + uses: rancher-eio/read-vault-secrets@main + with: + secrets: | + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials username | DOCKER_USERNAME ; + secret/data/github/repo/${{ github.repository }}/dockerhub/rancher/credentials password | DOCKER_PASSWORD + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: suc-digests-* + merge-multiple: true + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create --tag ${{ env.IMAGE }}:${{ env.TAG_SUC }} $(printf '${{ env.IMAGE }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.IMAGE }}:${{ env.TAG_SUC }} + diff --git a/.golangci.json b/.golangci.json index f317e317..30179d61 100644 --- a/.golangci.json +++ b/.golangci.json @@ -3,17 +3,11 @@ "disable-all": true, "enable": [ "govet", - "golint", + "revive", "goimports", "misspell", "ineffassign", "gofmt" ] - }, - "run": { - "skip-files": [ - "/zz_generated_" - ], - "deadline": "5m" } } diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 5662be1c..e61b638e 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -8,10 +8,10 @@ RUN go install golang.org/x/lint/golint@latest && \ go install golang.org/x/tools/cmd/goimports@latest && \ rm -rf /go/src /go/pkg RUN if [ "${ARCH}" == "amd64" ]; then \ - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.54.2; \ + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.59.1; \ fi -ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS +ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS GOARCH GOOS ARCH GH_VERSION ENV DAPPER_SOURCE /go/src/github.com/rancher/system-agent/ ENV DAPPER_OUTPUT ./bin ./dist ENV DAPPER_DOCKER_SOCKET true diff --git a/main.go b/main.go index 7bea145b..7a0d8302 100644 --- a/main.go +++ b/main.go @@ -58,7 +58,7 @@ func main() { } } -func run(c *cli.Context) error { +func run(_ *cli.Context) error { topContext := signals.SetupSignalContext() logrus.Infof("Rancher System Agent version %s is starting", version.FriendlyVersion()) diff --git a/package/Dockerfile b/package/Dockerfile index b930c1c8..3c8f8538 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -1,3 +1,4 @@ FROM alpine:3.18 COPY bin/rancher-system-agent /usr/bin/ +RUN chmod +x /usr/bin/rancher-system-agent CMD ["rancher-system-agent"] diff --git a/package/Dockerfile.suc b/package/Dockerfile.suc index f5e90b27..16d8aa4e 100644 --- a/package/Dockerfile.suc +++ b/package/Dockerfile.suc @@ -18,5 +18,6 @@ COPY system-agent-uninstall.sh /opt/rancher-system-agent-suc/system-agent-uninst COPY bin/rancher-system-agent /opt/rancher-system-agent-suc COPY package/suc/run.sh /opt/rancher-system-agent-suc/run.sh COPY --from=kubectl /usr/bin/kubectl /usr/bin/ +RUN chmod +x /opt/rancher-system-agent-suc/rancher-system-agent ENTRYPOINT ["/opt/rancher-system-agent-suc/run.sh"] diff --git a/pkg/k8splan/watcher.go b/pkg/k8splan/watcher.go index cd86ee54..4ecdc069 100644 --- a/pkg/k8splan/watcher.go +++ b/pkg/k8splan/watcher.go @@ -140,7 +140,7 @@ func (w *watcher) start(ctx context.Context, strictVerify bool) { hasRunOnce := false - core.Secret().OnChange(ctx, "secret-watch", func(s string, secret *v1.Secret) (*v1.Secret, error) { + core.Secret().OnChange(ctx, "secret-watch", func(_ string, secret *v1.Secret) (*v1.Secret, error) { if secret == nil { logrus.Fatalf("[K8s] received nil secret that was nil, stopping") return nil, nil diff --git a/pkg/localplan/watcher.go b/pkg/localplan/watcher.go index e5154c35..b2ff449a 100644 --- a/pkg/localplan/watcher.go +++ b/pkg/localplan/watcher.go @@ -70,7 +70,7 @@ func (w *watcher) listFiles(ctx context.Context, force bool) error { return nil } -func (w *watcher) listFilesIn(ctx context.Context, base string, force bool) error { +func (w *watcher) listFilesIn(ctx context.Context, base string, _ bool) error { files := map[string]os.FileInfo{} if err := filepath.Walk(base, func(path string, info os.FileInfo, err error) error { if err != nil { diff --git a/scripts/gha-ci b/scripts/gha-ci new file mode 100755 index 00000000..19418c08 --- /dev/null +++ b/scripts/gha-ci @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +cd $(dirname $0) + +./build +./test +./validate +./validate-ci diff --git a/scripts/version b/scripts/version index 599a9851..8137c813 100755 --- a/scripts/version +++ b/scripts/version @@ -19,6 +19,10 @@ fi SUFFIX="-${ARCH}" +if [[ -n ${GH_VERSION} ]]; then + VERSION=${GH_VERSION} +fi + TAG=${TAG:-${VERSION}${SUFFIX}} REPO=${REPO:-rancher} From fd8e4adbaacdc651d00db75c47bc0088a917c35f Mon Sep 17 00:00:00 2001 From: Jiaqi Luo <6218999+jiaqiluo@users.noreply.github.com> Date: Mon, 1 Jul 2024 13:16:30 -0700 Subject: [PATCH 2/2] add the step for creating GH release, and fix typo in filename --- ...l-requset.yaml => ci-on-pull-request.yaml} | 0 .github/workflows/release.yaml | 38 +++++++++++++++++++ 2 files changed, 38 insertions(+) rename .github/workflows/{ci-on-pull-requset.yaml => ci-on-pull-request.yaml} (100%) diff --git a/.github/workflows/ci-on-pull-requset.yaml b/.github/workflows/ci-on-pull-request.yaml similarity index 100% rename from .github/workflows/ci-on-pull-requset.yaml rename to .github/workflows/ci-on-pull-request.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 346b13c1..463685a9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -264,3 +264,41 @@ jobs: run: | docker buildx imagetools inspect ${{ env.IMAGE }}:${{ env.TAG_SUC }} + github_release: + runs-on: ubuntu-latest + needs: + - merge + - merge-suc + permissions: + contents: write # needed for creating the GH release + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download binary + uses: actions/download-artifact@v4 + with: + pattern: binary-* + path: bin + + - name: Prepare release + run: | + mkdir tmp + for DIR in bin/binary-*; do + if [[ -d "$DIR" && -f "$DIR/rancher-system-agent" ]]; then + ARCH=$(echo "$DIR" | awk -F'-' '{print $NF}') + mv "$DIR/rancher-system-agent" "tmp/rancher-system-agent-$ARCH" + fi + done + cp install.sh system-agent-uninstall.sh tmp + mkdir dist + for file in tmp/*; do + sha256sum "$file" >> dist/sha256sum.txt + done + mv tmp/* dist + + - name: Create release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: + gh release create ${{ github.ref_name }} --verify-tag --generate-notes dist/*