WIP: KUTTL action

.github/actions/k3d/action.yaml

@@ -0,0 +1,85 @@
+name: k3d
+description: Start k3s using k3d
+inputs:
+  k3d-tag:
+    default: latest
+    required: true
+    description: >
+      Git tag from https://github.com/rancher/k3d/releases or "latest"
+  k3s-channel:
+    default: latest
+    required: true
+    description: >
+      https://rancher.com/docs/k3s/latest/en/upgrades/basic/#release-channels
+  prefetch-images:
+    required: true
+    description: >
+      Each line is the name of an image to fetch onto all Kubernetes nodes
+  prefetch-timeout:
+    default: 90s
+    required: true
+    description: >
+      Amount of time to wait for images to be fetched
+
+outputs:
+  kubernetes-version:
+    value: ${{ steps.k3s.outputs.server }}
+    description: >
+      Kubernetes server version, as reported by the Kubernetes API
+
+runs:
+  using: composite
+  steps:
+    - id: k3d
+      name: Install k3d
+      shell: bash
+      env:
+        K3D_TAG: ${{ inputs.k3d-tag }}
+      run: |
+        curl --fail --silent https://raw.githubusercontent.com/rancher/k3d/main/install.sh |
+          TAG="${K3D_TAG#latest}" bash
+        k3d version | awk '{ print "::set-output name=" tolower($1) "::" $3 }'
+
+    - id: k3s
+      name: Start k3s
+      shell: bash
+      run: |
+        k3d cluster create --image '+${{ inputs.k3s-channel }}' --no-lb --timeout=2m --wait
+        kubectl version --short | awk '{ print "::set-output name=" tolower($1) "::" $3 }'
+
+        docker exec $(k3d node list --output json | jq --raw-output 'first.name') \
+          k3s agent --help | awk '$1 == "--pause-image" {
+            match($0, /default: "[^"]*"/)
+            print "::set-output name=pause-image::" substr($0, RSTART+10, RLENGTH-11)
+          }'
+
+    - name: Prefetch container images
+      shell: bash
+      env:
+        INPUT_IMAGES:  ${{ inputs.prefetch-images  }}
+        INPUT_TIMEOUT: ${{ inputs.prefetch-timeout }}
+      run: |
+        jq <<< "$INPUT_IMAGES" --raw-input 'select(. != "")' |
+        jq --slurp \
+          --arg pause '${{ steps.k3s.outputs.pause-image }}' \
+          --argjson labels '{"name":"image-prefetch"}' \
+          --argjson name '"image-prefetch"' \
+        '{
+          apiVersion: "apps/v1", kind: "DaemonSet",
+          metadata: { name: $name, labels: $labels },
+          spec: {
+            selector: { matchLabels: $labels },
+            template: {
+              metadata: { labels: $labels },
+              spec: {
+                initContainers: to_entries | map({
+                  name: "c\(.key)", image: .value, command: ["true"],
+                }),
+                containers: [{ name: "pause", image: $pause }]
+              }
+            }
+          }
+        }' |
+        kubectl create --filename=-
+        kubectl rollout status daemonset.apps/image-prefetch --timeout "$INPUT_TIMEOUT" ||
+        kubectl describe daemonset.apps/image-prefetch

.github/workflows/test.yaml

@@ -40,12 +40,11 @@ jobs:
       - run: gzip envtest.coverage
       - uses: actions/upload-artifact@v3
         with:
-          name: "kubernetes-api=${{ matrix.kubernetes }}"
+          name: "~coverage~kubernetes-api=${{ matrix.kubernetes }}"
           path: envtest.coverage.gz
           retention-days: 1
 
   kubernetes-k3d:
-    if: "${{ github.repository == 'CrunchyData/postgres-operator' }}"
     runs-on: ubuntu-latest
     needs: [go-test]
     strategy:
@@ -57,43 +56,15 @@ jobs:
       - uses: actions/setup-go@v3
         with: { go-version: 1.x }
 
-      - name: Install k3d
-        # Git tag from https://github.com/rancher/k3d/releases or "latest"
-        env: { K3D_TAG: latest }
-        run: |
-          curl --fail --silent https://raw.githubusercontent.com/rancher/k3d/main/install.sh |
-            TAG="${K3D_TAG#latest}" bash && k3d version | head -n1
-
       - name: Start k3s
-        # https://rancher.com/docs/k3s/latest/en/upgrades/basic/#release-channels
-        env: { K3S_CHANNEL: "${{ matrix.kubernetes }}" }
-        run: k3d cluster create --image="+${K3S_CHANNEL}" --no-lb --timeout=2m --wait
+        uses: ./.github/actions/k3d
+        with:
+          k3s-channel: "${{ matrix.kubernetes }}"
+          prefetch-images: |
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-13.6-1
+            registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.38-0
+            registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.16-2
 
-      - name: Prefetch container images
-        run: |
-          {
-            echo '"registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-13.6-1"'
-            echo '"registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.38-0"'
-            echo '"registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.16-2"'
-          } |
-          jq --slurp --arg name 'image-prefetch' --argjson labels '{"name":"image-prefetch"}' '{
-            apiVersion: "apps/v1", kind: "DaemonSet",
-            metadata: { name: $name, labels: $labels },
-            spec: {
-              selector: { matchLabels: $labels },
-              template: {
-                metadata: { labels: $labels },
-                spec: {
-                  initContainers: to_entries | map({ name: "c\(.key)", command: ["true"], image: .value }),
-                  containers: [{ name: "pause", image: "k8s.gcr.io/pause:3.5" }]
-                }
-              }
-            }
-          }' |
-          kubectl create --filename=- && {
-            kubectl rollout status daemonset.apps/image-prefetch --timeout=90s ||
-            kubectl describe daemonset.apps/image-prefetch
-          }
       - run: make createnamespaces check-envtest-existing
         env:
           PGO_TEST_TIMEOUT_SCALE: 1.2
@@ -103,16 +74,90 @@ jobs:
       - run: gzip envtest-existing.coverage
       - uses: actions/upload-artifact@v3
         with:
-          name: "kubernetes-k3d=${{ matrix.kubernetes }}"
+          name: "~coverage~kubernetes-k3d=${{ matrix.kubernetes }}"
           path: envtest-existing.coverage.gz
           retention-days: 1
 
+  kuttl-k3d:
+    runs-on: ubuntu-latest
+    needs: [go-test]
+    strategy:
+      fail-fast: false
+      matrix:
+        kubernetes: [latest]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with: { go-version: 1.x }
+
+      - name: Start k3s
+        uses: ./.github/actions/k3d
+        with:
+          k3s-channel: "${{ matrix.kubernetes }}"
+          prefetch-images: |
+            registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi8-4.30-2
+            registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.38-2
+            registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.16-4
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.4-0
+            registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-14.4-3.1-0
+
+      - run: go mod download
+      - name: Build coverage executable
+        run: |
+          make build-postgres-operator \
+            GO_BUILD='go test -c --tags main_with_coverage --trimpath --coverpkg ./internal/...' \
+            PGO_VERSION='${{ github.sha }}-coverage'
+
+      # Start a Docker container with the working directory mounted.
+      - name: Start PGO
+        run: |
+          kubectl apply --server-side -k ./config/namespace
+          kubectl apply --server-side -k ./config/dev
+          hack/create-kubeconfig.sh postgres-operator pgo
+
+          docker run --detach --network host --read-only \
+            --volume "$(pwd):/mnt" --workdir '/mnt' --env 'PATH=/mnt/bin' \
+            --env 'KUBECONFIG=hack/.kube/postgres-operator/pgo' \
+            --env 'RELATED_IMAGE_PGADMIN=registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi8-4.30-2' \
+            --env 'RELATED_IMAGE_PGBACKREST=registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.38-2' \
+            --env 'RELATED_IMAGE_PGBOUNCER=registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.16-4' \
+            --env 'RELATED_IMAGE_POSTGRES_14=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.4-0' \
+            --env 'RELATED_IMAGE_POSTGRES_14_GIS_3.1=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-14.4-3.1-0' \
+            --name 'postgres-operator' ubuntu \
+            postgres-operator --test.coverprofile 'kuttl.coverage'
+
+      - run: make tools/kuttl
+      - run: make generate-kuttl
+        env:
+          KUTTL_PG_VERSION: '14'
+          KUTTL_POSTGIS_VERSION: '3.1'
+          KUTTL_PSQL_IMAGE: 'registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.4-0'
+      - run: |
+          make check-kuttl && exit
+          failed=$?
+          echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::'
+          exit $failed
+        env:
+          KUTTL_TEST: hack/tools/kuttl test --test='(cluster-start|pgbackrest-restore|pgbouncer)' --timeout=180 --parallel=1
+
+      - name: Stop PGO
+        run: docker stop 'postgres-operator' || true
+
+      # Upload coverage to GitHub
+      - run: gzip kuttl.coverage
+      - uses: actions/upload-artifact@v3
+        with:
+          name: "~coverage~kuttl-k3d=${{ matrix.kubernetes }}"
+          path: kuttl.coverage.gz
+          retention-days: 1
+
   coverage-report:
     if: ${{ success() || contains(needs.*.result, 'success') }}
     runs-on: ubuntu-latest
     needs:
       - kubernetes-api
       - kubernetes-k3d
+      - kuttl-k3d
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
@@ -143,6 +188,6 @@ jobs:
       - run: gzip total-coverage.html
       - uses: actions/upload-artifact@v3
         with:
-          name: coverage-report
+          name: coverage-report=html
           path: total-coverage.html.gz
           retention-days: 15

Makefile

@@ -47,12 +47,17 @@ ifeq ("$(PGO_BASEOS)", "ubi8")
     PACKAGER=microdnf
 endif
 
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL = /usr/bin/env bash -O globstar -o pipefail
+.SHELLFLAGS = -ec
+
 DEBUG_BUILD ?= false
 GO ?= go
 GO_BUILD = $(GO_CMD) build -trimpath
 GO_CMD = $(GO_ENV) $(GO)
 GO_TEST ?= $(GO) test
-KUTTL_TEST ?= kuttl test
+KUTTL_TEST ?= $(KUTTL) test
 
 # Disable optimizations if creating a debug build
 ifeq ("$(DEBUG_BUILD)", "true")
@@ -210,7 +215,7 @@ check-envtest-existing: createnamespaces
 # Expects operator to be running
 .PHONY: check-kuttl
 check-kuttl:
-	${PGO_KUBE_CLIENT} ${KUTTL_TEST} \
+	${KUTTL_TEST} \
 		--config testing/kuttl/kuttl-test.yaml
 
 .PHONY: generate-kuttl
@@ -234,14 +239,12 @@ check-generate: generate-crd generate-deepcopy generate-rbac
 	git diff --exit-code -- config/rbac
 	git diff --exit-code -- pkg/apis
 
-clean: clean-deprecated
+clean: clean-deprecated clean-tools
 	rm -f bin/postgres-operator
 	rm -f config/rbac/role.yaml
 	[ ! -d testing/kuttl/e2e-generated ] || rm -r testing/kuttl/e2e-generated
 	[ ! -d testing/kuttl/e2e-generated-other ] || rm -r testing/kuttl/e2e-generated-other
 	[ ! -d build/crd/generated ] || rm -r build/crd/generated
-	[ ! -d hack/tools/envtest ] || rm -r hack/tools/envtest
-	[ ! -n "$$(ls hack/tools)" ] || rm hack/tools/*
 	[ ! -d hack/.kube ] || rm -r hack/.kube
 
 clean-deprecated:
@@ -306,3 +309,32 @@ hack/tools/envtest:
 license: licenses
 licenses:
 	./bin/license_aggregator.sh ./cmd/...
+
+## Tools
+
+.PHONY: clean-tools tools
+clean-tools:
+	rm -rf hack/krew hack/tools/envtest
+	[ ! -n "$$(ls hack/tools)" ] || rm hack/tools/*
+
+KREW ?= hack/tools/krew
+tools: tools/krew
+tools/krew:
+	$(call go-get-tool,$(KREW),sigs.k8s.io/krew/cmd/krew@v0.4.3)
+
+KUTTL ?= hack/tools/kuttl
+tools: tools/kuttl
+tools/kuttl: tools/krew
+	$(call krew-get-tool,$(KUTTL),kuttl)
+
+# go-get-tool will 'go install' any package $2 and install it to $1.
+define go-get-tool
+@[ -f '$(1)' ] || { echo Downloading '$(2)'; GOBIN='$(abspath $(dir $(1)))' $(GO) install '$(2)'; }
+endef
+
+# krew-get-tool will 'krew install' any plugin $2 and link it to $1.
+define krew-get-tool
+@[ -f '$(1)' ] || { KREW_ROOT='hack/krew' $(KREW) install '$(2)'; }
+@[ -f '$(1)' ] || { T="$$(readlink 'hack/krew/bin/kubectl-$(notdir $(1))')" && \
+        [[ '$(1)' == hack/* ]] && ln -fs "..$${T#$(realpath hack)}" '$(1)'; }
+endef

cmd/postgres-operator/main_coverage_test.go

@@ -0,0 +1,28 @@
+//go:build main_with_coverage
+
+/*
+ Copyright 2022 Crunchy Data Solutions, Inc.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package main
+
+import "testing"
+
+// TestWithCoverage is a simple way to run the entire application when built by
+// "go test -c" with code coverage enabled. This file has a build constraint,
+// and should *not* be run with regular "go test".
+//
+// - https://www.elastic.co/blog/code-coverage-for-your-golang-system-tests
+//
+func TestWithCoverage(t *testing.T) { main() }