Taskfile.yml

version: '3'

vars:
  CLUSTER_NAME: '{{.CLUSTER_NAME | default "test-infra"}}'
  K8S_VERSION: '{{.K8S_VERSION | default "v1.32.0"}}'
  KIND_VERSION: '{{.KIND_VERSION | default "v0.30.0"}}'
  WAIT_TIMEOUT: '{{.WAIT_TIMEOUT | default "300s"}}'
  TOOLS: kind kubectl kustomize flux
  # Local kubeconfig file - location depends on context
  KUBECONFIG_FILE:
    sh: |
      if [ -f "cluster/kind-config.yaml" ] && [ -f "components/flux/kustomization.yaml" ]; then
        echo "./kubeconfig"  # In test-infra repo
      elif [ -n "$KUBECONFIG" ] && [ -f "$KUBECONFIG" ]; then
        echo "$KUBECONFIG"  # External repo: use KUBECONFIG env var if set and file exists
      else
        echo ".test-infra/kubeconfig"  # External repo: default fallback
      fi
  # Detect if we're running in the test-infra repo or externally
  REPO_DIR:
    sh: |
      if [ -f "cluster/kind-config.yaml" ] && [ -f "components/flux/kustomization.yaml" ]; then
        echo "."
      else
        echo ".test-infra"
      fi
  # Repository URL for cloning when used externally
  REPO_URL: '{{.REPO_URL | default "https://github.com/datum-cloud/test-infra.git"}}'
  REPO_REF: '{{.REPO_REF | default "main"}}'
  # All file paths relative to REPO_DIR
  KIND_CFG: '{{.REPO_DIR}}/cluster/kind-config.yaml'

env:
  KIND_VERSION: '{{.KIND_VERSION}}'
  KUBECONFIG: '{{.KUBECONFIG_FILE}}'

tasks:
  ensure-repo:
    desc: "Ensure test-infra repository is available (clone if needed)"
    silent: true
    cmds:
      - |
        if [ "{{.REPO_DIR}}" != "." ]; then
          if [ ! -d "{{.REPO_DIR}}" ]; then
            echo "📦 Cloning test-infra repository to {{.REPO_DIR}}..."
            git clone {{.REPO_URL}} {{.REPO_DIR}}
            cd {{.REPO_DIR}}
            git checkout {{.REPO_REF}}
            echo "✅ Repository cloned successfully (ref: {{.REPO_REF}})"
          else
            echo "✅ Repository already exists at {{.REPO_DIR}}"
            cd {{.REPO_DIR}}
            # Fetch all refs to ensure we have tags and branches
            git fetch origin --tags --force
            git fetch origin

            # Check if REPO_REF is a tag, branch, or commit
            if git rev-parse --verify "{{.REPO_REF}}" >/dev/null 2>&1; then
              current_ref=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || git rev-parse HEAD)
              if [ "$current_ref" != "{{.REPO_REF}}" ]; then
                echo "🔄 Switching from $current_ref to {{.REPO_REF}}..."
                git checkout {{.REPO_REF}}
              fi
            else
              echo "⚠️  Reference '{{.REPO_REF}}' not found, trying as remote branch..."
              git checkout -B {{.REPO_REF}} origin/{{.REPO_REF}} || {
                echo "❌ Failed to checkout '{{.REPO_REF}}'"
                exit 1
              }
            fi

            # Only pull if it's a branch (not a tag or detached HEAD)
            if git symbolic-ref -q HEAD >/dev/null; then
              git pull origin {{.REPO_REF}} 2>/dev/null || true
            fi

            echo "✅ Repository updated (ref: {{.REPO_REF}})"
          fi
        else
          echo "✅ Running in source repository"
        fi

  help:
    desc: "Show organized help with command groups"
    silent: true
    cmds:
      - echo "Available commands for test-infra:"
      - echo ""
      - echo "🚀 Cluster Management:"
      - echo "  cluster-up              Spin everything up"
      - echo "  cluster-down            Delete the cluster"
      - echo "  cluster-reset           Delete and recreate cluster from scratch"
      - echo ""
      - echo "🔍 Cluster Inspection:"
      - echo "  cluster-status          Show cluster status and health"
      - echo "  kubectl                 Run kubectl commands (pass args with --)"
      - echo "  k9s                     Launch k9s to browse test-infra cluster"
      - echo ""
      - echo "🔧 Setup & Infrastructure:"
      - echo "  ensure-tools            Install/verify deps"
      - echo "  create-kind             Create KIND cluster"
      - echo "  delete-kind             Delete KIND cluster"
      - echo ""
      - echo "📦 Components:"
      - echo "  install-components      All add-ons"
      - echo "  install-flux            Deploy Flux via Kustomize"
      - echo "  install-cert-manager    Deploy cert-manager + CSI driver"
      - echo "  install-kyverno         Deploy Kyverno policy engine"
      - echo "  install-envoy-gateway-operator  Deploy Envoy Gateway"
      - echo "  install-observability   [OPTIONAL] Deploy telemetry stack"
      - echo ""
      - echo "🔨 CI/Development:"
      - echo "  kind-load-image         Load images into KIND"
      - echo "  kind-save-image         Save image tarball artifact"
      - echo ""
      - 'echo "For detailed help: task --list"'
      - 'echo "To run a command: task <command-name>"'

  default:
    desc: "Show help by default"
    cmds:
      - task: help
  #------------------------------------------------------------------------------
  # high-level targets
  #------------------------------------------------------------------------------
  cluster-up:
    aliases:
      - up
    desc: "Spin everything up"
    silent: true
    cmds:
      - echo "🚀 Starting cluster deployment..."
      - echo ""
      - |
        # Set up error handling
        set -e
        trap 'echo "❌ Cluster deployment failed! Use '\''task cluster-status'\'' to check cluster state."' ERR
      - task: ensure-repo
      - task: ensure-tools
      - task: create-kind
      - task: install-components
      - echo ""
      - echo "🎉 Test infrastructure deployment complete!"
      - echo ""
      - echo "🌐 Available endpoints:"
      - 'echo "   • HTTPS Gateway: https://localhost:8443"'
      - 'echo "   • Grafana:       http://localhost:30000 (after installing observability)"'
      - echo ""
      - echo "📦 Optional add-ons:"
      - echo "   • task install-observability"  # Deploy full telemetry stack
      - echo ""
      - echo "🚀 Quick start:"
      - echo "   • export KUBECONFIG={{.KUBECONFIG_FILE}}"  # Connect to test-infra cluster
      - echo "   • task cluster-status"                # Check cluster health
      - echo "   • kubectl get pods --all-namespaces"  # List all pods
      - echo "   • curl -k https://localhost:8443"     # Test gateway
      - echo "   • task help"                          # See all commands

  cluster-down:
    desc: "Delete the cluster"
    aliases:
      - down
    cmds:
      - task: delete-kind

  cluster-reset:
    desc: "Delete and recreate cluster from scratch"
    silent: true
    cmds:
      - echo "🔄 Resetting cluster '{{.CLUSTER_NAME}}'..."
      - task: cluster-down
      - echo "⏳ Waiting before recreating cluster..."
      - sleep 3
      - task: cluster-up

  #------------------------------------------------------------------------------
  # cluster status and validation
  #------------------------------------------------------------------------------
  kubectl:
    desc: "Run kubectl commands against the test-infra cluster"
    silent: true
    cmds:
      - |
        if [ ! -f "{{.KUBECONFIG_FILE}}" ]; then
          echo "❌ Kubeconfig not found at {{.KUBECONFIG_FILE}}"
          echo "💡 Run 'task cluster-up' first to create the cluster"
          exit 1
        fi

        kubectl --kubeconfig {{.KUBECONFIG_FILE}} {{.CLI_ARGS}}

  k9s:
    desc: "Launch k9s to browse the test-infra cluster"
    silent: true
    cmds:
      - |
        if [ ! -f "{{.KUBECONFIG_FILE}}" ]; then
          echo "❌ Kubeconfig not found at {{.KUBECONFIG_FILE}}"
          echo "💡 Run 'task cluster-up' first to create the cluster"
          exit 1
        fi

        if ! command -v k9s >/dev/null 2>&1; then
          echo "❌ k9s is not installed"
          echo "💡 Install k9s from: https://k9scli.io/topics/install/"
          exit 1
        fi

        echo "🚀 Launching k9s for test-infra cluster..."
        echo "📝 Using kubeconfig: {{.KUBECONFIG_FILE}}"
        k9s --kubeconfig {{.KUBECONFIG_FILE}}

  cluster-status:
    desc: "Show cluster status and health"
    silent: true
    cmds:
      - echo "🔍 Cluster Status Check"
      - echo ""
      - |
        if kind get clusters | grep -q "^{{.CLUSTER_NAME}}$"; then
          echo "✅ KIND cluster '{{.CLUSTER_NAME}}' exists"
          echo "📊 Kubernetes version: $(kubectl version --short --client=false 2>/dev/null | grep 'Server Version' || echo 'Not accessible')"
          echo "🏗️  Nodes:"
          kubectl get nodes --no-headers 2>/dev/null | awk '{print "   • " $1 " (" $2 ")"}' || echo "   ❌ Cannot access cluster"
        else
          echo "❌ KIND cluster '{{.CLUSTER_NAME}}' does not exist"
          exit 0
        fi
      - echo ""
      - echo "📦 Components:"
      - |
        components=(
          "flux-system:Flux CD"
          "cert-manager:cert-manager"
          "kyverno:Kyverno"
          "envoy-gateway-system:Envoy Gateway"
        )
        for comp in "${components[@]}"; do
          ns="${comp%:*}"
          name="${comp#*:}"
          if kubectl get ns "$ns" >/dev/null 2>&1; then
            ready=$(kubectl get pods -n "$ns" --no-headers 2>/dev/null | awk '$3=="Running" {ready++} END {print ready+0}')
            total=$(kubectl get pods -n "$ns" --no-headers 2>/dev/null | wc -l)
            if [ "$ready" -eq "$total" ] && [ "$total" -gt 0 ]; then
              echo "   ✅ $name ($ready/$total pods ready)"
            else
              echo "   ⚠️  $name ($ready/$total pods ready)"
            fi
          else
            echo "   ❌ $name (not installed)"
          fi
        done


  #------------------------------------------------------------------------------
  # granular targets
  #------------------------------------------------------------------------------
  ensure-tools:
    desc: "Install/verify deps"
    silent: true
    cmds:
      - bash {{.REPO_DIR}}/hack/ensure-tools.sh {{.TOOLS}}

  create-kind:
    desc: "Create KIND cluster"
    silent: true
    cmds:
      - echo "➡️  Creating KIND cluster '{{.CLUSTER_NAME}}' ..."
      - |
        # Check if cluster already exists
        if kind get clusters | grep -q "^{{.CLUSTER_NAME}}$"; then
          echo "✅ KIND cluster '{{.CLUSTER_NAME}}' already exists"
          # Validate cluster is accessible
          if kubectl cluster-info >/dev/null 2>&1; then
            echo "✅ Cluster is accessible and ready"
          else
            echo "⚠️  Cluster exists but is not accessible - checking..."
            kubectl cluster-info 2>&1 || echo "❌ Cluster may need to be recreated"
          fi
        else
          echo "🚀 Creating new KIND cluster..."
          # Validate config file exists
          if [ ! -f "{{.KIND_CFG}}" ]; then
            echo "❌ KIND config file not found: {{.KIND_CFG}}"
            exit 1
          fi

          # Export REPO_DIR for envsubst to substitute audit policy path in KIND config
          # KIND doesn't natively support env var substitution, so we preprocess with envsubst
          export REPO_DIR="{{.REPO_DIR}}"

          # Process KIND config and create cluster (--name and --image override config values)
          if envsubst < {{.KIND_CFG}} | kind create cluster --name {{.CLUSTER_NAME}} --image kindest/node:{{.K8S_VERSION}} --config - --kubeconfig {{.KUBECONFIG_FILE}}; then
            echo "✅ KIND cluster created successfully"
            echo "📝 Test-infra cluster kubeconfig written to: {{.KUBECONFIG_FILE}}"

            # Wait for cluster to be ready
            echo "⏳ Waiting for cluster to be ready..."
            timeout_count=0
            while ! kubectl cluster-info >/dev/null 2>&1; do
              if [ $timeout_count -ge 30 ]; then
                echo "❌ Timeout waiting for cluster to be ready"
                exit 1
              fi
              echo "   Waiting for API server... ($timeout_count/30)"
              sleep 2
              timeout_count=$((timeout_count + 1))
            done

            echo "✅ Cluster is ready and accessible"
          else
            echo "❌ Failed to create KIND cluster"
            exit 1
          fi
        fi
      # Always install flux to the cluster so additional components can be
      # installed using it.
      - task: install-flux

  delete-kind:
    desc: "Delete KIND cluster"
    silent: true
    cmds:
      - echo "🗑️  Deleting KIND cluster '{{.CLUSTER_NAME}}' ..."
      - |
        if kind get clusters | grep -q "^{{.CLUSTER_NAME}}$"; then
          echo "🗑️  Cluster exists, proceeding with deletion..."
          if kind delete cluster --name {{.CLUSTER_NAME}}; then
            echo "✅ KIND cluster deleted successfully"
          else
            echo "⚠️  Error during cluster deletion, but continuing..."
          fi
        else
          echo "ℹ️  KIND cluster '{{.CLUSTER_NAME}}' does not exist"
        fi

        # Clean up any lingering containers
        echo "🧹 Cleaning up any leftover containers..."
        docker ps -a --filter "label=io.x-k8s.kind.cluster={{.CLUSTER_NAME}}" --format "{{.ID}}" | xargs -r docker rm -f 2>/dev/null || true

        # Clean up kubeconfig file if it exists
        if [ -f "{{.KUBECONFIG_FILE}}" ]; then
          echo "🧹 Removing test-infra cluster kubeconfig file..."
          rm -f {{.KUBECONFIG_FILE}}
        fi

        echo "✅ Cleanup complete"

  install-components:
    desc: "All add-ons"
    silent: true
    deps:
      - install-cert-manager
      - install-kyverno
      - install-envoy-gateway-operator
    cmds:
      - echo "✅ All cluster components installed successfully"

  install-cert-manager:
    desc: "Deploy cert-manager (+ CSI driver) via Kustomize, then wait"
    silent: true
    cmds:
      - echo "➡️  Reconciling cert-manager …"
      - kustomize build {{.REPO_DIR}}/components/cert-manager | kubectl apply -f -
      - echo "⏳ Waiting for cert-manager HelmReleases …"
      - kubectl -n cert-manager wait helmrelease/{cert-manager,cert-manager-csi-driver} --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
      - echo "✅ cert-manager and CSI driver are ready"

  install-flux:
    desc: "Deploy Flux via Kustomize, then wait for its controllers"
    silent: true
    cmds:
      - echo "➡️  Reconciling Flux …"
      - kustomize build {{.REPO_DIR}}/components/flux | kubectl apply -f -
      - echo "⏳ Waiting for Flux controllers …"
      - kubectl -n flux-system wait deployment/{source-controller,helm-controller,kustomize-controller,notification-controller} --for=condition=Available --timeout={{.WAIT_TIMEOUT}}
      - echo "⏳ Waiting for Flux CRDs to be ready …"
      - sleep 10
      - echo "✅ Flux is ready"

  install-kyverno:
    desc: "Deploy or upgrade Kyverno idempotently, then wait"
    silent: true
    cmds:
      - echo "➡️  Reconciling Kyverno …"
      - kustomize build {{.REPO_DIR}}/components/kyverno | kubectl apply --server-side --field-manager=kyverno-installer --force-conflicts -f -
      - echo "⏳ Waiting for Kyverno controllers …"
      - kubectl -n kyverno wait deployment/kyverno-{admission-controller,background-controller,cleanup-controller,reports-controller} --for=condition=Available --timeout={{.WAIT_TIMEOUT}}
      - echo "✅ Kyverno is ready"

  install-envoy-gateway-operator:
    desc: "Deploy Envoy Gateway Operator via Kustomize, then wait"
    silent: true
    cmds:
      - echo "➡️  Reconciling Envoy Gateway Operator …"
      - kustomize build {{.REPO_DIR}}/components/envoy-gateway-operator | kubectl apply -f -
      - echo "⏳ Waiting for Envoy Gateway Operator HelmRelease …"
      - kubectl -n flux-system wait helmrelease/envoy-gateway --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
      - echo "➡️  Applying Gateway Configuration (merged gateway setup) …"
      - kustomize build {{.REPO_DIR}}/components/envoy-gateway-operator/gateway-resources | kubectl apply -f -
      - echo "⏳ Waiting for Gateway to be ready …"
      - kubectl wait --for=condition=Programmed gateway/default-gateway -n envoy-gateway-system --timeout={{.WAIT_TIMEOUT}} || true
      - echo "✅ Envoy Gateway Operator and merged gateway are ready"
      - echo ""
      - echo "🌐 Gateway exposed ports:"
      - 'echo "   HTTPS Gateway: https://localhost:8443 (self-signed cert - use -k flag)"'
      - echo "   To find the actual NodePort assignments, run:"
      - echo "   kubectl get svc -n envoy-gateway-system -l app.kubernetes.io/name=envoy"

  install-observability:
    desc: "[OPTIONAL] Deploy complete observability stack (Victoria Metrics, Loki, Tempo, Grafana, OTel)"
    silent: true
    cmds:
      - task: ensure-repo
      - echo "➡️  Reconciling Observability Stack …"
      - kustomize build {{.REPO_DIR}}/components/observability | kubectl apply -f -
      - echo "⏳ Waiting for Grafana Operator HelmRelease …"
      - kubectl -n flux-system wait helmrelease/grafana-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
      - echo "⏳ Waiting for Victoria Metrics Stack HelmRelease …"
      - kubectl -n flux-system wait helmrelease/vm --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
      - echo "⏳ Waiting for Loki HelmRelease …"
      - kubectl -n flux-system wait helmrelease/loki --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
      - echo "⏳ Waiting for Tempo HelmRelease …"
      - kubectl -n flux-system wait helmrelease/tempo --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
      - echo "⏳ Waiting for OpenTelemetry Operator HelmRelease …"
      - kubectl -n flux-system wait helmrelease/opentelemetry-operator --for=condition=Ready --timeout={{.WAIT_TIMEOUT}}
      - |
        set -euo pipefail
        echo "⏳ Waiting for Prometheus CRDs to be Established…"
        for crd in \
          servicemonitors.monitoring.coreos.com \
          podmonitors.monitoring.coreos.com \
          probes.monitoring.coreos.com \
          prometheuses.monitoring.coreos.com \
          prometheusrules.monitoring.coreos.com \
          alertmanagers.monitoring.coreos.com \
          thanosrulers.monitoring.coreos.com
        do
          echo "$crd"
          kubectl wait --for=condition=Established "crd/${crd}" --timeout=120s || true
        done
      - echo "⏳ Waiting for vmagent & vmsingle to be Ready (best-effort)…"
      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmagent -n telemetry-system --timeout=120s || true
      - kubectl wait --for=condition=Available deploy -l app.kubernetes.io/name=vmsingle -n telemetry-system --timeout=120s || true
      - echo "⏳ Waiting for OpenTelemetryCollector CRD to be Established …"
      - kubectl wait --for=condition=Established crd/opentelemetrycollectors.opentelemetry.io --timeout={{.WAIT_TIMEOUT}}
      - echo "➡️  Applying OpenTelemetryCollector CR (waiting for the operator webhook) …"
      - |
        set -euo pipefail
        # HR-Ready + CRD-Established do NOT guarantee the operator's admission
        # webhook is serving with an injected caBundle — both happen
        # asynchronously after the Helm install. Wait for the operator
        # Deployment, then retry the apply so a cold-start caBundle-injection
        # lag can't fail the install with an x509 / "no endpoints" webhook error.
        kubectl -n telemetry-system wait --for=condition=Available \
          deploy -l app.kubernetes.io/name=opentelemetry-operator --timeout={{.WAIT_TIMEOUT}}
        for i in $(seq 1 30); do
          kubectl apply -f {{.REPO_DIR}}/components/observability/otel-collector/opentelemetry-collector.yaml && break
          [ "$i" = 30 ] && { echo "OpenTelemetryCollector apply failed after retries" >&2; exit 1; }
          echo "  operator webhook not ready yet, retrying ($i)…" >&2
          sleep 5
        done
      - echo "⏳ Waiting for OTel Collector DaemonSet …"
      - kubectl -n telemetry-system rollout status daemonset/otel-collector-collector --timeout={{.WAIT_TIMEOUT}}
      - echo "➡️  Applying Grafana Instance (after Operator CRDs are ready) …"
      - kubectl apply -f {{.REPO_DIR}}/components/observability/grafana-instance.yaml
      - echo "➡️  Applying Grafana Datasources …"
      - kustomize build {{.REPO_DIR}}/components/observability/datasources | kubectl apply -f -
      - echo "✅ Observability stack is ready"
      - echo ""
      - 'echo "📊 Access Grafana at: http://localhost:30000"'
      - 'echo "   Username: admin"'
      - 'echo "   Password: datum123"'
      - echo ""
      - echo "🌐 Grafana is automatically exposed via NodePort 30000"
      - echo "   No port-forwarding needed!"

  # ------------------------------------------------------------------
  # Helpers for CI
  # ------------------------------------------------------------------
  kind-load-image:
    desc: "Load one or more images into KIND"
    silent: true
    vars:
      IMAGES: '{{.IMAGES}}'
    preconditions:
      - sh: '[ -n "{{.IMAGES}}" ]'
        msg: 'ERROR: pass IMAGES="img1 img2 ..." as argument'
    cmds:
      - |
        for img in {{.IMAGES}}; do
          echo "➡️  Loading $img into kind '{{.CLUSTER_NAME}}'"
          docker pull $img || true
          kind load docker-image $img --name {{.CLUSTER_NAME}}
        done

  kind-save-image:
    desc: "Save an image tarball artifact"
    silent: true
    vars:
      IMAGE: '{{.IMAGE}}'
      TAR: '{{.TAR}}'
    preconditions:
      - sh: '[ -n "{{.IMAGE}}" ]'
        msg: 'ERROR: set IMAGE=<name> as argument'
      - sh: '[ -n "{{.TAR}}" ]'
        msg: 'ERROR: set TAR=<file> as argument'
    cmds:
      - docker save "{{.IMAGE}}" | gzip > "{{.TAR}}"