deploy datadog to kops eks cluster and upgrade argocd

.gitignore

@@ -4,3 +4,6 @@
 
 # Ignore vscode config files
 .vscode
+
+# kustomize charts
+kubernetes/*/*/charts/

infra/gcp/terraform/k8s-infra-prow-build/iam.tf

@@ -61,3 +61,19 @@ resource "google_iam_workload_identity_pool_provider" "eks_cluster" {
     allowed_audiences = ["sts.googleapis.com"]
   }
 }
+
+resource "google_iam_workload_identity_pool_provider" "eks_kops" {
+  project = module.project.project_id
+
+  display_name                       = "kops"
+  workload_identity_pool_id          = google_iam_workload_identity_pool.eks_cluster.workload_identity_pool_id
+  workload_identity_pool_provider_id = "kops"
+  attribute_mapping = {
+    "google.subject" = "assertion.sub"
+  }
+  oidc {
+    # From EKS cluster created in https://github.com/kubernetes/k8s.io/tree/main/infra/aws/terraform/kops-infra-ci
+    issuer_uri        = "https://oidc.eks.us-east-2.amazonaws.com/id/7283E85C59E9C4129CFD07BAC9378D44"
+    allowed_audiences = ["sts.googleapis.com"]
+  }
+}

kubernetes/apps/datadog.yaml

@@ -0,0 +1,31 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: datadog
+spec:
+  goTemplate: true
+  generators:
+    # targets all clusters
+    - clusters:
+        selector:
+          matchExpressions:
+            - key: clusterType
+              operator: Exists
+  template:
+    metadata:
+      name: "datadog-{{ .name }}"
+    spec:
+      destination:
+        namespace: datadog
+        server: "{{ .server }}"
+      project: default
+      source:
+        path: kubernetes/{{ .name }}/datadog
+        repoURL: https://github.com/kubernetes/k8s.io
+        targetRevision: main
+      syncPolicy:
+        # automated:
+        #   prune: true
+        #   selfHeal: true
+        syncOptions:
+          - CreateNamespace=true

kubernetes/apps/kustomization.yaml

@@ -3,6 +3,7 @@ kind: Kustomization
 resources:
   # - argocd.yaml This has been manually applied to fix sync issues
   - atlantis.yaml
+  - datadog.yaml
   - external-secrets.yaml
   - cert-manager.yaml
   - kyverno.yaml

kubernetes/eks-prow-kops/datadog/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: datadog
+
+helmCharts:
+  - name: datadog
+    repo: https://helm.datadoghq.com
+    releaseName: datadog
+    version: 3.118.0
+    kubeVersion: "1.29"
+    valuesFile: values.yaml
+
+resources:
+  - secrets.yaml

kubernetes/eks-prow-kops/datadog/secrets.yaml

@@ -0,0 +1,11 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: datadog-secret
+spec:
+  dataFrom:
+    - extract:
+        key: datadog-secrets
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: k8s-infra-prow-build

kubernetes/eks-prow-kops/datadog/values.yaml

@@ -0,0 +1,34 @@
+registry: public.ecr.aws/datadog
+datadog:
+  apiKeyExistingSecret: datadog-secret
+  appKeyExistingSecret: datadog-secret
+  site: us5.datadoghq.com
+  clusterName: k8s-infra-kops-prow-build
+  logs:
+    enabled: true
+    containerCollectAll: true
+  prometheusScrape:
+    enabled: true
+    serviceEndpoints: true
+  kubeStateMetricsCore:
+    enabled: true
+  networkMonitoring:
+    enabled: true
+  processAgent:
+    enabled: true
+    processCollection: true
+  sbom:
+    enabled: true
+    containerImage:
+      enabled: true
+      uncompressedLayersSupport: true
+    host:
+      enabled: true
+clusterAgent:
+  tokenExistingSecret: datadog-secret
+agents:
+  tolerations: # datadog supports arm64
+    - key: kubernetes.io/arch
+      operator: Equal
+      value: arm64
+      effect: NoSchedule

kubernetes/eks-prow-kops/helm/external-secrets.yaml

@@ -0,0 +1,55 @@
+extraObjects:
+  - apiVersion: external-secrets.io/v1beta1
+    kind: ClusterSecretStore
+    metadata:
+      name: k8s-infra-prow-build
+    spec:
+      provider:
+        gcpsm:
+          projectID: k8s-infra-prow-build
+
+  - apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: google-adc
+      namespace: external-secrets
+    data:
+      adc.json: |
+        {
+          "type": "external_account",
+          "audience": "//iam.googleapis.com/projects/773781448124/locations/global/workloadIdentityPools/prow-eks/providers/kops",
+          "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
+          "token_url": "https://sts.googleapis.com/v1/token",
+          "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/kubernetes-external-secrets@k8s-infra-prow-build.iam.gserviceaccount.com:generateAccessToken",
+          "credential_source": {
+            "file": "/var/run/secrets/google-iam-token/serviceaccount/token",
+            "format": {
+              "type": "text"
+            }
+          }
+        }
+
+extraVolumes:
+  - name: google-iam-token
+    projected:
+      defaultMode: 420
+      sources:
+        - serviceAccountToken:
+            audience: sts.googleapis.com
+            expirationSeconds: 86400
+            path: token
+  - name: google-adc
+    configMap:
+      name: google-adc
+
+extraEnv:
+  - name: GOOGLE_APPLICATION_CREDENTIALS
+    value: /etc/google/adc.json
+
+extraVolumeMounts:
+  - mountPath: /var/run/secrets/google-iam-token/serviceaccount
+    name: google-iam-token
+    readOnly: true
+  - mountPath: /etc/google
+    name: google-adc
+    readOnly: true

kubernetes/gke-utility/argocd/argocd-app-controller-sts.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: argocd-application-controller
+spec:
+  template:
+    spec:
+      containers:
+        - name: argocd-application-controller
+          env:
+            - name: AWS_ROLE_ARN
+              value: arn:aws:iam::468814281478:role/Prow-EKS-Admin
+            - name: AWS_WEB_IDENTITY_TOKEN_FILE
+              value: /var/run/secrets/aws-iam-token/serviceaccount/token
+            - name: AWS_REGION
+              value: us-east-2
+          volumeMounts:
+            - mountPath: /var/run/secrets/aws-iam-token/serviceaccount
+              name: aws-iam-token
+              readOnly: true
+      volumes:
+        # AWS IAM token needed to assume role to access the EKS clusters.
+        - name: aws-iam-token
+          projected:
+            defaultMode: 420
+            sources:
+              - serviceAccountToken:
+                  audience: sts.amazonaws.com
+                  expirationSeconds: 86400
+                  path: token

kubernetes/gke-utility/argocd/argocd-cm-rbac.yaml

@@ -4,3 +4,6 @@ metadata:
   name: argocd-rbac-cm
 data:
   policy.default: role:readonly
+  policy.csv: |
+    g, kubernetes:sig-k8s-infra-leads, role:admin
+  scopes: "[groups, email]"

kubernetes/gke-utility/argocd/argocd-cm.yaml

@@ -12,11 +12,13 @@ data:
       ignoreDifferences: |
         jqPathExpressions:
         - '.webhooks[]?.clientConfig.caBundle'
-  resource.exclusions: |
-    - apiGroups:
-        - cilium.io
-      kinds:
-        - CiliumIdentity
-      clusters:
-        - "*"
   kustomize.buildOptions: --load-restrictor LoadRestrictionsNone --enable-alpha-plugins
+  dex.config: |
+    connectors:
+      - type: authproxy
+        id: oauth2-proxy
+        name: OAuth2 Proxy
+        config:
+          userHeader: X-Auth-Request-Email
+          groupHeader: X-Auth-Request-Groups
+          userIDHeader: X-Auth-Request-User

kubernetes/gke-utility/argocd/argocd-server-dp.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: argocd-server
+spec:
+  template:
+    spec:
+      containers:
+        - name: argocd-server
+          env:
+            - name: AWS_ROLE_ARN
+              value: arn:aws:iam::468814281478:role/Prow-EKS-Admin
+            - name: AWS_WEB_IDENTITY_TOKEN_FILE
+              value: /var/run/secrets/aws-iam-token/serviceaccount/token
+            - name: AWS_REGION
+              value: us-east-2
+          volumeMounts:
+            - mountPath: /var/run/secrets/aws-iam-token/serviceaccount
+              name: aws-iam-token
+              readOnly: true
+      volumes:
+        # AWS IAM token needed to assume role to access the EKS clusters.
+        - name: aws-iam-token
+          projected:
+            defaultMode: 420
+            sources:
+              - serviceAccountToken:
+                  audience: sts.amazonaws.com
+                  expirationSeconds: 86400
+                  path: token

kubernetes/gke-utility/argocd/clusters.yaml

@@ -95,3 +95,27 @@ spec:
   secretStoreRef:
     kind: ClusterSecretStore
     name: k8s-infra-prow
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: eks-prow-kops
+  labels:
+    argocd.argoproj.io/secret-type: cluster
+    clusterType: prow
+    environment: prod
+    prowNamespace: test-pods
+    cloud: eks
+type: Opaque
+stringData:
+  name: eks-prow-kops
+  server: https://7283E85C59E9C4129CFD07BAC9378D44.gr7.us-east-2.eks.amazonaws.com
+  config: |
+    {
+      "awsAuthConfig": {
+        "clusterName": "k8s-infra-kops-prow-build"
+      },
+      "tlsClientConfig": {
+        "insecure": true
+      }
+    }

kubernetes/gke-utility/argocd/kustomization.yaml

@@ -3,7 +3,7 @@ kind: Kustomization
 namespace: argocd
 
 resources:
-  - github.com/argoproj/argo-cd/manifests/ha/cluster-install?ref=v2.11.2
+  - github.com/argoproj/argo-cd/manifests/ha/cluster-install?ref=v3.0.6
   - extras.yaml
   - clusters.yaml
 
@@ -21,3 +21,5 @@ patches:
     target:
       kind: NetworkPolicy
       name: argocd-redis-ha-server-network-policy
+  - path: argocd-server-dp.yaml
+  - path: argocd-app-controller-sts.yaml