Adding a secure local registry

Signed-off-by: ytimocin <ytimocin@microsoft.com>
radius-project · Jul 18, 2024 · 38a65b4 · 38a65b4
1 parent d5437ee
commit 38a65b4
Show file tree

Hide file tree

Showing 2 changed files with 130 additions and 58 deletions.
diff --git a/.github/workflows/functional-test-noncloud.yaml b/.github/workflows/functional-test-noncloud.yaml
@@ -47,26 +47,17 @@ on:
 
 env:
   # Go version
-  GOVER: "1.22.2"
+  GOVER: "1.22.5"
   # Helm version
-  HELM_VER: "v3.12.0"
+  HELM_VER: "v3.15.3"
   # KinD cluster version
   KIND_VER: "v0.23.0"
   # Dapr version
   DAPR_VER: "1.12.0"
   # Dapr dashboard version
   DAPR_DASHBOARD_VER: "0.14.0"
   # Kubectl version
-  KUBECTL_VER: "v1.25.0"
-
-  # Container registry for storing container images
-  CONTAINER_REGISTRY: "radius-registry:5000"
-  # Container registry for storing Bicep recipe artifacts
-  BICEP_RECIPE_REGISTRY: "radius-registry:5000"
-  # Local Docker registry name
-  LOCAL_REGISTRY_NAME: "radius-registry"
-  # Local Docker registry port
-  LOCAL_REGISTRY_PORT: "5000"
+  KUBECTL_VER: "v1.30.0"
 
   # The radius functional test timeout
   FUNCTIONALTEST_TIMEOUT: 15m
@@ -84,6 +75,13 @@ env:
   # The number of failed tests to report.
   ISSUE_CREATE_THRESHOLD: 2
 
+  # Local Docker registry name
+  LOCAL_REGISTRY_NAME: "radius-registry"
+  # Local Docker registry server
+  LOCAL_REGISTRY_SERVER: "localhost"
+  # Local Docker registry port
+  LOCAL_REGISTRY_PORT: "5000"
+
 jobs:
   build:
     name: Build Radius for test
@@ -142,7 +140,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-latest-m]
         name:
           [
             cli-noncloud,
@@ -208,37 +206,105 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
+      - name: Create certificates for local registry
+        run: |
+          create_openssl_cfg() {
+            CFG=$(
+              cat <<'EOF'
+          [req]
+          distinguished_name = subject
+          x509_extensions    = x509_ext
+          prompt             = no
+
+          [subject]
+          CN = localhost
+
+          [x509_ext]
+          basicConstraints        = critical, CA:TRUE
+          subjectKeyIdentifier    = hash
+          authorityKeyIdentifier  = keyid:always, issuer:always
+          keyUsage                = critical, cRLSign, digitalSignature, keyCertSign
+          nsComment               = "OpenSSL Generated Certificate"
+          subjectAltName          = @alt_names
+
+          [alt_names]
+          DNS.1 = radius-registry
+          DNS.2 = localhost
+          EOF
+            )
+            echo "$CFG"
+          }
+
+          temp_cert_dir=$(mktemp -d 2>/dev/null || mktemp -d -t 'temp_cert_dir')
+          # Set the temp_cert_dir as an environment variable
+          echo "TEMP_CERT_DIR=$temp_cert_dir" >> $GITHUB_ENV
+
+          # Go to the temp_cert_dir
+          pushd $temp_cert_dir
+          # Create
+          mkdir -p certs/${{ env.LOCAL_REGISTRY_SERVER }}
+
+          echo "==== Generate the openssl config"
+          create_openssl_cfg >req.cnf
+
+          echo "==== Create the self signed certificate certificate and client key files"
+          openssl req -x509 \
+            -nodes \
+            -days 365 \
+            -newkey rsa:4096 \
+            -keyout certs/${{ env.LOCAL_REGISTRY_SERVER }}/client.key \
+            -out certs/${{ env.LOCAL_REGISTRY_SERVER }}/client.crt \
+            -config req.cnf \
+            -sha256
+
+      - name: Add the certificate to the system trust store
+        run: |
+          sudo apt install ca-certificates
+          sudo cp $TEMP_CERT_DIR/certs/${{ env.LOCAL_REGISTRY_SERVER }}/client.crt /usr/local/share/ca-certificates/${{ env.LOCAL_REGISTRY_SERVER }}.crt
+          sudo cp $TEMP_CERT_DIR/certs/${{ env.LOCAL_REGISTRY_SERVER }}/client.crt /usr/local/share/ca-certificates/${{ env.LOCAL_REGISTRY_NAME }}.crt
+          sudo update-ca-certificates
+
       - name: Create local Docker registry
         run: |
-          # This is going to start an insecure registry on localhost:5000 on the host machine.
           if [ "$(docker inspect -f '{{.State.Running}}' "${{ env.LOCAL_REGISTRY_NAME }}" 2>/dev/null || true)" != 'true' ]; then
-            docker run \
-              -d --restart=always -p "127.0.0.1:${{ env.LOCAL_REGISTRY_PORT }}:5000" --network bridge --name "${{ env.LOCAL_REGISTRY_NAME }}" \
+            echo "==== Creating a docker registry"
+            echo "==== TEMP_CERT_DIR: $TEMP_CERT_DIR"
+
+            ls -l $TEMP_CERT_DIR/certs/${{ env.LOCAL_REGISTRY_SERVER }}
+
+            docker run -d \
+              -p ${{ env.LOCAL_REGISTRY_PORT }}:5000 \
+              --restart=always \
+              --name ${{ env.LOCAL_REGISTRY_NAME }} \
+              -v $TEMP_CERT_DIR/certs/${{ env.LOCAL_REGISTRY_SERVER }}:/certs \
+              -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/client.crt \
+              -e REGISTRY_HTTP_TLS_KEY=/certs/client.key \
               registry:2
-          fi
 
-      - name: Add insecure registry to Docker daemon
-        run: |
-          # Check if /etc/docker/daemon.json exists
-          if [ ! -f /etc/docker/daemon.json ]; then
-            echo "daemon.json doesn't exist. Creating one..."
-            echo '{}' | sudo tee /etc/docker/daemon.json
+              # Wait a few seconds for the registry to start
+              sleep 5
+
+              # Check if the registry is running and not restarting
+              if [ "$(docker inspect -f '{{.State.Running}}' "${{ env.LOCAL_REGISTRY_NAME }}" 2>/dev/null || true)" != 'true' ]; then
+                echo "==== Docker registry failed to start"
+                echo "==== Inspecting Docker container logs"
+                docker logs ${{ env.LOCAL_REGISTRY_NAME }}
+                echo "==== Inspecting Docker container state"
+                docker inspect ${{ env.LOCAL_REGISTRY_NAME }}
+                exit 1
+              else
+                echo "==== Docker registry started successfully"
+              fi
+          else
+            # This shouldn't be the case.
+            echo "==== Docker registry already running"
           fi
 
-          # Add insecure registries to /etc/docker/daemon.json
-          echo '{"insecure-registries": ["radius-registry:5000"]}' | sudo tee /etc/docker/daemon.json
-          sudo systemctl daemon-reload
-          sudo systemctl restart docker
-
-      - name: Add radius-registry to /etc/hosts
-        run: |
-          sudo sh -c 'echo "127.0.0.1 radius-registry" >> /etc/hosts'
-
       - name: Build and Push container images
         run: |
           make build && make docker-build && make docker-push
         env:
-          DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
+          DOCKER_REGISTRY: "${{ env.LOCAL_REGISTRY_SERVER }}:${{ env.LOCAL_REGISTRY_PORT }}"
           DOCKER_TAG_VERSION: ${{ env.REL_VERSION }}
 
       - name: Install rad CLI
@@ -261,17 +327,17 @@ jobs:
           chmod +x ./kind
 
           # Create kind cluster with containerd registry config dir enabled
-          cat <<EOF | ./kind create cluster --config=-
+          cat <<EOF | kind create cluster --config=-
           kind: Cluster
           apiVersion: kind.x-k8s.io/v1alpha4
+          nodes:
+          - role: control-plane
+            extraMounts:
+              - containerPath: "/etc/containerd/certs.d/${{ env.LOCAL_REGISTRY_NAME }}"
+                hostPath: "$TEMP_CERT_DIR/certs/${{ env.LOCAL_REGISTRY_SERVER }}"
           containerdConfigPatches:
           - |-
             [plugins."io.containerd.grpc.v1.cri".registry]
-              [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-                [plugins."io.containerd.grpc.v1.cri".registry.mirrors."radius-registry:5000"]
-                  endpoint = ["http://radius-registry:5000"]
-                [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"]
-                  endpoint = ["http://radius-registry:5000"]
               config_path = "/etc/containerd/certs.d"
           EOF
 
@@ -283,21 +349,23 @@ jobs:
           #
           # We want a consistent name that works from both ends, so we tell containerd to
           # alias localhost:${reg_port} to the registry container when pulling images
-          LOCALHOST_REGISTRY_DIR="/etc/containerd/certs.d/localhost:${{ env.LOCAL_REGISTRY_PORT }}"
-          RADIUS_REGISTRY_DIR="/etc/containerd/certs.d/radius-registry:5000"
+          LOCALHOST_DIR="/etc/containerd/certs.d/${{ env.LOCAL_REGISTRY_SERVER }}:${{ env.LOCAL_REGISTRY_PORT }}"
+          RADIUS_DIR="/etc/containerd/certs.d/${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"
+
           for node in $(kind get nodes); do
-            # LOCALHOST_REGISTRY_DIR
-            docker exec "${node}" mkdir -p "${LOCALHOST_REGISTRY_DIR}"
-            cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${LOCALHOST_REGISTRY_DIR}/hosts.toml"
-          [host."http://${{ env.LOCAL_REGISTRY_NAME }}:5000"]
+            
+            # LOCALHOST_DIR
+            docker exec "${node}" mkdir -p "${LOCALHOST_DIR}"
+            cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${LOCALHOST_DIR}/hosts.toml"
+          [host."http://${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"]
             capabilities = ["pull", "resolve", "push"]
             skip_verify = true
           EOF
 
-              # RADIUS_REGISTRY_DIR
-              docker exec "${node}" mkdir -p "${RADIUS_REGISTRY_DIR}"
-              cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${RADIUS_REGISTRY_DIR}/hosts.toml"
-          [host."http://radius-registry:5000"]
+            # RADIUS_DIR
+            docker exec "${node}" mkdir -p "${RADIUS_DIR}"
+            cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${RADIUS_DIR}/hosts.toml"
+          [host."http://${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"]
             capabilities = ["pull", "resolve", "push"]
             skip_verify = true
           EOF
@@ -320,7 +388,7 @@ jobs:
             namespace: kube-public
           data:
             localRegistryHosting.v1: |
-              host: "localhost:${{ env.LOCAL_REGISTRY_PORT }}"
+              host: "${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"
               help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
           EOF
 
@@ -353,10 +421,11 @@ jobs:
           echo "*** Installing Radius to Kubernetes ***"
           rad install kubernetes \
             --chart ${{ env.RADIUS_CHART_LOCATION }} \
-            --set rp.image=${{ env.CONTAINER_REGISTRY }}/applications-rp,rp.tag=${{ env.REL_VERSION }} \
-            --set controller.image=${{ env.CONTAINER_REGISTRY }}/controller,controller.tag=${{ env.REL_VERSION }} \
-            --set ucp.image=${{ env.CONTAINER_REGISTRY }}/ucpd,ucp.tag=${{ env.REL_VERSION }} \
-            --set de.image=${{ env.DE_IMAGE }},de.tag=${{ env.DE_TAG }}
+            --set rp.image=${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}/applications-rp,rp.tag=${{ env.REL_VERSION }} \
+            --set controller.image=${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}/controller,controller.tag=${{ env.REL_VERSION }} \
+            --set ucp.image=${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}/ucpd,ucp.tag=${{ env.REL_VERSION }} \
+            --set de.image=${{ env.DE_IMAGE }},de.tag=${{ env.DE_TAG }} \
+            --set-file global.rootCA.cert=$TEMP_CERT_DIR/certs/${{ env.LOCAL_REGISTRY_SERVER }}/client.crt
 
           echo "*** Create workspace, group and environment for test ***"
           rad workspace create kubernetes
@@ -377,8 +446,9 @@ jobs:
           which rad || { echo "cannot find rad"; exit 1; }
           make publish-test-bicep-recipes
         env:
+          BICEP_RECIPE_REGISTRY: "${{ env.LOCAL_REGISTRY_SERVER }}:${{ env.LOCAL_REGISTRY_PORT }}"
           BICEP_RECIPE_TAG_VERSION: ${{ env.REL_VERSION }}
-          INSECURE_REGISTRY: true
+          SSL_CERT_FILE: $TEMP_CERT_DIR/certs/${{ env.LOCAL_REGISTRY_SERVER }}/client.crt
 
       - name: Run functional tests
         run: |
@@ -392,11 +462,11 @@ jobs:
 
           make test-functional-${{ matrix.name }}
         env:
-          DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
+          DOCKER_REGISTRY: "${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"
           TEST_TIMEOUT: ${{ env.FUNCTIONALTEST_TIMEOUT }}
           RADIUS_CONTAINER_LOG_PATH: ${{ github.workspace }}/${{ env.RADIUS_CONTAINER_LOG_BASE }}
           RADIUS_SAMPLES_REPO_ROOT: ${{ github.workspace }}/samples
-          BICEP_RECIPE_REGISTRY: ${{ env.BICEP_RECIPE_REGISTRY }}
+          BICEP_RECIPE_REGISTRY: "${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"
           BICEP_RECIPE_TAG_VERSION: ${{ env.BICEP_RECIPE_TAG_VERSION }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GOTESTSUM_OPTS: "--junitfile ./dist/functional_test/results.xml"

diff --git a/test/functional-portable/cli/noncloud/cli_test.go b/test/functional-portable/cli/noncloud/cli_test.go
@@ -68,7 +68,9 @@ func verifyRecipeCLI(ctx context.Context, t *testing.T, test rp.RPTest) {
 	version := strings.TrimPrefix(testutil.GetBicepRecipeVersion(), "version=")
 	resourceType := "Applications.Datastores/redisCaches"
 	file := "../../../testrecipes/test-bicep-recipes/corerp-redis-recipe.bicep"
-	target := fmt.Sprintf("br:ghcr.io/radius-project/dev/test-bicep-recipes/redis-recipe:%s", generateUniqueTag())
+
+	// If BICEP_RECIPE_REGISTRY is set then we will use the given registry, otherwise we will use the default registry.
+	target := fmt.Sprintf("br:%s/dev/test-bicep-recipes/redis-recipe:%s", registry, generateUniqueTag())
 
 	recipeName := "recipeName"
 	recipeTemplate := fmt.Sprintf("%s/recipes/local-dev/rediscaches:%s", registry, version)