Skip to content

Commit

Permalink
Adding a secure local registry
Browse files Browse the repository at this point in the history 
Signed-off-by: ytimocin <ytimocin@microsoft.com>
  • Loading branch information
@ytimocin
ytimocin committed Jul 18, 2024
1 parent d330bed commit 5124784
Showing 1 changed file with 115 additions and 54 deletions.
169 changes: 115 additions & 54 deletions .github/workflows/functional-test-noncloud.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,26 +47,17 @@ on:

env:
# Go version
GOVER: "1.22.2"
GOVER: "1.22.5"
# Helm version
HELM_VER: "v3.12.0"
HELM_VER: "v3.15.3"
# KinD cluster version
KIND_VER: "v0.23.0"
# Dapr version
DAPR_VER: "1.12.0"
# Dapr dashboard version
DAPR_DASHBOARD_VER: "0.14.0"
# Kubectl version
KUBECTL_VER: "v1.25.0"

# Container registry for storing container images
CONTAINER_REGISTRY: "radius-registry:5000"
# Container registry for storing Bicep recipe artifacts
BICEP_RECIPE_REGISTRY: "radius-registry:5000"
# Local Docker registry name
LOCAL_REGISTRY_NAME: "radius-registry"
# Local Docker registry port
LOCAL_REGISTRY_PORT: "5000"
KUBECTL_VER: "v1.30.0"

# The radius functional test timeout
FUNCTIONALTEST_TIMEOUT: 15m
Expand All @@ -84,6 +75,13 @@ env:
# The number of failed tests to report.
ISSUE_CREATE_THRESHOLD: 2

# Local Docker registry name
LOCAL_REGISTRY_NAME: "radius-registry"
# Local Docker registry server
LOCAL_REGISTRY_SERVER: "localhost"
# Local Docker registry port
LOCAL_REGISTRY_PORT: "5000"

jobs:
build:
name: Build Radius for test
Expand Down Expand Up @@ -208,37 +206,97 @@ jobs:
restore-keys: |
${{ runner.os }}-go-
- name: Create certificates for local registry
run: |
create_openssl_cfg() {
CFG=$(
cat <<'EOF'
[req]
distinguished_name = subject
x509_extensions = x509_ext
prompt = no
[subject]
CN = localhost
[x509_ext]
basicConstraints = critical, CA:TRUE
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always, issuer:always
keyUsage = critical, cRLSign, digitalSignature, keyCertSign
nsComment = "OpenSSL Generated Certificate"
subjectAltName = @alt_names
[alt_names]
DNS.1 = radius-registry
EOF
)
echo "$CFG"
}
temp_cert_dir=$(mktemp -d 2>/dev/null || mktemp -d -t 'temp_cert_dir')
# Set the temp_cert_dir as an environment variable
echo "TEMP_CERT_DIR=$temp_cert_dir" >> $GITHUB_ENV
# Go to the temp_cert_dir
pushd $temp_cert_dir
# Create
mkdir -p certs/${{ env.LOCAL_REGISTRY_SERVER }}
echo "==== Generate the openssl config"
create_openssl_cfg >req.cnf
echo "==== Create the self signed certificate certificate and client key files"
openssl req -x509 \
-nodes \
-days 365 \
-newkey rsa:4096 \
-keyout certs/${{ env.LOCAL_REGISTRY_SERVER }}/client.key \
-out certs/${{ env.LOCAL_REGISTRY_SERVER }}/client.crt \
-config req.cnf \
-sha256
- name: Create local Docker registry
run: |
# This is going to start an insecure registry on localhost:5000 on the host machine.
if [ "$(docker inspect -f '{{.State.Running}}' "${{ env.LOCAL_REGISTRY_NAME }}" 2>/dev/null || true)" != 'true' ]; then
docker run \
-d --restart=always -p "127.0.0.1:${{ env.LOCAL_REGISTRY_PORT }}:5000" --network bridge --name "${{ env.LOCAL_REGISTRY_NAME }}" \
echo "==== Creating a docker registry"
echo "==== TEMP_CERT_DIR: $TEMP_CERT_DIR"
ls -l ${{ env.TEMP_CERT_DIR }}/certs/${{ env.LOCAL_REGISTRY_SERVER }}
docker run -d \
-p ${{ env.LOCAL_REGISTRY_PORT }}:5000 \
--restart=always \
--name ${{ env.LOCAL_REGISTRY_NAME }} \
-v ${{ env.TEMP_CERT_DIR }}/certs/${{ env.LOCAL_REGISTRY_SERVER }}:/certs \
-e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/client.crt \
-e REGISTRY_HTTP_TLS_KEY=/certs/client.key \
registry:2
fi
- name: Add insecure registry to Docker daemon
run: |
# Check if /etc/docker/daemon.json exists
if [ ! -f /etc/docker/daemon.json ]; then
echo "daemon.json doesn't exist. Creating one..."
echo '{}' | sudo tee /etc/docker/daemon.json
# Wait a few seconds for the registry to start
sleep 5
# Check if the registry is running and not restarting
if [ "$(docker inspect -f '{{.State.Running}}' "${{ env.LOCAL_REGISTRY_NAME }}" 2>/dev/null || true)" != 'true' ]; then
echo "==== Docker registry failed to start"
echo "==== Inspecting Docker container logs"
docker logs ${{ env.LOCAL_REGISTRY_NAME }}
echo "==== Inspecting Docker container state"
docker inspect ${{ env.LOCAL_REGISTRY_NAME }}
exit 1
else
echo "==== Docker registry started successfully"
fi
else
# This shouldn't be the case.
echo "==== Docker registry already running"
fi
# Add insecure registries to /etc/docker/daemon.json
echo '{"insecure-registries": ["radius-registry:5000"]}' | sudo tee /etc/docker/daemon.json
sudo systemctl daemon-reload
sudo systemctl restart docker
- name: Add radius-registry to /etc/hosts
run: |
sudo sh -c 'echo "127.0.0.1 radius-registry" >> /etc/hosts'
- name: Build and Push container images
run: |
make build && make docker-build && make docker-push
env:
DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
DOCKER_REGISTRY: "${{ env.LOCAL_REGISTRY_SERVER }}:${{ env.LOCAL_REGISTRY_PORT }}"
DOCKER_TAG_VERSION: ${{ env.REL_VERSION }}

- name: Install rad CLI
Expand All @@ -261,17 +319,17 @@ jobs:
chmod +x ./kind
# Create kind cluster with containerd registry config dir enabled
cat <<EOF | ./kind create cluster --config=-
cat <<EOF | kind create cluster --config=-
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
extraMounts:
- containerPath: "/etc/containerd/certs.d/${{ env.LOCAL_REGISTRY_NAME }}"
hostPath: "$TEMP_CERT_DIR/certs/${{ env.LOCAL_REGISTRY_SERVER }}"
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."radius-registry:5000"]
endpoint = ["http://radius-registry:5000"]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"]
endpoint = ["http://radius-registry:5000"]
config_path = "/etc/containerd/certs.d"
EOF
Expand All @@ -283,21 +341,23 @@ jobs:
#
# We want a consistent name that works from both ends, so we tell containerd to
# alias localhost:${reg_port} to the registry container when pulling images
LOCALHOST_REGISTRY_DIR="/etc/containerd/certs.d/localhost:${{ env.LOCAL_REGISTRY_PORT }}"
RADIUS_REGISTRY_DIR="/etc/containerd/certs.d/radius-registry:5000"
LOCALHOST_DIR="/etc/containerd/certs.d/${{ env.LOCAL_REGISTRY_SERVER }}:${{ env.LOCAL_REGISTRY_PORT }}"
RADIUS_DIR="/etc/containerd/certs.d/${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"
for node in $(kind get nodes); do
# LOCALHOST_REGISTRY_DIR
docker exec "${node}" mkdir -p "${LOCALHOST_REGISTRY_DIR}"
cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${LOCALHOST_REGISTRY_DIR}/hosts.toml"
[host."http://${{ env.LOCAL_REGISTRY_NAME }}:5000"]
# LOCALHOST_DIR
docker exec "${node}" mkdir -p "${LOCALHOST_DIR}"
cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${LOCALHOST_DIR}/hosts.toml"
[host."http://${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"]
capabilities = ["pull", "resolve", "push"]
skip_verify = true
EOF
# RADIUS_REGISTRY_DIR
docker exec "${node}" mkdir -p "${RADIUS_REGISTRY_DIR}"
cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${RADIUS_REGISTRY_DIR}/hosts.toml"
[host."http://radius-registry:5000"]
# RADIUS_DIR
docker exec "${node}" mkdir -p "${RADIUS_DIR}"
cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${RADIUS_DIR}/hosts.toml"
[host."http://${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"]
capabilities = ["pull", "resolve", "push"]
skip_verify = true
EOF
Expand All @@ -320,7 +380,7 @@ jobs:
namespace: kube-public
data:
localRegistryHosting.v1: |
host: "localhost:${{ env.LOCAL_REGISTRY_PORT }}"
host: "${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"
help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
EOF
Expand Down Expand Up @@ -353,9 +413,9 @@ jobs:
echo "*** Installing Radius to Kubernetes ***"
rad install kubernetes \
--chart ${{ env.RADIUS_CHART_LOCATION }} \
--set rp.image=${{ env.CONTAINER_REGISTRY }}/applications-rp,rp.tag=${{ env.REL_VERSION }} \
--set controller.image=${{ env.CONTAINER_REGISTRY }}/controller,controller.tag=${{ env.REL_VERSION }} \
--set ucp.image=${{ env.CONTAINER_REGISTRY }}/ucpd,ucp.tag=${{ env.REL_VERSION }} \
--set rp.image=${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}/applications-rp,rp.tag=${{ env.REL_VERSION }} \
--set controller.image=${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}/controller,controller.tag=${{ env.REL_VERSION }} \
--set ucp.image=${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}/ucpd,ucp.tag=${{ env.REL_VERSION }} \
--set de.image=${{ env.DE_IMAGE }},de.tag=${{ env.DE_TAG }}
echo "*** Create workspace, group and environment for test ***"
Expand All @@ -377,6 +437,7 @@ jobs:
which rad || { echo "cannot find rad"; exit 1; }
make publish-test-bicep-recipes
env:
BICEP_RECIPE_REGISTRY: "${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"
BICEP_RECIPE_TAG_VERSION: ${{ env.REL_VERSION }}
INSECURE_REGISTRY: true

Expand All @@ -392,11 +453,11 @@ jobs:
make test-functional-${{ matrix.name }}
env:
DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
DOCKER_REGISTRY: "${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"
TEST_TIMEOUT: ${{ env.FUNCTIONALTEST_TIMEOUT }}
RADIUS_CONTAINER_LOG_PATH: ${{ github.workspace }}/${{ env.RADIUS_CONTAINER_LOG_BASE }}
RADIUS_SAMPLES_REPO_ROOT: ${{ github.workspace }}/samples
BICEP_RECIPE_REGISTRY: ${{ env.BICEP_RECIPE_REGISTRY }}
BICEP_RECIPE_REGISTRY: "${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}"
BICEP_RECIPE_TAG_VERSION: ${{ env.BICEP_RECIPE_TAG_VERSION }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GOTESTSUM_OPTS: "--junitfile ./dist/functional_test/results.xml"
Expand Down

0 comments on commit 5124784

Please sign in to comment.