feat: Docker rootless (#451)

* feat: run in docker rootless environment Modified vault compose entries to disable memory locking feature of vault. This lets us run vault in a rootless docker environment Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: expanded vault changes to remaining non taf compose files Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: docker rootless Added user ID resolver in Makefile and docker sock mounts for portainer and security-spire-config container Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: docker rootless added user ID resolver in Makefile Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: docker rootless Added test for docker rootless mode in makefiles Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: docker rootless Fixed up left over lines Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: docker rootless Updated readme files Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: rootless docker made makefile resolve docker environment Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: rootless docker Updated compose files for root dir Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: docker rootless added total system memory resolver to Makefile Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: docker rootless added generated files Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: docker rootless Removed debug Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: docker rootless Fixed vault storage backend issue causing vault not to boot Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: docker rootless Commiting generated files Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: docker rootless Fixed makefile delayed-start gen syntax issues Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: change yaml multi line Indicator change yaml multi line Indicator Signed-off-by: weichou1229 <weichou1229@gmail.com> * feat: docker rootless Removed multiline formatting Signed-off-by: jakeiotechsys <jake@iotechsys.com> * feat: add default vault config add default vault config Signed-off-by: weichou1229 <weichou1229@gmail.com> --------- Signed-off-by: jakeiotechsys <jake@iotechsys.com> Signed-off-by: weichou1229 <weichou1229@gmail.com> Co-authored-by: weichou1229 <weichou1229@gmail.com>
edgexfoundry · Aug 27, 2024 · b8b33b4 · b8b33b4
1 parent f5ae8f9
commit b8b33b4
Show file tree

Hide file tree

Showing 22 changed files with 170 additions and 43 deletions.
diff --git a/Makefile b/Makefile
@@ -27,6 +27,12 @@ OPTIONS:=" arm64 no-secty app-sample zero-trust " # Must have spaces around word
 # This tool now only supports compose V2, aka "docker compose" as it has replaced to old docker-compose tool.
 DOCKER_COMPOSE=docker compose
 
+# Resolve user ID for rootless docker port mapping
+export USERID:=$(shell id -u)
+
+# Set default rootful docker socket path
+export DOCKER_SOCKET_PATH=/var/run/docker.sock
+
 ifeq (arm64, $(filter arm64,$(ARGS)))
 	ARM64=-arm64
 	ARM64_OPTION=arm64
@@ -52,6 +58,9 @@ endef
 .PHONY: $(OPTIONS)
 
 portainer:
+	@if [ -e /run/user/${USERID}/docker.sock ]; then \
+    	export DOCKER_SOCKET_PATH=/run/user/${USERID}/docker.sock; \
+    fi
 	${DOCKER_COMPOSE} -p portainer -f docker-compose-portainer.yml up -d
 
 portainer-down:

diff --git a/compose-builder/Makefile b/compose-builder/Makefile
@@ -36,6 +36,15 @@ BUS=
 NANOMQ=
 REGISTRY=
 
+# Resolve user ID for rootless docker port mapping
+export USERID:=$(shell id -u)
+
+# Set default rootful docker socket path
+export DOCKER_SOCKET_PATH=/var/run/docker.sock
+
+# Get total system memory in megabytes for vault config
+export TOTAL_SYSTEM_MEMORY:=$(shell grep MemTotal /proc/meminfo | awk '{print $$2}')m
+
 BROKER_YAML=add-mqtt-broker-mosquitto.yml
 TAF_BROKER_YAML=add-taf-mqtt-broker-mosquitto.yml
 
@@ -132,11 +141,19 @@ ifeq (keeper, $(filter keeper,$(ARGS)))
 	endif
 endif
 
-# When in delay-start mode, we have to make sure support serivces be delayed-start-compliant: i.e. the runtime-token configuration be added etc..
-ifeq (delayed-start, $(filter delayed-start,$(ARGS)))
-	ext_file_sup_notif:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)" ./gen_runtime_token_config_compose_ext.sh support-notifications)
-	ext_file_sup_sch:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)" ./gen_runtime_token_config_compose_ext.sh support-scheduler)
-	COMPOSE_FILES:=$(COMPOSE_FILES) -f $(ext_file_sup_notif) -f $(ext_file_sup_sch)
+# When in delay-start mode, ensure support services are delay-start-compliant by adding runtime-token configuration
+ifeq (delayed-start,$(filter delayed-start,$(ARGS)))
+    # Ensure Docker is running rootless, as security-spire-agent runs only in Docker rootless mode
+    ifneq (,$(wildcard /run/user/$(USERID)/docker.sock))
+        export DOCKER_SOCKET_PATH := /run/user/$(USERID)/docker.sock
+    endif
+
+    # Generate runtime token config for support services
+    ext_file_sup_notif := $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)" ./gen_runtime_token_config_compose_ext.sh support-notifications)
+    ext_file_sup_sch := $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)"  ./gen_runtime_token_config_compose_ext.sh support-scheduler)
+
+    # Add generated config files to COMPOSE_FILES
+    COMPOSE_FILES += -f $(ext_file_sup_notif) -f $(ext_file_sup_sch)
 endif
 
 # Add Device Services
@@ -1060,6 +1077,11 @@ help:
 	echo "See README.md in this folder"
 
 portainer:
+	@if [ ! -e /run/user/${USERID}/docker.sock ]; then \
+    	echo "Error: Docker socket not found at /run/user/${USERID}/docker.sock"; \
+    	echo "Please ensure Docker is running rootless."; \
+    	exit 1; \
+    fi
 	make -C ${RELEASE_FOLDER} portainer
 
 portainer-down:

diff --git a/compose-builder/README.md b/compose-builder/README.md
@@ -163,7 +163,7 @@ Usage: make <target> where target is:
 #### Portainer
 
 ```
-portainer       Runs Portainer independent of the EdgeX services
+portainer       Runs Portainer independent of the EdgeX services.
 portainer-down	Stops Portainer independent of the EdgeX services
 ```
 #### Build

diff --git a/compose-builder/add-delayed-start-services.yml b/compose-builder/add-delayed-start-services.yml
@@ -75,7 +75,7 @@ services:
       - spire-ca:/srv/spiffe/ca
       - spire-agent:/srv/spiffe/agent
       - /tmp/edgex/secrets/spiffe:/tmp/edgex/secrets/spiffe:z
-      - /var/run/docker.sock:/var/run/docker.sock
+      - ${DOCKER_SOCKET_PATH}:/var/run/docker.sock
     depends_on:
       - security-spire-server
     pid: host

diff --git a/compose-builder/add-security.yml b/compose-builder/add-security.yml
@@ -104,18 +104,34 @@ services:
       - edgex-network
     ports:
       - "127.0.0.1:8200:8200"
-    cap_add:
-      - "IPC_LOCK"
+    deploy:
+      resources:
+        limits:
+          memory: "${TOTAL_SYSTEM_MEMORY}"
+    memswap_limit: "${TOTAL_SYSTEM_MEMORY}"
     tmpfs:
       - /vault/config
-    entrypoint: ["/edgex-init/vault_wait_install.sh"]
+    entrypoint: [ "/edgex-init/vault_wait_install.sh" ]
     env_file:
       - common-sec-stage-gate.env
     command: server
     environment:
       VAULT_ADDR: http://edgex-vault:8200
       VAULT_CONFIG_DIR: /vault/config
       VAULT_UI: "true"
+      SKIP_SETCAP: "true"
+      VAULT_LOCAL_CONFIG: |
+        listener "tcp" { 
+          address = "edgex-vault:8200" 
+          tls_disable = "1" 
+          cluster_address = "edgex-vault:8201" 
+        } 
+        backend "file" {
+          path = "/vault/file"
+        }
+        default_lease_ttl = "168h" 
+        max_lease_ttl = "720h"
+        disable_mlock = true
     volumes:
       - edgex-init:/edgex-init:ro
       - vault-file:/vault/file

diff --git a/docker-compose-arm64.yml b/docker-compose-arm64.yml
@@ -1293,19 +1293,22 @@ services:
         bind:
           create_host_path: true
   vault:
-    cap_add:
-      - IPC_LOCK
     command:
       - server
     container_name: edgex-vault
     depends_on:
       security-bootstrapper:
         condition: service_started
         required: true
+    deploy:
+      resources:
+        limits:
+          memory: "34032716873728"
     entrypoint:
       - /edgex-init/vault_wait_install.sh
     environment:
       PROXY_SETUP_HOST: edgex-security-proxy-setup
+      SKIP_SETCAP: "true"
       STAGEGATE_BOOTSTRAPPER_HOST: edgex-security-bootstrapper
       STAGEGATE_BOOTSTRAPPER_STARTPORT: "54321"
       STAGEGATE_DATABASE_HOST: edgex-redis
@@ -1321,9 +1324,11 @@ services:
       STAGEGATE_WAITFOR_TIMEOUT: 60s
       VAULT_ADDR: http://edgex-vault:8200
       VAULT_CONFIG_DIR: /vault/config
+      VAULT_LOCAL_CONFIG: "listener \"tcp\" { \n  address = \"edgex-vault:8200\" \n  tls_disable = \"1\" \n  cluster_address = \"edgex-vault:8201\" \n} \nbackend \"file\" {\n  path = \"/vault/file\"\n}\ndefault_lease_ttl = \"168h\" \nmax_lease_ttl = \"720h\"\ndisable_mlock = true\n"
       VAULT_UI: "true"
     hostname: edgex-vault
     image: hashicorp/vault:1.14
+    memswap_limit: "34032716873728"
     networks:
       edgex-network: null
     ports:

diff --git a/docker-compose-portainer.yml b/docker-compose-portainer.yml
@@ -27,6 +27,6 @@ services:
     restart: always
     command: -H unix:///var/run/docker.sock
     volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:z
+      - ${DOCKER_SOCKET_PATH}:/var/run/docker.sock
       - portainer_data:/data
 
diff --git a/docker-compose-with-app-sample-arm64.yml b/docker-compose-with-app-sample-arm64.yml
@@ -1370,19 +1370,22 @@ services:
         bind:
           create_host_path: true
   vault:
-    cap_add:
-      - IPC_LOCK
     command:
       - server
     container_name: edgex-vault
     depends_on:
       security-bootstrapper:
         condition: service_started
         required: true
+    deploy:
+      resources:
+        limits:
+          memory: "34032716873728"
     entrypoint:
       - /edgex-init/vault_wait_install.sh
     environment:
       PROXY_SETUP_HOST: edgex-security-proxy-setup
+      SKIP_SETCAP: "true"
       STAGEGATE_BOOTSTRAPPER_HOST: edgex-security-bootstrapper
       STAGEGATE_BOOTSTRAPPER_STARTPORT: "54321"
       STAGEGATE_DATABASE_HOST: edgex-redis
@@ -1398,9 +1401,11 @@ services:
       STAGEGATE_WAITFOR_TIMEOUT: 60s
       VAULT_ADDR: http://edgex-vault:8200
       VAULT_CONFIG_DIR: /vault/config
+      VAULT_LOCAL_CONFIG: "listener \"tcp\" { \n  address = \"edgex-vault:8200\" \n  tls_disable = \"1\" \n  cluster_address = \"edgex-vault:8201\" \n} \nbackend \"file\" {\n  path = \"/vault/file\"\n}\ndefault_lease_ttl = \"168h\" \nmax_lease_ttl = \"720h\"\ndisable_mlock = true\n"
       VAULT_UI: "true"
     hostname: edgex-vault
     image: hashicorp/vault:1.14
+    memswap_limit: "34032716873728"
     networks:
       edgex-network: null
     ports:

diff --git a/docker-compose-with-app-sample.yml b/docker-compose-with-app-sample.yml
@@ -1370,19 +1370,22 @@ services:
         bind:
           create_host_path: true
   vault:
-    cap_add:
-      - IPC_LOCK
     command:
       - server
     container_name: edgex-vault
     depends_on:
       security-bootstrapper:
         condition: service_started
         required: true
+    deploy:
+      resources:
+        limits:
+          memory: "34032716873728"
     entrypoint:
       - /edgex-init/vault_wait_install.sh
     environment:
       PROXY_SETUP_HOST: edgex-security-proxy-setup
+      SKIP_SETCAP: "true"
       STAGEGATE_BOOTSTRAPPER_HOST: edgex-security-bootstrapper
       STAGEGATE_BOOTSTRAPPER_STARTPORT: "54321"
       STAGEGATE_DATABASE_HOST: edgex-redis
@@ -1398,9 +1401,11 @@ services:
       STAGEGATE_WAITFOR_TIMEOUT: 60s
       VAULT_ADDR: http://edgex-vault:8200
       VAULT_CONFIG_DIR: /vault/config
+      VAULT_LOCAL_CONFIG: "listener \"tcp\" { \n  address = \"edgex-vault:8200\" \n  tls_disable = \"1\" \n  cluster_address = \"edgex-vault:8201\" \n} \nbackend \"file\" {\n  path = \"/vault/file\"\n}\ndefault_lease_ttl = \"168h\" \nmax_lease_ttl = \"720h\"\ndisable_mlock = true\n"
       VAULT_UI: "true"
     hostname: edgex-vault
     image: hashicorp/vault:1.14
+    memswap_limit: "34032716873728"
     networks:
       edgex-network: null
     ports:

diff --git a/docker-compose-zero-trust-arm64.yml b/docker-compose-zero-trust-arm64.yml
@@ -1124,19 +1124,22 @@ services:
           selinux: z
           create_host_path: true
   vault:
-    cap_add:
-      - IPC_LOCK
     command:
       - server
     container_name: edgex-vault
     depends_on:
       security-bootstrapper:
         condition: service_started
         required: true
+    deploy:
+      resources:
+        limits:
+          memory: "34032716873728"
     entrypoint:
       - /edgex-init/vault_wait_install.sh
     environment:
       PROXY_SETUP_HOST: edgex-security-proxy-setup
+      SKIP_SETCAP: "true"
       STAGEGATE_BOOTSTRAPPER_HOST: edgex-security-bootstrapper
       STAGEGATE_BOOTSTRAPPER_STARTPORT: "54321"
       STAGEGATE_DATABASE_HOST: edgex-redis
@@ -1152,9 +1155,11 @@ services:
       STAGEGATE_WAITFOR_TIMEOUT: 60s
       VAULT_ADDR: http://edgex-vault:8200
       VAULT_CONFIG_DIR: /vault/config
+      VAULT_LOCAL_CONFIG: "listener \"tcp\" { \n  address = \"edgex-vault:8200\" \n  tls_disable = \"1\" \n  cluster_address = \"edgex-vault:8201\" \n} \nbackend \"file\" {\n  path = \"/vault/file\"\n}\ndefault_lease_ttl = \"168h\" \nmax_lease_ttl = \"720h\"\ndisable_mlock = true\n"
       VAULT_UI: "true"
     hostname: edgex-vault
     image: hashicorp/vault:1.14
+    memswap_limit: "34032716873728"
     networks:
       edgex-network: null
     ports:

diff --git a/docker-compose-zero-trust.yml b/docker-compose-zero-trust.yml
@@ -1124,19 +1124,22 @@ services:
           selinux: z
           create_host_path: true
   vault:
-    cap_add:
-      - IPC_LOCK
     command:
       - server
     container_name: edgex-vault
     depends_on:
       security-bootstrapper:
         condition: service_started
         required: true
+    deploy:
+      resources:
+        limits:
+          memory: "34032716873728"
     entrypoint:
       - /edgex-init/vault_wait_install.sh
     environment:
       PROXY_SETUP_HOST: edgex-security-proxy-setup
+      SKIP_SETCAP: "true"
       STAGEGATE_BOOTSTRAPPER_HOST: edgex-security-bootstrapper
       STAGEGATE_BOOTSTRAPPER_STARTPORT: "54321"
       STAGEGATE_DATABASE_HOST: edgex-redis
@@ -1152,9 +1155,11 @@ services:
       STAGEGATE_WAITFOR_TIMEOUT: 60s
       VAULT_ADDR: http://edgex-vault:8200
       VAULT_CONFIG_DIR: /vault/config
+      VAULT_LOCAL_CONFIG: "listener \"tcp\" { \n  address = \"edgex-vault:8200\" \n  tls_disable = \"1\" \n  cluster_address = \"edgex-vault:8201\" \n} \nbackend \"file\" {\n  path = \"/vault/file\"\n}\ndefault_lease_ttl = \"168h\" \nmax_lease_ttl = \"720h\"\ndisable_mlock = true\n"
       VAULT_UI: "true"
     hostname: edgex-vault
     image: hashicorp/vault:1.14
+    memswap_limit: "34032716873728"
     networks:
       edgex-network: null
     ports:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1293,19 +1293,22 @@ services:
         bind:
           create_host_path: true
   vault:
-    cap_add:
-      - IPC_LOCK
     command:
       - server
     container_name: edgex-vault
     depends_on:
       security-bootstrapper:
         condition: service_started
         required: true
+    deploy:
+      resources:
+        limits:
+          memory: "34032716873728"
     entrypoint:
       - /edgex-init/vault_wait_install.sh
     environment:
       PROXY_SETUP_HOST: edgex-security-proxy-setup
+      SKIP_SETCAP: "true"
       STAGEGATE_BOOTSTRAPPER_HOST: edgex-security-bootstrapper
       STAGEGATE_BOOTSTRAPPER_STARTPORT: "54321"
       STAGEGATE_DATABASE_HOST: edgex-redis
@@ -1321,9 +1324,11 @@ services:
       STAGEGATE_WAITFOR_TIMEOUT: 60s
       VAULT_ADDR: http://edgex-vault:8200
       VAULT_CONFIG_DIR: /vault/config
+      VAULT_LOCAL_CONFIG: "listener \"tcp\" { \n  address = \"edgex-vault:8200\" \n  tls_disable = \"1\" \n  cluster_address = \"edgex-vault:8201\" \n} \nbackend \"file\" {\n  path = \"/vault/file\"\n}\ndefault_lease_ttl = \"168h\" \nmax_lease_ttl = \"720h\"\ndisable_mlock = true\n"
       VAULT_UI: "true"
     hostname: edgex-vault
     image: hashicorp/vault:1.14
+    memswap_limit: "34032716873728"
     networks:
       edgex-network: null
     ports:

diff --git a/taf/docker-compose-taf-arm64.yml b/taf/docker-compose-taf-arm64.yml
@@ -2331,19 +2331,22 @@ services:
         bind:
           create_host_path: true
   vault:
-    cap_add:
-      - IPC_LOCK
     command:
       - server
     container_name: edgex-vault
     depends_on:
       security-bootstrapper:
         condition: service_started
         required: true
+    deploy:
+      resources:
+        limits:
+          memory: "34032716873728"
     entrypoint:
       - /edgex-init/vault_wait_install.sh
     environment:
       PROXY_SETUP_HOST: edgex-security-proxy-setup
+      SKIP_SETCAP: "true"
       STAGEGATE_BOOTSTRAPPER_HOST: edgex-security-bootstrapper
       STAGEGATE_BOOTSTRAPPER_STARTPORT: "54321"
       STAGEGATE_DATABASE_HOST: edgex-redis
@@ -2359,9 +2362,11 @@ services:
       STAGEGATE_WAITFOR_TIMEOUT: 60s
       VAULT_ADDR: http://edgex-vault:8200
       VAULT_CONFIG_DIR: /vault/config
+      VAULT_LOCAL_CONFIG: "listener \"tcp\" { \n  address = \"edgex-vault:8200\" \n  tls_disable = \"1\" \n  cluster_address = \"edgex-vault:8201\" \n} \nbackend \"file\" {\n  path = \"/vault/file\"\n}\ndefault_lease_ttl = \"168h\" \nmax_lease_ttl = \"720h\"\ndisable_mlock = true\n"
       VAULT_UI: "true"
     hostname: edgex-vault
     image: hashicorp/vault:1.14
+    memswap_limit: "34032716873728"
     networks:
       edgex-network: null
     ports: