2025-03-03 ZeroTier - health checking - alternative proposal

Dockerfile

@@ -3,7 +3,7 @@ ARG ALPINE_VERSION=3.21
 ARG ZT_COMMIT=185a3a2c76e6bf1b1c0415871f43076638eb007c
 ARG ZT_VERSION=1.14.2
 
-FROM ${ALPINE_IMAGE}:${ALPINE_VERSION} as builder
+FROM ${ALPINE_IMAGE}:${ALPINE_VERSION} AS builder
 
 ARG ZT_COMMIT
 
@@ -27,9 +27,9 @@ LABEL org.opencontainers.image.title="zerotier" \
       org.opencontainers.image.licenses="MIT" \
       org.opencontainers.image.source="https://github.com/zyclonite/zerotier-docker"
 
-COPY --from=builder /src/zerotier-one /scripts/entrypoint.sh /usr/sbin/
+COPY --from=builder /src/zerotier-one /scripts/entrypoint.sh /scripts/healthcheck.sh /usr/sbin/
 
-RUN apk add --no-cache --purge --clean-protected libc6-compat libstdc++ \
+RUN apk add --no-cache --purge --clean-protected libc6-compat libstdc++ tzdata \
   && mkdir -p /var/lib/zerotier-one \
   && ln -s /usr/sbin/zerotier-one /usr/sbin/zerotier-idtool \
   && ln -s /usr/sbin/zerotier-one /usr/sbin/zerotier-cli \
@@ -40,3 +40,6 @@ EXPOSE 9993/udp
 ENTRYPOINT ["entrypoint.sh"]
 
 CMD ["-U"]
+
+HEALTHCHECK --interval=60s --timeout=8s --retries=2 --start-period=60s \
+  CMD ["/bin/sh", "/usr/sbin/healthcheck.sh"]

Dockerfile.router

@@ -13,7 +13,7 @@ ENV LOG_PATH=/var/log/supervisor
 
 COPY scripts/entrypoint-router.sh /usr/sbin/
 
-RUN apk add --no-cache --purge --clean-protected iptables iptables-legacy tzdata \
+RUN apk add --no-cache --purge --clean-protected iptables iptables-legacy \
   && rm -rf /var/cache/apk/*
 
 EXPOSE 9993/udp

README-router.md

@@ -17,6 +17,25 @@ This implementation extends the concept so that you have a choice of:
 * Permitting clients on your LAN to initiate connections with remote devices reachable across your ZeroTier Cloud network; or
 * Both of the above (ie a full router).
 
+### router vs client
+
+Both containers run the **same** [ZeroTier One client](https://github.com/zerotier/ZeroTierOne) software. No special parameters are passed to the ZeroTier One client that somehow cause it to become a router.
+
+Layer Three routing is simply the process of the *host* (not the *container*) making decisions about forwarding IP datagrams (aka "packets") between network interfaces. Routing is enabled by two things:
+
+1. Turning on the `net.ipv4.ip_forward` kernel parameter; and
+2. A set of *iptables* filters.
+
+The ZeroTier router container manages the insertion and withdrawal of the necessary *iptables* filters.
+
+The kernel parameter is usually enabled when Docker is installed. However, if IPv4 forwarding is not enabled when the Zerotier router container launches, it reports this problem in the container's log and falls back to behaving as the client. In that situation, you may need to enable the kernel parameter yourself:
+
+```
+$ sudo sysctl net.ipv4.ip_forward=1
+```
+
+Otherwise, the client and router containers are fully interchangeable and can utilise the same persistent store.
+
 ### Command line example
 
 ``` console
@@ -64,7 +83,7 @@ services:
       - PUID=999
       - PGID=994
       - ZEROTIER_ONE_LOCAL_PHYS=eth0
-      - ZEROTIER_ONE_USE_IPTABLES_NFT=true
+      - ZEROTIER_ONE_USE_IPTABLES_NFT=false
       - ZEROTIER_ONE_GATEWAY_MODE=inbound
     # - ZEROTIER_ONE_NETWORK_IDS=«yourDefaultNetworkID(s)»
 ```
@@ -80,26 +99,9 @@ Note:
 
 ### Environment variables
 
-* `TZ` – timezone support. Example:
-
-	``` yaml
-	environment:
-	- TZ=Australia/Sydney
-	```
-
-	Defaults to `Etc/UTC` if omitted.
-
-* `PUID` + `PGID` – user and group IDs for ownership of persistent store. Example:
-
-	``` yaml
-	environment:
-	- PUID=1000
-	- PGID=1000
-	```
-
-	If omitted, `PUID` defaults to user ID 999, while `PGID` defaults to group ID 994.
+All the variables supported by the client container are also supported by the router container. Please see the main [README](./README.md).
 
-	These variables are only used to ensure consistent ownership of persistent storage on each launch. They do not affect how the container *runs.* Absent a `user:` directive, the container runs as root and does not downgrade its privileges.
+The variables listed below are unique to the router container:
 
 * `ZEROTIER_ONE_LOCAL_PHYS` - a space-separated list of physical interfaces that should be configured to participate in NAT-based routing. Examples:
 
@@ -153,7 +155,7 @@ Note:
 	2. A line-count of zero. This means the container has not been able to configure net-filter rules on the host. If that happens, try the opposite setting for this environment variable (eg `true` instead of `false`).
 	3. A non-zero line-count. That means the container has been able to propagate net-filter rules into the host's tables, which is what you want. The actual number is not important, just something other than zero.
 
-	The container will always come up. Once you've authorised the client in ZeroTier Central, it will be able to join your ZeroTier Cloud network. Tests like `ping` and `traceroute` that you run on the same host will always work. However, if the container is not able to propagate its net-filter rules into the host's tables, traffic *beyond* the host where the container is running will not work properly. The problem is quite subtle so it's always a good idea to check that the host has the expected net-filters.
+	The container will always come up. Once you've approved the host in ZeroTier Central, it will be able to join your ZeroTier Cloud network. Tests like `ping` and `traceroute` that you run on the same host will always work. However, if the container is not able to propagate its net-filter rules into the host's tables, traffic *beyond* the host where the container is running will not work properly. The problem is quite subtle so it's always a good idea to check that the host has the expected net-filters.
 
 * `ZEROTIER_ONE_GATEWAY_MODE` - controls the traffic direction. Examples:
 
@@ -180,39 +182,6 @@ Note:
 
 	Defaults to `inbound` if omitted. Note that you will probably need one or more static routes configured in your local LAN router so that traffic originating in a local host which is not running the ZeroTier client can be directed to the gateway host.
 
-* `ZEROTIER_ONE_NETWORK_IDS` – a space-separated list of ZeroTier network IDs.
-
-	This variable is *only* effective on first launch. There is no default if it is omitted. Examples:
-
-	- to join a single network:
-
-		``` yaml
-		environment:
-		- ZEROTIER_ONE_NETWORK_IDS=aaaaaaaaaaaaaaaa
-		```
-
-		Equivalent of running the following command after the container first starts:
-
-		```
-		$ docker exec zerotier zerotier-cli join aaaaaaaaaaaaaaaa
-		```
-
-	- to join a multiple networks:
-
-		``` yaml
-		environment:
-		- ZEROTIER_ONE_NETWORK_IDS=aaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbb
-		```
-
-		Equivalent of running the following commands after the container first starts:
-
-		```
-		$ docker exec zerotier zerotier-cli join aaaaaaaaaaaaaaaa
-		$ docker exec zerotier zerotier-cli join bbbbbbbbbbbbbbbb
-		```
-
-	It does not matter whether you use this environment variable or the `join` command, you still need to use ZeroTier Central to approve the computer for each network it joins.
-
 ### Managed route(s)
 
 For each ZeroTier container that is configured as a router, ZeroTier needs at least one *Managed Route*.

README.md

@@ -12,25 +12,156 @@ This is a container based on a lightweight Alpine Linux image and a copy of Zero
 
 To run this container in the correct way requires some special options to give it special permissions and allow it to persist its files. Here's an example (tested on Fedora CoreOS):
 
-    docker run --name zerotier-one --device=/dev/net/tun --net=host \
-      --cap-add=NET_ADMIN --cap-add=SYS_ADMIN \
-      -v /var/lib/zerotier-one:/var/lib/zerotier-one zyclonite/zerotier
-
+``` console
+$ docker run --name zerotier-one --device=/dev/net/tun \
+  --network=host -d \
+  --cap-add=NET_ADMIN --cap-add=SYS_ADMIN \
+  --env TZ=Etc/UTC --env PUID=$(id -u) --env PGID=$(id -g) \
+  --env ZEROTIER_ONE_NETWORK_IDS=«yourDefaultNetworkID(s)» \
+  -v /var/lib/zerotier-one:/var/lib/zerotier-one \
+  zyclonite/zerotier
+```
 
 This runs zyclonite/zerotier in a container with special network admin permissions and with access to the host's network stack (no network isolation) and /dev/net/tun to create tun/tap devices. This will allow it to create zt# interfaces on the host the way a copy of ZeroTier One running on the host would normally be able to.
 
 In other words that basically does the same thing that running zerotier-one directly on the host would do, except it runs in a container. Since Fedora CoreOS has no package management this is the preferred way of distributing software for it.
 
 It also mounts /var/lib/zerotier-one to /var/lib/zerotier-one inside the container, allowing your service container to persist its state across restarts of the container itself. If you don't do this it'll generate a new identity every time. You can put the actual data somewhere other than /var/lib/zerotier-one if you want.
 
-To join a zerotier network you can use
+To join a zerotier network you can use any of the following methods, or a combination thereof:
+
+1. The [`ZEROTIER_ONE_NETWORK_IDS`](#joinVar) environment variable. This, however, only works on first launch when the container's persistent store does not exist.
+
+2. The command line:
+
+	``` console
+	$ docker exec zerotier-one zerotier-cli join «networkID»
+	```
+
+3. Create an empty file with the network as name:
+
+	```
+	/var/lib/zerotier-one/networks.d/«networkID».conf
+	```
+
+	and then restart the container.
+
+#### compose file example
+
+``` yaml
+---
+
+services:
+  zerotier:
+    image: zyclonite/zerotier
+    container_name: zerotier-one
+    devices:
+      - /dev/net/tun
+    network_mode: host
+    volumes:
+      - '/var/lib/zerotier-one:/var/lib/zerotier-one'
+    cap_add:
+      - NET_ADMIN
+      - SYS_ADMIN
+    restart: unless-stopped
+    environment:
+      - TZ=Etc/UTC
+      - PUID=999
+      - PGID=994
+    # - ZEROTIER_ONE_NETWORK_IDS=«yourDefaultNetworkID(s)»
+``` 
+
+#### Environment variables
+
+* `TZ` – timezone support. Example:
+
+	``` yaml
+	environment:
+	- TZ=Australia/Sydney
+	```
+
+	Defaults to `Etc/UTC` if omitted.
+
+* `PUID` + `PGID` – user and group IDs for ownership of persistent store. Example:
+
+	``` yaml
+	environment:
+	- PUID=1000
+	- PGID=1000
+	```
+
+	If omitted, `PUID` defaults to user ID 999, while `PGID` defaults to group ID 994.
+
+	These variables are only used to ensure consistent ownership of persistent storage on each launch. They do not affect how the container *runs.* Absent a `user:` directive, the container runs as root and does not downgrade its privileges.
+
+* <a name="joinVar"></a>
+`ZEROTIER_ONE_NETWORK_IDS` – a space-separated list of ZeroTier network IDs.
+
+	This variable is *only* effective on first launch. There is no default if it is omitted. Examples:
+
+	- to join a single network:
+
+		``` yaml
+		environment:
+		- ZEROTIER_ONE_NETWORK_IDS=aaaaaaaaaaaaaaaa
+		```
+
+		This is the equivalent of running the following command after the container first starts:
+
+		```
+		$ docker exec zerotier zerotier-cli join aaaaaaaaaaaaaaaa
+		```
+
+	- to join a multiple networks:
+
+		``` yaml
+		environment:
+		- ZEROTIER_ONE_NETWORK_IDS=aaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbb
+		```
+
+		This is the equivalent of running the following commands after the container starts for the first time:
+
+		```
+		$ docker exec zerotier zerotier-cli join aaaaaaaaaaaaaaaa
+		$ docker exec zerotier zerotier-cli join bbbbbbbbbbbbbbbb
+		```
+
+	It does not matter whether you use this environment variable or the `join` command, you still need to use ZeroTier Central to approve the host for each network it joins.
+
+<a name="healthCheck"></a>
+##### Health Checking
+
+The container (both client and router) runs a health-checking service. It works like this:
+
+1. The (internal) path:
+
+	```
+	/var/lib/zerotier-one/networks.d/
+	```
+
+	contains zero or more files matching the pattern:
+
+	```
+	«networkID».conf
+	```
+
+	Each `.conf` file indicates a ZeroTier network which the container has been configured to join. The *count* of those files (which may be zero or more) is the "expected network count".
+
+2. If the host's routing table does not contain the same number of direct routes to ZeroTier-associated interfaces as the *expected network count,* the container reports "unhealthy".
+
+A container state of "unhealthy" only tells you that *something* is wrong. It does not tell you *what* is wrong. It is simply a hints to dig deeper.
+
+One of the most common reasons for the container to report "unhealthy" is that you have instructed the ZeroTier client (running inside the container) to join a network for which it is not authorised. 
+
+###### Why Health Checking checks routes
 
-    docker exec zerotier-one zerotier-cli join 8056c2e21c000001
+The `zerotier-cli` command running inside the container may report that a ZeroTier Network is "OK" even if the host is aware that the associated network interface is not actually functioning.
 
+An entry in the host's routing table is a pinnacle artefact. A direct route to an interface will only be added to the host's routing table if the interface exists and is in a functioning state.
 
-or create an empty file with the network as name
+Each fully-functioning ZeroTier network that a client joins results in exactly one network interface and, therefore, exactly one direct route to that interface in the host's routing table.
 
-    /var/lib/zerotier-one/networks.d/8056c2e21c000001.conf
+Route insertion and withdrawal is both sensitive to network conditions and efficient so counting routes leads to fewer Type I (the container reporting "healthy" when it is not) and Type II (the container reporting "unhealthy" when it is not) errors. 
 
 #### Router mode
 

docker-compose-router.yml

@@ -1,4 +1,5 @@
-version: '3'
+---
+
 services:
   zerotier:
     image: "zyclonite/zerotier:router"
@@ -20,4 +21,6 @@ services:
       - ZEROTIER_ONE_LOCAL_PHYS=eth0
       - ZEROTIER_ONE_USE_IPTABLES_NFT=false
       - ZEROTIER_ONE_GATEWAY_MODE=inbound
-    # - ZEROTIER_ONE_NETWORK_IDS=yourNetworkID
+    # - ZEROTIER_ONE_NETWORK_IDS=«yourDefaultNetworkID(s)»
+    # - ZEROTIER_ONE_CHK_SPECIFIC_NETWORKS=«yourNetworkID(s)toCheck»
+    # - ZEROTIER_ONE_CHK_MIN_ROUTES_FOR_HEALTH=1

docker-compose.yml

@@ -1,4 +1,5 @@
-version: '3'
+---
+
 services:
   zerotier:
     image: zyclonite/zerotier
@@ -11,3 +12,11 @@ services:
     cap_add:
       - NET_ADMIN
       - SYS_ADMIN
+    restart: unless-stopped
+    environment:
+      - TZ=Etc/UTC
+      - PUID=999
+      - PGID=994
+    # - ZEROTIER_ONE_NETWORK_IDS=«yourDefaultNetworkID(s)»
+    # - ZEROTIER_ONE_CHK_SPECIFIC_NETWORKS=«yourNetworkID(s)toCheck»
+    # - ZEROTIER_ONE_CHK_MIN_ROUTES_FOR_HEALTH=1

scripts/entrypoint-router.sh

@@ -11,15 +11,21 @@ NETWORKS_DIR="${CONFIG_DIR}/networks.d"
 
 # set up network auto-join if (a) the networks directory does not exist
 # and (b) the ZEROTIER_ONE_NETWORK_IDS environment variable is non-null.
-if [ ! -d "${NETWORKS_DIR}" -a -n "${ZEROTIER_ONE_NETWORK_IDS}" ] ; then
-	echo "Assuming container first run."
-	mkdir -p "${NETWORKS_DIR}"
-	for NETWORK_ID in ${ZEROTIER_ONE_NETWORK_IDS} ; do
-		echo "Configuring auto-join of network ID: ${NETWORK_ID}"
-		touch "${NETWORKS_DIR}/${NETWORK_ID}.conf"
-		echo "You will need to authorize this host at:"
-		echo "   https://my.zerotier.com/network/${NETWORK_ID}"
-	done
+if [ ! -d "${NETWORKS_DIR}" ] ; then
+	echo "$(date) - assuming container first run."
+	if [ -n "${ZEROTIER_ONE_NETWORK_IDS}" ] ; then
+		mkdir -p "${NETWORKS_DIR}"
+		for NETWORK_ID in ${ZEROTIER_ONE_NETWORK_IDS} ; do
+			echo "  Configuring auto-join of network ID: ${NETWORK_ID}"
+			touch "${NETWORKS_DIR}/${NETWORK_ID}.conf"
+			echo "  You will need to authorize this host at:"
+			echo "     https://my.zerotier.com/network/${NETWORK_ID}"
+		done
+	else
+		echo " ZEROTIER_ONE_NETWORK_IDS not set. You will need to join"
+		echo " networks using zerotier-cli, and then approve this"
+		echo " host in ZeroTier Central."
+	fi
 fi
 
 # make sure permissions are always as expected (self-repair)