Fix clock sync flakiness for docker-based clusters (#1472)

* Fix clock sync flakiness for debian and ubuntu
replicatedhq · Nov 8, 2024 · 8286a4e · 8286a4e
1 parent f8cfdd1
commit 8286a4e
Show file tree

Hide file tree

Showing 7 changed files with 34 additions and 14 deletions.
diff --git a/dev/distros/dockerfiles/almalinux-8.Dockerfile b/dev/distros/dockerfiles/almalinux-8.Dockerfile
@@ -1,5 +1,8 @@
 FROM almalinux:8
 
+# Only required for systemd-timesyncd
+RUN dnf install -y epel-release
+
 # Install necessary packages
 RUN dnf install -y \
   sudo \
@@ -13,7 +16,7 @@ RUN dnf install -y \
   ipvsadm \
   kmod \
   iproute \
-  chrony \
+  systemd-timesyncd \
   expect \
   vim \
   --allowerasing

diff --git a/dev/distros/dockerfiles/centos-9.Dockerfile b/dev/distros/dockerfiles/centos-9.Dockerfile
@@ -1,5 +1,8 @@
 FROM quay.io/centos/centos:stream9
 
+# Only required for systemd-timesyncd
+RUN dnf install -y epel-release
+
 # Install necessary packages
 RUN dnf install -y \
   sudo \
@@ -13,7 +16,7 @@ RUN dnf install -y \
   ipvsadm \
   kmod \
   iproute \
-  chrony \
+  systemd-timesyncd \
   expect \
   vim \
   --allowerasing

diff --git a/dev/distros/dockerfiles/debian-bookworm.Dockerfile b/dev/distros/dockerfiles/debian-bookworm.Dockerfile
@@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y \
   ipvsadm \
   kmod \
   iproute2 \
-  chrony \
+  systemd-timesyncd \
   expect \
   vim
 

diff --git a/dev/distros/dockerfiles/debian-bullseye.Dockerfile b/dev/distros/dockerfiles/debian-bullseye.Dockerfile
@@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y \
   ipvsadm \
   kmod \
   iproute2 \
-  chrony \
+  systemd-timesyncd \
   expect \
   vim
 

diff --git a/dev/distros/dockerfiles/ubuntu-jammy.Dockerfile b/dev/distros/dockerfiles/ubuntu-jammy.Dockerfile
@@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y \
   ipvsadm \
   kmod \
   iproute2 \
-  chrony \
+  systemd-timesyncd \
   expect \
   vim
 

diff --git a/dev/distros/entrypoint.sh b/dev/distros/entrypoint.sh
@@ -17,13 +17,27 @@ systemctl mask \
 # A unique machine ID is required for multi-node clusters in k0s <= v1.29
 # https://github.com/k0sproject/k0s/blob/443e28b75d216e120764136b4513e6237cea7cc5/docs/external-runtime-deps.md#a-unique-machine-id-for-multi-node-setups
 if [ ! -f "/etc/machine-id.persistent" ]; then
-    dbus-uuidgen --ensure=/etc/machine-id.persistent
+    tr -dc 'a-z0-9' < /dev/urandom | head -c32 > /etc/machine-id.persistent
 fi
+mkdir -p /var/lib/dbus
 ln -sf /etc/machine-id.persistent /etc/machine-id
 ln -sf /etc/machine-id.persistent /var/lib/dbus/machine-id
 
-# Sync time
-chronyc -a makestep
+# Override timesyncd config to allow it to run in containers
+mkdir -p /etc/systemd/system/systemd-timesyncd.service.d/
+cat > /etc/systemd/system/systemd-timesyncd.service.d/override.conf << EOF
+[Unit]
+ConditionVirtualization=
+EOF
+
+# Wait for systemd then start timesyncd
+(
+    until systemctl is-system-running -q; do
+        sleep 1
+    done
+    systemctl enable systemd-timesyncd
+    systemctl start systemd-timesyncd
+) &
 
 # Launch the system
 exec /sbin/init
diff --git a/e2e/cluster/docker/container.go b/e2e/cluster/docker/container.go
@@ -200,7 +200,7 @@ func (c *Container) WaitForSystemd() {
 			c.t.Fatalf("timeout waiting for systemd to start: %v: %s: %s", err, stdout, stderr)
 		case <-tick:
 			status, _, _ := c.Exec([]string{"systemctl is-system-running"})
-			c.t.Logf("systemd stdout: %s", strings.TrimSpace(status))
+			c.t.Logf("systemd stdout: %s", status)
 			if strings.TrimSpace(status) == "running" {
 				return
 			}
@@ -214,12 +214,12 @@ func (c *Container) WaitForClockSync() {
 	for {
 		select {
 		case <-timeout:
-			stdout, stderr, err := c.Exec([]string{"chronyc tracking"})
-			c.t.Fatalf("timeout waiting for chronyd to start: %v: %s: %s", err, stdout, stderr)
+			stdout, stderr, err := c.Exec([]string{"timedatectl show -p NTP -p NTPSynchronized"})
+			c.t.Fatalf("timeout waiting for clock sync: %v: %s: %s", err, stdout, stderr)
 		case <-tick:
-			status, _, _ := c.Exec([]string{"chronyc tracking | grep 'Leap status'"})
-			c.t.Logf("chronyd stdout: %s", strings.TrimSpace(status))
-			if strings.Contains(status, "Normal") {
+			status, _, _ := c.Exec([]string{"timedatectl show -p NTP -p NTPSynchronized"})
+			c.t.Logf("timedatectl stdout: %s", status)
+			if strings.Contains(status, "NTP=yes") && strings.Contains(status, "NTPSynchronized=yes") {
 				return
 			}
 		}