use a systemd unit to manage ethtool options

Author: Timothy J Fontaine

instances/k8smaster/cloud_init/bootstrap.template.yaml

@@ -79,6 +79,12 @@ write_files:
     content: |
       ${kubelet_service_content}
 
+  - path: "/root/services/ethtool-disable-offload@.service"
+    permissions: "0600"
+    encoding: "gzip+base64"
+    content: |
+      ${ethtool_service_content}
+
 # Kube certs, tokens
   - path: "/etc/kubernetes/ssl/ca.pem"
     permissions: "0600"

instances/k8smaster/datasources.tf

@@ -57,6 +57,10 @@ data "template_file" "kubelet-service" {
   }
 }
 
+data "template_file" "ethtool-offload-service" {
+  template = "${file("${path.module}/scripts/ethtool-disable-offload.service")}"
+}
+
 data "template_file" "kube-controller-manager" {
   template = "${file("${path.module}/manifests/kube-controller-manager.yaml")}"
 
@@ -132,6 +136,7 @@ data "template_file" "kube_master_cloud_init_file" {
     master_kubeconfig_template_content       = "${base64gzip(data.template_file.master-kubeconfig.rendered)}"
     kube_scheduler_template_content          = "${base64gzip(data.template_file.kube-scheduler.rendered)}"
     kubelet_service_content                  = "${base64gzip(data.template_file.kubelet-service.rendered)}"
+    ethtool_service_content                  = "${base64gzip(data.template_file.ethtool-offload-service.rendered)}"
     ca-pem-content                           = "${base64gzip(var.root_ca_pem)}"
     ca-key-content                           = "${base64gzip(var.root_ca_key)}"
     api-server-key-content                   = "${base64gzip(var.api_server_private_key_pem)}"

instances/k8smaster/scripts/ethtool-disable-offload.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=ethtool-disable-offload
+Requires=network.target
+After=network.target
+
+[Service]
+ExecStart=/sbin/ethtool -K %i tx off
+Type=oneshot
+
+[Install]
+WantedBy=multi-user.target

instances/k8smaster/scripts/setup.template.sh

@@ -32,7 +32,12 @@ fi
 BROADCOM_DRIVER=$(lsmod | grep bnxt_en | awk '{print $1}')
 if [[ -n "$${BROADCOM_DRIVER}" ]]; then
    echo "Disabling hardware TX checksum offloading"
-   ethtool --offload $(ip -o -4 route show to default | awk '{print $5}') tx off
+   #ethtool --offload $(ip -o -4 route show to default | awk '{print $5}') tx off
+   mv /root/services/ethtool-disable-offload@.service /etc/systemd/system
+   systemctl daemon-reload
+   DEV=$(ip -o -4 route show to default | awk '{print $5}')
+   systemctl enable ethtool-disable-offload@$DEV
+   systemctl start ethtool-disable-offload@$DEV
 fi
 
 # Download etcdctl client

instances/k8sworker/cloud_init/bootstrap.template.yaml

@@ -36,6 +36,12 @@ write_files:
     content: |
       ${kubelet_service_content}
 
+  - path: "/root/services/ethtool-disable-offload@.service"
+    permissions: "0600"
+    encoding: "gzip+base64"
+    content: |
+      ${ethtool_service_content}
+
 # Kube certs
   - path: "/etc/kubernetes/ssl/ca.pem"
     permissions: "0600"

instances/k8sworker/datasources.tf

@@ -66,6 +66,10 @@ data "template_file" "kubelet-service" {
   }
 }
 
+data "template_file" "ethtool-offload-service" {
+  template = "${file("${path.module}/scripts/ethtool-disable-offload.service")}"
+}
+
 data "template_file" "kube_worker_cloud_init_file" {
   template = "${file("${path.module}/cloud_init/bootstrap.template.yaml")}"
 
@@ -76,6 +80,7 @@ data "template_file" "kube_worker_cloud_init_file" {
     kube_proxy_template_content        = "${base64gzip(data.template_file.kube-proxy.rendered)}"
     worker_kubeconfig_template_content = "${base64gzip(data.template_file.worker-kubeconfig.rendered)}"
     kubelet_service_content            = "${base64gzip(data.template_file.kubelet-service.rendered)}"
+    ethtool_service_content            = "${base64gzip(data.template_file.ethtool-offload-service.rendered)}"
     ca-pem-content                     = "${base64gzip(var.root_ca_pem)}"
     ca-key-content                     = "${base64gzip(var.root_ca_key)}"
     api-server-key-content             = "${base64gzip(var.api_server_private_key_pem)}"

instances/k8sworker/scripts/ethtool-disable-offload.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=ethtool-disable-offload
+Requires=network.target
+After=network.target
+
+[Service]
+ExecStart=/sbin/ethtool -K %i tx off
+Type=oneshot
+
+[Install]
+WantedBy=multi-user.target

instances/k8sworker/scripts/setup.template.sh

@@ -33,7 +33,11 @@ fi
 BROADCOM_DRIVER=$(lsmod | grep bnxt_en | awk '{print $1}')
 if [[ -n "$${BROADCOM_DRIVER}" ]]; then
    echo "Disabling hardware TX checksum offloading"
-   ethtool --offload $(ip -o -4 route show to default | awk '{print $5}') tx off
+   mv /root/services/ethtool-disable-offload@.service /etc/systemd/system
+   systemctl daemon-reload
+   DEV=$(ip -o -4 route show to default | awk '{print $5}')
+   systemctl enable ethtool-disable-offload@$DEV
+   systemctl start ethtool-disable-offload@$DEV
 fi
 
 ## Setup NVMe drives and mount at /var/lib/docker