Add weights to x86 and graviton pod placement

spack · Jul 27, 2023 · 2bc853b · 2bc853b
1 parent 1051e05
commit 2bc853b
Show file tree

Hide file tree

Showing 6 changed files with 119 additions and 10 deletions.
diff --git a/k8s/production/runners/protected/graviton/2/release.yaml b/k8s/production/runners/protected/graviton/2/release.yaml
@@ -65,8 +65,9 @@ spec:
             service_account = "runner"
 
             [runners.kubernetes.affinity]
-              # Schedule this pod on graviton 2 or 3 nodes
               [runners.kubernetes.affinity.node_affinity]
+
+              # Schedule this pod on graviton 2 or 3 nodes
               [runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution]
                 [[runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms]]
                   [[runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
@@ -77,11 +78,27 @@ spec:
                       key = "spack.io/pipeline"
                       operator = "Exists"
 
+              # Weight this pod towards graviton 2 nodes over graviton 3 nodes
+              [runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 2
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/graviton"
+                    operator = "In"
+                    values = ["2"]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 1
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/graviton"
+                    operator = "In"
+                    values = ["3"]
+
               # Place pod close to other pipeline pods if possible ("pack" the pods tightly)
+              # This takes precedence over the above weights, prioritizing pod packing
               # Docs: https://docs.gitlab.com/runner/executors/kubernetes.html#define-nodes-where-pods-are-scheduled
               [runners.kubernetes.affinity.pod_affinity]
                 [[runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution]]
-                weight = 1
+                weight = 4
                 [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term]
                   topology_key = "topology.kubernetes.io/zone"
                   [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term.label_selector]

diff --git a/k8s/production/runners/protected/x86_64/v2/release.yaml b/k8s/production/runners/protected/x86_64/v2/release.yaml
@@ -64,8 +64,9 @@ spec:
             service_account = "runner"
 
             [runners.kubernetes.affinity]
-              # Schedule this pod on any node with x86_64 >= v2
               [runners.kubernetes.affinity.node_affinity]
+
+              # Schedule this pod on any node with x86_64 >= v2
               [runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution]
                 [[runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms]]
                   [[runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
@@ -76,11 +77,32 @@ spec:
                       key = "spack.io/pipeline"
                       operator = "Exists"
 
+              # Weight this pod towards x86-64-v2 nodes
+              [runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 3
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/x86_64"
+                    operator = "In"
+                    values = ["v2"]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 2
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/x86_64"
+                    operator = "In"
+                    values = ["v3"]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 1
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/x86_64"
+                    operator = "In"
+                    values = ["v4"]
+
               # Place pod close to other pipeline pods if possible ("pack" the pods tightly)
               # Docs: https://docs.gitlab.com/runner/executors/kubernetes.html#define-nodes-where-pods-are-scheduled
               [runners.kubernetes.affinity.pod_affinity]
                 [[runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution]]
-                weight = 1
+                weight = 4
                 [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term]
                   topology_key = "topology.kubernetes.io/zone"
                   [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term.label_selector]

diff --git a/k8s/production/runners/protected/x86_64/v3/release.yaml b/k8s/production/runners/protected/x86_64/v3/release.yaml
@@ -76,11 +76,26 @@ spec:
                       key = "spack.io/pipeline"
                       operator = "Exists"
 
+              # Weight this pod towards x86-64-v3 nodes
+              [runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 2
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/x86_64"
+                    operator = "In"
+                    values = ["v3"]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 1
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/x86_64"
+                    operator = "In"
+                    values = ["v4"]
+
               # Place pod close to other pipeline pods if possible ("pack" the pods tightly)
               # Docs: https://docs.gitlab.com/runner/executors/kubernetes.html#define-nodes-where-pods-are-scheduled
               [runners.kubernetes.affinity.pod_affinity]
                 [[runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution]]
-                weight = 1
+                weight = 4
                 [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term]
                   topology_key = "topology.kubernetes.io/zone"
                   [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term.label_selector]

diff --git a/k8s/production/runners/public/graviton/2/release.yaml b/k8s/production/runners/public/graviton/2/release.yaml
@@ -65,8 +65,9 @@ spec:
             service_account = "runner"
 
             [runners.kubernetes.affinity]
-              # Schedule this pod on graviton 2 or 3 nodes
               [runners.kubernetes.affinity.node_affinity]
+
+              # Schedule this pod on graviton 2 or 3 nodes
               [runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution]
                 [[runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms]]
                   [[runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
@@ -77,11 +78,27 @@ spec:
                       key = "spack.io/pipeline"
                       operator = "Exists"
 
+              # Weight this pod towards graviton 2 nodes over graviton 3 nodes
+              [runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 2
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/graviton"
+                    operator = "In"
+                    values = ["2"]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 1
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/graviton"
+                    operator = "In"
+                    values = ["3"]
+
               # Place pod close to other pipeline pods if possible ("pack" the pods tightly)
+              # This takes precedence over the above weights, prioritizing pod packing
               # Docs: https://docs.gitlab.com/runner/executors/kubernetes.html#define-nodes-where-pods-are-scheduled
               [runners.kubernetes.affinity.pod_affinity]
                 [[runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution]]
-                weight = 1
+                weight = 4
                 [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term]
                   topology_key = "topology.kubernetes.io/zone"
                   [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term.label_selector]

diff --git a/k8s/production/runners/public/x86_64/v2/release.yaml b/k8s/production/runners/public/x86_64/v2/release.yaml
@@ -64,8 +64,9 @@ spec:
             service_account = "runner"
 
             [runners.kubernetes.affinity]
-              # Schedule this pod on any node with x86_64 >= v2
               [runners.kubernetes.affinity.node_affinity]
+
+              # Schedule this pod on any node with x86_64 >= v2
               [runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution]
                 [[runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms]]
                   [[runners.kubernetes.affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
@@ -76,11 +77,33 @@ spec:
                       key = "spack.io/pipeline"
                       operator = "Exists"
 
+              # Weight this pod towards x86-64-v2 nodes
+              [runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 3
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/x86_64"
+                    operator = "In"
+                    values = ["v2"]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 2
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/x86_64"
+                    operator = "In"
+                    values = ["v3"]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 1
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/x86_64"
+                    operator = "In"
+                    values = ["v4"]
+
               # Place pod close to other pipeline pods if possible ("pack" the pods tightly)
+              # This takes precedence over the above weights, prioritizing pod packing
               # Docs: https://docs.gitlab.com/runner/executors/kubernetes.html#define-nodes-where-pods-are-scheduled
               [runners.kubernetes.affinity.pod_affinity]
                 [[runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution]]
-                weight = 1
+                weight = 4
                 [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term]
                   topology_key = "topology.kubernetes.io/zone"
                   [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term.label_selector]

diff --git a/k8s/production/runners/public/x86_64/v3/release.yaml b/k8s/production/runners/public/x86_64/v3/release.yaml
@@ -76,11 +76,26 @@ spec:
                       key = "spack.io/pipeline"
                       operator = "Exists"
 
+              # Weight this pod towards x86-64-v3 nodes
+              [runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 2
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/x86_64"
+                    operator = "In"
+                    values = ["v3"]
+                [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms]]
+                  weight = 1
+                  [[runners.kubernetes.affinity.node_affinity.preferred_during_scheduling_ignored_during_execution.node_selector_terms.match_expressions]]
+                    key = "spack.io/x86_64"
+                    operator = "In"
+                    values = ["v4"]
+
               # Place pod close to other pipeline pods if possible ("pack" the pods tightly)
               # Docs: https://docs.gitlab.com/runner/executors/kubernetes.html#define-nodes-where-pods-are-scheduled
               [runners.kubernetes.affinity.pod_affinity]
                 [[runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution]]
-                weight = 1
+                weight = 4
                 [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term]
                   topology_key = "topology.kubernetes.io/zone"
                   [runners.kubernetes.affinity.pod_affinity.preferred_during_scheduling_ignored_during_execution.pod_affinity_term.label_selector]