Merge to main: Bump soperator version to 1.23.2

.github/workflows/one_job.yml

@@ -288,14 +288,23 @@ jobs:
           echo "Building and pushing image of the nfs-server ${NFS_VERSION}"
           make docker-build-and-push DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS}" UNSTABLE="${UNSTABLE}" IMAGE_NAME=nfs-server DOCKERFILE=nfs-server/nfs.dockerfile IMAGE_VERSION=${NFS_VERSION}
 
+          CUDA_VERSION_12="12.9.0"
+          CUDA_VERSION_13="13.0.2"
+          CUDA_VERSIONS=("${CUDA_VERSION_12}" "${CUDA_VERSION_13}")
+
           echo "Removing previous jail rootfs tar archive"
           rm -f images/jail_rootfs*.tar
 
-          echo "Building tarball for jail"
-          make docker-build-jail DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS}" UNSTABLE="${UNSTABLE}" IMAGE_VERSION=${IMAGE_VERSION}
+          for CUDA_VERSION in "${CUDA_VERSIONS[@]}"; do
+            CUDA_TAG="cuda${CUDA_VERSION}"
+
+            echo "Building tarball for jail (${CUDA_TAG})"
+            make docker-build-jail DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS}" UNSTABLE="${UNSTABLE}" IMAGE_VERSION=${IMAGE_VERSION} CUDA_VERSION="${CUDA_VERSION}"
 
-          echo "Building and pushing image of the populate_jail ${IMAGE_VERSION}"
-          make docker-build-and-push DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS}" UNSTABLE="${UNSTABLE}" IMAGE_NAME=populate_jail DOCKERFILE=populate_jail/populate_jail.dockerfile IMAGE_VERSION=${IMAGE_VERSION}
+            echo "Building and pushing image of the populate_jail ${IMAGE_VERSION}-${CUDA_TAG}"
+            DOCKER_BUILD_ARGS_WITH_CUDA="${DOCKER_BUILD_ARGS} --build-arg CUDA_VERSION=${CUDA_VERSION}"
+            make docker-build-and-push DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS_WITH_CUDA}" UNSTABLE="${UNSTABLE}" IMAGE_NAME=populate_jail DOCKERFILE=populate_jail/populate_jail.dockerfile IMAGE_VERSION="${IMAGE_VERSION}-${CUDA_TAG}"
+          done
 
           echo "Removing jail rootfs tar archive to speedup further docker builds."
           rm -f images/jail_rootfs*.tar

Makefile

@@ -36,6 +36,9 @@ CHART_STORAGECLASSES		  = $(CHART_PATH)/storageclasses
 
 SLURM_VERSION		  		= 25.05.4
 UBUNTU_VERSION		  		?= noble
+CUDA12_VERSION              ?= 12.9.0
+CUDA13_VERSION              ?= 13.0.2
+CUDA_VERSION                ?= 12.9.0
 NFS_VERSION_BASE          	= $(shell cat NFS_VERSION)
 VERSION_BASE           		= $(shell cat VERSION)
 
@@ -252,7 +255,10 @@ sync-version: yq ## Sync versions from file
 	@$(YQ) -i ".images.slurmd = \"$(IMAGE_REPO)/worker_slurmd:$(IMAGE_VERSION)\"" "helm/slurm-cluster/values.yaml"
 	@$(YQ) -i ".images.sshd = \"$(IMAGE_REPO)/login_sshd:$(IMAGE_VERSION)\"" "helm/slurm-cluster/values.yaml"
 	@$(YQ) -i ".images.munge = \"$(IMAGE_REPO)/munge:$(IMAGE_VERSION)\"" "helm/slurm-cluster/values.yaml"
-	@$(YQ) -i ".images.populateJail = \"$(IMAGE_REPO)/populate_jail:$(IMAGE_VERSION)\"" "helm/slurm-cluster/values.yaml"
+	@$(YQ) -i ".images.populateJailRepository = \"$(IMAGE_REPO)/populate_jail\"" "helm/slurm-cluster/values.yaml"
+	@$(YQ) -i ".images.populateJailTag = \"$(IMAGE_VERSION)\"" "helm/slurm-cluster/values.yaml"
+	@$(YQ) -i ".images.populateJailCudaVersions.\"12\" = \"$(CUDA12_VERSION)\"" "helm/slurm-cluster/values.yaml"
+	@$(YQ) -i ".images.populateJailCudaVersions.\"13\" = \"$(CUDA13_VERSION)\"" "helm/slurm-cluster/values.yaml"
 	@$(YQ) -i ".images.soperatorExporter = \"$(IMAGE_REPO)/soperator-exporter:$(IMAGE_VERSION)\"" "helm/slurm-cluster/values.yaml"
 	@$(YQ) -i ".images.sConfigController = \"$(IMAGE_REPO)/sconfigcontroller:$(OPERATOR_IMAGE_TAG)\"" "helm/slurm-cluster/values.yaml"
 	@$(YQ) -i ".images.mariaDB = \"docker-registry1.mariadb.com/library/mariadb:11.4.3\"" "helm/slurm-cluster/values.yaml"
@@ -412,7 +418,8 @@ ifneq ($(HAS_AMD64),)
 		-t "$(IMAGE_REPO)/jail:${IMAGE_VERSION}-amd64" \
 		-f images/jail/jail.dockerfile \
 		--build-arg SLURM_VERSION="${SLURM_VERSION}" \
-		--output type=tar,dest=images/jail_rootfs_amd64.tar \
+		--build-arg CUDA_VERSION="${CUDA_VERSION}" \
+		--output type=tar,dest=images/jail_rootfs_cuda$(CUDA_VERSION)_amd64.tar \
 		--progress=plain \
 		$(DOCKER_BUILD_ARGS) \
 		.
@@ -424,7 +431,8 @@ ifneq ($(HAS_ARM64),)
 		-t "$(IMAGE_REPO)/jail:${IMAGE_VERSION}-arm64" \
 		-f images/jail/jail.dockerfile \
 		--build-arg SLURM_VERSION="${SLURM_VERSION}" \
-		--output type=tar,dest=images/jail_rootfs_arm64.tar \
+		--build-arg CUDA_VERSION="${CUDA_VERSION}" \
+		--output type=tar,dest=images/jail_rootfs_cuda$(CUDA_VERSION)_arm64.tar \
 		--progress=plain \
 		$(DOCKER_BUILD_ARGS) \
 		.

VERSION

@@ -1 +1 @@
-1.23.1
+1.23.2

ansible/roles/common-packages/defaults/main.yml

@@ -6,7 +6,7 @@ common_packages:
   - build-essential=12.10ubuntu1
   - ca-certificates=20240203
   - curl=8.5.0-2ubuntu10.6
-  - gnupg=2.4.4-2ubuntu17.3
+  - gnupg=2.4.4-2ubuntu17.4
   - jq=1.7.1-3ubuntu0.24.04.1
   - flex=2.6.4-8.2build1
   - kmod=31+20240202-2ubuntu7.1
@@ -23,11 +23,11 @@ common_packages:
   - libdrm-dev=2.4.122-1~ubuntu0.24.04.2
   - numactl=2.0.18-1build1
   - pkg-config=1.8.1-2build1
-  - rdma-core=50.0-2ubuntu0.2
+  - rdma-core=2404mlnx51-1.2404066
   - software-properties-common=0.99.49.3
   - squashfs-tools=1:4.6.1-1build1 # it's required for enroot
   - sudo=1.9.15p5-3ubuntu5.24.04.1
-  - ibverbs-utils=50.0-2ubuntu0.2
+  - ibverbs-utils=2404mlnx51-1.2404066
   - iproute2=6.1.0-1ubuntu6.2 # it's required for Slurm
   - libncurses-dev=6.4+20240113-1ubuntu2 # it's required for Slurm
   - libpmix-dev=5.0.1-4.1build1

ansible/roles/dcgmi/defaults/main.yml

@@ -2,5 +2,5 @@
 
 # Map CUDA *major* version -> dcgmi package name
 dcgmi_cuda_package_map:
-  "12": "datacenter-gpu-manager-4-cuda12=1:4.4.2-1"
-  "13": "datacenter-gpu-manager-4-cuda13=1:4.4.2-1"
+  "12": "datacenter-gpu-manager-4-cuda12=1:4.5.0-1"
+  "13": "datacenter-gpu-manager-4-cuda13=1:4.5.0-1"

config/manager/kustomization.yaml

@@ -3,4 +3,4 @@ resources:
 images:
   - name: controller
     newName: cr.eu-north1.nebius.cloud/soperator/slurm-operator
-    newTag: 1.23.1
+    newTag: 1.23.2

config/manager/manager.yaml

@@ -84,7 +84,7 @@ spec:
             value: "false"
           - name: SLURM_OPERATOR_WATCH_NAMESPACES
             value: "*"
-        image: controller:1.23.1
+        image: controller:1.23.2
         imagePullPolicy: Always
         name: manager
         securityContext:

config/soperatorchecks/kustomization.yaml

@@ -4,7 +4,7 @@ resources:
 images:
   - name: controller
     newName: cr.eu-north1.nebius.cloud/soperator/soperatorchecks
-    newTag: 1.23.1
+    newTag: 1.23.2
 patches:
   # Protect the /metrics endpoint by putting it behind auth.
   # If you want your controller-manager to expose the /metrics

fluxcd/environment/local/helmrelease.yaml

@@ -8,7 +8,7 @@ spec:
   chart:
     spec:
       chart: helm-soperator-fluxcd
-      version: "1.23.1"
+      version: "1.23.2"
       sourceRef:
         kind: HelmRepository
         name: soperator-fluxcd

fluxcd/environment/local/slurmCluster.yml

@@ -2,7 +2,7 @@ slurmCluster:
   enabled: true
   interval: 5m
   timeout: 5m
-  version: 1.23.1
+  version: 1.23.2
   namespace: soperator
   releaseName: soperator
   overrideValues:

fluxcd/environment/local/values.yaml

@@ -10,7 +10,7 @@ ns:
 soperator:
   enabled: true
   namespace: soperator-system
-  version: "1.23.1"
+  version: "1.23.2"
   interval: 5m
   timeout: 5m
   releaseName: soperator-controller
@@ -56,7 +56,7 @@ slurmCluster:
   enabled: true
   interval: 5m
   timeout: 5m
-  version: 1.23.1
+  version: 1.23.2
   namespace: soperator
   releaseName: soperator
   overrideValues:
@@ -265,10 +265,10 @@ backup:
 # NFS Server - disabled for local dev
 nfsServer:
   enabled: false
-  version: 1.23.1
+  version: 1.23.2
   overrideValues:
     image:
-      tag: 1.23.1
+      tag: 1.23.2
 # Notifier - disabled for local dev
 notifier:
   enabled: false

fluxcd/environment/nebius-cloud/dev/bootstrap/flux-kustomization.yaml

@@ -11,7 +11,7 @@ spec:
     name: nebius-cloud
   postBuild:
     substitute:
-      soperator_version: 1.23.1
+      soperator_version: 1.23.2
   path: "./fluxcd/enviroment/nebius-cloud/dev"
   prune: true
   timeout: 1m

fluxcd/environment/nebius-cloud/prod/bootstrap/flux-kustomization.yaml

@@ -10,7 +10,7 @@ spec:
     name: nebius-cloud
   postBuild:
     substitute:
-      soperator_version: 1.23.1
+      soperator_version: 1.23.2
   path: "./fluxcd/enviroment/nebius-cloud/prod"
   prune: false
   timeout: 1m

helm/nodeconfigurator/Chart.yaml

@@ -13,9 +13,9 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.23.1
+version: 1.23.2
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.23.1"
+appVersion: "1.23.2"

helm/nodeconfigurator/values.yaml

@@ -28,7 +28,7 @@ rebooter:
   env: []
   image:
     repository: "cr.eu-north1.nebius.cloud/soperator/rebooter"
-    tag: "1.23.1"
+    tag: "1.23.2"
     pullPolicy: IfNotPresent
   nodeSelector: {}
   resources: {}

helm/nodesets/Chart.yaml

@@ -13,9 +13,9 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.23.1
+version: 1.23.2
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.23.1"
+appVersion: "1.23.2"

helm/nodesets/values.yaml

@@ -17,10 +17,10 @@ priorityClasses:
 images:
   munge:
     repository: "cr.eu-north1.nebius.cloud/soperator/munge"
-    tag: "1.23.1-noble-slurm25.05.4"
+    tag: "1.23.2-noble-slurm25.05.4"
   slurmd:
     repository: "cr.eu-north1.nebius.cloud/soperator/worker_slurmd"
-    tag: "1.23.1-noble-slurm25.05.4"
+    tag: "1.23.2-noble-slurm25.05.4"
 # NodeSets configuration
 # Each NodeSet defines a group of worker nodes with specific characteristics
 # Example GPU NodeSet

helm/slurm-cluster-storage/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: helm-slurm-cluster-storage
 description: A Helm chart for Kubernetes
 type: application
-version: "1.23.1"
-appVersion: "1.23.1"
+version: "1.23.2"
+appVersion: "1.23.2"

helm/slurm-cluster/Chart.yaml

@@ -2,6 +2,6 @@ apiVersion: v2
 name: helm-slurm-cluster
 description: A Helm chart for Kubernetes
 type: application
-version: "1.23.1"
-appVersion: "1.23.1"
+version: "1.23.2"
+appVersion: "1.23.2"
 kubeVersion: ">=1.29.0-0"

helm/slurm-cluster/templates/slurm-cluster-cr.yaml

@@ -101,9 +101,20 @@ spec:
     - name: {{ .name | quote }}
       {{- omit . "name" "createPVC" "storageClassName" "size" | toYaml | nindent 6 }}
     {{- end }}
-  secrets: {{ toYaml .Values.secrets | nindent 4 }}
+  secrets:
+    {{- toYaml .Values.secrets | nindent 4 }}
+  {{ $populateJail := "" -}}
+  {{- if and .Values.images.populateJail (ne .Values.images.populateJail "") -}}
+  {{- $populateJail = .Values.images.populateJail -}}
+  {{- else -}}
+  {{- $cudaMajorVersion := default 12 .Values.cudaMajorVersion -}}
+  {{- $cudaVersion := required "populateJail cuda version for the selected major version must be provided." (index .Values.images.populateJailCudaVersions (printf "%v" $cudaMajorVersion)) -}}
+  {{- $populateJailRepo := required "populateJail repository must be provided." .Values.images.populateJailRepository -}}
+  {{- $populateJailTag := required "populateJail tag must be provided." .Values.images.populateJailTag -}}
+  {{- $populateJail = printf "%s:%s-cuda%s" $populateJailRepo $populateJailTag $cudaVersion -}}
+  {{- end -}}
   populateJail:
-    image: {{ required "populateJail image" .Values.images.populateJail | quote }}
+    image: {{ required "populateJail image" $populateJail | quote }}
     imagePullPolicy: {{ default "IfNotPresent" .Values.populateJail.imagePullPolicy | quote }}
     appArmorProfile: {{ default "unconfined" .Values.populateJail.appArmorProfile | quote }}
     k8sNodeFilterName: {{ required "Populate Jail job k8s node filter name must be provided." .Values.populateJail.k8sNodeFilterName | quote }}

helm/slurm-cluster/values.yaml

@@ -13,6 +13,8 @@ useDefaultAppArmorProfile: false
 maintenance: "none"
 # Slurm cluster type. Can be now gpu or cpu
 clusterType: gpu
+# CUDA major version used for populate-jail image selection.
+cudaMajorVersion: 12
 # partitionConfiguration define partition configuration of slurm worker nodes
 # https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION
 partitionConfiguration:
@@ -530,15 +532,20 @@ sConfigController:
     interval: "30s"
     scrapeTimeout: "28s"
 images:
-  slurmctld: "cr.eu-north1.nebius.cloud/soperator/controller_slurmctld:1.23.1-noble-slurm25.05.4"
-  slurmrestd: "cr.eu-north1.nebius.cloud/soperator/slurmrestd:1.23.1-noble-slurm25.05.4"
-  slurmd: "cr.eu-north1.nebius.cloud/soperator/worker_slurmd:1.23.1-noble-slurm25.05.4"
-  sshd: "cr.eu-north1.nebius.cloud/soperator/login_sshd:1.23.1-noble-slurm25.05.4"
-  munge: "cr.eu-north1.nebius.cloud/soperator/munge:1.23.1-noble-slurm25.05.4"
-  populateJail: "cr.eu-north1.nebius.cloud/soperator/populate_jail:1.23.1-noble-slurm25.05.4"
-  slurmdbd: "cr.eu-north1.nebius.cloud/soperator/controller_slurmdbd:1.23.1-noble-slurm25.05.4"
-  soperatorExporter: "cr.eu-north1.nebius.cloud/soperator/soperator-exporter:1.23.1-noble-slurm25.05.4"
-  sConfigController: cr.eu-north1.nebius.cloud/soperator/sconfigcontroller:1.23.1
+  slurmctld: "cr.eu-north1.nebius.cloud/soperator/controller_slurmctld:1.23.2-noble-slurm25.05.4"
+  slurmrestd: "cr.eu-north1.nebius.cloud/soperator/slurmrestd:1.23.2-noble-slurm25.05.4"
+  slurmd: "cr.eu-north1.nebius.cloud/soperator/worker_slurmd:1.23.2-noble-slurm25.05.4"
+  sshd: "cr.eu-north1.nebius.cloud/soperator/login_sshd:1.23.2-noble-slurm25.05.4"
+  munge: "cr.eu-north1.nebius.cloud/soperator/munge:1.23.2-noble-slurm25.05.4"
+  populateJail: ""
+  populateJailRepository: "cr.eu-north1.nebius.cloud/soperator/populate_jail"
+  populateJailTag: "1.23.2-noble-slurm25.05.4"
+  populateJailCudaVersions:
+    "12": "12.9.0"
+    "13": "13.0.2"
+  slurmdbd: "cr.eu-north1.nebius.cloud/soperator/controller_slurmdbd:1.23.2-noble-slurm25.05.4"
+  soperatorExporter: "cr.eu-north1.nebius.cloud/soperator/soperator-exporter:1.23.2-noble-slurm25.05.4"
+  sConfigController: cr.eu-north1.nebius.cloud/soperator/sconfigcontroller:1.23.2
   mariaDB: docker-registry1.mariadb.com/library/mariadb:11.4.3
 # Configuration for slurm scripts.
 slurmScripts:

helm/soperator-activechecks/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: helm-soperator-activechecks
 description: A Helm chart for Kubernetes
 type: application
-version: "1.23.1"
-appVersion: "1.23.1"
+version: "1.23.2"
+appVersion: "1.23.2"

helm/soperator-activechecks/templates/_helpers.tpl

@@ -56,16 +56,29 @@ Create the name of the service account to use
 Pyxis format for active check image.
 */}}
 {{- define "activecheck.image.pyxis" -}}
+{{- include "activecheck.image.resolve" . -}}
 {{- .Values.activeCheckImage -}}
 {{- end -}}
 
+{{/*
+Resolve active check image when not explicitly set.
+*/}}
+{{- define "activecheck.image.resolve" -}}
+{{- if not .Values.activeCheckImage -}}
+{{- $cudaMajorVersion := default 12 .Values.cudaMajorVersion -}}
+{{- $tag := required "activeCheck image tag for the selected CUDA major version must be provided." (index .Values.images.activeCheckImageTags (printf "%v" $cudaMajorVersion)) -}}
+{{- $repo := required "activeCheck image repository must be provided." .Values.images.activeCheckImageRepository -}}
+{{- $_ := set .Values "activeCheckImage" (printf "%s:%s" $repo $tag) -}}
+{{- end -}}
+{{- end -}}
+
 
 {{/*
 Docker format for active check image.
 Converts from format "reg#repo:tag" to format "reg/repo:tag".
 */}}
 {{- define "activecheck.image.docker" -}}
-{{- .Values.activeCheckImage | replace "#" "/" -}}
+{{- include "activecheck.image.pyxis" . | replace "#" "/" -}}
 {{- end -}}
 
 {{/*

helm/soperator-activechecks/values.yaml

@@ -1,4 +1,6 @@
 slurmClusterRefName: "soperator"
+# CUDA major version used for active check image selection.
+cudaMajorVersion: 12
 jobContainer:
   env:
     - name: "K8S_POD_NAME"
@@ -19,11 +21,15 @@ jobContainer:
       persistentVolumeClaim:
         claimName: "jail-pvc"
 images:
-  slurmJob: "cr.eu-north1.nebius.cloud/soperator/slurm_check_job:1.23.1-noble-slurm25.05.4"
-  k8sJob: "cr.eu-north1.nebius.cloud/soperator/k8s_check_job:1.23.1-noble-slurm25.05.4"
-  munge: "cr.eu-north1.nebius.cloud/soperator/munge:1.23.1-noble-slurm25.05.4"
-# Pyxis syntax with '#' separator for registry
-activeCheckImage: "cr.eu-north1.nebius.cloud#soperator/active_checks:12.9.0-ubuntu24.04-nccl_tests2.16.4-3935b93"
+  slurmJob: "cr.eu-north1.nebius.cloud/soperator/slurm_check_job:1.23.2-noble-slurm25.05.4"
+  k8sJob: "cr.eu-north1.nebius.cloud/soperator/k8s_check_job:1.23.2-noble-slurm25.05.4"
+  munge: "cr.eu-north1.nebius.cloud/soperator/munge:1.23.2-noble-slurm25.05.4"
+  activeCheckImageRepository: "cr.eu-north1.nebius.cloud#soperator/active_checks"
+  activeCheckImageTags:
+    "12": "12.9.0-ubuntu24.04-nccl_tests2.16.4-3935b93"
+    "13": "13.0.2-ubuntu24.04-nccl_tests2.17.6-3935b93"
+# Optional override in Pyxis ("reg#repo:tag") format.
+activeCheckImage: ""
 checks:
   allReducePerfNCCLInDocker:
     suspend: true

helm/soperator-crds/Chart.yaml

@@ -2,6 +2,6 @@ apiVersion: v2
 name: helm-soperator-crds
 description: A Helm chart for Kubernetes
 type: application
-version: 1.23.1
-appVersion: "1.23.1"
+version: 1.23.2
+appVersion: "1.23.2"
 kubeVersion: ">=1.29.0-0"