From 0527e16fbed3f8928439ab968641be6af76f7ce2 Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <harmenstoppels@gmail.com>
Date: Tue, 1 Mar 2022 11:46:26 +0100
Subject: [PATCH] Back to mpich...

---
 environments/daint/spack.yaml | 23 +++++++++++++----------
 experimental/Make.inc         |  2 +-
 experimental/Makefile         |  4 ++--
 3 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/environments/daint/spack.yaml b/environments/daint/spack.yaml
index ec761d306f3e68..6ac63e1c751738 100644
--- a/environments/daint/spack.yaml
+++ b/environments/daint/spack.yaml
@@ -10,12 +10,11 @@ spack:
       - llvm +clang +compiler-rt +cuda +libcxx +lld +lldb +omp_as_runtime +polly targets=amdgpu,bpf,nvptx
       - pika +cuda
       - sirius +cuda
-      - openmpi +cuda
   packages:
     all:
       variants: [cuda_arch=60]
       providers:
-        mpi: [openmpi]
+        mpi: [mpich]
         blas: [openblas]
         lapack: [openblas]
         fftw-api: [fftw]
@@ -36,15 +35,15 @@ spack:
       externals:
       - spec: m4@1.4.18
         prefix: /usr
-    openmpi:
+    mpich:
       variants:
-      - +thread_multiple
-      - +cuda
-      - ~gpfs
-      - ~internal-hwloc
-      - ~lustre
-      - fabrics=auto
-      - schedulers=auto
+      - device=ch4
+      - netmod=ofi
+      - pmi=pmi2
+      - +fortran
+      - ~hydra
+      - +slurm
+      - ~verbs
     perl:
       externals:
       - spec: perl@5.26.1~cpanm+shared+threads
@@ -53,6 +52,10 @@ spack:
       externals:
       - spec: pkg-config@0.29.2
         prefix: /usr
+    slurm:
+      externals:
+      - spec: slurm@21-11-8-1
+        prefix: /usr
   'compilers:':
   - compiler:
       spec: gcc@11.2.0
diff --git a/experimental/Make.inc b/experimental/Make.inc
index 253c90cc3bf812..0d156c7bf2b01e 100644
--- a/experimental/Make.inc
+++ b/experimental/Make.inc
@@ -8,7 +8,7 @@ DOWNSTREAM_STORE:=$(CURDIR)/store
 SRUN:=srun --pty -Ajenscscs -Cmc -N1
 
 # `srun` command for distributed builds
-SRUN_BUILD:=$(SRUN) -n8 -c32 --overcommit --oversubscribe
+SRUN_BUILD:=$(SRUN) -n8 -c72 --overcommit --oversubscribe
 
 GIT:=git
 CURL:=curl
diff --git a/experimental/Makefile b/experimental/Makefile
index 68c4caee172da9..fd9d6fc21477bd 100644
--- a/experimental/Makefile
+++ b/experimental/Makefile
@@ -34,7 +34,7 @@ spack.lock: spack.yaml deps/spack
 	$(SRUN) $(SPACK) \
 		-e $(CURDIR) \
 		-c "config:install_tree:root:$(DOWNSTREAM_STORE)" \
-		concretize --reuse
+		concretize -f
 
 # Distributed build using overlayfs.
 # Spack believes it's installing to UPSTREAM_STORE, but in reality it's installing new
@@ -55,7 +55,7 @@ build: spack.lock deps/usr/bin/overlayfs
 				-c "config:install_tree:root:$(UPSTREAM_STORE)" \
 				-c "modules:default:enable::[]" \
 				-e $(CURDIR) \
-				install -j32 && touch "$@"
+				install -j72 && touch "$@"
 
 # Create a layer, including the new database index
 store.tar.zst: exclude.txt build deps/usr/bin/zstd