From f6eb95d33907713049a814d07b9fa118b030df45 Mon Sep 17 00:00:00 2001
From: Harmen Stoppels <harmenstoppels@gmail.com>
Date: Fri, 25 Feb 2022 11:58:29 +0100
Subject: [PATCH] just add cray-mpich to the database directly

---
 environments/daint/spack.yaml      | 28 +++++-----------------------
 experimental/.gitignore            |  1 +
 experimental/Makefile              | 23 ++++++++++++++++-------
 experimental/bin/add_cray_mpich.py | 17 +++++++++++++++++
 4 files changed, 39 insertions(+), 30 deletions(-)
 create mode 100755 experimental/bin/add_cray_mpich.py

diff --git a/environments/daint/spack.yaml b/environments/daint/spack.yaml
index 3e9ad533ec5b76..c60648a633ee2d 100644
--- a/environments/daint/spack.yaml
+++ b/environments/daint/spack.yaml
@@ -3,37 +3,19 @@ spack:
   concretization: separately
   specs:
   - matrix:
-    - - target=broadwell
-      - target=haswell
+    - - target=haswell
     - - cuda
-      - gromacs
+      - gromacs +cuda +lapack +blas +mpi
       - julia ^llvm ~clang ~compiler-rt ~libcxx ~lld ~lldb ~polly ^libgit2 ~mmap
       - llvm +clang +compiler-rt +cuda +libcxx +lld +lldb +omp_as_runtime +polly targets=amdgpu,bpf,nvptx
-      - openmpi +cuda
       - pika +cuda
-      - 'sirius +cuda ^intel-mkl@2020:'
+      - sirius +cuda
   packages:
     all:
       variants: [cuda_arch=60]
       providers:
-        mpi: [openmpi]
-        blas: [intel-mkl]
-        lapack: [intel-mkl]
-        fftw-api: [intel-mkl]
-        pkgconfig: [pkg-config]
-    openmpi:
-      variants:
-      - +thread_multiple
-      - ~gpfs
-      - ~internal-hwloc
-      - ~lustre
-      - fabrics=auto
-      - schedulers=auto
-    pkg-config:
-      externals:
-      - spec: pkg-config@0.29.2
-        prefix: /usr
-  compilers:
+        mpi: [cray-mpich]
+  'compilers:':
   - compiler:
       spec: gcc@11.2.0
       paths:
diff --git a/experimental/.gitignore b/experimental/.gitignore
index 56d7efffd063d2..4ee1b1a247fbe7 100644
--- a/experimental/.gitignore
+++ b/experimental/.gitignore
@@ -2,3 +2,4 @@
 /spack.lock
 /spack.yaml
 /.spack
+add_cray_mpich
diff --git a/experimental/Makefile b/experimental/Makefile
index b24e4666f6be45..5f480d98456d86 100644
--- a/experimental/Makefile
+++ b/experimental/Makefile
@@ -29,19 +29,29 @@ deps/spack:
 spack.yaml: deps/spack
 	cp $(CURDIR)/../environments/daint/spack.yaml $@
 
+# Ensure that cray-mpich is in the database so it can use --reuse'd.
+add_cray_mpich: bin/add_cray_mpich.py spack.yaml deps/spack
+	$(SRUN) $(SPACK) \
+		-e $(CURDIR) \
+		-c "config:install_tree:root:$(DOWNSTREAM_STORE)" \
+		python $< && touch "$@"
+
 # Concretize
-spack.lock: spack.yaml deps/spack
-	$(SRUN) $(SPACK) -e $(CURDIR) concretize
+spack.lock: spack.yaml deps/spack add_cray_mpich
+	$(SRUN) $(SPACK) -e $(CURDIR) concretize --reuse
 
 # Distributed build using overlayfs.
 # Spack believes it's installing to UPSTREAM_STORE, but in reality it's installing new
 # packages to `DOWNSTREAM_STORE`.
-build: spack.lock deps/usr/bin/overlayfs
+build: spack.lock deps/usr/bin/overlayfs add_cray_mpich
 	# Not sure if really required, but to avoid having to bind mount
 	# the merged dir over the bottom layer, add a level of indirection with a
 	# symlink from $(UPSTREAM_STORE) to $(UPSTREAM_STORE)-current
-	mkdir -p $(UPSTREAM_STORE)-current $(DOWNSTREAM_STORE) work merged
-	ln -s $(UPSTREAM_STORE)-current $(UPSTREAM_STORE) || true
+	if [ "$$(realpath $(UPSTREAM_STORE)-current)" != "$$(realpath $(UPSTREAM_STORE))" ]; then \
+		echo "Make sure `$(UPSTREAM_STORE)-current` is a symlink to `$(UPSTREAM_STORE)`" \
+		exit 1 \
+	fi
+	mkdir -p $(UPSTREAM_STORE)-current $(UPSTREAM_STORE) $(DOWNSTREAM_STORE) work merged
 	$(SRUN_BUILD) \
 		$(OVERLAY_SH) $(OVERLAYFS) $(UPSTREAM_STORE)-current $(DOWNSTREAM_STORE) $(CURDIR)/work $(CURDIR)/merged \
 		$(UNSHARE) -rm \
@@ -50,8 +60,7 @@ build: spack.lock deps/usr/bin/overlayfs
 				-c "config:install_tree:root:$(UPSTREAM_STORE)" \
 				-c "modules:default:enable::[]" \
 				-e $(dir $<) \
-				install -j32
-	touch "$@"
+				install -j32 && touch "$@"
 
 # Create a layer, including the new database index
 store.tar.zst: exclude.txt build deps/usr/bin/zstd
diff --git a/experimental/bin/add_cray_mpich.py b/experimental/bin/add_cray_mpich.py
new file mode 100755
index 00000000000000..e7bb6d1a45ae97
--- /dev/null
+++ b/experimental/bin/add_cray_mpich.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env spack-python
+
+# this just adds cray-mpich as a package to the store.
+# currently very piz daint-specific.
+
+from spack.spec import Spec
+import spack.store
+import spack.compilers
+import spack.environment
+
+for compiler in spack.compilers.all_compilers():
+    if compiler.name == 'gcc':
+        spec = Spec('cray-mpich@7.7.18 target=x86_64')
+        spec.external_path = '/opt/cray/pe/mpt/7.7.18/gni/mpich-gnu'
+        spec.compiler = compiler.spec
+        spec.concretize()
+        spack.store.db.add(spec, directory_layout=None, explicit=True)