Back to openmpi...

environments/daint/spack.yaml

@@ -10,11 +10,49 @@ spack:
       - llvm +clang +compiler-rt +cuda +libcxx +lld +lldb +omp_as_runtime +polly targets=amdgpu,bpf,nvptx
       - pika +cuda
       - sirius +cuda
+      - openmpi +cuda
   packages:
     all:
       variants: [cuda_arch=60]
       providers:
-        mpi: [cray-mpich]
+        mpi: [openmpi]
+        blas: [openblas]
+        lapack: [openblas]
+        fftw-api: [fftw]
+        pkgconfig: [pkg-config]
+  packages:
+    autoconf:
+      externals:
+      - spec: autoconf@2.69
+        prefix: /usr
+    automake:
+      externals:
+      - spec: automake@1.15.1
+        prefix: /usr
+    libtool:
+      externals:
+      - spec: libtool@2.4.6
+        prefix: /usr
+    m4:
+      externals:
+      - spec: m4@1.4.18
+        prefix: /usr
+    openmpi:
+      variants:
+      - +thread_multiple
+      - ~gpfs
+      - ~internal-hwloc
+      - ~lustre
+      - fabrics=auto
+      - schedulers=auto
+    perl:
+      externals:
+      - spec: perl@5.26.1~cpanm+shared+threads
+        prefix: /usr
+    pkg-config:
+      externals:
+      - spec: pkg-config@0.29.2
+        prefix: /usr
   'compilers:':
   - compiler:
       spec: gcc@11.2.0

experimental/Makefile

@@ -29,15 +29,8 @@ deps/spack:
 spack.yaml: deps/spack
 	cp $(CURDIR)/../environments/daint/spack.yaml $@
 
-# Ensure that cray-mpich is in the database so it can use --reuse'd.
-add_cray_mpich: bin/add_cray_mpich.py spack.yaml deps/spack
-	$(SRUN) $(SPACK) \
-		-e $(CURDIR) \
-		-c "config:install_tree:root:$(DOWNSTREAM_STORE)" \
-		python $< && touch "$@"
-
 # Concretize
-spack.lock: spack.yaml deps/spack add_cray_mpich
+spack.lock: spack.yaml deps/spack
 	$(SRUN) $(SPACK) \
 		-e $(CURDIR) \
 		-c "config:install_tree:root:$(DOWNSTREAM_STORE)" \
@@ -46,7 +39,7 @@ spack.lock: spack.yaml deps/spack add_cray_mpich
 # Distributed build using overlayfs.
 # Spack believes it's installing to UPSTREAM_STORE, but in reality it's installing new
 # packages to `DOWNSTREAM_STORE`.
-build: spack.lock deps/usr/bin/overlayfs add_cray_mpich
+build: spack.lock deps/usr/bin/overlayfs
 	# Not sure if really required, but to avoid having to bind mount
 	# the merged dir over the bottom layer, add a level of indirection with a
 	# symlink from $(UPSTREAM_STORE) to $(UPSTREAM_STORE)-current
@@ -57,12 +50,11 @@ build: spack.lock deps/usr/bin/overlayfs add_cray_mpich
 	mkdir -p $(UPSTREAM_STORE)-current $(UPSTREAM_STORE) $(DOWNSTREAM_STORE) work merged
 	$(SRUN_BUILD) \
 		$(OVERLAY_SH) $(OVERLAYFS) $(UPSTREAM_STORE)-current $(DOWNSTREAM_STORE) $(CURDIR)/work $(CURDIR)/merged \
-		$(UNSHARE) -rm \
-			$(BIND_SH) $(CURDIR)/merged $(UPSTREAM_STORE) \
+		bwrap --dev-bind / / --bind $(CURDIR)/merged $(UPSTREAM_STORE) --chdir $(CURDIR) \
 			$(SPACK) \
 				-c "config:install_tree:root:$(UPSTREAM_STORE)" \
 				-c "modules:default:enable::[]" \
-				-e $(dir $<) \
+				-e $(CURDIR) \
 				install -j32 && touch "$@"
 
 # Create a layer, including the new database index

experimental/bin/overlay.sh

@@ -30,6 +30,6 @@ done
 
 # Execute the actual command
 
-env --chdir="$PWD" "$@"
+"$@"
 
 fusermount -u "$merge" > /dev/null 2>&1 || true