Setup cscs/main branch

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "daily"

.github/workflows/mirror.yaml

@@ -0,0 +1,26 @@
+name: mirror
+
+on:
+  workflow_dispatch:
+    inputs:
+      upstream_commit:
+        description: 'Spack commit SHA'
+        required: true
+
+jobs:
+  bump_develop:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Make checkout actions setup auth
+        uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
+        with:
+          fetch-depth: 1
+          persist-credentials: true
+          ref: cscs/main
+          ssh-key: ${{ secrets.DEPLOY_KEY }}
+
+      - name: Bump develop branch
+        run: |
+          git fetch https://github.com/spack/spack.git "${{ github.event.inputs.upstream_commit }}"
+          git push origin "${{ github.event.inputs.upstream_commit }}:refs/heads/develop"

.github/workflows/shellcheck.yaml

@@ -0,0 +1,13 @@
+name: ShellCheck
+
+on: push
+
+jobs:
+  test:
+    name: ShellCheck
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
+      - name: Test
+        run: shellcheck scripts/*.sh

.gitignore

@@ -0,0 +1,6 @@
+.spack-env/
+bad_sha
+good_sha
+env.diff
+**/__pycache__
+generated-configs/

README.md

@@ -0,0 +1,33 @@
+Fork of [Spack][spack] with a slower update cadence to the [`develop` branch][cscs-develop].
+
+[`cscs-eths/spack` ↔️ `spack/spack` comparison][compare]
+
+## Setup Spack from this fork
+
+```console
+git clone -b develop -c feature.manyFiles=true https://github.com/eth-cscs/spack.git
+cd spack/bin
+./spack install zlib
+```
+
+## Updates
+
+Updates work more or less automatic. A scheduled pipeline bisects the upstream
+`develop` branch to the latest working version and opens an issue with a request to
+update to the corresponding commit SHA.
+
+Then a [Github action][action] is run by hand with the corresponding commit SHA to sync
+the repository.
+
+## What's in this branch?
+
+1. [`config/<name>`](config/) contains config for system `<name>`;
+2. [`environments/<name>/spack.yaml`](environments/) is an environment file with a list of
+   packages part of the available applications of CSCS. These environment files include
+   the system config files;
+3. [`patches`](patches/) contains some patches for Spack itself that are not upstreamed.
+
+[action]: https://github.com/eth-cscs/spack/actions/workflows/mirror.yaml
+[compare]: https://github.com/eth-cscs/spack/compare/develop...spack:develop
+[cscs-develop]: https://github.com/eth-cscs/spack/tree/develop
+[spack]: https://github.com/spack/spack

config/dom/compilers.yaml

@@ -0,0 +1,31 @@
+compilers:
+- compiler:
+    spec: gcc@9.3.0
+    paths:
+      cc: cc
+      cxx: CC
+      f77: ftn
+      fc: ftn
+    flags: {}
+    operating_system: cnl7
+    target: any
+    modules:
+    - PrgEnv-gnu
+    - gcc/9.3.0
+    environment: {}
+    extra_rpaths: []
+- compiler:
+    spec: gcc@10.3.0
+    paths:
+      cc: cc
+      cxx: CC
+      f77: ftn
+      fc: ftn
+    flags: {}
+    operating_system: cnl7
+    target: any
+    modules:
+    - PrgEnv-gnu
+    - gcc/10.3.0
+    environment: {}
+    extra_rpaths: []

config/dom/packages.yaml

@@ -0,0 +1,15 @@
+packages:
+  all:
+    target: [haswell]
+    compiler: [gcc, clang]
+    providers:
+      blas: [openblas]
+      lapack: [openblas]
+      mpi: [cray-mpich, openmpi]
+      scalapack: [netlib-scalapack]
+  cray-mpich:
+    buildable: false
+    externals:
+    - spec: cray-mpich@7.7.17
+      modules:
+      - cray-mpich/7.7.17

config/gha-ubuntu-20.04/compilers.yaml

@@ -0,0 +1 @@
+compilers: []

config/gha-ubuntu-20.04/packages.yaml

@@ -0,0 +1,9 @@
+packages:
+  all:
+    target: [x86_64]
+    compiler: [gcc, clang]
+    providers:
+      blas: [openblas]
+      lapack: [openblas]
+      mpi: [mpich]
+      scalapack: [netlib-scalapack]

environments/cscs/spack.yaml

@@ -0,0 +1,12 @@
+spack:
+  view: false
+  concretization: separately
+  specs:
+  - gromacs
+  - pika
+  - sirius
+  packages:
+    llvm:
+      variants: ~clang~libcxx~lldb~lld~compiler-rt~polly
+    libgit2:
+      variants: ~mmap

experimental/.gitignore

@@ -0,0 +1,4 @@
+/deps
+/spack.lock
+/spack.yaml
+/.spack

experimental/Make.inc

@@ -0,0 +1,26 @@
+# This is where the system spack db lives
+UPSTREAM_STORE:=/apps/daint/UES/jenscscs/store
+
+# Where to build new packages
+DOWNSTREAM_STORE:=$(CURDIR)/store
+
+# Basic `srun` command (for concretization)
+SRUN:=srun --pty -Ajenscscs -Cmc -N1
+
+# `srun` command for distributed builds
+SRUN_BUILD:=$(SRUN) -n8 -c32 --overcommit --oversubscribe
+
+GIT:=git
+CURL:=curl
+UNSHARE:=unshare
+
+# Managed executables
+SPACK:=$(CURDIR)/deps/spack/bin/spack
+OVERLAYFS:=$(CURDIR)/deps/usr/bin/overlayfs
+ZSTD:=$(CURDIR)/deps/usr/bin/zstd
+BIND_SH:=$(CURDIR)/bin/bind.sh
+OVERLAY_SH:=$(CURDIR)/bin/overlay.sh
+
+# Make tools available and make sure Spack config is local
+export SPACK_USER_CACHE_PATH:=$(CURDIR)/.spack
+export SPACK_USER_CONFIG_PATH:=$(CURDIR)/.spack

experimental/Makefile

@@ -0,0 +1,63 @@
+include $(CURDIR)/Make.inc
+
+.PHONY: clean
+
+all: build
+
+# Install overlayfs from the official binaries distributed on Github, cause
+# unclear if available by default.
+deps/usr/bin/overlayfs:
+	mkdir -p $(dir $@)
+	$(CURL) -Lfso $(notdir $@) https://github.com/containers/fuse-overlayfs/releases/download/v1.8.2/fuse-overlayfs-x86_64
+	echo "d7488446d5a17586a4aa3cc15e21d675a04a04f553e031c7789ed5e83ec4d829 overlayfs" | sha256sum --check
+	mv $(notdir $@) $@
+	chmod +x $@
+
+deps/usr/bin/zstd:
+	mkdir -p $(dir $@)
+	$(CURL) -Lfso zstd.tar.gz https://github.com/JuliaBinaryWrappers/Zstd_jll.jl/releases/download/Zstd-v1.5.2%2B0/Zstd.v1.5.2.x86_64-linux-gnu.tar.gz
+	echo "9d0fff908e3e0273cc8360a3fb4696e4227bc5cd0dd10bd3a4c9d295b9e4e25e zstd.tar.gz" | sha256sum --check
+	tar -xf zstd.tar.gz -C deps/usr
+	rm -f zstd.tar.gz
+
+# Install our hopefully working fork of Spack
+deps/spack:
+	mkdir -p $(dir $@)
+	$(GIT) clone --depth=1 --branch=develop https://github.com/eth-cscs/spack.git $@
+
+# Setup the environment
+spack.yaml: deps/spack
+	cp $(CURDIR)/../environments/cscs/spack.yaml $@
+	$(SPACK) -e $(CURDIR) config add 'packages:all:target:[haswell]'
+
+# Concretize
+spack.lock: spack.yaml deps/spack
+	$(SRUN) $(SPACK) -e $(CURDIR) concretize
+
+# Distributed build using overlayfs.
+# Spack believes it's installing to UPSTREAM_STORE, but in reality it's installing new
+# packages to `DOWNSTREAM_STORE`.
+build: spack.lock deps/usr/bin/overlayfs
+	mkdir -p $(UPSTREAM_STORE) $(DOWNSTREAM_STORE) work merged
+	$(SRUN_BUILD) \
+		$(OVERLAY_SH) $(OVERLAYFS) $(UPSTREAM_STORE) $(DOWNSTREAM_STORE) $(CURDIR)/work $(CURDIR)/merged \
+		$(UNSHARE) -rm \
+			$(BIND_SH) $(CURDIR)/merged $(UPSTREAM_STORE) \
+			$(SPACK) \
+				-c "config:install_tree:root:$(UPSTREAM_STORE)" \
+				-c "modules:default:enable::[]" \
+				-e $(dir $<) \
+				install -j32
+	touch "$@"
+
+# Create a layer, including the new database index
+store.tar.zst: exclude.txt build deps/usr/bin/zstd
+	tar -I "$(ZSTD) -9 -T0" -X $< -cf $@ -C $(DOWNSTREAM_STORE) .
+
+# Installing comes down to merging `DOWNSTREAM_STORE` with `UPSTREAM_STORE`
+install: store.tar.zst build deps/usr/bin/zstd
+	tar -I $(ZSTD) -xf $< -C $(UPSTREAM_STORE)
+
+clean:
+	rm -rf -- deps build $(DOWNSTREAM_STORE) work merged .spack spack.yaml \
+	          spack.lock .spack-env store.tar.zst

experimental/bin/bind.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -e
+
+if [ "$#" -lt 2 ]; then
+    echo "$0 [src] [dst] [args...]"
+    exit 1
+fi
+
+src="$1"; shift
+dst="$1"; shift
+
+mount --bind "$src" "$dst"
+
+env --chdir="$PWD" "$@"

experimental/bin/overlay.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -e
+
+# Print usage when input is incorrect
+if [ "$#" -lt 5 ] || ! command -v "$1" > /dev/null 2>&1 ; then
+    echo "$0 [overlayfs command] [lower] [upper] [work] [merge] [args...]"
+    exit 1
+fi
+
+# Swallow required args
+overlayfscmd="$1"; shift
+lower="$1";        shift
+upper="$1";        shift
+work="$1";         shift
+merge="$1";        shift
+
+# Defensively unmount
+fusermount -u "$merge" > /dev/null 2>&1 || true
+
+# First process runs the overlay mount
+if [ "$SLURM_LOCALID" = 0 ] || [ -z "${SLURM_LOCALID+x}" ]; then
+    "$overlayfscmd" -o "auto_unmount,big_writes,max_write=1048576,threaded=0,lowerdir=$lower,upperdir=$upper,workdir=$work" "$merge"
+fi
+
+# Other processes wait for it to come into existence
+while ! mountpoint -q "$merge"; do
+    sleep 1
+done
+
+# Execute the actual command
+
+env --chdir="$PWD" "$@"
+
+fusermount -u "$merge" > /dev/null 2>&1 || true

experimental/exclude.txt

@@ -0,0 +1,7 @@
+./bin
+./.spack-db/failures
+./.spack-db/index_verifier
+./.spack-db/lock
+./.spack-db/prefix_failures
+./.spack-db/prefix_lock
+.wh.*

experimental/readme.md

@@ -0,0 +1,42 @@
+Build an environment in an overlay directory on top of an existing "upstream" database
+of installed packages.
+
+## System requirements
+- Linux 4.18+
+- `fusermount`
+- `unshare`
+
+## Usage
+
+It's advisable to build from a clean shell
+
+```console
+$ env -i TERM=$TERM PATH=/usr/bin:/bin sh
+```
+
+Then setup an environment and build it with:
+
+```console
+make spack.yaml # you may want to modify the specs
+make build
+```
+
+this builds in `./store` and afterwards compresses that to `store.tar.zst`.
+
+Next,
+
+```console
+make install
+```
+
+merges the new installs with the upstream database.
+
+## Local testing and overriding variables
+
+When you don't have slurm available, you can disable it with
+
+```
+env -i make SRUN= SRUN_BUILD= UPSTREAM_STORE=/path/to/your/spack/store build
+```
+
+See the full list of variables in [Make.inc](Make.inc)