pipelines: simplify and lint aws-pcluster-* (spack#47989)

mcuma · Dec 10, 2024 · b50dbb8 · b50dbb8
1 parent 30c0035
commit b50dbb8
Show file tree

Hide file tree

Showing 3 changed files with 59 additions and 163 deletions.
diff --git a/lib/spack/spack/util/spack_yaml.py b/lib/spack/spack/util/spack_yaml.py
@@ -416,10 +416,12 @@ def dump_config(data, stream, *, default_flow_style=False, blame=False):
     if blame:
         handler = ConfigYAML(yaml_type=YAMLType.ANNOTATED_SPACK_CONFIG_FILE)
         handler.yaml.default_flow_style = default_flow_style
+        handler.yaml.width = maxint
         return _dump_annotated(handler, data, stream)
 
     handler = ConfigYAML(yaml_type=YAMLType.SPACK_CONFIG_FILE)
     handler.yaml.default_flow_style = default_flow_style
+    handler.yaml.width = maxint
     return handler.dump(data, stream)
 
 

diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
@@ -1,6 +1,5 @@
 spack:
   view: false
-
   definitions:
   - apps:
     - gromacs
@@ -9,20 +8,18 @@ spack:
     - openfoam
     - quantum-espresso
     - wrf
-
   - targets:
-      - 'target=neoverse_v1'
-      - 'target=neoverse_n1'
-
+    - target=neoverse_v1
+    - target=neoverse_n1
   specs:
   - matrix:
-      - [$apps]
-      - [$targets]
+    - [$apps]
+    - [$targets]
   ci:
     pipeline-gen:
     - build-job:
-        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07", "entrypoint": [""] }
-        tags: ["aarch64"]
+        image: {name: ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07, entrypoint: ['']}
+        tags: [aarch64]
         before_script:
         - - . "./share/spack/setup-env.sh"
           - . /etc/profile.d/modules.sh
@@ -31,27 +28,12 @@ spack:
           - export PATH=/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-aarch64/gcc-7.3.1/binutils-2.37-2yxz3xsjfmesxujxtlrgcctxlyilynmp/bin:$PATH
     - signing-job:
         before_script:
-          # Do not distribute Intel & ARM binaries
-          - - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
-            - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
-
+            # Do not distribute Intel & ARM binaries
+        - - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
+          - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
   cdash:
     build-group: AWS Packages
-
   compilers:
-  - compiler:
-      environment: {}
-      extra_rpaths: []
-      flags: {}
-      modules: []
-      operating_system: amzn2
-      paths:
-        cc: /usr/bin/gcc
-        cxx: /usr/bin/g++
-        f77: /usr/bin/gfortran
-        fc: /usr/bin/gfortran
-      spec: gcc@=7.3.1
-      target: aarch64
   - compiler:
       environment: {}
       extra_rpaths: []
@@ -65,41 +47,34 @@ spack:
         fc: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-aarch64/gcc-7.3.1/gcc-12.4.0-v6wxye6ijzrxnzxftcwnpu3psohsjl2b/bin/gfortran
       spec: gcc@=12.4.0
       target: aarch64
-
   packages:
     acfl:
       require:
-        - one_of: ["%gcc target=aarch64"]
-          message: "Clang based compilers need GCC libraries and they should be made available for the wide range of CPUs they actually support.
-                  Edit $SPACK_ROOT/etc/spack/packages.yaml to change this default."
+      - '%gcc target=aarch64'
     gromacs:
       require:
-        - one_of:
-            - "gromacs@2024.3 %arm ^fftw ^openmpi"
-            - "gromacs@2024.3 %gcc ^armpl-gcc ^openmpi"
+      - gromacs@2024.3 %gcc ^armpl-gcc ^openmpi
     libfabric:
       buildable: true
       externals:
-        - prefix: /opt/amazon/efa/
-          spec: libfabric@1.17.0
-      require: ['fabrics=shm,efa']
+      - prefix: /opt/amazon/efa/
+        spec: libfabric@1.17.0
+      require:
+      - fabrics=shm,efa
     llvm:
       variants: ~lldb
     mpas-model:
       require:
-        - one_of:
-            - "precision=single make_target=llvm %arm ^parallelio+pnetcdf"
-            - "precision=single %gcc ^parallelio+pnetcdf"
+      - precision=single %gcc ^parallelio+pnetcdf
     mpich:
-      require: "mpich pmi=pmi2 device=ch4 netmod=ofi +slurm"
+      require:
+      - mpich pmi=pmi2 device=ch4 netmod=ofi +slurm
     nvhpc:
       require:
-        - one_of:
-            - "nvhpc %gcc target=aarch64"
-          message: "NVHPC should be built with GCC and should be made available for the wide range of CPUs they actually support.
-                  Edit $SPACK_ROOT/etc/spack/packages.yaml to change this default."
+      - nvhpc %gcc target=aarch64
     openfoam:
-      require: "openfoam %gcc ^scotch@6.0.9"
+      require:
+      - openfoam ^scotch@6.0.9
     openmpi:
       variants: ~atomics ~cuda ~cxx ~cxx_exceptions ~internal-hwloc ~java +legacylaunchers ~lustre ~memchecker +pmi +romio ~singularity +vt +wrapper-rpath fabrics=ofi schedulers=slurm
       require: '@4:'
@@ -108,19 +83,15 @@ spack:
     #   require:
     #     - one_of: ["palace cxxflags=\"-include cstdint\" ^fmt@9.1.0"]
     pmix:
-      require: "pmix@3:"
+      require: 'pmix@3:'
     quantum-espresso:
-      require: "quantum-espresso@6.6 %gcc ^armpl-gcc"
+      require:
+      - quantum-espresso@6.6 %gcc ^armpl-gcc
     slurm:
       buildable: false
       externals:
-        - prefix: /opt/slurm
-          spec: slurm@22.05.8 +pmix
-    wrf:
-      require:
-        - one_of:
-            - "wrf%arm"
-            - "wrf%gcc"
+      - prefix: /opt/slurm
+        spec: slurm@22.05.8 +pmix
     all:
       compiler: [gcc, arm, nvhpc, clang]
       providers:

diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml
@@ -1,31 +1,28 @@
 spack:
   view: false
-
   definitions:
   - apps:
     - gromacs %oneapi
     - lammps %oneapi
-    # earliest oneapi version with fix does not run on AmazonLinux2, see https://github.com/spack/spack/pull/46457
-    # - mpas-model %oneapi
+        # earliest oneapi version with fix does not run on AmazonLinux2, see https://github.com/spack/spack/pull/46457
+        # - mpas-model %oneapi
     - openfoam %gcc
-    - palace %oneapi ^superlu-dist%oneapi # hack: force fortran-rt provider through superlu-dist
-    # TODO: Find out how to make +ipo cmake flag work.
-    # - quantum-espresso %oneapi
+    - palace %oneapi ^superlu-dist%oneapi     # hack: force fortran-rt provider through superlu-dist
+        # TODO: Find out how to make +ipo cmake flag work.
+        # - quantum-espresso %oneapi
     - openmpi %oneapi
     - wrf %oneapi
-
   - targets:
-      - 'target=x86_64_v4'
-      - 'target=x86_64_v3'
-
+    - target=x86_64_v4
+    - target=x86_64_v3
   specs:
   - matrix:
-      - [$apps]
-      - [$targets]
+    - [$apps]
+    - [$targets]
   ci:
     pipeline-gen:
     - build-job:
-        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07", "entrypoint": [""] }
+        image: {name: ghcr.io/spack/pcluster-amazonlinux-2:v2024-10-07, entrypoint: ['']}
         before_script:
         - - . "./share/spack/setup-env.sh"
           - . /etc/profile.d/modules.sh
@@ -34,13 +31,11 @@ spack:
           - export PATH=/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-7.3.1/binutils-2.37-qvccg7zpskturysmr4bzbsfrx34kvazo/bin:$PATH
     - signing-job:
         before_script:
-          # Do not distribute Intel & ARM binaries
-          - - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
-            - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
-
+            # Do not distribute Intel & ARM binaries
+        - - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
+          - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
   cdash:
     build-group: AWS Packages
-
   compilers:
   - compiler:
       environment: {}
@@ -69,131 +64,59 @@ spack:
         fc: /home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-amzn2-x86_64_v3/gcc-12.4.0/intel-oneapi-compilers-2024.1.0-f5u3psfhbwscasajkn324igtupn3blop/compiler/2024.1/bin/ifx
       spec: oneapi@=2024.1.0
       target: x86_64
-
   packages:
-    cpio:
-      require:
-        - one_of:
-            - "cflags=-std=c18 target=x86_64_v4"
-            - "cflags=-std=c18 target=x86_64_v3"
-          when: "%intel"
     gettext:
       # Newer gettext cannot build with gcc@12 and old AL2 glibc headers
       # Older gettext versions do not build correctly with oneapi.
       require:
-        - one_of:
-            - '@:0.20'
-            - '%oneapi'
+      - one_of: ['@:0.20 %gcc', '%oneapi']
     gromacs:
       require:
-        - one_of:
-            - "+intel_provided_gcc ^intel-oneapi-mkl target=x86_64_v4"
-            - "+intel_provided_gcc ^intel-oneapi-mkl target=x86_64_v3"
-          when: "%intel"
-        - one_of:
-            - "+intel_provided_gcc target=x86_64_v4 ^intel-oneapi-mkl"
-            - "+intel_provided_gcc target=x86_64_v3 ^intel-oneapi-mkl"
-          when: "%oneapi"
-    intel-oneapi-compilers:
-      require: "intel-oneapi-compilers %gcc target=x86_64_v3"
+      - +intel_provided_gcc ^intel-oneapi-mkl
     intel-oneapi-mpi:
       variants: +external-libfabric generic-names=True
     lammps:
       require:
-        - one_of:
-            - "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package +intel fft=mkl ^intel-oneapi-mkl target=x86_64_v4"
-            - "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl target=x86_64_v3"
-          when: "%intel"
-        - one_of:
-            - "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package +intel fft=mkl ^intel-oneapi-mkl target=x86_64_v4"
-            - "lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl target=x86_64_v3"
-          when: "%oneapi"
-    libidn2:
-      require:
-        - one_of:
-            - "cflags=-std=c18 target=x86_64_v4"
-            - "cflags=-std=c18 target=x86_64_v3"
-          when: "%intel"
+      - lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl
+      - one_of: [+intel target=x86_64_v4, target=x86_64_v3]
     libfabric:
       buildable: true
       externals:
-        - prefix: /opt/amazon/efa/
-          spec: libfabric@1.17.0
-      require: ['fabrics=shm,efa']
-    libunistring:
+      - prefix: /opt/amazon/efa/
+        spec: libfabric@1.17.0
       require:
-        - one_of:
-            - "cflags=-std=c18 target=x86_64_v4"
-            - "cflags=-std=c18 target=x86_64_v3"
-          when: "%intel"
+      - fabrics=shm,efa
     mpas-model:
       require:
-        - one_of:
-            - "precision=single ^parallelio+pnetcdf target=x86_64_v4"
-            - "precision=single ^parallelio+pnetcdf target=x86_64_v3"
-          when: "%intel"
-        - one_of:
-            - "precision=single ^parallelio+pnetcdf target=x86_64_v4"
-            - "precision=single ^parallelio+pnetcdf target=x86_64_v3"
-          when: "%oneapi"
+      - spec: precision=single ^parallelio+pnetcdf
+        when: '%oneapi'
     mpich:
       require:
-        - one_of:
-            - "mpich pmi=pmi2 device=ch4 netmod=ofi +slurm target=x86_64_v4"
-            - "mpich pmi=pmi2 device=ch4 netmod=ofi +slurm target=x86_64_v3"
+      - mpich pmi=pmi2 device=ch4 netmod=ofi +slurm
     openfoam:
       require:
-        - one_of:
-            - "openfoam %gcc ^scotch@6.0.9 target=x86_64_v4"
-            - "openfoam %gcc ^scotch@6.0.9 target=x86_64_v3"
+      - openfoam %gcc ^scotch@6.0.9
     openmpi:
       variants: ~atomics ~cuda ~cxx ~cxx_exceptions ~internal-hwloc ~java +legacylaunchers ~lustre ~memchecker +pmi +romio ~singularity +vt +wrapper-rpath fabrics=ofi schedulers=slurm
       require:
-        - one_of:
-            - "openmpi @4: target=x86_64_v4"
-            - "openmpi @4: target=x86_64_v3"
+      - 'openmpi @4:'
     palace:
       require:
-        - one_of:
-            - "palace ^fmt@9.1.0 target=x86_64_v4"
-            - "palace ^fmt@9.1.0 target=x86_64_v3"
-          when: "%oneapi"
-        - one_of:
-            - "palace ^fmt@9.1.0"
-          when: "%gcc"
+      - palace ^fmt@9.1.0
     pmix:
       require:
-        - one_of:
-            - "pmix@3: target=x86_64_v4"
-            - "pmix@3: target=x86_64_v3"
+      - 'pmix@3:'
     quantum-espresso:
       require:
-        - one_of:
-            - "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v4"
-            - "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v3"
-          when: "%intel"
-        - one_of:
-            - "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v4"
-            - "quantum-espresso@6.6 ^intel-oneapi-mkl+cluster target=x86_64_v3"
-          when: "%oneapi"
+      - quantum-espresso@6.6 ^intel-oneapi-mkl+cluster
     slurm:
       buildable: false
       externals:
-        - prefix: /opt/slurm
-          spec: slurm@22.05.8 +pmix
+      - prefix: /opt/slurm
+        spec: slurm@22.05.8 +pmix
     wrf:
       require:
-        - one_of:
-            - "wrf@4 build_type=dm+sm target=x86_64_v4"
-            - "wrf@4 build_type=dm+sm target=x86_64_v3"
-            - "wrf@4.2.2 +netcdf_classic fflags=\"-fp-model fast=2 -no-heap-arrays -no-prec-div -no-prec-sqrt -fno-common\" build_type=dm+sm target=x86_64_v3"
-          when: "%intel"
-        - one_of:
-            - "wrf@4 build_type=dm+sm target=x86_64_v4"
-            - "wrf@4 build_type=dm+sm target=x86_64_v3"
-            - "wrf@4.2.2 +netcdf_classic fflags=\"-fp-model fast=2 -no-heap-arrays -no-prec-div -no-prec-sqrt -fno-common\" build_type=dm+sm target=x86_64_v3"
-          when: "%oneapi"
-
+      - wrf@4 build_type=dm+sm
     all:
       compiler: [oneapi, gcc]
       permissions: