Splitting up CI matrix for pip llnl

.gitlab/llnl-ci-pip.yml

@@ -9,83 +9,53 @@ stages:
   - test
   - cleanup
 
+### MACHINES
+.on_quartz:
+  variables:
+    HOST: "quartz"
+    VENV_ENV_NAME: "quartz-env"
+
+.on_ruby:
+  variables:
+    HOST: "ruby"
+    VENV_ENV_NAME: "ruby-env"
+
+.on_lassen:
+  variables:
+    HOST: "lassen"
+    VENV_ENV_NAME: "lassen-env"
+
+### SCRIPTS
 .final_steps:
   script: &finalize
     - chgrp -fR exaworks ${PIP_WORK_DIR} || true
     - chmod -fR 02770 ${PIP_WORK_DIR}    || true
 
-pip-env-setup:
-  stage: setup
-  parallel:
-    matrix:
-      - HOST : [ "quartz" ]
-        VENV_ENV_NAME: [ "quartz-env" ]
-        RUNNER_TYPE: "shell"
-      - HOST : [ "ruby" ]
-        VENV_ENV_NAME: [ "ruby-env" ]
-        RUNNER_TYPE: "shell"
-      # - HOST : [ "lassen" ]
-      #   ENVIRONMENT: [ "BATCH" ]
-      #   VENV_ENV_NAME: [ "lassen-env" ]
-      #   RUNNER_TYPE: "batch"
+.job_tags:
   tags:
     - $HOST
     - $RUNNER_TYPE
+
+.pip_env_setup:
+  stage: setup
   script:
     - mkdir -p ${PIP_WORK_DIR}
     - test -d ${PIP_WORK_DIR}/${VENV_ENV_NAME} && exit 0
     - python3 -m venv ${PIP_WORK_DIR}/${VENV_ENV_NAME}
     - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate
     - pip install -U pip setuptools wheel
     - pip cache purge
+    - *finalize
 
-pip-build:
+.pip_build:
   stage: build
-  needs:
-    - job: pip-env-setup
-  parallel:
-    matrix:
-      - HOST : [ "quartz" ]
-        VENV_ENV_NAME: [ "quartz-env" ]
-        RUNNER_TYPE: "shell"
-      - HOST : [ "ruby" ]
-        VENV_ENV_NAME: [ "ruby-env" ]
-        RUNNER_TYPE: "shell"
-      # - HOST : [ "lassen" ]
-      #   ENVIRONMENT: [ "BATCH" ]
-      #   VENV_ENV_NAME: [ "lassen-env" ]
-      #   RUNNER_TYPE: "batch"
-  tags:
-    - $HOST
-    - $RUNNER_TYPE
   script:
     - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate
     - pip install --no-cache-dir -r ${PIP_REQUIREMENTS}
     - *finalize
 
-pip-tests:
+.pip_test:
   stage: test
-  needs: 
-    - job: pip-build
-  parallel:
-    matrix:
-      - HOST : [ "quartz" ]
-        VENV_ENV_NAME: [ "quartz-env" ]
-        RUNNER_TYPE: "batch"
-        LLNL_SLURM_SCHEDULER_PARAMETERS: [ "--nodes=1 -t 30" ]
-      - HOST : [ "ruby" ]
-        VENV_ENV_NAME: [ "ruby-env" ]
-        RUNNER_TYPE: "batch"
-        LLNL_SLURM_SCHEDULER_PARAMETERS: [ "--nodes=1 -t 30" ]
-      # - HOST : [ "lassen" ]
-      #   ENVIRONMENT: [ "BATCH" ]
-      #   VENV_ENV_NAME: [ "lassen-env" ]
-      #   RUNNER_TYPE: "batch"
-        # LLNL_LSF_SCHEDULER_PARAMETERS: [ "-nnodes 1 -W 30" ]
-        # LSB_JOB_STARTER: [ "ENVIRONMENT=BATCH /usr/tcetmp/bin/bsub_job_starter %USRCMD" ]
-  tags:
-    - $HOST
-    - $RUNNER_TYPE
   script:
     - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate
     - export run_id=${CI_PIPELINE_ID}
@@ -102,30 +72,62 @@ pip-tests:
     - python3 ci/tests/test.py -e
     - deactivate
 
-.pip-env-cleanup: &pip-env-cleanup
-  stage: cleanup
-  tags:
-    - ruby
-    - shell
-  script:
-    - source ${PIP_WORK_DIR}/ruby-env/bin/activate
-    - pip cache purge
-    - *finalize
+### JOBS
+# QUARTZ
+pip_setup_quartz:
+  variables:
+    RUNNER_TYPE: "shell"
+  extends: [.on_quartz, .job_tags, .pip_env_setup]
+
+pip_build_quartz:
+  variables:
+    RUNNER_TYPE: "shell"
+  needs: [pip_setup_quartz]
+  extends: [.on_quartz, .job_tags, .pip_build]
+
+pip_test_quartz:
+  variables:
+    RUNNER_TYPE: "batch"
+    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 30"
+  needs: [pip_build_quartz]
+  extends: [.on_quartz, .job_tags, .pip_test]
+
+# RUBY
+pip_setup_ruby:
+  variables:
+    RUNNER_TYPE: "shell"
+  extends: [.on_ruby, .job_tags, .pip_env_setup]
+
+pip_build_ruby:
+  variables:
+    RUNNER_TYPE: "shell"
+  needs: [pip_setup_ruby]
+  extends: [.on_ruby, .job_tags, .pip_build]
+
+pip_test_ruby:
+  variables:
+    RUNNER_TYPE: "batch"
+    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 30"
+  needs: [pip_build_ruby]
+  extends: [.on_ruby, .job_tags, .pip_test]
+
+# LASSEN
+pip_setup_lassen:
+  variables:
+    RUNNER_TYPE: "shell"
+  extends: [.on_lassen, .job_tags, .pip_env_setup]
+
+pip_build_lassen:
+  variables:
+    RUNNER_TYPE: "shell"
+  needs: [pip_setup_lassen]
+  extends: [.on_lassen, .job_tags, .pip_build]
 
-pip-cleanup-on-build-failure:
-  needs:
-    - job: pip-build
-  when: on_failure
-  <<: *pip-env-cleanup
-
-pip-cleanup-on-test-failure:
-  needs:
-    - job: pip-tests
-  when: on_failure
-  <<: *pip-env-cleanup
-
-pip-cleanup-on-success:
-  needs:
-    - job: pip-tests
-  when: on_success
-  <<: *pip-env-cleanup
+pip_test_lassen:
+  variables:
+    RUNNER_TYPE: "batch"
+    LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -W 30"
+    LSB_JOB_STARTER: "ENVIRONMENT=BATCH /usr/tcetmp/bin/bsub_job_starter %USRCMD"
+    EVIRONMENT: "BATCH"
+  needs: [pip_build_lassen]
+  extends: [.on_lassen, .job_tags, .pip_test]