Merge pull request #192 from ExaWorks/refactor/llnl-ci-pip

Splitting up CI matrix for pip llnl
ExaWorks · Aug 4, 2023 · 8a67441 · 8a67441
2 parents 0194cbf + 0acf5fb
commit 8a67441
Show file tree

Hide file tree

Showing 5 changed files with 184 additions and 106 deletions.
diff --git a/.gitlab/llnl-ci-conda.yml b/.gitlab/llnl-ci-conda.yml
@@ -3,11 +3,15 @@ variables:
   CONDA_REQUIREMENTS: .gitlab/conda-requirements.yml
   RADICAL_PILOT_DBURL: ${MONGODB_CONNECTION_STRING}?tlsAllowInvalidCertificates=true
 
+
 stages:
   - setup
   - build
   - test
+  - cleanup
+
 
+### MACHINES
 .on_quartz:
   variables:
     HOST: "quartz"
@@ -20,16 +24,18 @@ stages:
     CONDA_ENV_NAME: "ruby-env"
     CONDA_ARCH: "x86_64"
 
-.job_tags:
-  tags:
-    - $HOST
-    - $RUNNER_TYPE
 
+### SCRIPTS
 .final_steps:
   script: &finalize
     - chgrp -fR exaworks ${CONDA_WORK_DIR} || true
     - chmod -fR 02770 ${CONDA_WORK_DIR}    || true
 
+.job_tags:
+  tags:
+    - $HOST
+    - $RUNNER_TYPE
+
 .conda_setup:
   stage: setup
   script:
@@ -78,56 +84,67 @@ stages:
     - python3 ci/tests/test.py -e
     - *finalize
 
+### JOBS
 
-
-
+# QUARTZ
 conda_setup_quartz:
   variables:
     RUNNER_TYPE: "shell"
-  extends: [.on_quartz, .job_tags, .conda_setup]
+  extends: [ .on_quartz, .job_tags, .conda_setup ]
 
 conda_env_setup_quartz:
   variables:
     RUNNER_TYPE: "shell"
-  needs: [ "conda_setup_quartz" ]
-  extends: [.on_quartz, .job_tags, .conda_env_setup]
+  needs: [ conda_setup_quartz ]
+  extends: [ .on_quartz, .job_tags, .conda_env_setup ]
 
 conda_build_quartz:
   variables:
     RUNNER_TYPE: "shell"
-  needs: [ "conda_env_setup_quartz" ]
-  extends: [.on_quartz, .job_tags, .conda_build]
+  needs: [ conda_env_setup_quartz ]
+  extends: [ .on_quartz, .job_tags, .conda_build ]
 
 conda_tests_quartz:
   variables:
     RUNNER_TYPE: "batch"
     LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 60"
-  needs: [ "conda_build_quartz" ]
-  extends: [.on_quartz, .job_tags, .conda_tests]
-
-
+  needs: [ conda_build_quartz ]
+  extends: [ .on_quartz, .job_tags, .conda_tests ]
 
 
+# RUBY
 conda_setup_ruby:
   variables:
     RUNNER_TYPE: "shell"
-  extends: [.on_ruby, .job_tags, .conda_setup]
+  extends: [ .on_ruby, .job_tags, .conda_setup ]
 
 conda_env_setup_ruby:
   variables:
     RUNNER_TYPE: "shell"
-  needs: [ "conda_setup_ruby" ]
-  extends: [.on_ruby, .job_tags, .conda_env_setup]
+  needs: [ conda_setup_ruby ]
+  extends: [ .on_ruby, .job_tags, .conda_env_setup ]
 
 conda_build_ruby:
   variables:
     RUNNER_TYPE: "shell"
-  needs: [ "conda_env_setup_ruby" ]
-  extends: [.on_ruby, .job_tags, .conda_build]
+  needs: [ conda_env_setup_ruby ]
+  extends: [ .on_ruby, .job_tags, .conda_build ]
 
 conda_tests_ruby:
   variables:
     RUNNER_TYPE: "batch"
     LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 60"
-  needs: [ "conda_build_ruby" ]
-  extends: [.on_ruby, .job_tags, .conda_tests]
+  needs: [ conda_build_ruby ]
+  extends: [ .on_ruby, .job_tags, .conda_tests ]
+
+
+# ALWAYS
+conda_cleanup_all:
+  stage: cleanup
+  when: always
+  variables:
+    RUNNER_TYPE: "shell"
+  extends: [ .on_quartz, .job_tags ]
+  script:
+   - *finalize
+
diff --git a/.gitlab/llnl-ci-pip.yml b/.gitlab/llnl-ci-pip.yml
@@ -1,91 +1,64 @@
 variables:
   PIP_WORK_DIR: ${SDK_WORK_DIR}/pip
-  PIP_REQUIREMENTS: .gitlab/venv-requirements.txt
+  PIP_REQUIREMENTS: .gitlab/pip-requirements.txt
   RADICAL_PILOT_DBURL: ${MONGODB_CONNECTION_STRING}?tlsAllowInvalidCertificates=true
 
+
 stages:
   - setup
   - build
   - test
   - cleanup
 
+
+### MACHINES
+.on_quartz:
+  variables:
+    HOST: "quartz"
+    VENV_ENV_NAME: "quartz-env"
+
+.on_ruby:
+  variables:
+    HOST: "ruby"
+    VENV_ENV_NAME: "ruby-env"
+
+.on_lassen:
+  variables:
+    HOST: "lassen"
+    VENV_ENV_NAME: "lassen-env"
+
+
+### SCRIPTS
 .final_steps:
   script: &finalize
     - chgrp -fR exaworks ${PIP_WORK_DIR} || true
     - chmod -fR 02770 ${PIP_WORK_DIR}    || true
 
-pip-env-setup:
-  stage: setup
-  parallel:
-    matrix:
-      - HOST : [ "quartz" ]
-        VENV_ENV_NAME: [ "quartz-env" ]
-        RUNNER_TYPE: "shell"
-      - HOST : [ "ruby" ]
-        VENV_ENV_NAME: [ "ruby-env" ]
-        RUNNER_TYPE: "shell"
-      # - HOST : [ "lassen" ]
-      #   ENVIRONMENT: [ "BATCH" ]
-      #   VENV_ENV_NAME: [ "lassen-env" ]
-      #   RUNNER_TYPE: "batch"
+.job_tags:
   tags:
     - $HOST
     - $RUNNER_TYPE
+
+.pip_env_setup:
+  stage: setup
   script:
     - mkdir -p ${PIP_WORK_DIR}
     - test -d ${PIP_WORK_DIR}/${VENV_ENV_NAME} && exit 0
     - python3 -m venv ${PIP_WORK_DIR}/${VENV_ENV_NAME}
     - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate
     - pip install -U pip setuptools wheel
     - pip cache purge
+    - *finalize
 
-pip-build:
+.pip_build:
   stage: build
-  needs:
-    - job: pip-env-setup
-  parallel:
-    matrix:
-      - HOST : [ "quartz" ]
-        VENV_ENV_NAME: [ "quartz-env" ]
-        RUNNER_TYPE: "shell"
-      - HOST : [ "ruby" ]
-        VENV_ENV_NAME: [ "ruby-env" ]
-        RUNNER_TYPE: "shell"
-      # - HOST : [ "lassen" ]
-      #   ENVIRONMENT: [ "BATCH" ]
-      #   VENV_ENV_NAME: [ "lassen-env" ]
-      #   RUNNER_TYPE: "batch"
-  tags:
-    - $HOST
-    - $RUNNER_TYPE
   script:
     - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate
     - pip install --no-cache-dir -r ${PIP_REQUIREMENTS}
     - *finalize
 
-pip-tests:
+.pip_test:
   stage: test
-  needs: 
-    - job: pip-build
-  parallel:
-    matrix:
-      - HOST : [ "quartz" ]
-        VENV_ENV_NAME: [ "quartz-env" ]
-        RUNNER_TYPE: "batch"
-        LLNL_SLURM_SCHEDULER_PARAMETERS: [ "--nodes=1 -t 30" ]
-      - HOST : [ "ruby" ]
-        VENV_ENV_NAME: [ "ruby-env" ]
-        RUNNER_TYPE: "batch"
-        LLNL_SLURM_SCHEDULER_PARAMETERS: [ "--nodes=1 -t 30" ]
-      # - HOST : [ "lassen" ]
-      #   ENVIRONMENT: [ "BATCH" ]
-      #   VENV_ENV_NAME: [ "lassen-env" ]
-      #   RUNNER_TYPE: "batch"
-        # LLNL_LSF_SCHEDULER_PARAMETERS: [ "-nnodes 1 -W 30" ]
-        # LSB_JOB_STARTER: [ "ENVIRONMENT=BATCH /usr/tcetmp/bin/bsub_job_starter %USRCMD" ]
-  tags:
-    - $HOST
-    - $RUNNER_TYPE
   script:
     - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate
     - export run_id=${CI_PIPELINE_ID}
@@ -102,30 +75,79 @@ pip-tests:
     - python3 ci/tests/test.py -e
     - deactivate
 
-.pip-env-cleanup: &pip-env-cleanup
+### JOBS
+
+# QUARTZ
+pip_setup_quartz:
+  variables:
+    RUNNER_TYPE: "shell"
+  extends: [ .on_quartz, .job_tags, .pip_env_setup ]
+
+pip_build_quartz:
+  variables:
+    RUNNER_TYPE: "shell"
+  needs: [ pip_setup_quartz ]
+  extends: [ .on_quartz, .job_tags, .pip_build ]
+
+pip_test_quartz:
+  variables:
+    RUNNER_TYPE: "batch"
+    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 30"
+  needs: [ pip_build_quartz ]
+  extends: [ .on_quartz, .job_tags, .pip_test ]
+
+
+# RUBY
+pip_setup_ruby:
+  variables:
+    RUNNER_TYPE: "shell"
+  extends: [ .on_ruby, .job_tags, .pip_env_setup ]
+
+pip_build_ruby:
+  variables:
+    RUNNER_TYPE: "shell"
+  needs: [ pip_setup_ruby ]
+  extends: [ .on_ruby, .job_tags, .pip_build ]
+
+pip_test_ruby:
+  variables:
+    RUNNER_TYPE: "batch"
+    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 30"
+  needs: [ pip_build_ruby ]
+  extends: [ .on_ruby, .job_tags, .pip_test ]
+
+
+# LASSEN
+pip_setup_lassen:
+  variables:
+    RUNNER_TYPE: "shell"
+  extends: [ .on_lassen, .job_tags, .pip_env_setup ]
+
+pip_build_lassen:
+  variables:
+    RUNNER_TYPE: "shell"
+  needs: [ pip_setup_lassen ]
+  extends: [ .on_lassen, .job_tags, .pip_build ]
+  before_script:
+    - .gitlab/llnl-lassen-openssl-parsl.sh ${PIP_WORK_DIR}/${VENV_ENV_NAME}
+
+pip_test_lassen:
+  variables:
+    RUNNER_TYPE: "batch"
+    LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -W 30"
+    LSB_JOB_STARTER: "ENVIRONMENT=BATCH /usr/tcetmp/bin/bsub_job_starter %USRCMD"
+    EVIRONMENT: "BATCH"
+  needs: [ pip_build_lassen ]
+  extends: [ .on_lassen, .job_tags, .pip_test ]
+
+
+# ALWAYS
+pip_cleanup_all:
   stage: cleanup
-  tags:
-    - ruby
-    - shell
+  when: always
+  variables:
+    RUNNER_TYPE: "shell"
+  extends: [ .on_quartz, .job_tags ]
   script:
-    - source ${PIP_WORK_DIR}/ruby-env/bin/activate
-    - pip cache purge
-    - *finalize
+   - *finalize
 
-pip-cleanup-on-build-failure:
-  needs:
-    - job: pip-build
-  when: on_failure
-  <<: *pip-env-cleanup
-
-pip-cleanup-on-test-failure:
-  needs:
-    - job: pip-tests
-  when: on_failure
-  <<: *pip-env-cleanup
-
-pip-cleanup-on-success:
-  needs:
-    - job: pip-tests
-  when: on_success
-  <<: *pip-env-cleanup
diff --git a/.gitlab/llnl-lassen-openssl-parsl.sh b/.gitlab/llnl-lassen-openssl-parsl.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -e
+
+VENV_NAME="$1"
+
+OPENSSL_VERSION="1.1.1v"
+OPENSSL_PREFIX="${VENV_NAME}/../lassen-openssl"
+
+if [[ -d "$OPENSSL_PREFIX" ]]; then
+    echo 'OpenSSL for Lassen is already installed'
+    exit
+fi
+
+source "${VENV_NAME}/bin/activate"
+pip install -U pip setuptools wheel
+
+curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
+tar xvf openssl-${OPENSSL_VERSION}.tar.gz
+cd openssl-${OPENSSL_VERSION}
+./config no-shared no-ssl2 no-ssl3 -fPIC --prefix="${OPENSSL_PREFIX}"
+make && make install
+cd ..
+
+CFLAGS="-I${OPENSSL_PREFIX}/include" LDFLAGS="-L${OPENSSL_PREFIX}/lib" \
+pip install --no-cache-dir --no-binary :all: cryptography==3.3.2
+
+pip cache purge
+rm -rf openssl-${OPENSSL_VERSION}*
+
diff --git a/.gitlab/pip-requirements.txt b/.gitlab/pip-requirements.txt
@@ -0,0 +1,14 @@
+# parsl dependencies to avoid rust installation
+bcrypt<4.0.0
+cryptography==3.3.2
+globus_sdk<3.1
+
+# core components
+parsl
+radical.pilot
+maestrowf
+
+# tests
+pytest
+pytest-tap
+