Add CentOS to CI (#45)

WRF-CMake · Mar 15, 2020 · c7074f3 · c7074f3
1 parent eda7671
commit c7074f3
Show file tree

Hide file tree

Showing 16 changed files with 252 additions and 88 deletions.
diff --git a/.ci/azure-pipelines/matrix.yml b/.ci/azure-pipelines/matrix.yml
@@ -3,13 +3,17 @@
 
 parameters:
   templates:
-    Linux: unix.yml
+    Ubuntu: unix.yml
+    CentOS: unix.yml
     macOS: unix.yml
     Windows: windows.yml
 
-  Linux:
+  Ubuntu:
     CC: gcc-9
     FC: gfortran-9
+  CentOS:
+    CC: gcc
+    FC: gfortran
   macOS:
     CC: gcc-9
     FC: gfortran-9
@@ -31,7 +35,7 @@ parameters:
 
   # Loop/matrix parameters.
   # Note that these may be overridden by the including yml (typically azure-pipelines.yml).
-  OS_NAMES: [Windows, Linux, macOS]
+  OS_NAMES: [Windows, Ubuntu, CentOS, macOS]
   BUILD_SYSTEMS: [Make, CMake]
   BUILD_TYPES: [Debug, Release]
   MODES: [serial, smpar, dmpar, dm_sm]
@@ -60,9 +64,9 @@ jobs:
       - ${{ if eq(parameters.WATS_DIFF, 'true') }}:
         - template: wats_diff.yml
           parameters:
-            # For Windows, since there is no Make-based reference, compare against Linux.
+            # For Windows, since there is no Make-based reference, compare against Ubuntu.
             ${{ if eq(OS_NAME, 'Windows') }}:
-              OS_NAME_LEFT: Linux
+              OS_NAME_LEFT: Ubuntu
             ${{ if ne(OS_NAME, 'Windows') }}:
               OS_NAME_LEFT: ${{ OS_NAME }}
             OS_NAME_RIGHT: ${{ OS_NAME }}
@@ -78,7 +82,7 @@ jobs:
     - ${{ each MODE in parameters.MODES }}:
       - template: wats_diff.yml
         parameters:
-          OS_NAME_LEFT: Linux
+          OS_NAME_LEFT: Ubuntu
           OS_NAME_RIGHT: macOS
           BUILD_SYSTEM_LEFT: Make
           BUILD_SYSTEM_RIGHT: Make

diff --git a/.ci/azure-pipelines/release.yml b/.ci/azure-pipelines/release.yml
@@ -9,7 +9,7 @@ steps:
     displayName: Create distribution package
 
 - ${{ if not(eq(parameters.OS_NAME, 'Windows')) }}:
-  - bash: bash .ci/unix/delocate.sh
+  - bash: $(run) .ci/unix/delocate.sh
     displayName: Delocate
 
   - task: ArchiveFiles@2
@@ -21,9 +21,14 @@ steps:
       archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.SourceBranchName)-$(NESTING)_nesting-$(MODE)-x64-$(OS_NAME)-$(BUILD_TYPE).tar.xz' 
     displayName: Create distribution package
 
-- pwsh: |
-    dir $(Build.ArtifactStagingDirectory) -r | % { if ($_.Name -cne $_.Name.ToLower()) { ren $_.FullName $_.Name.ToLower() } }
-  displayName: Rename to lowercase
+- bash: |
+    set -ex
+    cd "$(Build.ArtifactStagingDirectory)"
+    # CentOS packages will run on most glibc-based Linux distributions
+    find . -name '*CentOS*' -exec bash -c 'mv $0 ${0/CentOS/Linux}' {} \;
+    # Rename all files to lowercase
+    for file in $(ls); do mv ${file} $(echo ${file} | tr '[:upper:]' '[:lower:]'); done
+  displayName: Rename package files
 
 # not strictly needed
 - task: PublishBuildArtifacts@1

diff --git a/.ci/azure-pipelines/unix.yml b/.ci/azure-pipelines/unix.yml
@@ -8,37 +8,31 @@ jobs:
   timeoutInMinutes: 0
 
   pool:
-    ${{ if eq(parameters.OS_NAME, 'Linux') }}:
+    ${{ if or(eq(parameters.OS_NAME, 'Ubuntu'), eq(parameters.OS_NAME, 'CentOS')) }}:
       vmImage: ubuntu-latest
     ${{ if eq(parameters.OS_NAME, 'macOS') }}:
       vmImage: macOS-10.14
 
-  ${{ if eq(parameters.OS_NAME, 'Linux') }}:
-    container:
-      image: ubuntu:16.04
-      options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
-
   variables:
     ${{ insert }}: ${{ parameters }}
-
+    ${{ if eq(parameters.OS_NAME, 'Ubuntu') }}:
+      IMAGE: ubuntu:16.04
+      run: .ci/unix/run-in-docker.sh
+    ${{ if eq(parameters.OS_NAME, 'CentOS') }}:
+      IMAGE: quay.io/pypa/manylinux2010_x86_64:latest
+      run: .ci/unix/run-in-docker.sh
+
   steps:
-  - ${{ if eq(parameters.OS_NAME, 'Linux') }}:
-    # https://github.com/Microsoft/azure-pipelines-agent/issues/2043#issuecomment-489692810
-    - script: |
-        /tmp/docker exec -t -u 0 ci-container \
-          sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
-      displayName: Set up sudo
-
-  - script: .ci/unix/dump-vm-specs.sh
+  - script: $(run) .ci/unix/dump-vm-specs.sh
     displayName: Dump VM specs
 
-  - script: printenv
+  - script: $(run) printenv | sort
     displayName: Dump environment variables
 
-  - script: .ci/unix/setup-dependencies.sh
+  - script: $(run) .ci/unix/setup-dependencies.sh
     displayName: Setup dependencies
 
-  - script: .ci/unix/install-wrf.sh
+  - script: $(run) .ci/unix/install-wrf.sh
     displayName: Install WRF
 
   - ${{ if eq(parameters.WATS_RUN, 'true') }}:

diff --git a/.ci/azure-pipelines/wats_diff.yml b/.ci/azure-pipelines/wats_diff.yml
@@ -6,7 +6,7 @@ jobs:
   displayName: 'WATS ${{ parameters.OS_NAME_LEFT }}/${{ parameters.OS_NAME_RIGHT }} ${{ parameters.BUILD_SYSTEM_LEFT }}/${{ parameters.BUILD_SYSTEM_RIGHT }}: ${{ parameters.BUILD_TYPE }}, ${{ parameters.MODE }}'
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-latest
 
   dependsOn:
   - ${{ parameters.OS_NAME_LEFT }}_${{ parameters.BUILD_SYSTEM_LEFT }}_${{ parameters.BUILD_TYPE }}_${{ parameters.MODE }}
@@ -16,14 +16,10 @@ jobs:
     ${{ insert }}: ${{ parameters }}
 
   steps:
-  - bash: bash .ci/unix/use-conda.sh
+  - bash: .ci/unix/use-conda.sh
     displayName: Enable Conda
 
-  - bash: |
-      set -ex
-      curl -L --retry 3 https://github.com/$WATS_REPO/archive/$WATS_BRANCH.tar.gz | tar xz
-      mv wats-$WATS_BRANCH wats
-      conda env update -n base -f wats/environment.yml
+  - bash: .ci/unix/install-wats.sh
     displayName: Install WATS
 
   - task: DownloadBuildArtifacts@0

diff --git a/.ci/azure-pipelines/wats_plots.yml b/.ci/azure-pipelines/wats_plots.yml
@@ -12,7 +12,7 @@ jobs:
   timeoutInMinutes: 0
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-latest
 
   dependsOn:
   - ${{ each OS_NAME in parameters.OS_NAMES }}:
@@ -27,32 +27,28 @@ jobs:
     ${{ insert }}: ${{ parameters.vars }}
 
   steps:
-  - bash: bash .ci/unix/use-conda.sh
+  - bash: .ci/unix/use-conda.sh
     displayName: Enable Conda
 
-  - bash: |
-      set -ex
-      curl -L --retry 3 https://github.com/$WATS_REPO/archive/$WATS_BRANCH.tar.gz | tar xz
-      mv wats-$WATS_BRANCH wats
-      conda env update -n base -f wats/environment.yml
+  - bash: .ci/unix/install-wats.sh
     displayName: Install WATS
 
-  # Plot: Linux/Make/Debug/serial against all others
+  # Plot: Ubuntu/Make/Debug/serial against all others
 
   - task: DownloadBuildArtifacts@0
-    displayName: Retrieve reference Linux/Make/Debug/serial WATS outputs
+    displayName: Retrieve reference Ubuntu/Make/Debug/serial WATS outputs
     inputs:
       downloadPath: wats_outputs
       downloadType: specific
-      itemPattern: 'wats_Linux_Make_Debug_serial/**'
+      itemPattern: 'wats_Ubuntu_Make_Debug_serial/**'
 
   - ${{ each OS_NAME in parameters.OS_NAMES }}:
     - ${{ each BUILD_TYPE in parameters.BUILD_TYPES }}:
       - ${{ each MODE in parameters.MODES }}:
         - ${{ each BUILD_SYSTEM in parameters.BUILD_SYSTEMS }}:
           # For Windows, since there is no Make-based build, skip that combination.
           # Also, skip the reference combination.
-          - ${{ if not(or(and(eq(OS_NAME, 'Windows'), eq(BUILD_SYSTEM, 'Make')), and(eq(OS_NAME, 'Linux'), eq(BUILD_SYSTEM, 'Make'), eq(MODE, 'serial'), eq(BUILD_TYPE, 'Debug')))) }}:
+          - ${{ if not(or(and(eq(OS_NAME, 'Windows'), eq(BUILD_SYSTEM, 'Make')), and(eq(OS_NAME, 'Ubuntu'), eq(BUILD_SYSTEM, 'Make'), eq(MODE, 'serial'), eq(BUILD_TYPE, 'Debug')))) }}:
 
             - task: DownloadBuildArtifacts@0
               displayName: Retrieve trial ${{ OS_NAME }}/${{ BUILD_SYSTEM }}/${{ BUILD_TYPE }}/${{ MODE }} WATS outputs
@@ -67,22 +63,22 @@ jobs:
                     set -ex
                     O=wats_outputs
                     W="python wats/wats/plots.py compute --stats-dir wats_stats/${{ DOMAIN }}/T${{ TIME }} --filter ${{ DOMAIN }} --time-idx ${{ TIME }} --append"
-                    $W $O/wats_Linux_Make_Debug_serial $O/wats_${{ OS_NAME }}_${{ BUILD_SYSTEM }}_${{ BUILD_TYPE }}_${{ MODE }}
-                  displayName: 'Run WATS: compute plot stats Linux/Make/Debug/serial vs ${{ OS_NAME }}/${{ BUILD_SYSTEM }}/${{ BUILD_TYPE }}/${{ MODE }} for domain ${{ DOMAIN }} - T${{ TIME }}'
+                    $W $O/wats_Ubuntu_Make_Debug_serial $O/wats_${{ OS_NAME }}_${{ BUILD_SYSTEM }}_${{ BUILD_TYPE }}_${{ MODE }}
+                  displayName: 'Run WATS: compute plot stats Ubuntu/Make/Debug/serial vs ${{ OS_NAME }}/${{ BUILD_SYSTEM }}/${{ BUILD_TYPE }}/${{ MODE }} for domain ${{ DOMAIN }} - T${{ TIME }}'
                 
             - script: |
                 rm -rf wats_outputs/wats_${{ OS_NAME }}_${{ BUILD_SYSTEM }}_${{ BUILD_TYPE }}_${{ MODE }}
               displayName: Remove trial ${{ OS_NAME }}/${{ BUILD_SYSTEM }}/${{ BUILD_TYPE }}/${{ MODE }} WATS outputs 
 
   - script: |
-      rm -rf wats_outputs/wats_Linux_Make_Debug_serial
-    displayName: Remove reference Linux/Make/Debug/serial WATS outputs 
+      rm -rf wats_outputs/wats_Ubuntu_Make_Debug_serial
+    displayName: Remove reference Ubuntu/Make/Debug/serial WATS outputs 
 
   - ${{ each DOMAIN in parameters.DOMAINS }}:   
     - ${{ each TIME in parameters.TIMES }}:
       - script: |
           python wats/wats/plots.py plot --stats-dir wats_stats/${{ DOMAIN }}/T${{ TIME }} --plots-dir wats_plots/single_ref/${{ DOMAIN }}/T${{ TIME }}
-        displayName: 'Run WATS: create Linux/Make/Debug/serial reference plots for domain ${{ DOMAIN }} - T${{ TIME }}'
+        displayName: 'Run WATS: create Ubuntu/Make/Debug/serial reference plots for domain ${{ DOMAIN }} - T${{ TIME }}'
 
   - script: |
       rm -rf wats_stats/

diff --git a/.ci/azure-pipelines/wats_run.yml b/.ci/azure-pipelines/wats_run.yml
@@ -2,14 +2,10 @@
 # Copyright 2019 M. Riechert and D. Meyer. Licensed under the MIT License.
 
 steps:
-- bash: bash .ci/unix/use-conda.sh
+- bash: $(run) .ci/unix/use-conda.sh
   displayName: Enable Conda
 
-- bash: |
-    set -ex
-    curl -L --retry 3 https://github.com/$WATS_REPO/archive/$WATS_BRANCH.tar.gz | tar xz
-    mv wats-$WATS_BRANCH wats
-    conda env update -n base -f wats/environment.yml
+- bash: $(run) .ci/unix/install-wats.sh
   displayName: Install WATS
 
 - task: UniversalPackages@0
@@ -26,21 +22,7 @@ steps:
     find . -type f
   displayName: List WPS output files
 
-- bash: |
-    set -ex
-    if [[ $MODE == dm* ]]; then mpi_flag="--mpi"; fi
-    if [[ $OS_NAME == macOS ]]; then
-      # Work around Open MPI issue
-      # https://github.com/open-mpi/ompi/issues/6518
-      # https://github.com/open-mpi/ompi/issues/5798
-      # https://www.mail-archive.com/devel@lists.open-mpi.org/msg20760.html
-      export OMPI_MCA_btl=self,tcp
-      # Disable new shared memory component of Open MPI to work around issue
-      # https://github.com/open-mpi/ompi/issues/7516
-      export PMIX_MCA_gds=hash
-    fi
-    if [[ $BUILD_SYSTEM == "CMake" ]]; then dir_suffix="build/install"; fi
-    python wats/wats/main.py run --mode $WATS_MODE --wrf-dir ./$dir_suffix --wps-dir ../WPS/$dir_suffix --wps-case-output-dir wats_wps_outputs/00 --work-dir wats_work $mpi_flag
+- bash: $(run) .ci/unix/run-wats.sh
   displayName: Run WATS
 
 - task: PublishBuildArtifacts@1

diff --git a/.ci/azure-pipelines/windows.yml b/.ci/azure-pipelines/windows.yml
@@ -8,7 +8,7 @@ jobs:
   timeoutInMinutes: 0
 
   pool:
-      vmImage: windows-2019
+      vmImage: windows-latest
 
   variables:
     ${{ insert }}: ${{ parameters }}

diff --git a/.ci/unix/delocate.sh b/.ci/unix/delocate.sh
@@ -10,12 +10,37 @@ cd $SCRIPTDIR/../..
 
 if [ "$(uname)" == "Darwin" ]; then
 
-    pip3 install delocate
+    pip install delocate
     delocate-listdeps --all --depending build/install/main
     delocate-path build/install/main
     delocate-listdeps --all --depending build/install/main
 
+elif [ "$(lsb_release -i -s)" == "CentOS" ]; then
+
+    # Assumes we're in the manylinux Docker image.
+    pys=(/opt/python/cp*)
+    # Use the newest Python available in the image (last item).
+    py=${pys[@]:(-1)}/bin/python
+
+    root_dir=$(pwd)
+    tmp_dir=$(mktemp -d)
+    cd $tmp_dir
+
+    echo "from setuptools import setup; setup(name='app', packages=['main'], package_data={'main': ['*.exe']})" > setup.py
+    ln -s $root_dir/build/install/main main
+    $py setup.py bdist_wheel
+
+    # CentOS uses /usr/lib64 but some manually installed dependencies end up in /usr/lib
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib
+
+    auditwheel repair dist/*.whl --no-update-tags
+    cd wheelhouse
+    unzip *.whl
+
+    # /bin/cp as cp is aliased to 'cp -i' and would ask before overwriting
+    /bin/cp -r main/. $root_dir/build/install/main
+
 else
-    echo "Unknown OS: $(uname)"
+    echo "Unsupported OS: $(uname)"
     exit 1
 fi
diff --git a/.ci/unix/dump-vm-specs.sh b/.ci/unix/dump-vm-specs.sh
@@ -15,7 +15,11 @@ if [ "$(uname)" == "Darwin" ]; then
     sudo scutil --get LocalHostName || true
 elif [ "$(uname)" == "Linux" ]; then
     if [ "$(which lsb_release)" == "" ]; then
-        sudo apt install -y lsb-release
+        if [ -f /etc/redhat-release ]; then
+            sudo yum install -y redhat-lsb-core
+        else
+            sudo apt install -y lsb-release
+        fi
     fi
     lsb_release -a
     free -m

diff --git a/.ci/unix/install-wats.sh b/.ci/unix/install-wats.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# WRF-CMake (https://github.com/WRF-CMake/wrf).
+# Copyright 2018 M. Riechert and D. Meyer. Licensed under the MIT License.
+
+set -ex
+
+SCRIPTDIR=$(dirname "$0")
+cd $SCRIPTDIR/../..
+
+curl -L --retry 3 https://github.com/$WATS_REPO/archive/$WATS_BRANCH.tar.gz | tar xz
+mv wats-$WATS_BRANCH wats
+conda env update -n base -f wats/environment.yml
diff --git a/.ci/unix/install-wrf.sh b/.ci/unix/install-wrf.sh
@@ -78,7 +78,7 @@ elif [ $BUILD_SYSTEM == 'Make' ]; then
             *) echo "Invalid: $MODE"; exit 1 ;;
         esac
 
-        if [ "$(lsb_release -c -s)" == "trusty" ]; then
+        if [ "$(lsb_release -c -s)" == "trusty" -o "$(lsb_release -i -s)" == "CentOS" ]; then
             export HDF5=/usr
             export NETCDF=/usr
         else

diff --git a/.ci/unix/run-in-docker.sh b/.ci/unix/run-in-docker.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# WRF-CMake (https://github.com/WRF-CMake/wrf).
+# Copyright 2018 M. Riechert and D. Meyer. Licensed under the MIT License.
+
+set -e
+
+SCRIPTDIR=$(dirname "$0")
+ROOTDIR=$(realpath $SCRIPTDIR/../..)
+cd $ROOTDIR
+
+container=wrf-ci
+
+# Create container if it doesn't exist
+if [ ! "$(docker ps -q -f name=$container)" ]; then
+    echo "Creating Docker container $container"
+    set -x
+    docker run --name $container -t -d -v $ROOTDIR:$ROOTDIR -w $ROOTDIR -e DOCKER=1 $IMAGE
+    set +x
+
+    echo "Installing sudo inside container"
+    if [[ $OS_NAME == CentOS ]]; then
+      docker exec $container sh -c "yum install -y sudo"
+    else
+      docker exec $container sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
+    fi
+fi
+
+echo "Running inside container: $@"
+host_envs=$(env | cut -f1 -d= | sed 's/^/-e /' | grep -v -e PATH -e HOME)
+# Use login shell so that ~/.bash_profile is read.
+# use-conda.sh appends to that file to modify the PATH.
+docker exec $host_envs $container bash --login -c "$@"