Merge pull request #2968 from bilke/gitlab-ci

Add GitLab CI config

.gitlab-ci.yml

@@ -1,18 +1,49 @@
+stages:
+  - pre-checks
+  - build
+  - checks
+  - gui
+  - package
+
+
 variables:
-  GIT_STRATEGY: fetch
-  OPENMPI_VERSIONS: 'off 2.1.6 3.1.4 4.0.1'
+  # Docker image names
+  CONTAINER_GCC_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/$CI_PROJECT_NAMESPACE/gcc:$CI_COMMIT_REF_SLUG
+  CONTAINER_GCC_GUI_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/$CI_PROJECT_NAMESPACE/gcc-gui:$CI_COMMIT_REF_SLUG
+  CONTAINER_CLANG_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/$CI_PROJECT_NAMESPACE/clang:$CI_COMMIT_REF_SLUG
+  XUNIT_TO_JUNIT_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/xunit-to-junit
+  PRECOMMIT_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/pre-commit
+  WEB_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/web
+  # Build config
+  BUILD_TYPE: Release
+  BUILD_PROCESSES: "" # Empty string: builds all processes
 
-build-container:
-  tags:
-    - docker-shell
-  stage: build
-  script:
-    - git submodule update --init ThirdParty/container-maker
-    - virtualenv .venv
-    - source .venv/bin/activate
-    - pip install -r ThirdParty/container-maker/requirements.txt
-    - export PYTHONPATH="${PYTHONPATH}:${PWD}/ThirdParty/container-maker"
-    - python ThirdParty/container-maker/ogscm/cli.py -B -C -R --ogs . --pm system --cvode --ompi $OPENMPI_VERSIONS
-  artifacts:
-    paths:
-      - _out/images/*.sif
+include:
+  - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'
+  # extends
+  - local: '/scripts/ci/extends/defaults.yml'
+  - local: '/scripts/ci/extends/rules-master-manual.yml'
+  - local: '/scripts/ci/extends/template-build-linux.yml'
+  - local: '/scripts/ci/extends/template-build-win.yml'
+  - local: '/scripts/ci/extends/container-maker-setup.yml'
+  - local: '/scripts/ci/extends/vs2019-environment.yml'
+  - local: '/scripts/ci/extends/test-artifacts.yml'
+  # jobs
+  - local: '/scripts/ci/jobs/pre-commit.yml'
+  - local: '/scripts/ci/jobs/build_image.yml'
+  - local: '/scripts/ci/jobs/build-linux.yml'
+  - local: '/scripts/ci/jobs/build-linux-petsc.yml'
+  - local: '/scripts/ci/jobs/build-linux-frontend.yml'
+  - local: '/scripts/ci/jobs/build-docs.yml'
+  - local: '/scripts/ci/jobs/build-win.yml'
+  - local: '/scripts/ci/jobs/build-mac.yml'
+  - local: '/scripts/ci/jobs/checks.yml'
+  - local: '/scripts/ci/jobs/container.yml'
+  - local: '/scripts/ci/jobs/build-gui-linux.yml'
+  - local: '/scripts/ci/jobs/build-gui-win.yml'
+  - local: '/scripts/ci/jobs/build-gui-mac.yml'
+  - local: '/scripts/ci/jobs/check-header.yml'
+  - local: '/scripts/ci/jobs/tests-large.yml'
+  - local: '/scripts/ci/jobs/web.yml'
+  - local: '/scripts/ci/jobs/clang-sanitizer.yml'
+  - local: '/scripts/ci/jobs/clang-tidy.yml'

Applications/Utils/Tests.cmake

@@ -340,8 +340,7 @@ if(OGS_USE_NETCDF)
         EXECUTABLE_ARGS -i sresa1b_ncar_ccsm3-example.nc -o ${Data_BINARY_DIR}/FileConverter/sresa1b_ncar_ccsm3-example.asc -v pr -t 0 --dim1 2 --dim2 1 --timestep-first 0 --timestep-last 0 --images
         REQUIREMENTS NOT OGS_USE_MPI
         TESTER diff
-        DIFF_DATA
-        sresa1b_ncar_ccsm3-example.asc
+        DIFF_DATA sresa1b_ncar_ccsm3-example0.asc
     )
 endif()
 

CMakeLists.txt

@@ -26,7 +26,11 @@ option(OGS_ENABLE_AVX2 "Enable the use of AVX2 instructions" OFF)
 option(OGS_USE_CONAN "Should Conan package manager be used?" ON)
 set(OGS_CONAN_BUILD "missing" CACHE STRING "Possible values: all, missing, \
     never or list of libs to build")
-option(OGS_DISABLE_CCACHE "Disables ccache compiler cache." OFF)
+if(WIN32)
+    option(OGS_DISABLE_CLCACHE "Disables clcache compiler cache." OFF)
+else()
+    option(OGS_DISABLE_CCACHE "Disables ccache compiler cache." OFF)
+endif()
 if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.16)
     option(OGS_USE_UNITY_BUILDS "Enables Unity builds for faster compilation." ON)
 endif()
@@ -69,6 +73,7 @@ include(ConanSetup)
 include(CompilerSetup)
 include(Find)
 include(CCacheSetup)
+include(CLCacheSetup)
 include(DocumentationSetup)
 include(test/Test)
 if(OGS_COVERAGE AND NOT IS_SUBPROJECT)

Jenkinsfile

@@ -412,8 +412,6 @@ pipeline {
             MSVC_NUMBER = '16'
             MSVC_VERSION = '2019'
             OMP_NUM_THREADS = '1'
-            CC = 'clcache'
-            CXX = 'clcache'
           }
           steps {
             script {

MaterialLib/SolidModels/MFront/DruckerPrager.mfront

@@ -90,4 +90,3 @@ lam.setGlossaryName("EquivalentPlasticStrain");
         dflam_ddeel = theta * (nF | D) / D(0, 0);
     }
 }
-

Tests/CMakeLists.txt

@@ -92,8 +92,10 @@ endif()
 # cmake-format: off
 # Add make-target tests which runs the testrunner
 if(IS_CI AND NOT OGS_COVERAGE)
+    set(OGS_CI_TESTRUNNER_REPEAT 3 CACHE STRING
+        "The number of testrunner repeats for the tests target")
     set(TESTRUNNER_ADDITIONAL_ARGUMENTS ${TESTRUNNER_ADDITIONAL_ARGUMENTS}
-        --gtest_shuffle --gtest_repeat=3)
+        --gtest_shuffle --gtest_repeat=${OGS_CI_TESTRUNNER_REPEAT})
 endif()
 set(TESTRUNNER_ADDITIONAL_ARGUMENTS ${TESTRUNNER_ADDITIONAL_ARGUMENTS}
     -l warn

scripts/ci/extends/container-maker-setup.yml

@@ -0,0 +1,7 @@
+.container-maker-setup:
+  before_script:
+    - git submodule update --init ThirdParty/container-maker
+    - virtualenv .venv
+    - source .venv/bin/activate
+    - pip install -r ThirdParty/container-maker/requirements.txt
+    - export PYTHONPATH="${PYTHONPATH}:${PWD}/ThirdParty/container-maker"

scripts/ci/extends/defaults.yml

@@ -0,0 +1,3 @@
+.defaults:
+  # Disable artifacts passing between stages
+  dependencies: []

scripts/ci/extends/rules-master-manual.yml

@@ -0,0 +1,5 @@
+.rules-master-manual:
+  rules:
+    - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
+    - when: manual
+      allow_failure: true

scripts/ci/extends/template-build-dockerfile.yml

@@ -0,0 +1,24 @@
+# not working until https://gitlab.com/gitlab-org/gitlab-runner/issues/1809
+# is implemented
+# Usage:
+#
+# build pre-commit image:
+#   extends: .template-build-dockerfile
+#   variables:
+#     DOCKERFILE: scripts/docker/Dockerfile.pre-commit
+#     IMAGE_REF: $PRECOMMIT_IMAGE:latest
+.template-build-dockerfile:
+  stage: .pre
+  only:
+    changes:
+      - $DOCKERFILE
+      - scripts/ci/extends/template-build-dockerfile.yml
+  tags:
+    - docker-shell
+  variables:
+    GIT_LFS_SKIP_SMUDGE: "1"
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker pull $IMAGE_REF || true
+    - docker build --cache-from $IMAGE_REF -t $IMAGE_REF -f $DOCKERFILE scripts/docker
+    - docker push $IMAGE_REF

scripts/ci/extends/template-build-linux.yml

@@ -0,0 +1,32 @@
+.template-build-linux:
+  stage: build
+  tags:
+    - docker
+  extends:
+    - .defaults
+    - .test-artifacts
+
+  variables:
+    CCACHE_DIR: "$CI_PROJECT_DIR/.ccache"
+
+  before_script:
+    - git lfs install
+    - git lfs fetch
+    - git lfs checkout
+    - mkdir -p $BUILD_DIR
+    - cd $BUILD_DIR
+
+  script:
+    - >
+      cmake .. -G Ninja
+      $CMAKE_ARGS
+      -DCMAKE_BUILD_TYPE=$BUILD_TYPE
+      -DOGS_BUILD_PROCESSES=$BUILD_PROCESSES
+    - $BUILD_CMD_PREFIX cmake --build . > >(tee make.output)
+    - $BUILD_CMD_PREFIX cmake --build . --target tests
+    - $BUILD_CMD_PREFIX cmake --build . --target ctest
+    - cp Testing/**/Test.xml Tests/ctest.xml
+
+  cache:
+    paths:
+      - $CCACHE_DIR

scripts/ci/extends/template-build-win.yml

@@ -0,0 +1,23 @@
+.template-build-win:
+  stage: build
+  tags:
+    - windows
+  extends:
+    - .defaults
+    - .vs2019-environment
+    - .test-artifacts
+
+  variables:
+
+  script:
+    - mkdir -p $BUILD_DIR
+    - cd $BUILD_DIR
+    - >
+      cmake .. -G Ninja
+      $env:CMAKE_ARGS.Split(" ")
+      -DCMAKE_BUILD_TYPE="$env:BUILD_TYPE"
+      -DOGS_BUILD_PROCESSES="$env:BUILD_PROCESSES"
+    - cmake --build . | Tee-Object -FilePath make.output
+    - cmake --build . --target tests
+    - cmake --build . --target ctest
+    - cp Testing/**/Test.xml Tests/ctest.xml

scripts/ci/extends/test-artifacts.yml

@@ -0,0 +1,10 @@
+.test-artifacts:
+  artifacts:
+    paths:
+      - $BUILD_DIR/Tests/ctest.xml
+      - $BUILD_DIR/Tests/testrunner.xml
+      - $BUILD_DIR/make.output
+    expire_in: 1 week
+    reports:
+      junit:
+        - $BUILD_DIR/Tests/testrunner.xml

scripts/ci/extends/vs2019-environment.yml

@@ -0,0 +1,13 @@
+.vs2019-environment:
+  before_script:
+    # Load VS environment
+    - |
+      pushd $env:VS160COMNTOOLS
+      cmd /c "VsDevCmd.bat -arch=amd64 -host_arch=amd64&set" |
+      foreach {
+        if ($_ -match "=") {
+          $v = $_.split("=", 2); set-item -force -path "ENV:\$($v[0])"  -value "$($v[1])"
+        }
+      }
+      popd
+      Write-Host "`nVisual Studio Command Prompt variables set." -ForegroundColor Yellow

scripts/ci/jobs/build-docs.yml

@@ -0,0 +1,17 @@
+build docs:
+  stage: build
+  image:
+    name: $CONTAINER_GCC_IMAGE
+
+  before_script:
+    - mkdir -p build
+    - cd build
+
+  script:
+    - cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DOGS_USE_CONAN=OFF -DOGS_BUILD_PROCESSES=GroundwaterFlow
+    - cmake --build . --target doc > >(tee make-docs.output)
+
+  artifacts:
+    expose_as: 'Doxygen'
+    paths: ['build/docs/']
+    expire_in: 1 week

scripts/ci/jobs/build-gui-linux.yml

@@ -0,0 +1,27 @@
+build gui linux:
+  extends: .template-build-linux
+  stage: gui
+  tags:
+    - envinf1
+    - docker
+  needs:
+    - job: build linux
+      artifacts: false
+  rules:
+    - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
+    - changes:
+      - Applications/DataExplorer/**/*
+    - when: manual
+      allow_failure: true
+
+  variables:
+    BUILD_DIR: "build-gui-linux"
+    CMAKE_ARGS: >-
+      -DOGS_USE_CONAN=ON
+      -DOGS_USE_PCH=OFF
+      -DOGS_BUILD_GUI=ON
+      -DOGS_BUILD_UTILS=ON
+      -DOGS_USE_NETCDF=ON
+
+  image:
+    name: $CONTAINER_GCC_GUI_IMAGE

scripts/ci/jobs/build-gui-mac.yml

@@ -0,0 +1,23 @@
+build gui mac:
+  extends: .template-build-linux
+  stage: gui
+  needs:
+    - job: build mac
+      artifacts: false
+  tags:
+    - mac
+  rules:
+    - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
+    - changes:
+      - Applications/DataExplorer/**/*
+    - when: manual
+      allow_failure: true
+
+  variables:
+    BUILD_DIR: "build-gui-mac"
+    CMAKE_ARGS: >-
+      -DOGS_USE_CONAN=OFF
+      -DOGS_USE_PCH=OFF
+      -DOGS_BUILD_GUI=ON
+      -DOGS_BUILD_UTILS=ON
+      -DOGS_USE_NETCDF=ON

scripts/ci/jobs/build-gui-win.yml

@@ -0,0 +1,22 @@
+build gui win:
+  extends: .template-build-win
+  stage: gui
+  needs:
+    - job: build win
+      artifacts: false
+  rules:
+    - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
+    - changes:
+      - Applications/DataExplorer/**/*
+    - when: manual
+      allow_failure: true
+
+  variables:
+    BUILD_DIR: build-gui-win
+    CMAKE_ARGS: >-
+      -DOGS_USE_PYTHON=ON
+      -DOGS_BUILD_GUI=ON
+      -DOGS_BUILD_UTILS=ON
+      -DOGS_BUILD_SWMM=ON
+      -DOGS_USE_NETCDF=ON
+      -DOGS_CI_TESTRUNNER_REPEAT=1

scripts/ci/jobs/build-linux-frontend.yml

@@ -0,0 +1,22 @@
+build linux frontend:
+  extends: .template-build-linux
+  tags:
+    - frontend1
+  rules:
+    - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
+    - when: manual
+      allow_failure: true
+
+  variables:
+    BUILD_DIR: "build"
+    CCACHE_DIR: "/data/ogs/.ccache"
+    CMAKE_ARGS: "-DOGS_USE_CONAN=OFF -DOGS_DISABLE_CCACHE=OFF"
+
+  before_script:
+    - source scripts/env/eve/cli.sh
+    - mkdir -p $BUILD_DIR
+    - cd $BUILD_DIR
+
+  cache:
+    paths:
+      # do not cache CCACHE_DIR

scripts/ci/jobs/build-linux-petsc.yml

@@ -0,0 +1,9 @@
+build linux petsc:
+  extends: .template-build-linux
+  tags:
+    - envinf1
+    - docker-shell
+
+  variables:
+    BUILD_DIR: "build-petsc"
+    CMAKE_ARGS: "-DOGS_USE_CONAN=OFF -DOGS_USE_PETSC=ON -DBUILD_SHARED_LIBS=ON"

scripts/ci/jobs/build-linux.yml

@@ -0,0 +1,10 @@
+build linux:
+  extends: .template-build-linux
+  tags:
+    - envinf1
+    - docker
+  variables:
+    BUILD_DIR: "build"
+    CMAKE_ARGS: "-DOGS_USE_CONAN=OFF"
+  image:
+    name: $CONTAINER_GCC_IMAGE

scripts/ci/jobs/build-mac.yml

@@ -0,0 +1,8 @@
+build mac:
+  extends: .template-build-linux
+
+  tags:
+    - mac
+
+  variables:
+    BUILD_DIR: "build-mac"