CI: rework pipeline: short/extended based on labels (#11324)

* CI: reduce VM resources requests to improve scheduling

* CI: Reduce default jobs; add labels(ci-full/extended) to run more test

* CI: use jobs dependencies instead of stages

* precommit one-job

* CI: Use Kubevirt VM to run Molecule and Vagrant jobs

.gitlab-ci.yml

@@ -1,12 +1,9 @@
 ---
 stages:
   - build
-  - unit-tests
-  - moderator
+  - test
   - deploy-part1
-  - deploy-part2
-  - deploy-part3
-  - deploy-special
+  - deploy-extended
 
 variables:
   KUBESPRAY_VERSION: v2.25.0
@@ -43,15 +40,26 @@ before_script:
 
 .job: &job
   tags:
-    - packet
+    - ffci
   image: $PIPELINE_IMAGE
   artifacts:
     when: always
     paths:
       - cluster-dump/
+  needs:
+    - pipeline-image
+
+.job-moderated:
+  extends: .job
+  needs:
+    - pipeline-image
+    - ci-not-authorized
+    - check-galaxy-version  # lint
+    - pre-commit            # lint
+    - vagrant-validate      # lint
 
 .testcases: &testcases
-  <<: *job
+  extends: .job-moderated
   retry: 1
   interruptible: true
   before_script:
@@ -61,12 +69,12 @@ before_script:
   script:
     - ./tests/scripts/testcases_run.sh
   after_script:
-    - chronic ./tests/scripts/testcases_cleanup.sh
+    - ./tests/scripts/testcases_cleanup.sh
 
 # For failfast, at least 1 job must be defined in .gitlab-ci.yml
 # Premoderated with manual actions
 ci-not-authorized:
-  stage: moderator
+  stage: build
   before_script: []
   after_script: []
   rules:
@@ -87,8 +95,8 @@ ci-not-authorized:
   script:
     - exit $CI_OK_TO_TEST
   tags:
-    - light
-
+    - ffci
+  needs: []
 
 include:
   - .gitlab-ci/build.yml

.gitlab-ci/build.yml

@@ -5,7 +5,7 @@
     paths:
       - image-cache
   tags:
-    - packet
+    - ffci
   stage: build
   image:
     name: gcr.io/kaniko-project/executor:debug

.gitlab-ci/lint.yml

@@ -1,40 +1,35 @@
 ---
-generate-pre-commit:
-  image: 'mikefarah/yq@sha256:bcb889a1f9bdb0613c8a054542d02360c2b1b35521041be3e1bd8fbd0534d411'
-  stage: build
-  before_script: []
+pre-commit:
+  stage: test
+  tags:
+  - ffci
+  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:aaf2c7b38b22286f2d381c11673bec571c28f61dd086d11b43a1c9444a813cef'
+  variables:
+    PRE_COMMIT_HOME: /pre-commit-cache
   script:
-    - >
-      yq -r < .pre-commit-config.yaml '.repos[].hooks[].id' |
-      sed 's/^/      - /' |
-      cat .gitlab-ci/pre-commit-dynamic-stub.yml - > pre-commit-generated.yml
-  artifacts:
+  - pre-commit run --all-files
+  cache:
+    key: pre-commit-all
     paths:
-      - pre-commit-generated.yml
-
-run-pre-commit:
-  stage: unit-tests
-  trigger:
-    include:
-      - artifact: pre-commit-generated.yml
-        job: generate-pre-commit
-    strategy: depend
+    - /pre-commit-cache
+  needs: []
 
 vagrant-validate:
   extends: .job
-  stage: unit-tests
-  tags: [light]
+  stage: test
+  tags: [ffci]
   variables:
     VAGRANT_VERSION: 2.3.7
   script:
-    - ./tests/scripts/vagrant-validate.sh
+  - ./tests/scripts/vagrant-validate.sh
   except: ['triggers', 'master']
 
 
 # TODO: convert to pre-commit hook
 check-galaxy-version:
-  stage: unit-tests
-  tags: [light]
+  needs: []
+  stage: test
+  tags: [ffci]
   image: python:3
   script:
-    - tests/scripts/check_galaxy_version.sh
+  - tests/scripts/check_galaxy_version.sh

.gitlab-ci/molecule.yml

@@ -1,30 +1,42 @@
 ---
-
 .molecule:
-  tags: [c3.small.x86]
+  tags: [ffci-vm-med]
   only: [/^pr-.*$/]
   except: ['triggers']
-  image: $PIPELINE_IMAGE
+  image: quay.io/kubespray/vm-kubespray-ci:v6
   services: []
   stage: deploy-part1
+  needs: []
+  # - ci-not-authorized
+  variables:
+    VAGRANT_DEFAULT_PROVIDER: "libvirt"
   before_script:
-    - tests/scripts/rebase.sh
-    - ./tests/scripts/vagrant_clean.sh
+  - groups
+  - python3 -m venv citest
+  - source citest/bin/activate
+  - vagrant plugin expunge --reinstall --force --no-tty
+  - vagrant plugin install vagrant-libvirt
+  - pip install --no-compile --no-cache-dir pip -U
+  - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
+  - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
+  - ./tests/scripts/rebase.sh
+  - ./tests/scripts/vagrant_clean.sh
   script:
-    - ./tests/scripts/molecule_run.sh
+  - ./tests/scripts/molecule_run.sh
   after_script:
-    - chronic ./tests/scripts/molecule_logs.sh
+  - ./tests/scripts/molecule_logs.sh
   artifacts:
     when: always
     paths:
-      - molecule_logs/
+    - molecule_logs/
 
 # CI template for periodic CI jobs
 # Enabled when PERIODIC_CI_ENABLED var is set
+
 .molecule_periodic:
   only:
     variables:
-      - $PERIODIC_CI_ENABLED
+    - $PERIODIC_CI_ENABLED
   allow_failure: true
   extends: .molecule
 
@@ -34,50 +46,50 @@ molecule_full:
 molecule_no_container_engines:
   extends: .molecule
   script:
-    - ./tests/scripts/molecule_run.sh -e container-engine
+  - ./tests/scripts/molecule_run.sh -e container-engine
   when: on_success
 
 molecule_docker:
   extends: .molecule
   script:
-    - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd
+  - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd
   when: on_success
 
 molecule_containerd:
   extends: .molecule
   script:
-    - ./tests/scripts/molecule_run.sh -i container-engine/containerd
+  - ./tests/scripts/molecule_run.sh -i container-engine/containerd
   when: on_success
 
 molecule_cri-o:
   extends: .molecule
-  stage: deploy-part2
+  stage: deploy-part1
   script:
-    - ./tests/scripts/molecule_run.sh -i container-engine/cri-o
+  - ./tests/scripts/molecule_run.sh -i container-engine/cri-o
   allow_failure: true
   when: on_success
 
-# Stage 3 container engines don't get as much attention so allow them to fail
-molecule_kata:
-  extends: .molecule
-  stage: deploy-part3
-  script:
-    - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers
-  when: manual
-# FIXME: this test is broken (perma-failing)
+# # Stage 3 container engines don't get as much attention so allow them to fail
+# molecule_kata:
+#   extends: .molecule
+#   stage: deploy-extended
+#   script:
+#     - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers
+#   when: manual
+# # FIXME: this test is broken (perma-failing)
 
 molecule_gvisor:
   extends: .molecule
-  stage: deploy-part3
+  stage: deploy-extended
   script:
-    - ./tests/scripts/molecule_run.sh -i container-engine/gvisor
+  - ./tests/scripts/molecule_run.sh -i container-engine/gvisor
   when: manual
 # FIXME: this test is broken (perma-failing)
 
 molecule_youki:
   extends: .molecule
-  stage: deploy-part3
+  stage: deploy-extended
   script:
-    - ./tests/scripts/molecule_run.sh -i container-engine/youki
+  - ./tests/scripts/molecule_run.sh -i container-engine/youki
   when: manual
 # FIXME: this test is broken (perma-failing)