From 89ec81f72ffcdded19e0cefe5992466132f55419 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Thu, 30 Nov 2023 17:44:24 -0300 Subject: [PATCH] jobs-builder: RIP With the migration of CI to Github actions the jobs-builder no longer is used. Fixes #553 Signed-off-by: Wainer dos Santos Moschetta --- .github/workflows/publish_jobs.yml | 37 -- README.md | 3 - jobs-builder/README.md | 89 ---- jobs-builder/jjb.conf.template | 20 - jobs-builder/jobs/cc.yaml | 424 ------------------ jobs-builder/jobs/dependencies.yaml | 157 ------- jobs-builder/jobs/include/cc-ci.groovy | 142 ------ .../jobs/include/cc-ci_entrypoint.sh.inc | 20 - .../jobs/include/ci_entrypoint.sh.inc | 16 - .../include/operator-ci_entrypoint.sh.inc | 15 - jobs-builder/jobs/include/os2node.yaml.inc | 21 - jobs-builder/jobs/pr.yaml | 191 -------- jobs-builder/publish_jobs.sh | 84 ---- jobs-builder/trash | 15 - 14 files changed, 1234 deletions(-) delete mode 100644 .github/workflows/publish_jobs.yml delete mode 100644 jobs-builder/README.md delete mode 100644 jobs-builder/jjb.conf.template delete mode 100644 jobs-builder/jobs/cc.yaml delete mode 100644 jobs-builder/jobs/dependencies.yaml delete mode 100644 jobs-builder/jobs/include/cc-ci.groovy delete mode 100644 jobs-builder/jobs/include/cc-ci_entrypoint.sh.inc delete mode 100644 jobs-builder/jobs/include/ci_entrypoint.sh.inc delete mode 100644 jobs-builder/jobs/include/operator-ci_entrypoint.sh.inc delete mode 100644 jobs-builder/jobs/include/os2node.yaml.inc delete mode 100644 jobs-builder/jobs/pr.yaml delete mode 100755 jobs-builder/publish_jobs.sh delete mode 100644 jobs-builder/trash diff --git a/.github/workflows/publish_jobs.yml b/.github/workflows/publish_jobs.yml deleted file mode 100644 index c4713c4a..00000000 --- a/.github/workflows/publish_jobs.yml +++ /dev/null @@ -1,37 +0,0 @@ -# This workflow publishs all Jenkins jobs when a PR is merged. -# ---- -on: - push: - branches: - - main - paths: - # Run only when the definitions of jobs changed. - - 'jobs-builder/jobs/**' - # Or when the trash file changed. - - 'jobs-builder/trash' -name: Publish Jenkins jobs -jobs: - publish: - runs-on: ubuntu-20.04 - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Install the Jenkins Job Builder - run: | - pip install --user --upgrade jinja2 - pip install --user jenkins-job-builder==4.3.0 - - name: Run the publish script - env: - JENKINS_USER: ${{ secrets.JENKINS_USER }} - JENKINS_PASSWORD: ${{ secrets.JENKINS_PASSWORD }} - working-directory: jobs-builder - run: | - cp jjb.conf.template jjb.conf - if [[ -z "$JENKINS_USER" || -z "$JENKINS_PASSWORD" ]]; then - echo "ERROR: Missing the secrets JENKINS_USER and/or JENKINS_PASSWORD" >&2 - exit 1 - fi - sed -i -e "s/user=XXX/user=$JENKINS_USER/" \ - -e "s/password=XXX/password=$JENKINS_PASSWORD/" jjb.conf - ./publish_jobs.sh -c jjb.conf diff --git a/README.md b/README.md index 5bbf4c86..c5d9b919 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,3 @@ The Jenkins CI jobs can be controlled (such as re-triggered or skipped) in a num control trigger phrases are listed on [a community repo wiki page](https://github.com/kata-containers/community/wiki/Controlling-the-CI). -# Jobs builder - -See [README](jobs-builder/README.md) for information about the use of Jenkins Job Builder (JJB) to make the CI jobs. diff --git a/jobs-builder/README.md b/jobs-builder/README.md deleted file mode 100644 index 5edfaa67..00000000 --- a/jobs-builder/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Overview - -Manage the Jenkins jobs with help of the [Jenkins Job Builder](https://jenkins-job-builder.readthedocs.io/en/latest/index.html) (JJB). - -The JJB converts jobs and views from YAML representations into the Jenkins XML -configuration files. Also the tool is able to manage jobs, for example, publish -the generated jobs and views in a running Jenkins instance. - -# Getting started - -First of all, you need to install Jenkins Job Builder in your environment. The -instructions can be found [here](https://jenkins-job-builder.readthedocs.io/en/latest/installation.html). - -To use the Jenkins Job Builder a configuration file is needed which contains -the Jenkins URL, user and token API to manage Jenkins, among other information -that sets the tool's behavior. That file can copied from the `jjb.conf.template` -template then filling out the fields marked with *XXX*. - -Bootstrapping your environment: -```bash -$ pip install --user jenkins-job-builder -$ cp jjb.conf.template jjb.conf -$ sed -i 's/user=XXX/user=my_user/' jjb.conf -$ sed -i 's/password=XXX/password=my_user_token/' jjb.conf -``` - -# Managing the jobs - -Use the `publish_jobs.sh` to update all the Jenkins Job Builder managed jobs. - -Example of use: -```bash -$ ./publish_jobs.sh -c jjb.conf -``` - -If you only want to check the jobs can be generated but not actually publish -them all then do: -```bash -$ ./publish_jobs.sh -c jjb.conf -t -``` - -The `publish_jobs.sh` can also delete unwanted and/or not needed anymore jobs. You -just need to list their names in the `trash` file. - -Run `./publish_jobs.sh -h` to see all the available options of the script. - -# Checking your changes on a local Jenkins - -Often you will need to see how the jobs look like on the Jenkins UI in order -to check that the generated configurations are correct. The easiest way to -accomplish that is to publish the jobs on a local Jenkins instance. Use the -following instructions to setup a sandbox locally, but beware that it won't be -exactly alike the instance on production, although for the purpose of checking -the configurations it works out. - - 1. Start the Jenkins container - -The following command creates the Jenkins container, and the instance service -will be accessible through the port 8080 on the local host. - -```bash -$ docker run --rm -p 8080:8080 --name=jenkins-container -d jenkins/jenkins -``` - - 2. Give an initial configuration - -Using your web browser, access Jenkins from [http://localhost:8080](http://localhost:8080). - -The first displayed page asks for the initial administrator password, which can -be obtained with the following command: - -```bash -$ docker exec jenkins-container cat /var/jenkins_home/secrets/initialAdminPassword -``` - -Paste that token on the "administrator password" field then continue with the -setup. You will be asked to install plugins (select to install all) and finally -to create an account. - - 3. Create the API token - -Access your new user account (on the top-right menus) then go to "Configure". -Click "Add new Token" then on the "Generate" button. Save the generated token. - - 4. Create the JJB configuration file - -Now you need to create the `jjb.conf` as explained on [Getting started](#getting-started). Use -the user name and API token created on the previous steps, and don't forget to set -the `url` property to `http://localhost:8080`. diff --git a/jobs-builder/jjb.conf.template b/jobs-builder/jjb.conf.template deleted file mode 100644 index 52b6bae0..00000000 --- a/jobs-builder/jjb.conf.template +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) 2020 Red Hat, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Jenkins Job Builder configuration file (template). -# -[job_builder] -ignore_cache=True -keep_descriptions=False -include_path=.:scripts:~/git/ -recursive=False -exclude=.*:manual:./development -allow_duplicates=False -update=all - -[jenkins] -user=XXX -password=XXX -url=http://jenkins.katacontainers.io -query_plugins_info=False diff --git a/jobs-builder/jobs/cc.yaml b/jobs-builder/jobs/cc.yaml deleted file mode 100644 index 0fcb71c7..00000000 --- a/jobs-builder/jobs/cc.yaml +++ /dev/null @@ -1,424 +0,0 @@ -# Copyright (c) 2022 Red Hat, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -# This file contains the configurations to generate the pull request jobs -# for Confidential Containers project. -# ---- -### -# Define shareable YAML snippets. -### -- cc_jobs_common_properties: &cc_jobs_common_properties - name: 'common_job_properties' - project-type: freestyle - disabled: false - concurrent: {concurrent_toggle} - logrotate: - daysToKeep: 30 - numToKeep: 30 - # Convert the os variable to label name. - node: !include-jinja2: include/os2node.yaml.inc - wrappers: - - ansicolor: - colormap: "xterm" - - credentials-binding: - - text: - credential-id: quay_kata-containers-cc_auth_bot_creds - variable: REGISTRY_CREDENTIAL_ENCODED - - openstack: - single-use: True - - timestamps - - timeout: - timeout: 20 - type: no-activity - - workspace-cleanup -- cc_jobs_common_publishers: &cc_jobs_common_publishers - name: 'default_publishers' - publishers: - - post-tasks: - - matches: - - log-text: .* - operator: OR - script: | - #!/bin/bash - - export GOPATH=$WORKSPACE/go - export GOROOT="/usr/local/go" - export PATH="${{GOPATH}}/bin:/usr/local/go/bin:/usr/sbin:/usr/local/bin:${{PATH}}" - - cd $GOPATH/src/github.com/kata-containers/tests - .ci/teardown.sh "$WORKSPACE/artifacts" - - archive: - artifacts: "artifacts/*" -### -# Define jobs templates. -### -- job-template: - # Use to generate pull request (PR) jobs. - # - # Arguments: - # repo - the repository name. - # os - the node Operating System in - format. - # arch - the node architecture (e.g x86_64, s390x, ppc64le, and so on). - # ci_job - the CI job type as defined in https://github.com/kata-containers/tests/blob/main/.ci/ci_job_flags.sh - # - name: "{repo}-CCv0-{os}-{arch}-{ci_job}-PR" - <<: *cc_jobs_common_properties - # Allow concurrent jobs by default. Specify `false` on the project definition otherwise. - concurrent_toggle: true - tee: "" - description: - !j2: | - 
-         Pull Request (PR) job.
-         OS="{{ os }}"
-         arch="{{ arch }}"
-         CI_JOB="{{ ci_job }}"
-         repo="{{ repo }}"
-         type="PR"
-
- scm: - - git: - url: https://github.com/kata-containers/{repo} - branches: - - '${{sha1}}' - refspec: '+refs/pull/${{ghprbPullId}}/*:refs/remotes/origin/pr/${{ghprbPullId}}/*' - wipe-workspace: false - properties: - - github: - url: https://github.com/kata-containers/{repo} - triggers: - - github-pull-request: - auth-id: 'katacontainers' - github-hooks: true - # Trigger only on commenting phrase in the pull request. - only-trigger-phrase: true - # The expected phrase will be like "/(re)test-" - trigger-phrase: - !j2: | - {% if not tee -%} - .*(\n|^|\s)/(re)?test(-{{ os.split("-")[0] }})?(\n|$|\s)+.* - {%- else -%} - .*(\n|^|\s)/(re)?test(-{{ os.split("-")[0] }})?(-{{ tee }})?(\n|$|\s)+.* - {%- endif %} - # Skip on commenting phrase. - skip-build-phrase: '.*\[skip\W+ci\].*' - cron: 'H/5 * * * *' - # List of organizations whose members are allowed to build. - org-list: - - kata-containers - # Members of allowed organizations will have admin rights. - allow-whitelist-orgs-as-admins: true - # Branches allowed to be tested. - white-list-target-branches: - - CCv0 - # Branches disallowed to be tested. - black-list-target-branches: - - main - - master - - stable-.* - cancel-builds-on-update: true - # Commit Status Context - status-context: - !j2: 'jenkins-ci-{{ os }}-{{ arch }}-{{ ci_job.lower() }}' - # Commit Status Build Triggered - triggered-status: Build triggered - # Commit Status Build Started - started-status: Build running - builders: - - shell: - !include-jinja2: include/cc-ci_entrypoint.sh.inc - <<: *cc_jobs_common_publishers - -- job-template: - # Use to generate pull request (PR) jobs for - # confidential-containers/operator. - # - # Arguments: - # os - the node Operating System in - - # format. - # arch - the node architecture (e.g x86_64, s390x, ppc64le, - # and so on). - # container_runtime - the container runtime (e.g. containerd, crio) - # runtimeclass - the runtimeclass as configured by the operator - # (e.g. kata-qemu, kata-clh, kata-qemu-tdx). - name: "confidential-containers-operator-main-{os}-{arch}-{container_runtime}_{runtimeclass}-PR" - <<: *cc_jobs_common_properties - # Allow concurrent jobs by default. Specify `false` on the project definition otherwise. - concurrent_toggle: true - description: - !j2: | - 
-          Pull Request (PR) job for the confidential-containers/operator repository.
-          OS="{{ os }}"
-          arch="{{ arch }}"
-          container runtime="{{ container_runtime }}"
-          runtimeclass="{{ runtimeclass }}"
-          type="PR"
-
- scm: - - git: - url: https://github.com/confidential-containers/operator - branches: - - '${{sha1}}' - refspec: '+refs/pull/${{ghprbPullId}}/*:refs/remotes/origin/pr/${{ghprbPullId}}/*' - wipe-workspace: false - properties: - - github: - url: https://github.com/confidential-containers/operator - triggers: - - github-pull-request: - auth-id: 'Confidential Containers external jobs' - github-hooks: true - # Trigger only on commenting phrase in the pull request. - only-trigger-phrase: true - # The expected phrase will be like "/(re)test-" - trigger-phrase: - !j2: '.*(\n|^|\s)/(re)?test(-{{ runtimeclass }})?(-{{ os.split("-")[0] }})?(\n|$|\s)+.*' - # Skip on commenting phrase. - skip-build-phrase: '.*\[skip\W+ci\].*' - cron: 'H/5 * * * *' - # List of organizations whose members are allowed to build. - org-list: - - confidential-containers - # Members of allowed organizations will have admin rights. - allow-whitelist-orgs-as-admins: true - # Branches allowed to be tested. - white-list-target-branches: - - main - # Branches disallowed to be tested. - black-list-target-branches: - - master - - stable-.* - cancel-builds-on-update: true - # Commit Status Context - status-context: - !j2: 'tests-e2e-{{ os }}-{{ arch }}-{{ container_runtime }}_{{ runtimeclass }}' - # Commit Status Build Triggered - triggered-status: Build triggered - # Commit Status Build Started - started-status: Build running - builders: - - shell: - !include-jinja2: include/operator-ci_entrypoint.sh.inc - -- job-template: - # Use to generate baseline jobs for Kata Containers. - # - # Arguments: - # os - the node Operating System in - format. - # arch - the node architecture (e.g x86_64, s390x, ppc64le, and so on). - # ci_job - the CI job type as defined in https://github.com/kata-containers/tests/blob/main/.ci/ci_job_flags.sh - # maintainers - a list of maintainer (use the github's mentions @somebody). - # - name: "kata-containers-CCv0-{os}-{arch}-{ci_job}-baseline" - <<: *cc_jobs_common_properties - # Allow concurrent jobs by default. Specify `false` on the project definition otherwise. - concurrent_toggle: true - description: - !j2: | - 
-         Baseline job.
-         /scheduled
-         status="Stable"
-         OS="{{ os }}"
-         arch="{{ arch }}"
-         branch="CCv0"
-         CI_JOB="{{ ci_job }}"
-         repo="kata-containers"
-         type="baseline"
-         Maintainers:
-           {% for i in maintainers -%}
-           - {{ i }}
-           {% endfor %}
-
- scm: - - git: - url: https://github.com/kata-containers/kata-containers - branches: - - '*/CCv0' - wipe-workspace: false - triggers: - - timed: 'H 0 * * *' - builders: - - shell: - !include-jinja2: include/cc-ci_entrypoint.sh.inc - <<: *cc_jobs_common_publishers - -- job-template: - # Use to generate baseline jobs for the operator. - # - # Arguments: - # os - the node Operating System in - format. - # arch - the node architecture (e.g x86_64, s390x, ppc64le, and so on). - # container_runtime - the container runtime (e.g. containerd, crio, and so on). - # runtimeclass - the kata runtimeclass (e.g. kata-qemu, kata-clh, and so on). - # maintainers - a list of maintainer (use the github's mentions @somebody). - # - name: "confidential-containers-operator-main-{os}-{arch}-{container_runtime}_{runtimeclass}-baseline" - <<: *cc_jobs_common_properties - # Allow concurrent jobs by default. Specify `false` on the project definition otherwise. - concurrent_toggle: true - description: - !j2: | - 
-         Baseline job.
-         /scheduled
-         status="Stable"
-         OS="{{ os }}"
-         arch="{{ arch }}"
-         branch="main"
-         repo="confidential-containers/operator"
-         type="baseline"
-         Maintainers:
-           {% for i in maintainers -%}
-           - {{ i }}
-           {% endfor %}
-
- scm: - - git: - url: https://github.com/confidential-containers/operator - branches: - - '*/main' - wipe-workspace: false - triggers: - - timed: 'H 0 * * *' - builders: - - shell: - !include-jinja2: include/operator-ci_entrypoint.sh.inc - -### -# Define the projects -### -- project: - name: "Generate jobs for Confidential Containers" - repo: - - kata-containers - - tests - os: - - ubuntu-20.04 - arch: - - x86_64 - baremetal: "false" - ci_job: - - CC_CRI_CONTAINERD - - CC_CRI_CONTAINERD_CLOUD_HYPERVISOR - - CC_CRI_CONTAINERD_K8S - maintainers: - - '@wainersm' - jobs: - - '{repo}-CCv0-{os}-{arch}-{ci_job}-PR' - - 'kata-containers-CCv0-{os}-{arch}-{ci_job}-baseline' -- project: - name: "Generate jobs for the Confidential Containers Operator" - os: - - ubuntu-20.04 - arch: - - x86_64 - baremetal: "false" - container_runtime: - - containerd - runtimeclass: - - kata-qemu - - kata-clh - maintainers: - - '@wainersm' - - '@fidencio' - jobs: - - 'confidential-containers-operator-main-{os}-{arch}-{container_runtime}_{runtimeclass}-PR' - - 'confidential-containers-operator-main-{os}-{arch}-{container_runtime}_{runtimeclass}-baseline' -- project: - # The node label is selected based on this project name. Be careful if you - # need to change the name. - name: "Generate jobs for the Confidential Containers Operator on AMD SEV" - os: - - ubuntu-20.04_sev - arch: - - x86_64 - baremetal: "true" - container_runtime: - - containerd - runtimeclass: - - kata-qemu-sev - maintainers: - - '@ryansavino' - jobs: - - 'confidential-containers-operator-main-{os}-{arch}-{container_runtime}_{runtimeclass}-PR' - - 'confidential-containers-operator-main-{os}-{arch}-{container_runtime}_{runtimeclass}-baseline' -- project: - name: "Generate jobs for Confidential Containers on AMD SEV" - repo: - - kata-containers - - tests - os: - - ubuntu-20.04_sev - arch: - - x86_64 - baremetal: "true" - tee: "sev" - ci_job: - - CC_SEV_CRI_CONTAINERD_K8S - maintainers: - - '@ryansavino' - jobs: - - '{repo}-CCv0-{os}-{arch}-{ci_job}-PR' - - 'kata-containers-CCv0-{os}-{arch}-{ci_job}-baseline' -- project: - # The node label is selected based on this project name. Be careful if you - # need to change the name. - name: "Generate jobs for the Confidential Containers Operator on AMD SEV-SNP" - os: - - ubuntu-20.04_snp - arch: - - x86_64 - baremetal: "true" - container_runtime: - - containerd - runtimeclass: - - kata-qemu-snp - maintainers: - - '@UnmeshDeodhar' - jobs: - - 'confidential-containers-operator-main-{os}-{arch}-{container_runtime}_{runtimeclass}-PR' - - 'confidential-containers-operator-main-{os}-{arch}-{container_runtime}_{runtimeclass}-baseline' -- project: - name: "Generate jobs for Confidential Containers on AMD SEV-SNP" - repo: - - kata-containers - - tests - os: - - ubuntu-20.04_snp - arch: - - x86_64 - baremetal: "true" - tee: "snp" - ci_job: - - CC_SNP_CRI_CONTAINERD_K8S - maintainers: - - '@UnmeshDeodhar' - jobs: - - '{repo}-CCv0-{os}-{arch}-{ci_job}-PR' - - 'kata-containers-CCv0-{os}-{arch}-{ci_job}-baseline' - -### -# Define stand-alone jobs. -### -- job: - project-type: pipeline - name: confidential-containers-ci - concurrent: false - description: | - Confidential Containers CI job. - dsl: !include-raw: include/cc-ci.groovy - sandbox: true - triggers: - - pollscm: - cron: "@hourly" -### -# Define views. -### -- view: - name: Daily CCv0 baseline - view-type: list - regex: '.*\(operator-main\|CCv0\)-.*-baseline' diff --git a/jobs-builder/jobs/dependencies.yaml b/jobs-builder/jobs/dependencies.yaml deleted file mode 100644 index c6045234..00000000 --- a/jobs-builder/jobs/dependencies.yaml +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright (c) 2021 Red Hat, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -# This file contains the configurations to generate the Kernel and QEMU jobs. -# ---- -### -# Define shareable YAML snippets. -### -- common_job_properties: &common_job_properties - name: 'common_job_properties' - project-type: freestyle - disabled: false - concurrent: true - logrotate: - daysToKeep: 30 - numToKeep: 5 - # Convert the os variable to label name. - node: !include-jinja2: include/os2node.yaml.inc - scm: - - git: - url: https://github.com/kata-containers/kata-containers - branches: - - '*/{branch}' - wrappers: - - ansicolor: - colormap: "xterm" - - openstack: - single-use: True - - timestamps - - timeout: - timeout: 20 - type: no-activity -- default_publishers: &default_publishers - name: 'default_publishers' - publishers: - - archive: - artifacts: "artifacts/*" -### -# Define jobs templates. -### -- job-template: - name: kata-containers-{branch}-kernel-{flavor}-{arch} - <<: *common_job_properties - maintainers: - - Kata Containers CI team (#kata-ci at Slack) - description: - !j2: | -

Build Kernel for usage in Kata Containers CI jobs.

-

You should not use these artifacts on production.

- - Maintainers: -
    - {% for i in maintainers -%} -
  • {{ i }}
    • - {% endfor %} -
- triggers: - - timed: 'H 0-23/6 * * 1-5' - builders: - - shell: - !j2: | - #!/bin/bash - set -o errexit - set -o nounset - set -o pipefail - [ -n "$BASH_VERSION" ] && set -o errtrace - {% raw %} - [ -n "${DEBUG:-}" ] && set -o xtrace - {% endraw %} - - export GOPATH="${WORKSPACE}/go" - export GOROOT="/usr/local/go" - export PATH="$PATH:$GOPATH/bin:$GOROOT/bin" - - source ci/lib.sh - export branch="{{branch}}" - export target_branch="$branch" - {% if flavor == "experimental" -%} - export build_type="experimental" - {% endif %} - clone_tests_repo - ci/install_go.sh - cd "$tests_repo_dir" - ./.ci/install_kata_kernel.sh - ./.ci/ci_cache_components.sh -k - <<: *default_publishers -- job-template: - name: kata-containers-{branch}-qemu-{arch} - <<: *common_job_properties - maintainers: - - Kata Containers CI team (#kata-ci at Slack) - description: - # Passing a Jinja2 template. - !j2: | -

Build QEMU for usage in Kata Containers CI jobs.

-

You should not use these artifacts on production.

- - Maintainers: -
    - {% for i in maintainers -%} -
  • {{ i }}
    • - {% endfor %} -
- triggers: - - github - builders: - - shell: | - #!/bin/bash - set -o errexit - set -o nounset - set -o pipefail - [ -n "$BASH_VERSION" ] && set -o errtrace - [ -n "${{DEBUG:-}}" ] && set -o xtrace - - export GOPATH="$WORKSPACE/go" - export GOROOT="/usr/local/go" - export PATH="$PATH:$GOPATH/bin:$GOROOT/bin" - - source ci/lib.sh - export branch="{branch}" - export target_branch="$branch" - clone_tests_repo - ci/install_go.sh - - cd "$tests_repo_dir" - ./.ci/setup_env_ubuntu.sh "default" - ./cmd/container-manager/manage_ctr_mgr.sh docker install -f - ./.ci/install_qemu.sh - ./.ci/ci_cache_components.sh -q - <<: *default_publishers -### -# Define the projects -### -- project: - name: "Create Kernel cache jobs" - flavor: - - vanilla - - experimental - branch: - - main - arch: - - x86_64 - os: ubuntu1804 - jobs: - - "kata-containers-{branch}-kernel-{flavor}-{arch}" -- project: - name: "Create QEMU cache jobs" - branch: - - main - - stable-2.3 - arch: - - x86_64 - os: ubuntu1804 - jobs: - - "kata-containers-{branch}-qemu-{arch}" diff --git a/jobs-builder/jobs/include/cc-ci.groovy b/jobs-builder/jobs/include/cc-ci.groovy deleted file mode 100644 index cd2a0057..00000000 --- a/jobs-builder/jobs/include/cc-ci.groovy +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) 2023 Red Hat, Inc. -// SPDX-License-Identifier: Apache-2.0 -// -// The kata-containers, kata-containers/tests, and confidential-containers/operator repositories are monitored by polling, -// in case of changes are detected it will wait for the runtime-payload (and other images) to show up on the registry, -// afterwards tests jobs are triggered. - -// Define jobs configurations -def jobsConfig = [ - 'cc-ci-ubuntu-20.04-x86_64-containerd_kata-qemu': [runtimeClass: 'kata-qemu', - node: 'ubuntu_20.04', - arch: 'x86_64', - baremetal: false], - 'cc-ci-ubuntu-20.04-x86_64-containerd_kata-clh': [runtimeClass: 'kata-clh', - node: 'ubuntu_20.04', - arch: 'x86_64', - baremetal: false], -// 'cc-ci-ubuntu-20.04-x86_64-containerd_kata-qemu-sev': [runtimeClass: 'kata-qemu-sev', -// node: 'amd-ubuntu-2004_op-ci', -// arch: 'x86_64', -// baremetal: true] -] - -def jobs = [:] -def jobsArches = jobsConfig.collect { "$it.value.arch" }.unique() -// The new runtime-payload image -def payloadNewImg = "confidential-containers/runtime-payload-ci" -// The new runtime-payload image tag (the -arch suffix is omitted) -def payloadNewImgTag = "" -// The confidential-containers/operator repository latest commit SHA-1 -def operatorCommit = "" -// The kata-containers repositories branch it should monitor -def kataRepoBranch = "CCv0" -// The amount of time in minutes it should wait for the images be built. -def waitImagesTimeout = 90 - -// Keep polling the repositories for new changes. -node("amd-ubuntu-2004_op-ci") { - def kataCommit = "" - - stage("Checkout SCM") { - dir("kata-containers") { - checkout(poll: true, - scm: [$class: 'GitSCM', - branches: [[name: kataRepoBranch]], - extensions: [], - userRemoteConfigs: [[url: 'https://github.com/kata-containers/kata-containers']]]) - - kataCommit = sh(returnStdout: true, script: 'git rev-list --max-count=1 HEAD').trim() - } - dir("tests") { - checkout(poll: true, - scm: [$class: 'GitSCM', - branches: [[name: kataRepoBranch]], - extensions: [], - userRemoteConfigs: [[url: 'https://github.com/kata-containers/tests']]]) - } - dir("operator") { - checkout(poll: true, - scm: [$class: 'GitSCM', - branches: [[name: 'main']], - extensions: [], - userRemoteConfigs: [[url: 'https://github.com/confidential-containers/operator']]]) - operatorCommit = sh(returnStdout: true, script: 'git rev-list --max-count=1 HEAD').trim() - } - } - - stage("Wait for images") { - // This job should be triggered just after a push to the repositories and the images might not be built, so - // keep polling the registry until they show up. - - // TODO: add support for s390x. - payloadNewImgTag = "kata-containers-" + kataCommit - timeout(time: waitImagesTimeout, unit: 'MINUTES') { - sh """ - for arch in ${jobsArches.join(' ')}; do - payload_img_tag="${payloadNewImgTag}-\$arch" - tag="" - while [ -z "\$tag" ]; do - sleep 60 - tag=\$(curl -s https://quay.io/api/v1/repository/${payloadNewImg} | grep "\$payload_img_tag") || true - done - done - """ - } - } -} - -// Define the tests jobs which are the operator CI ones. -jobsConfig.each { key, c -> - jobs[key] = { - node(c['node']) { - def arch = c['arch'] - def undoFlag = "" - - if (c['baremetal']) { - undoFlag="-u" - } - stage("Bootstrap test node") { - sh "sudo apt-get update -y" - sh "sudo apt-get install -y ansible python-is-python3" - } - stage("Checkout operator source") { - dir("operator") { - checkout(poll: true, - scm: [$class: 'GitSCM', - branches: [[name: operatorCommit]], - extensions: [], - userRemoteConfigs: [[url: 'https://github.com/confidential-containers/operator']]]) - } - } - stage("Prepare the operator deploy") { - // Update the deployment files to leverage the new runtime-payload and other images. - - dir("operator") { - sh """ - [ -f "./kustomize" ] || \ - curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash - cd config/samples/ccruntime/default - ../../../../kustomize edit set image \ - quay.io/confidential-containers/runtime-payload=quay.io/${payloadNewImg}:${payloadNewImgTag}-${arch} - cat kustomization.yaml - """ - } - } - stage("Run tests") { - withCredentials([string(credentialsId: 'quay_kata-containers-cc_auth_bot_creds', - variable: 'REGISTRY_CREDENTIAL_ENCODED')]) { - dir("operator") { - sh """ - export PATH="$PATH:/usr/local/bin" - ./tests/e2e/run-local.sh -r ${c['runtimeClass']} ${undoFlag} - """ - } - } - } - } - } -} - -// Run the tests jobs. -parallel jobs diff --git a/jobs-builder/jobs/include/cc-ci_entrypoint.sh.inc b/jobs-builder/jobs/include/cc-ci_entrypoint.sh.inc deleted file mode 100644 index 6dd04a74..00000000 --- a/jobs-builder/jobs/include/cc-ci_entrypoint.sh.inc +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2022 Red Hat, Inc. -# SPDX-License-Identifier: Apache-2.0 -# -{# Wrapper shell script to call kata-containers/tests/.ci/ci_entry_point.sh - with expected environment variables exported. - This is a jinja2 template. -#} - -set -e -set -x - -curl -OL https://raw.githubusercontent.com/kata-containers/tests/CCv0/.ci/ci_entry_point.sh -{% if baremetal == "true" -%} -export BAREMETAL=true -{% endif %} -export DEBUG=true -export CI_JOB="{{ ci_job }}" -bash -x ci_entry_point.sh "$GIT_URL" diff --git a/jobs-builder/jobs/include/ci_entrypoint.sh.inc b/jobs-builder/jobs/include/ci_entrypoint.sh.inc deleted file mode 100644 index 6410fc20..00000000 --- a/jobs-builder/jobs/include/ci_entrypoint.sh.inc +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2022 Red Hat, Inc. -# SPDX-License-Identifier: Apache-2.0 -# -# Wrapper shell script to call kata-containers/tests/.ci/ci_entry_point.sh -# with expected environment variables exported. -# - -set -e -set -x - -curl -OL https://raw.githubusercontent.com/kata-containers/tests/main/.ci/ci_entry_point.sh -export DEBUG=true -export CI_JOB="{ci_job}" -bash -x ci_entry_point.sh "$GIT_URL" diff --git a/jobs-builder/jobs/include/operator-ci_entrypoint.sh.inc b/jobs-builder/jobs/include/operator-ci_entrypoint.sh.inc deleted file mode 100644 index 1a02dd2a..00000000 --- a/jobs-builder/jobs/include/operator-ci_entrypoint.sh.inc +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2023 Red Hat, Inc. -# SPDX-License-Identifier: Apache-2.0 -# -{# Run the Operator CI tests. - This is a jinja2 template. -#} - -sudo apt-get update -y -sudo apt-get install -y ansible python-is-python3 -cd tests/e2e -export PATH="$PATH:/usr/local/bin" -./run-local.sh -r "{{ runtimeclass }}" {%+ if baremetal == "true" %}-u{% endif %} - diff --git a/jobs-builder/jobs/include/os2node.yaml.inc b/jobs-builder/jobs/include/os2node.yaml.inc deleted file mode 100644 index 89081889..00000000 --- a/jobs-builder/jobs/include/os2node.yaml.inc +++ /dev/null @@ -1,21 +0,0 @@ -{# -# Copyright (c) 2020 Red Hat, Inc. -# SPDX-License-Identifier: Apache-2.0 -# -# Convert OS name to node label string. -#} -{%- if os == "centos-8-stream" -%} -centos_8_stream -{%- elif os in ["fedora35", "fedora-35"] -%} -fedora35_azure -{%- elif os == "ubuntu1804" -%} -ubuntu1804_azure || ubuntu1804-azure -{%- elif os == "ubuntu-20.04" -%} -ubuntu_20.04 -{%- elif os == "ubuntu-20.04_sev" -%} -amd-ubuntu-2004 -{%- elif os == "ubuntu-20.04_snp" -%} -amd-milan-ubuntu-2004 -{%- elif os == "ubuntu-20.04-ARM" -%} -arm_node || arm-ubuntu-2004 -{%- endif %} diff --git a/jobs-builder/jobs/pr.yaml b/jobs-builder/jobs/pr.yaml deleted file mode 100644 index b339df2f..00000000 --- a/jobs-builder/jobs/pr.yaml +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright (c) 2022 Red Hat, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -# This file contains the configurations to generate the pull request and -# baseline jobs. -# ---- -### -# Define shareable YAML snippets. -### -- main_jobs_common_properties: &main_jobs_common_properties - name: 'common_job_properties' - project-type: freestyle - disabled: - !j2: '{% if os == "centos-8-stream" -%}true{% else %}false{%- endif %}' - concurrent: {concurrent_toggle} - logrotate: - daysToKeep: 30 - numToKeep: 30 - # Convert the os variable to label name. - node: !include-jinja2: include/os2node.yaml.inc - wrappers: - - ansicolor: - colormap: "xterm" - - openstack: - single-use: True - - timestamps - - timeout: - timeout: 20 - type: no-activity -- main_jobs_common_publishers: &main_jobs_common_publishers - name: 'default_publishers' - publishers: - - post-tasks: - - matches: - - log-text: .* - operator: OR - script: | - #!/bin/bash - - export GOPATH=$WORKSPACE/go - export GOROOT="/usr/local/go" - export PATH="${{GOPATH}}/bin:/usr/local/go/bin:/usr/sbin:/usr/local/bin:${{PATH}}" - - cd $GOPATH/src/github.com/kata-containers/tests - .ci/teardown.sh "$WORKSPACE/artifacts" - - archive: - artifacts: "artifacts/*" -### -# Define jobs templates. -### -- job-template: - # Use to generate baseline jobs. - # - # Arguments: - # branch - the repository branch. - # os - the node Operating System in - format. - # arch - the node architecture (e.g x86_64, s390x, ppc64le, and so on). - # ci_job - the CI job type as defined in https://github.com/kata-containers/tests/blob/main/.ci/ci_job_flags.sh - # maintainers - a list of maintainer (use the github's mentions @somebody). - # - name: "kata-containers-{branch}-{os}-{arch}-{ci_job}-baseline" - <<: *main_jobs_common_properties - # Allow concurrent jobs by default. Specify `false` on the project definition otherwise. - concurrent_toggle: true - description: - !j2: | - 
-         Baseline job.
-         /scheduled
-         status="Stable"
-         OS="{{ os }}"
-         arch="{{ arch }}"
-         CI_JOB="{{ ci_job }}"
-         repo="kata-containers"
-         type="baseline"
-         Maintainers:
-           {% for i in maintainers -%}
-           - {{ i }}
-           {% endfor %}
-
- scm: - - git: - url: https://github.com/kata-containers/kata-containers - branches: - - '*/{branch}' - wipe-workspace: false - triggers: - - timed: 'H 0 * * *' - builders: - - shell: - !include-raw: include/ci_entrypoint.sh.inc - <<: *main_jobs_common_publishers - -- job-template: - # Use to generate pull request (PR) jobs. - # - # Arguments: - # repo - the repository name. - # os - the node Operating System in - format. - # arch - the node architecture (e.g x86_64, s390x, ppc64le, and so on). - # ci_job - the CI job type as defined in https://github.com/kata-containers/tests/blob/main/.ci/ci_job_flags.sh - # - name: "{repo}-{os}-{arch}-{ci_job}-PR" - <<: *main_jobs_common_properties - # Allow concurrent jobs by default. Specify `false` on the project definition otherwise. - concurrent_toggle: true - description: - !j2: | - 
-         Pull Request (PR) job.
-         OS="{{ os }}"
-         arch="{{ arch }}"
-         CI_JOB="{{ ci_job }}"
-         repo="{{ repo }}"
-         type="PR"
-
- scm: - - git: - url: https://github.com/kata-containers/{repo} - branches: - - '${{sha1}}' - refspec: '+refs/pull/${{ghprbPullId}}/*:refs/remotes/origin/pr/${{ghprbPullId}}/*' - wipe-workspace: false - properties: - - github: - url: https://github.com/kata-containers/{repo} - triggers: - - github-pull-request: - auth-id: 'katacontainers' - github-hooks: true - # Trigger only on commenting phrase in the pull request. - only-trigger-phrase: true - # The expected phrase will be like "/(re)test-" - trigger-phrase: - !j2: '.*(\n|^|\s)/(re)?test(-{{ os.split("-")[0] }})?(\n|$|\s)+.*' - # Skip on commenting phrase. - skip-build-phrase: '.*\[skip\W+ci\].*' - cron: 'H/5 * * * *' - # List of organizations whose members are allowed to build. - org-list: - - kata-containers - # Members of allowed organizations will have admin rights. - allow-whitelist-orgs-as-admins: true - # Branches allowed to be tested. - white-list-target-branches: - - main - - stable-2.* - - stable-3.* - # Branches disallowed to be tested. - black-list-target-branches: - - master - - stable-1.* - cancel-builds-on-update: true - # Commit Status Context - status-context: - !j2: 'jenkins-ci-{{ os }}-{{ arch }}-{{ ci_job.lower() }}' - # Commit Status Build Triggered - triggered-status: Build triggered - # Commit Status Build Started - started-status: Build running - builders: - - shell: - !include-raw: include/ci_entrypoint.sh.inc - <<: *main_jobs_common_publishers - -### -# Define the projects -### -- project: - name: "Generate jobs for CRIO_K8S" - repo: - - kata-containers - - tests - branch: - - main - os: - - fedora-35 - - centos-8-stream - arch: - - x86_64 - ci_job: - - CRIO_K8S - - CRIO_K8S_MINIMAL - maintainers: - - '@snir911' - - '@c3d' - jobs: - - 'kata-containers-{branch}-{os}-{arch}-{ci_job}-baseline' - - '{repo}-{os}-{arch}-{ci_job}-PR' diff --git a/jobs-builder/publish_jobs.sh b/jobs-builder/publish_jobs.sh deleted file mode 100755 index ce82e1bf..00000000 --- a/jobs-builder/publish_jobs.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2020 Red Hat, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Use this script to publish the jobs on Jenkins. -# -set -o errexit -set -o nounset -set -o pipefail - -[ -n "$BASH_VERSION" ] && set -o errtrace - -[ -n "${DEBUG:-}" ] && set -o xtrace - -script_dir="$(realpath $(dirname $0))" -jobs_dir="$script_dir/jobs" -readonly cmd="jenkins-jobs" -test_only=0 -config_file="" - -function die -{ - local msg="$*" - echo "ERROR: $msg" >&2 - exit 1 -} - -function usage -{ - cat <<-EOF - This script uses the Jenkins Job Builder to manage the Kata Containers - CI jobs on Jenkins. - - It needs the jenkins-jobs command installed as well as a configuration - file that contains information about how to access the Jenkins - instance. See the README.md for further details. - - Usage $0: -c CONFIG [-t] [-h], where: - -c Path to the configuration file. - -h Print this message. - -t Do not publish jobs, only test they can be generated. - EOF -} - -function parse_args -{ - while getopts "c:ht" opt; do - case ${opt} in - c) config_file="${OPTARG}" ;; - h) usage; exit 0 ;; - t) test_only=1 ;; - *) usage; exit 1 ;; - esac - done - - if [ -z "$config_file" ]; then - usage - die "missing the config file" - fi -} - -function main -{ - parse_args "$@" - command -v "$cmd" || die "$cmd command is needed" - - # First test it can generate the jobs. - $cmd test "$jobs_dir" || die "some jobs cannot be generated" - - if [ $test_only -eq 0 ]; then - # Going to update the managed jobs. - $cmd --conf "$config_file" update "$jobs_dir" - - # Going to delete jobs. - jobs_to_delete=$(grep -v "^#" "${script_dir}/trash") - if [ -n "$jobs_to_delete" ]; then - $cmd --conf "$config_file" delete -j ${jobs_to_delete} - fi - fi -} - -main "$@" diff --git a/jobs-builder/trash b/jobs-builder/trash deleted file mode 100644 index e2869da7..00000000 --- a/jobs-builder/trash +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2023 Red Hat -# -# SPDX-License-Identifier: Apache-2.0 -# -# List of jobs that should be deleted from Jenkins. -# -# Use to delete either unwanted generated jobs or jobs that no longer are needed. -# The job can be removed from this list eventually if it is not generated anymore and it -# was deleted at least one time. -kata-containers-CCv0-ubuntu-20.04-x86_64-CC_SKOPEO_CRI_CONTAINERD-PR -kata-containers-CCv0-ubuntu-20.04-x86_64-CC_SKOPEO_CRI_CONTAINERD_CLOUD_HYPERVISOR-PR -kata-containers-CCv0-ubuntu-20.04-x86_64-CC_SKOPEO_CRI_CONTAINERD_K8S-PR -tests-CCv0-ubuntu-20.04-x86_64-CC_SKOPEO_CRI_CONTAINERD-PR -tests-CCv0-ubuntu-20.04-x86_64-CC_SKOPEO_CRI_CONTAINERD_CLOUD_HYPERVISOR-PR -tests-CCv0-ubuntu-20.04-x86_64-CC_SKOPEO_CRI_CONTAINERD_K8S-PR