Skip to content

Commit

Permalink
Improve build and release system (#5991)
Browse files Browse the repository at this point in the history
* Separate artifacts for container builds

Signed-off-by: Jakub Scholz <www@scholzj.com>

* Rework the existing pipelines and add release and CVE pieplines

Signed-off-by: Jakub Scholz <www@scholzj.com>

* Review comments & Add docker-images Makefile

Signed-off-by: Jakub Scholz <www@scholzj.com>
  • Loading branch information
scholzj authored Dec 6, 2021
1 parent 59c669d commit c3bd3a8
Show file tree
Hide file tree
Showing 120 changed files with 1,175 additions and 487 deletions.
2 changes: 1 addition & 1 deletion .azure/acceptance-pipeline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ pr:
- '*'

jobs:
- template: 'templates/jobs/acceptance_jobs.yaml'
- template: 'templates/jobs/system-tests/acceptance_jobs.yaml'


143 changes: 99 additions & 44 deletions .azure/build-pipeline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,57 +4,112 @@ trigger:
include:
- 'main'
- 'release-*'
tags:
include:
- '*'
pr:
autoCancel: true
branches:
include:
- '*'
# Jobs
jobs:
- job: 'strimzi_build_and_test'
displayName: 'Build & Test'
# Strategy for the job
strategy:
matrix:
'java-11':
image: 'Ubuntu-18.04'
jdk_version: '11'
jdk_path: '/usr/lib/jvm/java-11-openjdk-amd64'
# Set timeout for jobs
timeoutInMinutes: 120
# Base system
pool:
vmImage: $(image)
# Environment variables
variables:
- template: "templates/steps/default_variables.yaml"
# Pipeline steps
steps:
- template: "templates/steps/general_steps.yaml"

- bash: ".azure/scripts/build.sh"
env:
BUILD_REASON: $(Build.Reason)
BRANCH: $(Build.SourceBranch)
DOCKER_USER: $(QUAY_USER)
DOCKER_PASS: $(QUAY_PASS)
GPG_PASSPHRASE: $(GPG_PASSPHRASE)
GPG_SIGNING_KEY: $(GPG_SIGNING_KEY)
NEXUS_USERNAME: $(NEXUS_USERNAME)
NEXUS_PASSWORD: $(NEXUS_PASSWORD)
GITHUB_DEPLOY_KEY: $(GITHUB_DEPLOY_KEY)
MVN_ARGS: '-B -Dsurefire.rerunFailingTestsCount=5 -Dfailsafe.rerunFailingTestsCount=2'
displayName: "Build & Test"
- task: PublishTestResults@2
inputs:
testResultsFormat: JUnit
testResultsFiles: '**/TEST-*.xml'
testRunTitle: "Unit & Integration tests"
condition: always()
# Stages
stages:
# Build Strimzi Java code and does basic checks of the PR (uncommitted files, release files etc.)
- stage: build_strimzi
displayName: Build Strimzi
jobs:
- template: 'templates/jobs/build/build_strimzi.yaml'

# Runs Strimzi unit and integration tests
- stage: test_strimzi
displayName: Strimzi Unit & IT tests
dependsOn:
- build_strimzi
jobs:
- template: 'templates/jobs/build/test_strimzi.yaml'
parameters:
artifactSource: 'current'
artifactProject: 'strimzi'
artifactPipeline: ''
artifactRunVersion: ''
artifactRunId: ''

# Builds Strimzi docs
- stage: build_docs
displayName: Build docs
dependsOn:
- build_strimzi
jobs:
- template: 'templates/jobs/build/build_docs.yaml'

# Builds Strimzi containers
- stage: build_containers
displayName: Build Containers
dependsOn:
- build_strimzi
jobs:
- template: 'templates/jobs/build/build_containers.yaml'
parameters:
artifactSource: 'current'
artifactProject: 'strimzi'
artifactPipeline: ''
artifactRunVersion: ''
artifactRunId: ''
architectures: ['amd64', 'arm64']

# Push Strimzi containers -> run only on main branch
- stage: push_containers
displayName: Push Containers
dependsOn:
- build_strimzi
- test_strimzi
- build_containers
- build_docs
condition: and(succeeded(), eq(variables['build.sourceBranch'], 'refs/heads/main'))
jobs:
- template: 'templates/jobs/build/push_containers.yaml'
parameters:
dockerTag: 'latest'
artifactSource: 'current'
artifactProject: 'strimzi'
artifactPipeline: ''
artifactRunVersion: ''
artifactRunId: ''
architectures: ['amd64', 'arm64']

# Publish Strimzi docs to the website -> run only on main branch
- stage: public_docs
displayName: Publish Docs
dependsOn:
- build_strimzi
- test_strimzi
- build_containers
- build_docs
condition: and(succeeded(), eq(variables['build.sourceBranch'], 'refs/heads/main'))
jobs:
- template: 'templates/jobs/build/publish_docs.yaml'
parameters:
dockerTag: 'latest'
artifactSource: 'current'
artifactProject: 'strimzi'
artifactPipeline: ''
artifactRunVersion: ''
artifactRunId: ''

# Deploy Strimzi Java artifacts -> run only on main branch (where it deploys to OSS snapshot repos)
# or release branches (where it deploys to OSS staging repos)
- stage: deploy_strimzi_java
displayName: Deploy Java artifacts
dependsOn:
- build_strimzi
- test_strimzi
- build_containers
- build_docs
condition: and(succeeded(), or(eq(variables['build.sourceBranch'], 'refs/heads/main'), startsWith(variables['build.sourceBranch'], 'refs/heads/release-')))
jobs:
- template: 'templates/jobs/build/deploy_strimzi_java.yaml'
parameters:
dockerTag: 'latest'
artifactSource: 'current'
artifactProject: 'strimzi'
artifactPipeline: ''
artifactRunVersion: ''
artifactRunId: ''
90 changes: 90 additions & 0 deletions .azure/cve-pipeline.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# Triggers
trigger: none
pr: none

# Parameters
parameters:
- name: releaseVersion
displayName: Release Version
type: string
- name: useSuffix
displayName: Build suffixed images
type: boolean
default: true
- name: releaseSuffix
displayName: Release Suffix
type: number
- name: sourcePipelineId
displayName: Pipeline ID of the source build
type: number
# 16 is the pipeline ID of the Operators build pipeline which provides the artifacts in Azure
default: 16
values:
- 16
- name: sourceBuildId
displayName: Build ID of the source build
type: number

# Stages
stages:
- stage: container_build
displayName: Prepare Container
jobs:
- template: 'templates/jobs/build/build_containers.yaml'
parameters:
artifactSource: 'specific'
artifactProject: 'strimzi'
artifactPipeline: '${{ parameters.sourcePipelineId }}'
artifactRunVersion: 'specific'
artifactRunId: '${{ parameters.sourceBuildId }}'
architectures: ['amd64', 'arm64']
- stage: containers_publish_with_suffix
displayName: Publish Containers for ${{ parameters.releaseVersion }}-${{ parameters.releaseSuffix }}
dependsOn:
- container_build
condition: and(succeeded(), startsWith(variables['build.sourceBranch'], 'refs/heads/release-'))
jobs:
- template: 'templates/jobs/build/push_containers.yaml'
parameters:
dockerTag: '${{ parameters.releaseVersion }}-${{ parameters.releaseSuffix }}'
artifactSource: 'current'
artifactProject: 'strimzi'
artifactPipeline: ''
artifactRunVersion: ''
artifactRunId: ''
architectures: ['amd64', 'arm64']
- stage: manual_validation
displayName: Validate container before pushing container as ${{ parameters.releaseVersion }}
dependsOn:
- containers_publish_with_suffix
condition: and(succeeded(), startsWith(variables['build.sourceBranch'], 'refs/heads/release-'))
jobs:
- job: waitForValidation
displayName: Wait for container image validation
pool: server
timeoutInMinutes: 4320 # job times out in 3 days
steps:
- task: ManualValidation@0
timeoutInMinutes: 4310 # task times out in 3 days
inputs:
notifyUsers: |
github@scholzj.com
xstejs24@gmail.com
instructions: 'Please validate the container image'
onTimeout: 'reject'
- stage: containers_publish
displayName: Publish Containers for ${{ parameters.releaseVersion }}
dependsOn:
- manual_validation
- containers_publish_with_suffix
condition: and(succeeded(), startsWith(variables['build.sourceBranch'], 'refs/heads/release-'))
jobs:
- template: 'templates/jobs/build/push_containers.yaml'
parameters:
dockerTag: '${{ parameters.releaseVersion }}'
artifactSource: 'current'
artifactProject: 'strimzi'
artifactPipeline: ''
artifactRunVersion: ''
artifactRunId: ''
architectures: ['amd64', 'arm64']
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ pr:
-
# Regression tests are split into 6 jobs because of timeout set to 360 minutes for each job
jobs:
- template: 'templates/jobs/feature_gates_regression_namespace_rbac_jobs.yaml'
- template: 'templates/jobs/system-tests/feature_gates_regression_namespace_rbac_jobs.yaml'

2 changes: 1 addition & 1 deletion .azure/feature-gates-regression-pipeline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@ pr:
-
# Regression tests are split into 6 jobs because of timeout set to 360 minutes for each job
jobs:
- template: 'templates/jobs/feature_gates_regression_jobs.yaml'
- template: 'templates/jobs/system-tests/feature_gates_regression_jobs.yaml'
2 changes: 1 addition & 1 deletion .azure/helm-acceptance-pipeline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ pr:
- '*'

jobs:
- template: 'templates/jobs/acceptance_helm_jobs.yaml'
- template: 'templates/jobs/system-tests/acceptance_helm_jobs.yaml'


2 changes: 1 addition & 1 deletion .azure/helm-namespace-rbac-scope-acceptance-pipeline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ pr:
- '*'

jobs:
- template: 'templates/jobs/acceptance_helm_namespace_rbac_jobs.yaml'
- template: 'templates/jobs/system-tests/acceptance_helm_namespace_rbac_jobs.yaml'



2 changes: 1 addition & 1 deletion .azure/namespace-rbac-scope-acceptance-pipeline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ pr:
- '*'

jobs:
- template: 'templates/jobs/acceptance_namespace_rbac_jobs.yaml'
- template: 'templates/jobs/system-tests/acceptance_namespace_rbac_jobs.yaml'


2 changes: 1 addition & 1 deletion .azure/namespace-rbac-scope-regression-pipeline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ pr:
-
# Regression tests are split into 6 jobs because of timeout set to 360 minutes for each job
jobs:
- template: 'templates/jobs/regression_namespace_rbac_jobs.yaml'
- template: 'templates/jobs/system-tests/regression_namespace_rbac_jobs.yaml'

2 changes: 1 addition & 1 deletion .azure/regression-pipeline.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@ pr:
-
# Regression tests are split into 6 jobs because of timeout set to 360 minutes for each job
jobs:
- template: 'templates/jobs/regression_jobs.yaml'
- template: 'templates/jobs/system-tests/regression_jobs.yaml'
66 changes: 66 additions & 0 deletions .azure/release-pipeline.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# Triggers
trigger: none
pr: none

# Parameters
parameters:
- name: releaseVersion
displayName: Release Version
type: string
- name: useSuffix
displayName: Build suffixed images
type: boolean
default: true
- name: releaseSuffix
displayName: Release Suffix
type: number
- name: sourcePipelineId
displayName: Pipeline ID of the source build
type: number
# 16 is the pipeline ID of the Operators build pipeline which provides the artifacts in Azure
default: 16
values:
- 16
- name: sourceBuildId
displayName: Build ID of the source build
type: number

# Stages
stages:
- stage: prepare_release_artifacts
displayName: Prepare release artifacts for ${{ parameters.releaseVersion }}
condition: startsWith(variables['build.sourceBranch'], 'refs/heads/release-')
jobs:
- template: 'templates/jobs/build/release_artifacts.yaml'
parameters:
releaseVersion: '${{ parameters.releaseVersion }}'
- stage: containers_publish_with_suffix
displayName: Publish Containers for ${{ parameters.releaseVersion }}-${{ parameters.releaseSuffix }}
dependsOn:
- prepare_release_artifacts
condition: and(succeeded(), startsWith(variables['build.sourceBranch'], 'refs/heads/release-'))
jobs:
- template: 'templates/jobs/build/push_containers.yaml'
parameters:
dockerTag: '${{ parameters.releaseVersion }}-${{ parameters.releaseSuffix }}'
artifactSource: 'specific'
artifactProject: 'strimzi'
artifactPipeline: '${{ parameters.sourcePipelineId }}'
artifactRunVersion: 'specific'
artifactRunId: '${{ parameters.sourceBuildId }}'
architectures: ['amd64', 'arm64']
- stage: containers_publish
displayName: Publish Containers for ${{ parameters.releaseVersion }}
dependsOn:
- containers_publish_with_suffix
condition: and(succeeded(), startsWith(variables['build.sourceBranch'], 'refs/heads/release-'))
jobs:
- template: 'templates/jobs/build/push_containers.yaml'
parameters:
dockerTag: '${{ parameters.releaseVersion }}'
artifactSource: 'specific'
artifactProject: 'strimzi'
artifactPipeline: '${{ parameters.sourcePipelineId }}'
artifactRunVersion: 'specific'
artifactRunId: '${{ parameters.sourceBuildId }}'
architectures: ['amd64', 'arm64']
Loading

0 comments on commit c3bd3a8

Please sign in to comment.