Skip to content

refactor/download pingcap oci artifact 2 #4138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wuhuizuo wants to merge 15 commits into
base: main
Choose a base branch
from
Draft

refactor/download pingcap oci artifact 2 #4138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
969896c
refactor(pipelines): add utils container to pipeline pod templates
wuhuizuo Feb 1, 2026
c38de9f
refactor: add utils container to 9.0-beta pod yamls and switch to OCI…
wuhuizuo Feb 1, 2026
e636781
refactor(oci): extend utils container to remaining 9.0-beta pod manif…
wuhuizuo Feb 1, 2026
99675e2
refactor(oci): migrate remaining 9.0-beta integration pipelines to OC…
wuhuizuo Feb 1, 2026
3d8816e
refactor(pod): add utils container for 8.4 integration mysql pod
wuhuizuo Feb 1, 2026
e6377de
refactor(oci): migrate 8.4 integration pipelines to OCI artifact down…
wuhuizuo Feb 1, 2026
f77b96b
refactor(oci): migrate 8.3 integration pipelines to OCI artifact down…
wuhuizuo Feb 2, 2026
33a69df
refactor(oci): migrate 8.2 integration pipelines to OCI artifact down…
wuhuizuo Feb 2, 2026
5efcac2
refactor(oci): migrate 6.5.7 integration pipelines to OCI artifact do…
wuhuizuo Feb 2, 2026
b891f94
refactor(oci): migrate pd copr integration to OCI tikv artifact
wuhuizuo Feb 2, 2026
14897e5
refactor(oci): migrate tidb-test ghpr integration mysql to OCI artifa…
wuhuizuo Feb 2, 2026
cb4235d
pipelines(tidb): download OCI artifacts and cache
wuhuizuo Feb 3, 2026
4fc0e33
pipelines(tidb): Use OCI artifacts and cache workspace
wuhuizuo Feb 3, 2026
a1f4851
pipelines(tidb): Use OCI artifacts and caching
wuhuizuo Feb 3, 2026
21a8106
pipelines(tidb): add k8s agent and simplify stages
wuhuizuo Feb 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions pipelines/pingcap-qe/tidb-test/release-6.0/ghpr_integration_mysql_test.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh'
final GIT_FULL_REPO_NAME = 'PingCAP-QE/tidb-test'
final POD_TEMPLATE_FILE = 'pipelines/pingcap-qe/tidb-test/release-6.0/pod-ghpr_integration_mysql_test.yaml'
final REFS = readJSON(text: params.JOB_SPEC).refs
final OCI_TAG_PD = component.computeArtifactOciTagFromPR('pd', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref)
final OCI_TAG_TIKV = component.computeArtifactOciTagFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref)

pipeline {
agent {
Expand All @@ -17,7 +19,7 @@ pipeline {
}
}
environment {
FILE_SERVER_URL = 'http://fileserver.pingcap.net'
OCI_ARTIFACT_HOST = 'hub-zot.pingcap.net/mirrors/hub'
}
options {
timeout(time: 70, unit: 'MINUTES')
Expand Down Expand Up @@ -66,12 +68,14 @@ pipeline {
dir('tidb') {
cache(path: "./bin", includes: '**/*', key: "ws/${BUILD_TAG}/dependencies") {
sh label: 'tidb-server', script: 'ls bin/tidb-server || make'
sh label: 'download binary', script: """
chmod +x ${WORKSPACE}/scripts/artifacts/*.sh
${WORKSPACE}/scripts/artifacts/download_pingcap_artifact.sh --pd=${REFS.base_ref} --tikv=${REFS.base_ref}
mv third_bin/* bin/
ls -alh bin/
"""
container("utils") {
dir("bin") {
sh label: 'download binary', script: """
${WORKSPACE}/scripts/artifacts/download_pingcap_oci_artifact.sh --pd=${OCI_TAG_PD} --tikv=${OCI_TAG_TIKV}
ls -alh .
"""
}
}
}
}
dir('tidb-test') {
Expand Down
18 changes: 11 additions & 7 deletions pipelines/pingcap-qe/tidb-test/release-6.2/ghpr_integration_mysql_test.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh'
final GIT_FULL_REPO_NAME = 'PingCAP-QE/tidb-test'
final POD_TEMPLATE_FILE = 'pipelines/pingcap-qe/tidb-test/release-6.2/pod-ghpr_integration_mysql_test.yaml'
final REFS = readJSON(text: params.JOB_SPEC).refs
final OCI_TAG_PD = component.computeArtifactOciTagFromPR('pd', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref)
final OCI_TAG_TIKV = component.computeArtifactOciTagFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref)

pipeline {
agent {
Expand All @@ -17,7 +19,7 @@ pipeline {
}
}
environment {
FILE_SERVER_URL = 'http://fileserver.pingcap.net'
OCI_ARTIFACT_HOST = 'hub-zot.pingcap.net/mirrors/hub'
}
options {
timeout(time: 45, unit: 'MINUTES')
Expand Down Expand Up @@ -66,12 +68,14 @@ pipeline {
dir('tidb') {
cache(path: "./bin", includes: '**/*', key: "ws/${BUILD_TAG}/dependencies") {
sh label: 'tidb-server', script: 'ls bin/tidb-server || make'
sh label: 'download binary', script: """
chmod +x ${WORKSPACE}/scripts/artifacts/*.sh
${WORKSPACE}/scripts/artifacts/download_pingcap_artifact.sh --pd=${REFS.base_ref} --tikv=${REFS.base_ref}
mv third_bin/* bin/
ls -alh bin/
"""
container("utils") {
dir("bin") {
sh label: 'download binary', script: """
${WORKSPACE}/scripts/artifacts/download_pingcap_oci_artifact.sh --pd=${OCI_TAG_PD} --tikv=${OCI_TAG_TIKV}
ls -alh .
"""
}
}
}
}
dir('tidb-test') {
Expand Down
10 changes: 10 additions & 0 deletions pipelines/pingcap-qe/tidb-test/release-6.2/pod-ghpr_integration_mysql_test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,16 @@ spec:
limits:
memory: 256Mi
cpu: 100m
- name: utils
image: ghcr.io/pingcap-qe/cd/utils/release:v2025.10.12-7-gfdd779c
tty: true
resources:
requests:
memory: 256Mi
cpu: 100m
limits:
cpu: "1"
memory: 4Gi
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
Expand Down
117 changes: 52 additions & 65 deletions pipelines/pingcap/tidb/latest/pull_br_integration_test.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,41 +8,40 @@ final GIT_FULL_REPO_NAME = 'pingcap/tidb'
final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh'
final POD_TEMPLATE_FILE = 'pipelines/pingcap/tidb/latest/pod-pull_br_integration_test.yaml'
final REFS = readJSON(text: params.JOB_SPEC).refs
final OCI_TAG_PD = component.computeArtifactOciTagFromPR('pd', REFS.base_ref, REFS.pulls[0].title, 'master')
final OCI_TAG_TIKV = component.computeArtifactOciTagFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, 'master')
final OCI_TAG_TIFLASH = component.computeArtifactOciTagFromPR('tiflash', REFS.base_ref, REFS.pulls[0].title, 'master')

prow.setPRDescription(REFS)
pipeline {
agent {
kubernetes {
namespace K8S_NAMESPACE
yamlFile POD_TEMPLATE_FILE
defaultContainer 'golang'
}
}
agent none
environment {
FILE_SERVER_URL = 'http://fileserver.pingcap.net'
OCI_ARTIFACT_HOST = 'hub-zot.pingcap.net/mirrors/hub'
}
options {
timeout(time: 90, unit: 'MINUTES')
parallelsAlwaysFailFast()
}
stages {
stage('Checkout') {
options { timeout(time: 5, unit: 'MINUTES') }
stage('Checkout & Prepare') {
agent {
kubernetes {
namespace K8S_NAMESPACE
yamlFile POD_TEMPLATE_FILE
defaultContainer 'golang'
}
}
options { timeout(time: 15, unit: 'MINUTES') }
steps {
dir("tidb") {
dir(REFS.repo) {
cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) {
retry(2) {
script {
prow.checkoutRefs(REFS)
}
}
}
}
}
}
stage('Prepare') {
steps {
dir('tidb') {

cache(path: "./bin", includes: '**/*', key: prow.getCacheKey('binary', REFS, 'br-integration-test')) {
// build br.test for integration test
// only build binarys if not exist, use the cached binarys if exist
Expand All @@ -53,6 +52,37 @@ pipeline {
./bin/tidb-server -V
"""
}
dir("bin") {
container("utils") {
script {
retry(2) {
sh label: "download tidb components", script: """
rm -rf ./*
${WORKSPACE}/scripts/artifacts/download_pingcap_oci_artifact.sh \\
--pd=${OCI_TAG_PD} \\
--tikv=${OCI_TAG_TIKV} \\
--tiflash=${OCI_TAG_TIFLASH}
"""
}
}
}
sh """
mv tiflash tiflash_dir
ln -s tiflash_dir/tiflash tiflash

ls -alh .
./pd-server -V
./tikv-server -V
./tiflash --version
"""
}
}

// cache workspace for matrix pods
cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}") {
sh """
touch rev-${REFS.pulls[0].sha}
"""
}
}
}
Expand All @@ -72,57 +102,14 @@ pipeline {
}
}
stages {
stage('Prepare') {
steps {
dir("tidb") {
cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS)) {
sh 'ls -lh'
}
}
dir("third_party_download") {
script {
def otherComponentBranch = component.computeBranchFromPR('other', REFS.base_ref, REFS.pulls[0].title, 'master')
retry(2) {
sh label: "download third_party", script: """
chmod +x ${WORKSPACE}/tidb/br/tests/*.sh
${WORKSPACE}/tidb/br/tests/download_integration_test_binaries.sh ${otherComponentBranch}
rm -rf bin/ && mkdir -p bin
mv third_bin/* bin/
ls -alh bin/
./bin/pd-server -V
./bin/tikv-server -V
./bin/tiflash --version
"""
}
}
}
dir('tidb') {
sh label: "check all tests added to group", script: """#!/usr/bin/env bash
chmod +x br/tests/*.sh
./br/tests/run_group_br_tests.sh others
"""
// build br.test for integration test
// only build binarys if not exist, use the cached binarys if exist
cache(path: "./bin", includes: '**/*', key: prow.getCacheKey('binary', REFS, 'br-integration-test')) {
sh label: "prepare", script: """
[ -f ./bin/tidb-server ] || (make failpoint-enable && make && make failpoint-disable)
[ -f ./bin/br.test ] || make build_for_br_integration_test
ls -alh ./bin
./bin/tidb-server -V
"""
}
sh label: "prepare", script: """
cp -r ../third_party_download/bin/* ./bin/
ls -alh ./bin
"""
}
}
}
stage("Test") {
environment { CODECOV_TOKEN = credentials('codecov-token-tidb') }
options { timeout(time: 60, unit: 'MINUTES') }
steps {
dir('tidb') {
dir(REFS.repo) {
cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}") {
sh "ls rev-${REFS.pulls[0].sha}"
}
sh label: "TEST_GROUP ${TEST_GROUP}", script: """#!/usr/bin/env bash
chmod +x br/tests/*.sh
./br/tests/run_group_br_tests.sh ${TEST_GROUP}
Expand All @@ -139,7 +126,7 @@ pipeline {
archiveArtifacts artifacts: "log-${TEST_GROUP}.tar.gz", fingerprint: true
}
success {
dir('tidb'){
dir(REFS.repo) {
sh label: "upload coverage", script: """
ls -alh /tmp/group_cover
gocovmerge /tmp/group_cover/cov.* > coverage.txt
Expand Down
61 changes: 32 additions & 29 deletions pipelines/pingcap/tidb/latest/pull_integration_e2e_test.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
final K8S_NAMESPACE = "jenkins-tidb"
final POD_TEMPLATE_FILE = 'pipelines/pingcap/tidb/latest/pod-pull_integration_e2e_test.yaml'
final REFS = readJSON(text: params.JOB_SPEC).refs
final OCI_TAG_PD = component.computeArtifactOciTagFromPR('pd', REFS.base_ref, REFS.pulls[0].title, 'master')
final OCI_TAG_TIKV = component.computeArtifactOciTagFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, 'master')
final OCI_TAG_TIFLASH = component.computeArtifactOciTagFromPR('tiflash', REFS.base_ref, REFS.pulls[0].title, 'master')
final OCI_TAG_TICDC_NEW = (REFS.base_ref == "feature/materialized_view" ? "release-8.5" : component.computeArtifactOciTagFromPR('ticdc', REFS.base_ref, REFS.pulls[0].title, 'master'))

prow.setPRDescription(REFS)
pipeline {
Expand All @@ -17,15 +21,15 @@ pipeline {
}
}
environment {
FILE_SERVER_URL = 'http://fileserver.pingcap.net'
OCI_ARTIFACT_HOST = 'hub-zot.pingcap.net/mirrors/hub' // cache mirror for us-docker.pkg.dev/pingcap-testing-account/hub
}
options {
timeout(time: 60, unit: 'MINUTES')
}
stages {
stage('Checkout') {
steps {
dir('tidb') {
dir(REFS.repo) {
cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) {
retry(2) {
script {
Expand All @@ -36,40 +40,39 @@ pipeline {
}
}
}
stage('Prepare') {
steps {
dir('tidb') {
script {
// Computes the branch name for downloading binaries based on the PR target branch and title.
def otherComponentBranch = component.computeBranchFromPR('other', REFS.base_ref, REFS.pulls[0].title, 'master')
retry(3) {
sh label: 'download binary', script: """
cd tests/integrationtest2 && ./download_integration_test_binaries.sh ${otherComponentBranch}
ls -alh third_bin/
./third_bin/tikv-server -V
./third_bin/pd-server -V
./third_bin/tiflash --version
./third_bin/cdc version
"""
}
}
}
}
}
stage('Tests') {
options { timeout(time: 45, unit: 'MINUTES') }
steps {
dir('tidb') {
sh label: 'test', script: """
cd tests/integrationtest2 && ./run-tests.sh
"""
dir("${REFS.repo}/tests/integrationtest2") {
dir("third_bin") {
container("utils") {
script {
retry(2) {
sh label: "download tidb components", script: """
${WORKSPACE}/scripts/artifacts/download_pingcap_oci_artifact.sh \
--pd=${OCI_TAG_PD} \
--tikv=${OCI_TAG_TIKV} \
--tiflash=${OCI_TAG_TIFLASH} \
--ticdc-new=${OCI_TAG_TICDC_NEW}
"""
}
}
}
sh '''
mv tiflash tiflash_dir
ln -s `pwd`/tiflash_dir/tiflash tiflash

./tikv-server -V
./pd-server -V
./tiflash --version
'''
}
sh './run-tests.sh'
}
}
post{
failure {
script {
archiveArtifacts(artifacts: 'tidb/tests/integrationtest2/logs/*.log', allowEmptyArchive: true)
}
archiveArtifacts(artifacts: 'tidb/tests/integrationtest2/logs/*.log', allowEmptyArchive: true)
}
}
}
Expand Down
Loading