Skip to content

Commit f8df526

Browse files
breskebybreskeby
authored
[Build] Breakup check and bwc build jobs further down (#129279) (#129482)
* [Build] Breakup BWC Jobs per version as we do for check tasks * Break checks into 6 parts
1 parent 8efeec2 commit f8df526

File tree

13 files changed

+197
-3
lines changed

13 files changed

+197
-3
lines changed

‎.buildkite/pipelines/intake.template.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,15 @@ steps:
4848
image: family/elasticsearch-ubuntu-2004
4949
machineType: n1-standard-32
5050
buildDirectory: /dev/shm/bk
51+
- label: part6
52+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart6
53+
timeout_in_minutes: 300
54+
agents:
55+
provider: gcp
56+
image: family/elasticsearch-ubuntu-2004
57+
machineType: n1-standard-32
58+
buildDirectory: /dev/shm/bk
59+
5160
- group: bwc-snapshots
5261
steps:
5362
- label: "{{matrix.BWC_VERSION}} / bwc-snapshots"

‎.buildkite/pipelines/intake.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,14 @@ steps:
4949
image: family/elasticsearch-ubuntu-2004
5050
machineType: n1-standard-32
5151
buildDirectory: /dev/shm/bk
52+
- label: part6
53+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart6
54+
timeout_in_minutes: 300
55+
agents:
56+
provider: gcp
57+
image: family/elasticsearch-ubuntu-2004
58+
machineType: n1-standard-32
59+
buildDirectory: /dev/shm/bk
5260
- group: bwc-snapshots
5361
steps:
5462
- label: "{{matrix.BWC_VERSION}} / bwc-snapshots"

‎.buildkite/pipelines/lucene-snapshot/run-tests.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,14 @@ steps:
4848
image: family/elasticsearch-ubuntu-2004
4949
machineType: custom-32-98304
5050
buildDirectory: /dev/shm/bk
51+
- label: part6
52+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart6
53+
timeout_in_minutes: 300
54+
agents:
55+
provider: gcp
56+
image: family/elasticsearch-ubuntu-2004
57+
machineType: custom-32-98304
58+
buildDirectory: /dev/shm/bk
5159
- group: bwc-snapshots
5260
steps:
5361
- label: "{{matrix.BWC_VERSION}} / bwc-snapshots"

‎.buildkite/pipelines/periodic-platform-support.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ steps:
5151
- checkPart3
5252
- checkPart4
5353
- checkPart5
54+
- checkPart6
5455
- checkRestCompat
5556
agents:
5657
provider: gcp
@@ -77,6 +78,7 @@ steps:
7778
- checkPart3
7879
- checkPart4
7980
- checkPart5
81+
- checkPart6
8082
- checkRestCompat
8183
agents:
8284
provider: aws

‎.buildkite/pipelines/periodic.template.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ steps:
5151
- checkPart3
5252
- checkPart4
5353
- checkPart5
54+
- checkPart6
5455
- checkRestCompat
5556
agents:
5657
provider: gcp
@@ -95,6 +96,7 @@ steps:
9596
- checkPart3
9697
- checkPart4
9798
- checkPart5
99+
- checkPart6
98100
- checkRestCompat
99101
agents:
100102
provider: gcp

‎.buildkite/pipelines/periodic.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -774,6 +774,7 @@ steps:
774774
- checkPart3
775775
- checkPart4
776776
- checkPart5
777+
- checkPart6
777778
- checkRestCompat
778779
agents:
779780
provider: gcp
@@ -818,6 +819,7 @@ steps:
818819
- checkPart3
819820
- checkPart4
820821
- checkPart5
822+
- checkPart6
821823
- checkRestCompat
822824
agents:
823825
provider: gcp

‎.buildkite/pipelines/pull-request/bwc-snapshots.yml

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@ config:
66
steps:
77
- group: bwc-snapshots
88
steps:
9-
- label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
10-
key: "bwc-snapshots"
11-
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest
9+
- label: "{{matrix.BWC_VERSION}} / Part 1 / bwc-snapshots"
10+
key: "bwc-snapshots-part1"
11+
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTestPart1
1212
timeout_in_minutes: 300
1313
matrix:
1414
setup:
@@ -18,3 +18,64 @@ steps:
1818
image: family/elasticsearch-ubuntu-2004
1919
machineType: n1-standard-32
2020
buildDirectory: /dev/shm/bk
21+
- label: "{{matrix.BWC_VERSION}} / Part 2 / bwc-snapshots"
22+
key: "bwc-snapshots-part2"
23+
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTestPart2
24+
timeout_in_minutes: 300
25+
matrix:
26+
setup:
27+
BWC_VERSION: $SNAPSHOT_BWC_VERSIONS
28+
agents:
29+
provider: gcp
30+
image: family/elasticsearch-ubuntu-2004
31+
machineType: n1-standard-32
32+
buildDirectory: /dev/shm/bk
33+
- label: "{{matrix.BWC_VERSION}} / Part 3 / bwc-snapshots"
34+
key: "bwc-snapshots-part3"
35+
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTestPart3
36+
timeout_in_minutes: 300
37+
matrix:
38+
setup:
39+
BWC_VERSION: $SNAPSHOT_BWC_VERSIONS
40+
agents:
41+
provider: gcp
42+
image: family/elasticsearch-ubuntu-2004
43+
machineType: n1-standard-32
44+
buildDirectory: /dev/shm/bk
45+
- label: "{{matrix.BWC_VERSION}} / Part 4 / bwc-snapshots"
46+
key: "bwc-snapshots-part4"
47+
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTestPart5
48+
timeout_in_minutes: 300
49+
matrix:
50+
setup:
51+
BWC_VERSION: $SNAPSHOT_BWC_VERSIONS
52+
agents:
53+
provider: gcp
54+
image: family/elasticsearch-ubuntu-2004
55+
machineType: n1-standard-32
56+
buildDirectory: /dev/shm/bk
57+
- label: "{{matrix.BWC_VERSION}} / Part 5 / bwc-snapshots"
58+
key: "bwc-snapshots-part5"
59+
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTestPart5
60+
timeout_in_minutes: 300
61+
matrix:
62+
setup:
63+
BWC_VERSION: $SNAPSHOT_BWC_VERSIONS
64+
agents:
65+
provider: gcp
66+
image: family/elasticsearch-ubuntu-2004
67+
machineType: n1-standard-32
68+
buildDirectory: /dev/shm/bk
69+
- label: "{{matrix.BWC_VERSION}} / Part 6 / bwc-snapshots"
70+
key: "bwc-snapshots-part6"
71+
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTestPart6
72+
timeout_in_minutes: 300
73+
matrix:
74+
setup:
75+
BWC_VERSION: $SNAPSHOT_BWC_VERSIONS
76+
agents:
77+
provider: gcp
78+
image: family/elasticsearch-ubuntu-2004
79+
machineType: n1-standard-32
80+
buildDirectory: /dev/shm/bk
81+

‎.buildkite/pipelines/pull-request/part-6-arm.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
config:
2+
allow-labels: "test-arm"
3+
steps:
4+
- label: part-6-arm
5+
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart6
6+
timeout_in_minutes: 300
7+
agents:
8+
provider: aws
9+
imagePrefix: elasticsearch-ubuntu-2004-aarch64
10+
instanceType: m6g.8xlarge
11+
diskSizeGb: 350
12+
diskType: gp3
13+
diskName: /dev/sda1

‎.buildkite/pipelines/pull-request/part-6-fips.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
config:
2+
allow-labels:
3+
- Team:Security
4+
- test-fips
5+
steps:
6+
- label: part-6-fips
7+
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart6
8+
timeout_in_minutes: 300
9+
agents:
10+
provider: gcp
11+
image: family/elasticsearch-ubuntu-2004
12+
machineType: custom-32-98304
13+
buildDirectory: /dev/shm/bk

‎.buildkite/pipelines/pull-request/part-6-windows.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
config:
2+
allow-labels: "test-windows"
3+
steps:
4+
- label: part-6-windows
5+
command: .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
6+
timeout_in_minutes: 300
7+
agents:
8+
provider: gcp
9+
image: family/elasticsearch-windows-2022
10+
machineType: custom-32-98304
11+
diskType: pd-ssd
12+
diskSizeGb: 350
13+
env:
14+
GRADLE_TASK: checkPart6

‎.buildkite/pipelines/pull-request/part-6.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
steps:
2+
- label: part-6
3+
command: |
4+
.ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart6
5+
timeout_in_minutes: 300
6+
agents:
7+
provider: gcp
8+
image: family/elasticsearch-ubuntu-2004
9+
machineType: custom-32-98304
10+
buildDirectory: /dev/shm/bk

‎.buildkite/pipelines/pull-request/release-tests.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ steps:
1515
- checkPart3
1616
- checkPart4
1717
- checkPart5
18+
- checkPart6
1819
agents:
1920
provider: gcp
2021
image: family/elasticsearch-ubuntu-2004

‎build.gradle

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -354,27 +354,71 @@ allprojects {
354354
dependsOn 'assemble'
355355
}
356356
}
357+
358+
tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart4") { name ->
359+
if(name.endsWith("#bwcTestPart4")) {
360+
project.getTasks().register(name) {
361+
task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part4") })
362+
}
363+
}
364+
}
357365
} else if (project.path == ":x-pack:plugin" || project.path.contains("ql") || project.path.contains("smoke-test")) {
358366
tasks.register('checkPart3') {
359367
dependsOn 'check'
360368
withReleaseBuild {
361369
dependsOn 'assemble'
362370
}
363371
}
372+
373+
tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart3") { name ->
374+
if(name.endsWith("#bwcTestPart3")) {
375+
project.getTasks().register(name) {
376+
task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part3") })
377+
}
378+
}
379+
}
364380
} else if (project.path.contains("multi-node")) {
365381
tasks.register('checkPart5') {
366382
dependsOn 'check'
367383
withReleaseBuild {
368384
dependsOn 'assemble'
369385
}
370386
}
387+
tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart5") { name ->
388+
if(name.endsWith("#bwcTestPart5")) {
389+
project.getTasks().register(name) {
390+
task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part5") })
391+
}
392+
}
393+
}
371394
} else {
372395
tasks.register('checkPart2') {
373396
dependsOn 'check'
374397
withReleaseBuild {
375398
dependsOn 'assemble'
376399
}
377400
}
401+
tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart2") { name ->
402+
if(name.endsWith("#bwcTestPart2")) {
403+
project.getTasks().register(name) {
404+
task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part2") })
405+
}
406+
}
407+
}
408+
}
409+
} else if(project.path.startsWith(":qa:")) {
410+
tasks.register('checkPart6') {
411+
dependsOn 'check'
412+
withReleaseBuild {
413+
dependsOn 'assemble'
414+
}
415+
}
416+
tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart6") { name ->
417+
if(name.endsWith("#bwcTestPart6")) {
418+
project.getTasks().register(name) {
419+
task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part6") })
420+
}
421+
}
378422
}
379423
} else {
380424
tasks.register('checkPart1') {
@@ -383,6 +427,13 @@ allprojects {
383427
dependsOn 'assemble'
384428
}
385429
}
430+
tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart1") { name ->
431+
if(name.endsWith("#bwcTestPart1")) {
432+
project.getTasks().register(name) {
433+
task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part1") })
434+
}
435+
}
436+
}
386437
}
387438
tasks.register('functionalTests') {
388439
dependsOn 'check'

0 commit comments

Comments
 (0)