Skip to content

Commit 1bb1091

Browse files
brianseedersbrianseeders
authored
[CI] Slack alerts - Elasticsearch snapshot failures (#64724)
1 parent ba0670d commit 1bb1091

File tree

2 files changed

+150
-34
lines changed

2 files changed

+150
-34
lines changed

‎.ci/es-snapshots/Jenkinsfile_verify_es‎

Lines changed: 39 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -21,41 +21,46 @@ def SNAPSHOT_MANIFEST = "https://storage.googleapis.com/kibana-ci-es-snapshots-d
2121

2222
kibanaPipeline(timeoutMinutes: 120) {
2323
catchErrors {
24-
retryable.enable(2)
25-
withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) {
26-
parallel([
27-
'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
28-
'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
29-
'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
30-
'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
31-
'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
32-
'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
33-
'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
34-
'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
35-
'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
36-
'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
37-
'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
38-
'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
39-
'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
40-
'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
41-
'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
42-
]),
43-
'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
44-
'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
45-
'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
46-
'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
47-
'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
48-
'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
49-
'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
50-
'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
51-
'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
52-
'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
53-
'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
54-
]),
55-
])
56-
}
24+
slackNotifications.onFailure(
25+
title: ":broken_heart: *<${env.BUILD_URL}|[${SNAPSHOT_VERSION}] ES Snapshot Verification Failure>*",
26+
message: ":broken_heart: [${SNAPSHOT_VERSION}] ES Snapshot Verification Failure",
27+
) {
28+
retryable.enable(2)
29+
withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) {
30+
parallel([
31+
'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
32+
'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
33+
'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
34+
'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
35+
'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
36+
'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
37+
'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
38+
'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
39+
'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
40+
'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
41+
'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
42+
'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
43+
'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
44+
'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
45+
'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
46+
]),
47+
'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
48+
'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
49+
'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
50+
'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
51+
'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
52+
'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
53+
'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
54+
'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
55+
'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
56+
'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
57+
'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
58+
]),
59+
])
60+
}
5761

58-
promoteSnapshot(SNAPSHOT_VERSION, SNAPSHOT_ID)
62+
promoteSnapshot(SNAPSHOT_VERSION, SNAPSHOT_ID)
63+
}
5964
}
6065

6166
kibanaPipeline.sendMail()

‎vars/slackNotifications.groovy‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
def getFailedBuildBlocks() {
2+
def messages = [
3+
getFailedSteps(),
4+
getTestFailures(),
5+
]
6+
7+
return messages
8+
.findAll { !!it } // No blank strings
9+
.collect { markdownBlock(it) }
10+
}
11+
12+
def dividerBlock() {
13+
return [ type: "divider" ]
14+
}
15+
16+
def markdownBlock(message) {
17+
return [
18+
type: "section",
19+
text: [
20+
type: "mrkdwn",
21+
text: message,
22+
],
23+
]
24+
}
25+
26+
def contextBlock(message) {
27+
return [
28+
type: "context",
29+
elements: [
30+
[
31+
type: 'mrkdwn',
32+
text: message,
33+
]
34+
]
35+
]
36+
}
37+
38+
def getFailedSteps() {
39+
try {
40+
def steps = jenkinsApi.getFailedSteps()?.findAll { step ->
41+
step.displayName != 'Check out from version control'
42+
}
43+
44+
if (steps?.size() > 0) {
45+
def list = steps.collect { "• <${it.logs}|${it.displayName}>" }.join("\n")
46+
return "*Failed Steps*\n${list}"
47+
}
48+
} catch (ex) {
49+
buildUtils.printStacktrace(ex)
50+
print "Error retrieving failed pipeline steps for PR comment, will skip this section"
51+
}
52+
53+
return ""
54+
}
55+
56+
def getTestFailures() {
57+
def failures = testUtils.getFailures()
58+
if (!failures) {
59+
return ""
60+
}
61+
62+
def messages = []
63+
messages << "*Test Failures*"
64+
65+
def list = failures.collect { "• <${it.url}|${it.fullDisplayName}>" }.join("\n")
66+
return "*Test Failures*\n${list}"
67+
}
68+
69+
def sendFailedBuild(Map params = [:]) {
70+
def displayName = "${env.JOB_NAME} ${env.BUILD_DISPLAY_NAME}"
71+
72+
def config = [
73+
channel: '#kibana-operations',
74+
title: ":broken_heart: *<${env.BUILD_URL}|${displayName}>*",
75+
message: ":broken_heart: ${displayName}",
76+
color: 'danger',
77+
icon: ':jenkins:',
78+
username: 'Kibana Operations',
79+
context: contextBlock("${displayName} · <https://ci.kibana.dev/${env.JOB_BASE_NAME}/${env.BUILD_NUMBER}|ci.kibana.dev>"),
80+
] + params
81+
82+
def blocks = [markdownBlock(config.title)]
83+
getFailedBuildBlocks().each { blocks << it }
84+
blocks << dividerBlock()
85+
blocks << config.context
86+
87+
slackSend(
88+
channel: config.channel,
89+
username: config.username,
90+
iconEmoji: config.icon,
91+
color: config.color,
92+
message: config.message,
93+
blocks: blocks
94+
)
95+
}
96+
97+
def onFailure(Map options = [:], Closure closure) {
98+
// try/finally will NOT work here, because the build status will not have been changed to ERROR when the finally{} block executes
99+
catchError {
100+
closure()
101+
}
102+
103+
def status = buildUtils.getBuildStatus()
104+
if (status != "SUCCESS" && status != "UNSTABLE") {
105+
catchErrors {
106+
sendFailedBuild(options)
107+
}
108+
}
109+
}
110+
111+
return this

0 commit comments

Comments
 (0)