33library ' kibana-pipeline-library'
44kibanaLibrary. load()
55
6- def CI_GROUP_PARAM = params. CI_GROUP
6+ def TASK_PARAM = params . TASK ?: params. CI_GROUP
77
88// Looks like 'oss:ciGroup:1', 'oss:firefoxSmoke'
9- def JOB_PARTS = CI_GROUP_PARAM . split(' :' )
9+ def JOB_PARTS = TASK_PARAM . split(' :' )
1010def IS_XPACK = JOB_PARTS [0 ] == ' xpack'
11- def JOB = JOB_PARTS [ 1 ]
11+ def JOB = JOB_PARTS . size() > 1 ? JOB_PARTS [ 1 ] : JOB_PARTS [ 0 ]
1212def CI_GROUP = JOB_PARTS . size() > 2 ? JOB_PARTS [2 ] : ' '
1313def EXECUTIONS = params. NUMBER_EXECUTIONS . toInteger()
1414def AGENT_COUNT = getAgentCount(EXECUTIONS )
15-
16- def worker = getWorkerFromParams(IS_XPACK , JOB , CI_GROUP )
17-
18- def workerFailures = []
15+ def NEED_BUILD = JOB != ' jestIntegration' && JOB != ' apiIntegration'
1916
2017currentBuild. displayName + = trunc(" ${ params.GITHUB_OWNER} :${ params.branch_specifier} " , 24 )
2118currentBuild. description = " ${ params.CI_GROUP} <br />Agents: ${ AGENT_COUNT} <br />Executions: ${ params.NUMBER_EXECUTIONS} "
2219
2320kibanaPipeline(timeoutMinutes : 180 ) {
2421 def agents = [:]
22+ def workerFailures = []
23+
24+ def worker = getWorkerFromParams(IS_XPACK , JOB , CI_GROUP )
25+
2526 for (def agentNumber = 1 ; agentNumber <= AGENT_COUNT ; agentNumber++ ) {
26- def agentNumberInside = agentNumber
2727 def agentExecutions = floor(EXECUTIONS / AGENT_COUNT ) + (agentNumber <= EXECUTIONS % AGENT_COUNT ? 1 : 0 )
28+
2829 agents[" agent-${ agentNumber} " ] = {
29- catchErrors {
30- print " Agent ${ agentNumberInside} - ${ agentExecutions} executions"
31-
32- withEnv([
33- ' IGNORE_SHIP_CI_STATS_ERROR=true' ,
34- ]) {
35- workers. functional(' flaky-test-runner' , {
36- if (! IS_XPACK ) {
37- kibanaPipeline. buildOss()
38- if (CI_GROUP == ' 1' ) {
39- runbld(" ./test/scripts/jenkins_build_kbn_sample_panel_action.sh" , " Build kbn tp sample panel action for ciGroup1" )
40- }
41- } else {
42- kibanaPipeline. buildXpack()
43- }
44- }, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))()
45- }
46- }
30+ agentProcess(
31+ agentNumber : agentNumber,
32+ agentExecutions : agentExecutions,
33+ worker : worker,
34+ workerFailures : workerFailures,
35+ needBuild : NEED_BUILD ,
36+ isXpack : IS_XPACK ,
37+ ciGroup : CI_GROUP
38+ )
4739 }
4840 }
4941
@@ -59,14 +51,70 @@ kibanaPipeline(timeoutMinutes: 180) {
5951 }
6052}
6153
54+ def agentProcess (Map params = [:]) {
55+ def config = [
56+ agentNumber : 1 ,
57+ agentExecutions : 0 ,
58+ worker : {},
59+ workerFailures : [],
60+ needBuild : false ,
61+ isXpack : false ,
62+ ciGroup : null ,
63+ ] + params
64+
65+ catchErrors {
66+ print " Agent ${ config.agentNumber} - ${ config.agentExecutions} executions"
67+
68+ withEnv([
69+ ' IGNORE_SHIP_CI_STATS_ERROR=true' ,
70+ ]) {
71+ kibanaPipeline. withTasks([
72+ parallel : 20 ,
73+ ]) {
74+ task {
75+ if (config. needBuild) {
76+ if (! config. isXpack) {
77+ kibanaPipeline. buildOss()
78+ } else {
79+ kibanaPipeline. buildXpack()
80+ }
81+ }
82+
83+ for (def i = 0 ; i < config. agentExecutions; i++ ) {
84+ def taskNumber = i
85+ task({
86+ withEnv([
87+ " REMOVE_KIBANA_INSTALL_DIR=1" ,
88+ ]) {
89+ catchErrors {
90+ try {
91+ config. worker()
92+ } catch (ex) {
93+ config. workerFailures << " agent-${ config.agentNumber} -${ taskNumber} "
94+ throw ex
95+ }
96+ }
97+ }
98+ })
99+ }
100+ }
101+ }
102+ }
103+ }
104+ }
105+
62106def getWorkerFromParams (isXpack , job , ciGroup ) {
63107 if (! isXpack) {
64108 if (job == ' accessibility' ) {
65109 return kibanaPipeline. functionalTestProcess(' kibana-accessibility' , ' ./test/scripts/jenkins_accessibility.sh' )
66110 } else if (job == ' firefoxSmoke' ) {
67111 return kibanaPipeline. functionalTestProcess(' firefoxSmoke' , ' ./test/scripts/jenkins_firefox_smoke.sh' )
68- } else if (job == ' visualRegression' ) {
112+ } else if (job == ' visualRegression' ) {
69113 return kibanaPipeline. functionalTestProcess(' visualRegression' , ' ./test/scripts/jenkins_visual_regression.sh' )
114+ } else if (job == ' jestIntegration' ) {
115+ return kibanaPipeline. scriptTaskDocker(' Jest Integration Tests' , ' test/scripts/test/jest_integration.sh' )
116+ } else if (job == ' apiIntegration' ) {
117+ return kibanaPipeline. scriptTask(' API Integration Tests' , ' test/scripts/test/api_integration.sh' )
70118 } else {
71119 return kibanaPipeline. ossCiGroupProcess(ciGroup)
72120 }
@@ -76,45 +124,16 @@ def getWorkerFromParams(isXpack, job, ciGroup) {
76124 return kibanaPipeline. functionalTestProcess(' xpack-accessibility' , ' ./test/scripts/jenkins_xpack_accessibility.sh' )
77125 } else if (job == ' firefoxSmoke' ) {
78126 return kibanaPipeline. functionalTestProcess(' xpack-firefoxSmoke' , ' ./test/scripts/jenkins_xpack_firefox_smoke.sh' )
79- } else if (job == ' visualRegression' ) {
127+ } else if (job == ' visualRegression' ) {
80128 return kibanaPipeline. functionalTestProcess(' xpack-visualRegression' , ' ./test/scripts/jenkins_xpack_visual_regression.sh' )
81129 } else {
82130 return kibanaPipeline. xpackCiGroupProcess(ciGroup)
83131 }
84132}
85133
86- def getWorkerMap (agentNumber , numberOfExecutions , worker , workerFailures , maxWorkerProcesses = 12 ) {
87- def workerMap = [:]
88- def numberOfWorkers = Math . min(numberOfExecutions, maxWorkerProcesses)
89-
90- for (def i = 1 ; i <= numberOfWorkers; i++ ) {
91- def workerExecutions = floor(numberOfExecutions/ numberOfWorkers + (i <= numberOfExecutions% numberOfWorkers ? 1 : 0 ))
92-
93- workerMap[" agent-${ agentNumber} -worker-${ i} " ] = { workerNumber ->
94- for (def j = 0 ; j < workerExecutions; j++ ) {
95- print " Execute agent-${ agentNumber} worker-${ workerNumber} : ${ j} "
96- withEnv([
97- " REMOVE_KIBANA_INSTALL_DIR=1" ,
98- ]) {
99- catchErrors {
100- try {
101- worker(workerNumber)
102- } catch (ex) {
103- workerFailures << " agent-${ agentNumber} worker-${ workerNumber} -${ j} "
104- throw ex
105- }
106- }
107- }
108- }
109- }
110- }
111-
112- return workerMap
113- }
114-
115134def getAgentCount (executions ) {
116- // Increase agent count every 24 worker processess, up to 3 agents maximum
117- return Math . min(3 , 1 + floor(executions/ 24 ))
135+ // Increase agent count every 20 worker processess, up to 3 agents maximum
136+ return Math . min(3 , 1 + floor(executions/ 20 ))
118137}
119138
120139def trunc (str , length ) {
0 commit comments