Skip to content

Commit ae3827e

Browse files
harry671003harry671003
committed
Fix a flaky test
Signed-off-by: 🌲 Harry 🌊 John 🏔 <johrry@amazon.com>
1 parent 8309bbc commit ae3827e

File tree

1 file changed

+157
-133
lines changed

1 file changed

+157
-133
lines changed

‎pkg/alertmanager/distributor_test.go

Lines changed: 157 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -40,75 +40,82 @@ func TestDistributor_DistributeRequest(t *testing.T) {
4040
isRead bool
4141
isDelete bool
4242
expStatusCode int
43-
expectedTotalCalls int
43+
expectedMaxCalls int
44+
expectedMinCalls int
4445
headersNotPreserved bool
4546
route string
4647
// Paths where responses are merged, we need to supply a valid response body.
4748
// Note that the actual merging logic is tested elsewhere (merger_test.go).
4849
responseBody []byte
4950
}{
5051
{
51-
name: "Write /alerts, Simple AM request, all AM healthy",
52-
numAM: 4,
53-
numHappyAM: 4,
54-
replicationFactor: 3,
55-
expStatusCode: http.StatusOK,
56-
expectedTotalCalls: 3,
57-
route: "/alerts",
52+
name: "Write /alerts, Simple AM request, all AM healthy",
53+
numAM: 4,
54+
numHappyAM: 4,
55+
replicationFactor: 3,
56+
expStatusCode: http.StatusOK,
57+
expectedMinCalls: 3,
58+
expectedMaxCalls: 3,
59+
route: "/alerts",
5860
}, {
5961
name: "Write /alerts, Less than quorum AM available",
6062
numAM: 1,
6163
numHappyAM: 1,
6264
replicationFactor: 3,
6365
expStatusCode: http.StatusInternalServerError,
64-
expectedTotalCalls: 0,
66+
expectedMinCalls: 0,
67+
expectedMaxCalls: 0,
6568
headersNotPreserved: true, // There is nothing to preserve since it does not hit any AM.
6669
route: "/alerts",
6770
}, {
68-
name: "Write /alerts, Less than quorum AM succeed",
69-
numAM: 5,
70-
numHappyAM: 3, // Though we have 3 happy, it will hit >1 unhappy AM.
71-
replicationFactor: 3,
72-
expStatusCode: http.StatusInternalServerError,
73-
expectedTotalCalls: 3,
74-
route: "/alerts",
71+
name: "Write /alerts, Less than quorum AM succeed",
72+
numAM: 5,
73+
numHappyAM: 3, // Though we have 3 happy, it will hit >1 unhappy AM.
74+
replicationFactor: 3,
75+
expStatusCode: http.StatusInternalServerError,
76+
expectedMinCalls: 3,
77+
expectedMaxCalls: 3,
78+
route: "/alerts",
7579
}, {
76-
name: "Read /v1/alerts is sent to 3 AMs",
77-
numAM: 5,
78-
numHappyAM: 5,
79-
replicationFactor: 3,
80-
isRead: true,
81-
expStatusCode: http.StatusOK,
82-
expectedTotalCalls: 3,
83-
route: "/v1/alerts",
84-
responseBody: []byte(`{"status":"success","data":[]}`),
80+
name: "Read /v1/alerts is sent to 3 AMs",
81+
numAM: 5,
82+
numHappyAM: 5,
83+
replicationFactor: 3,
84+
isRead: true,
85+
expStatusCode: http.StatusOK,
86+
expectedMinCalls: 3,
87+
expectedMaxCalls: 3,
88+
route: "/v1/alerts",
89+
responseBody: []byte(`{"status":"success","data":[]}`),
8590
}, {
86-
name: "Read /v2/alerts is sent to 3 AMs",
87-
numAM: 5,
88-
numHappyAM: 5,
89-
replicationFactor: 3,
90-
isRead: true,
91-
expStatusCode: http.StatusOK,
92-
expectedTotalCalls: 3,
93-
route: "/v2/alerts",
94-
responseBody: []byte(`[]`),
91+
name: "Read /v2/alerts is sent to 3 AMs",
92+
numAM: 5,
93+
numHappyAM: 5,
94+
replicationFactor: 3,
95+
isRead: true,
96+
expStatusCode: http.StatusOK,
97+
expectedMinCalls: 3,
98+
expectedMaxCalls: 3,
99+
route: "/v2/alerts",
100+
responseBody: []byte(`[]`),
95101
}, {
96-
name: "Read /v2/alerts/groups is sent to 3 AMs",
97-
numAM: 5,
98-
numHappyAM: 5,
99-
replicationFactor: 3,
100-
isRead: true,
101-
expStatusCode: http.StatusOK,
102-
expectedTotalCalls: 3,
103-
route: "/v2/alerts/groups",
104-
responseBody: []byte(`[]`),
102+
name: "Read /v2/alerts/groups is sent to 3 AMs",
103+
numAM: 5,
104+
numHappyAM: 5,
105+
replicationFactor: 3,
106+
isRead: true,
107+
expStatusCode: http.StatusOK,
108+
expectedMinCalls: 3,
109+
expectedMaxCalls: 3,
110+
route: "/v2/alerts/groups",
111+
responseBody: []byte(`[]`),
105112
}, {
106113
name: "Read /v1/alerts/groups not supported",
107114
numAM: 5,
108115
numHappyAM: 5,
109116
replicationFactor: 3,
110117
expStatusCode: http.StatusNotFound,
111-
expectedTotalCalls: 0,
118+
expectedMaxCalls: 0,
112119
headersNotPreserved: true,
113120
route: "/v1/alerts/groups",
114121
}, {
@@ -117,128 +124,142 @@ func TestDistributor_DistributeRequest(t *testing.T) {
117124
numHappyAM: 5,
118125
replicationFactor: 3,
119126
expStatusCode: http.StatusNotFound,
120-
expectedTotalCalls: 0,
127+
expectedMinCalls: 0,
128+
expectedMaxCalls: 0,
121129
headersNotPreserved: true,
122130
route: "/alerts/groups",
123131
}, {
124-
name: "Read /v1/silences is sent to 3 AMs",
125-
numAM: 5,
126-
numHappyAM: 5,
127-
replicationFactor: 3,
128-
isRead: true,
129-
expStatusCode: http.StatusOK,
130-
expectedTotalCalls: 3,
131-
route: "/v1/silences",
132-
responseBody: []byte(`{"status":"success","data":[]}`),
132+
name: "Read /v1/silences is sent to 3 AMs",
133+
numAM: 5,
134+
numHappyAM: 5,
135+
replicationFactor: 3,
136+
isRead: true,
137+
expStatusCode: http.StatusOK,
138+
expectedMinCalls: 3,
139+
expectedMaxCalls: 3,
140+
route: "/v1/silences",
141+
responseBody: []byte(`{"status":"success","data":[]}`),
133142
}, {
134-
name: "Read /v2/silences is sent to 3 AMs",
135-
numAM: 5,
136-
numHappyAM: 5,
137-
replicationFactor: 3,
138-
isRead: true,
139-
expStatusCode: http.StatusOK,
140-
expectedTotalCalls: 3,
141-
route: "/v2/silences",
142-
responseBody: []byte(`[]`),
143+
name: "Read /v2/silences is sent to 3 AMs",
144+
numAM: 5,
145+
numHappyAM: 5,
146+
replicationFactor: 3,
147+
isRead: true,
148+
expStatusCode: http.StatusOK,
149+
expectedMinCalls: 3,
150+
expectedMaxCalls: 3,
151+
route: "/v2/silences",
152+
responseBody: []byte(`[]`),
143153
}, {
144-
name: "Write /silences is sent to only 1 AM",
145-
numAM: 5,
146-
numHappyAM: 5,
147-
replicationFactor: 3,
148-
expStatusCode: http.StatusOK,
149-
expectedTotalCalls: 1,
150-
route: "/silences",
154+
name: "Write /silences is sent to only 1 AM",
155+
numAM: 5,
156+
numHappyAM: 5,
157+
replicationFactor: 3,
158+
expStatusCode: http.StatusOK,
159+
expectedMinCalls: 1,
160+
expectedMaxCalls: 1,
161+
route: "/silences",
151162
}, {
152-
name: "Read /v1/silence/id is sent to 3 AMs",
153-
numAM: 5,
154-
numHappyAM: 5,
155-
replicationFactor: 3,
156-
isRead: true,
157-
expStatusCode: http.StatusOK,
158-
expectedTotalCalls: 3,
159-
route: "/v1/silence/id",
160-
responseBody: []byte(`{"status":"success","data":{"id":"aaa","updatedAt":"2020-01-01T00:00:00Z"}}`),
163+
name: "Read /v1/silence/id is sent to 3 AMs",
164+
numAM: 5,
165+
numHappyAM: 5,
166+
replicationFactor: 3,
167+
isRead: true,
168+
expStatusCode: http.StatusOK,
169+
expectedMinCalls: 3,
170+
expectedMaxCalls: 3,
171+
route: "/v1/silence/id",
172+
responseBody: []byte(`{"status":"success","data":{"id":"aaa","updatedAt":"2020-01-01T00:00:00Z"}}`),
161173
}, {
162-
name: "Read /v2/silence/id is sent to 3 AMs",
163-
numAM: 5,
164-
numHappyAM: 5,
165-
replicationFactor: 3,
166-
isRead: true,
167-
expStatusCode: http.StatusOK,
168-
expectedTotalCalls: 3,
169-
route: "/v2/silence/id",
170-
responseBody: []byte(`{"id":"aaa","updatedAt":"2020-01-01T00:00:00Z"}`),
174+
name: "Read /v2/silence/id is sent to 3 AMs",
175+
numAM: 5,
176+
numHappyAM: 5,
177+
replicationFactor: 3,
178+
isRead: true,
179+
expStatusCode: http.StatusOK,
180+
expectedMinCalls: 3,
181+
expectedMaxCalls: 3,
182+
route: "/v2/silence/id",
183+
responseBody: []byte(`{"id":"aaa","updatedAt":"2020-01-01T00:00:00Z"}`),
171184
},
172185
{
173186
name: "Write /silence/id not supported",
174187
numAM: 5,
175188
numHappyAM: 5,
176189
replicationFactor: 3,
177190
expStatusCode: http.StatusNotFound,
178-
expectedTotalCalls: 0,
191+
expectedMinCalls: 0,
192+
expectedMaxCalls: 0,
179193
headersNotPreserved: true,
180194
route: "/silence/id",
181195
}, {
182-
name: "Delete /silence/id is sent to only 1 AM",
183-
numAM: 5,
184-
numHappyAM: 5,
185-
replicationFactor: 3,
186-
isDelete: true,
187-
expStatusCode: http.StatusOK,
188-
expectedTotalCalls: 1,
189-
route: "/silence/id",
196+
name: "Delete /silence/id is sent to only 1 AM",
197+
numAM: 5,
198+
numHappyAM: 5,
199+
replicationFactor: 3,
200+
isDelete: true,
201+
expStatusCode: http.StatusOK,
202+
expectedMinCalls: 1,
203+
expectedMaxCalls: 1,
204+
route: "/silence/id",
190205
}, {
191-
name: "Read /status is sent to only 1 AM",
192-
numAM: 5,
193-
numHappyAM: 5,
194-
replicationFactor: 3,
195-
isRead: true,
196-
expStatusCode: http.StatusOK,
197-
expectedTotalCalls: 1,
198-
route: "/status",
206+
name: "Read /status is sent to only 1 AM",
207+
numAM: 5,
208+
numHappyAM: 5,
209+
replicationFactor: 3,
210+
isRead: true,
211+
expStatusCode: http.StatusOK,
212+
expectedMinCalls: 1,
213+
expectedMaxCalls: 1,
214+
route: "/status",
199215
}, {
200-
name: "Read /status should try all alert managers on error",
201-
numAM: 3,
202-
numHappyAM: 0,
203-
replicationFactor: 3,
204-
isRead: true,
205-
expStatusCode: http.StatusInternalServerError,
206-
expectedTotalCalls: 3,
207-
route: "/status",
216+
name: "Read /status should try all alert managers on error",
217+
numAM: 3,
218+
numHappyAM: 0,
219+
replicationFactor: 3,
220+
isRead: true,
221+
expStatusCode: http.StatusInternalServerError,
222+
expectedMinCalls: 3,
223+
expectedMaxCalls: 3,
224+
route: "/status",
208225
}, {
209-
name: "Read /status is sent to 3 AM when 2 are not happy",
210-
numAM: 3,
211-
numHappyAM: 1,
212-
replicationFactor: 3,
213-
isRead: true,
214-
expStatusCode: http.StatusOK,
215-
expectedTotalCalls: 3,
216-
route: "/status",
226+
name: "Read /status is sent to atleast 2 AM when 2 are not happy",
227+
numAM: 3,
228+
numHappyAM: 1,
229+
replicationFactor: 3,
230+
isRead: true,
231+
expStatusCode: http.StatusOK,
232+
expectedMinCalls: 2,
233+
expectedMaxCalls: 3,
234+
route: "/status",
217235
}, {
218236
name: "Write /status not supported",
219237
numAM: 5,
220238
numHappyAM: 5,
221239
replicationFactor: 3,
222240
expStatusCode: http.StatusNotFound,
223-
expectedTotalCalls: 0,
241+
expectedMinCalls: 0,
242+
expectedMaxCalls: 0,
224243
headersNotPreserved: true,
225244
route: "/status",
226245
}, {
227-
name: "Read /receivers is sent to only 1 AM",
228-
numAM: 5,
229-
numHappyAM: 5,
230-
replicationFactor: 3,
231-
isRead: true,
232-
expStatusCode: http.StatusOK,
233-
expectedTotalCalls: 1,
234-
route: "/receivers",
246+
name: "Read /receivers is sent to only 1 AM",
247+
numAM: 5,
248+
numHappyAM: 5,
249+
replicationFactor: 3,
250+
isRead: true,
251+
expStatusCode: http.StatusOK,
252+
expectedMinCalls: 1,
253+
expectedMaxCalls: 1,
254+
route: "/receivers",
235255
}, {
236256
name: "Write /receivers not supported",
237257
numAM: 5,
238258
numHappyAM: 5,
239259
replicationFactor: 3,
240260
expStatusCode: http.StatusNotFound,
241-
expectedTotalCalls: 0,
261+
expectedMinCalls: 0,
262+
expectedMaxCalls: 0,
242263
headersNotPreserved: true,
243264
route: "/receivers",
244265
},
@@ -282,7 +303,7 @@ func TestDistributor_DistributeRequest(t *testing.T) {
282303
// Since the response is sent as soon as the quorum is reached, when we
283304
// reach this point the 3rd AM may not have received the request yet.
284305
// To avoid flaky test we retry until we hit the desired state within a reasonable timeout.
285-
test.Poll(t, time.Second, c.expectedTotalCalls, func() interface{} {
306+
test.Poll(t, time.Second, true, func() interface{} {
286307
totalReqCount := 0
287308
for _, a := range ams {
288309
reqCount := a.requestsCount(route)
@@ -291,7 +312,10 @@ func TestDistributor_DistributeRequest(t *testing.T) {
291312
totalReqCount += reqCount
292313
}
293314

294-
return totalReqCount
315+
if totalReqCount <= c.expectedMaxCalls && totalReqCount >= c.expectedMinCalls {
316+
return true
317+
}
318+
return false
295319
})
296320
})
297321
}

0 commit comments

Comments
 (0)