Skip to content

Commit 547699f

Browse files
nihussmannnihussmann
committed
implement polling for node IP retrieval to prevent flaky tests
1 parent d678aea commit 547699f

File tree

6 files changed

+63
-36
lines changed

6 files changed

+63
-36
lines changed

‎src/main/groovy/com/cloudogu/gitops/utils/K8sClient.groovy

Lines changed: 49 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import jakarta.inject.Singleton
1313
class K8sClient {
1414
private static final String[] APPLY_FROM_STDIN = ['kubectl', 'apply', '-f-']
1515

16+
protected int SLEEPTIME = 1000
17+
1618
private CommandExecutor commandExecutor
1719
private FileSystemUtils fileSystemUtils
1820
private Provider<Config> configProvider
@@ -27,14 +29,50 @@ class K8sClient {
2729
this.configProvider = configProvider
2830
}
2931

30-
String getInternalNodeIp() {
31-
String node = waitForNode()
3232

33+
private String waitForOutput(String[] command, String[] additionalCommand = null, int maxTries = 120, String logMessage, String failureMessage) {
34+
int tryCount = 0
35+
String output = ""
36+
37+
log.debug(logMessage)
38+
while (output.isEmpty() && tryCount < maxTries) {
39+
if(!additionalCommand){
40+
output = commandExecutor.execute(command).stdOut
41+
}else{
42+
output = commandExecutor.execute(command,additionalCommand).stdOut
43+
}
44+
45+
if (output.isEmpty()) {
46+
tryCount++
47+
log.debug("Still waiting... (try $tryCount/$maxTries)")
48+
sleep(SLEEPTIME)
49+
}
50+
}
51+
52+
if (output.isEmpty()) {
53+
throw new RuntimeException(failureMessage)
54+
}
55+
56+
return output
57+
}
58+
59+
String waitForInternalNodeIp() {
60+
61+
String node = waitForNode()
3362
// For k3d this is either the host's IP or the IP address of the k3d API server's container IP (when --bind-localhost=false)
3463
// Note that this might return multiple InternalIP (IPV4 and IPV6) - we assume the first one is IPV4 (break after first)
3564
String[] command = ["kubectl", "get", "$node",
3665
"--template='{{range .status.addresses}}{{ if eq .type \"InternalIP\" }}{{.address}}{{break}}{{end}}{{end}}'"]
37-
return commandExecutor.execute(command).stdOut
66+
String output = waitForOutput(
67+
command,
68+
null,
69+
12,
70+
"Waiting for internal IP of node $node",
71+
"Failed to retrieve internal node IP"
72+
)
73+
74+
log.debug("Internal IP of node $node: $output")
75+
return output
3876
}
3977

4078
/**
@@ -44,25 +82,13 @@ class K8sClient {
4482
String[] command1 = ['kubectl', 'get', 'node', '-oname']
4583
String[] command2 = ['head', '-n1']
4684

47-
int maxTries = 120
48-
int sleepMillis = 1000
49-
int tryCount = 0
50-
String output = ""
51-
52-
log.debug("Waiting for first node of the cluster to become ready")
53-
while (output.isEmpty() && tryCount < maxTries) {
54-
output = commandExecutor.execute(command1, command2).stdOut
55-
56-
if (output.isEmpty()) {
57-
tryCount++
58-
log.debug("Still waiting for first node of the cluster to become ready (try $tryCount/$maxTries)")
59-
sleep(sleepMillis)
60-
}
61-
}
85+
String output = waitForOutput(
86+
command1, command2,
87+
120,
88+
"Waiting for first node of the cluster to become ready",
89+
"Failed waiting for node of the cluster to become ready"
90+
)
6291

63-
if (output.isEmpty()) {
64-
throw new RuntimeException("Back up waiting for first node of the cluster to become ready after $maxTries tries")
65-
}
6692
log.debug("First node of the cluster is ready: $output")
6793
return output
6894
}
@@ -323,9 +349,9 @@ class K8sClient {
323349
if (namespace) {
324350
commandAsList.add("-n $namespace" as String)
325351
}
326-
String[] command = commandAsList.toArray(new String [0])
352+
String[] command = commandAsList.toArray(new String[0])
327353
def result = commandExecutor.execute(command, false)
328-
if (!result.getStdErr().isEmpty()){
354+
if (!result.getStdErr().isEmpty()) {
329355
throw new RuntimeException("Failed to fetch data from resource [$resource/$name] in namespace [$namespace]: ${result.stdErr}")
330356
}
331357
log.debug("getAnnotation returns = ${result.stdOut}")

‎src/main/groovy/com/cloudogu/gitops/utils/NetworkingUtils.groovy

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ class NetworkingUtils {
2424
String findClusterBindAddress() {
2525
log.debug("Figuring out the address of the k8s cluster")
2626

27-
String potentialClusterBindAddress = k8sClient.getInternalNodeIp()
27+
String potentialClusterBindAddress = k8sClient.waitForInternalNodeIp()
2828
potentialClusterBindAddress = potentialClusterBindAddress.replaceAll("'", "")
2929

3030
String localAddress = localAddress

‎src/test/groovy/com/cloudogu/gitops/features/JenkinsTest.groovy

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ class JenkinsTest {
3535

3636
@BeforeEach
3737
void setup() {
38-
// getInternalNodeIp -> waitForNode()
38+
// waitForInternalNodeIp -> waitForNode()
3939
when(k8sClient.waitForNode()).thenReturn("node/${expectedNodeName}".toString())
4040
when(k8sClient.run(anyString(), anyString(), anyString(), anyMap(), any())).thenReturn('')
4141
}

‎src/test/groovy/com/cloudogu/gitops/utils/K8sClientForTest.groovy

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,6 @@ class K8sClientForTest extends K8sClient {
1515
}
1616
})
1717
commandExecutorForTest = commandExecutor
18+
this.SLEEPTIME = 1
1819
}
1920
}

‎src/test/groovy/com/cloudogu/gitops/utils/K8sClientTest.groovy

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@ class K8sClientTest {
1818
void 'Gets internal nodeIp'() {
1919
// waitForNode()
2020
k8sClient.commandExecutorForTest.enqueueOutput(new CommandExecutor.Output('', 'node/k3d-gitops-playground-server-0', 0))
21-
// getInternalNodeIp()
21+
// waitForInternalNodeIp()
2222
k8sClient.commandExecutorForTest.enqueueOutput(new CommandExecutor.Output('', '1.2.3.4', 0))
2323

24-
def actualNodeIp = k8sClient.getInternalNodeIp()
24+
def actualNodeIp = k8sClient.waitForInternalNodeIp()
2525

2626
assertThat(actualNodeIp).isEqualTo('1.2.3.4')
2727
assertThat(commandExecutor.actualCommands[1]).isEqualTo(
@@ -35,10 +35,10 @@ class K8sClientTest {
3535
k8sClient.commandExecutorForTest.enqueueOutput(new CommandExecutor.Output('', '', 0))
3636
k8sClient.commandExecutorForTest.enqueueOutput(new CommandExecutor.Output('', '', 0))
3737
k8sClient.commandExecutorForTest.enqueueOutput(new CommandExecutor.Output('', 'node/k3d-gitops-playground-server-0', 0))
38-
// getInternalNodeIp()
38+
// waitForInternalNodeIp()
3939
k8sClient.commandExecutorForTest.enqueueOutput(new CommandExecutor.Output('', '1.2.3.4', 0))
4040

41-
def actualNodeIp = k8sClient.getInternalNodeIp()
41+
def actualNodeIp = k8sClient.waitForInternalNodeIp()
4242

4343
assertThat(actualNodeIp).isEqualTo('1.2.3.4')
4444
}

‎src/test/groovy/com/cloudogu/gitops/utils/NetworkingUtilsTest.groovy

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@ class NetworkingUtilsTest {
1818
void 'clusterBindAddress: returns bind address for external cluster'() {
1919
def internalNodeIp = "1.2.3.4"
2020
def localIp = "5.6.7.8"
21-
// getInternalNodeIp -> waitForNode()
21+
// waitForInternalNodeIp -> waitForNode()
2222
k8sClient.commandExecutorForTest.enqueueOutput(new CommandExecutor.Output('', 'node/something', 0))
23-
// getInternalNodeIp -> actual exec
23+
// waitForInternalNodeIp -> actual exec
2424
k8sClient.commandExecutorForTest.enqueueOutput(new CommandExecutor.Output('', internalNodeIp, 0))
2525
commandExecutor.enqueueOutput(new CommandExecutor.Output('',
2626
"1.0.0.0 via w.x.y.z dev someDevice src ${localIp} uid 1000", 0))
@@ -35,9 +35,9 @@ class NetworkingUtilsTest {
3535
def internalNodeIp = networkingUtils.localAddress
3636
assertThat(internalNodeIp).isNotEmpty()
3737

38-
// getInternalNodeIp -> waitForNode(), don't care
38+
// waitForInternalNodeIp -> waitForNode(), don't care
3939
k8sClient.commandExecutorForTest.enqueueOutput(new CommandExecutor.Output('', 'node/something', 0))
40-
// getInternalNodeIp -> actual exec
40+
// waitForInternalNodeIp -> actual exec
4141
k8sClient.commandExecutorForTest.enqueueOutput(new CommandExecutor.Output('', internalNodeIp, 0))
4242

4343
def actualBindAddress = networkingUtils.findClusterBindAddress()
@@ -48,16 +48,16 @@ class NetworkingUtilsTest {
4848
@Test
4949
void 'clusterBindAddress: fails when no potential bind address'() {
5050
def internalNodeIp = ''
51-
// getInternalNodeIp -> waitForNode()
51+
// waitForInternalNodeIp -> waitForNode()
5252
k8sClient.commandExecutorForTest.enqueueOutput(new CommandExecutor.Output('', 'node/something', 0))
53-
// getInternalNodeIp -> actual exec
53+
// waitForInternalNodeIp -> actual exec
5454
k8sClient.commandExecutorForTest.enqueueOutput(new CommandExecutor.Output('', internalNodeIp, 0))
5555
commandExecutor.enqueueOutput(new CommandExecutor.Output('',
5656
"1.0.0.0 via w.x.y.z dev someDevice src 1.2.3.4 uid 1000", 0))
5757

5858
def exception = shouldFail(RuntimeException) {
5959
networkingUtils.findClusterBindAddress()
6060
}
61-
assertThat(exception.message).isEqualTo('Could not connect to kubernetes cluster: no cluster bind address')
61+
assertThat(exception.message).isEqualTo('Failed to retrieve internal node IP')
6262
}
6363
}

0 commit comments

Comments
 (0)