Skip to content

Owls85476 attempt to fix intermittent integration test issues in nightlies #2037

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Nov 6, 2020
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
switch to kubectl and improve the test code
  • Loading branch information
doxiao committed Nov 2, 2020
commit 2e08664e4036b32a61bb3ff757ef23d9de772d02
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.Callable;

import io.kubernetes.client.custom.V1Patch;
import io.kubernetes.client.openapi.models.V1ConfigMap;
Expand Down Expand Up @@ -44,6 +44,7 @@
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;

import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT;
Expand Down Expand Up @@ -83,6 +84,7 @@
import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort;
import static oracle.weblogic.kubernetes.actions.TestActions.patchDomainCustomResource;
import static oracle.weblogic.kubernetes.assertions.TestAssertions.appAccessibleInPod;
import static oracle.weblogic.kubernetes.assertions.TestAssertions.appAccessibleInPodKubectl;
import static oracle.weblogic.kubernetes.assertions.TestAssertions.appNotAccessibleInPod;
import static oracle.weblogic.kubernetes.assertions.TestAssertions.doesImageExist;
import static oracle.weblogic.kubernetes.assertions.TestAssertions.domainResourceImagePatched;
Expand Down Expand Up @@ -137,9 +139,9 @@ public static void initAll(@Namespaces(3) List<String> namespaces) {
.atMost(6, MINUTES).await();

// create a reusable quick retry policy
withQuickRetryPolicy = with().pollDelay(0, SECONDS)
.and().with().pollInterval(4, SECONDS)
.atMost(10, SECONDS).await();
withQuickRetryPolicy = with().pollDelay(1, SECONDS)
.and().with().pollInterval(2, SECONDS)
.atMost(15, SECONDS).await();

// get a new unique opNamespace
logger.info("Creating unique namespace for Operator");
Expand Down Expand Up @@ -334,7 +336,7 @@ public void testPatchAppV2() {
accountingThread =
new Thread(
() -> {
collectAppAvaiability(
collectAppAvailability(
domainNamespace,
appAvailability,
managedServerPrefix,
Expand Down Expand Up @@ -1060,56 +1062,74 @@ private void checkPodImagePatched(
namespace)));
}

private static void collectAppAvaiability(
private static void collectAppAvailability(
String namespace,
List<Integer> appAvailability,
String managedServerPrefix,
int replicaCount,
String internalPort,
String appPath
) {
boolean v2AppAvailable = false;
with().pollDelay(2, SECONDS)
.and().with().pollInterval(200, MILLISECONDS)
.atMost(15, MINUTES)
.await()
.conditionEvaluationListener(
condition -> logger.info("Waiting for patched application running on all managed servers in namespace {1} "
+ "(elapsed time {2}ms, remaining time {3}ms)",
namespace,
condition.getElapsedTimeInMS(),
condition.getRemainingTimeInMS()))
.until(assertDoesNotThrow(() -> checkContinuousAvailability(
namespace, appAvailability, managedServerPrefix, replicaCount, internalPort, appPath),
String.format(
"App is not available on all managed servers in namespace %s.",
namespace)));

// Access the pod periodically to check application's availability across the duration
// of patching the domain with newer version of the application.
while (!v2AppAvailable) {
v2AppAvailable = true;
}

private static Callable<Boolean> checkContinuousAvailability(
String namespace,
List<Integer> appAvailability,
String managedServerPrefix,
int replicaCount,
String internalPort,
String appPath) {
return () -> {
boolean v2AppAvailable = true;

for (int i = 1; i <= replicaCount; i++) {
v2AppAvailable = v2AppAvailable && appAccessibleInPod(
v2AppAvailable = v2AppAvailable && appAccessibleInPodKubectl(
namespace,
managedServerPrefix + i,
internalPort,
appPath,
managedServerPrefix + i,
internalPort,
appPath,
MII_APP_RESPONSE_V2 + i);
}

int count = 0;
for (int i = 1; i <= replicaCount; i++) {
if (appAccessibleInPod(
if (appAccessibleInPodKubectl(
namespace,
managedServerPrefix + i,
internalPort,
appPath,
"Hello World")) {
managedServerPrefix + i,
internalPort,
appPath,
"Hello World")) {
count++;
}
}
appAvailability.add(count);

if (count == 0) {
logger.info("XXXXXXXXXXX: application not available XXXXXXXX");
break;
} else {
logger.fine("YYYYYYYYYYY: application available YYYYYYYY count = " + count);
return true;
}
try {
TimeUnit.MILLISECONDS.sleep(200);
} catch (InterruptedException ie) {
// do nothing
}
}

logger.fine("YYYYYYYYYYY: application available YYYYYYYY count = " + count);
return v2AppAvailable;
};
}

private static boolean appAlwaysAvailable(List<Integer> appAvailability) {
for (Integer count: appAvailability) {
if (count == 0) {
Expand Down