Skip to content

OWLS-72816 Generate event and status when operator can't scale cluster past maximum #2097

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Dec 18, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
create event and update domain status condition when replicas value i…
…s not valid for the weblogic cluster
  • Loading branch information
alai8 committed Dec 10, 2020
commit be30b7269806047cb867af4b96849f6b1b19d03b
Original file line number Diff line number Diff line change
Expand Up @@ -655,6 +655,11 @@ public boolean wasInspectionRun() {
return inspectionRun;
}

@Override
public boolean isExplicitRecheck() {
return explicitRecheck;
}

private boolean shouldContinue() {
DomainPresenceInfo cachedInfo = getExistingDomainPresenceInfo(getNamespace(), getDomainUid());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ public interface EventConstants {
String DOMAIN_PROCESSING_FAILED_EVENT = "DomainProcessingFailed";
String DOMAIN_PROCESSING_RETRYING_EVENT = "DomainProcessingRetrying";
String DOMAIN_PROCESSING_ABORTED_EVENT = "DomainProcessingAborted";
String INVALID_REPLICAS_VALUE_EVENT = "InvalidReplicasValue";
String EVENT_NORMAL = "Normal";
String EVENT_WARNING = "Warning";
String WEBLOGIC_OPERATOR_COMPONENT = "weblogic.operator";
Expand All @@ -31,4 +32,6 @@ public interface EventConstants {
= "Retrying the processing of domain resource %s after one or more failed attempts";
String DOMAIN_PROCESSING_ABORTED_PATTERN
= "Aborting the processing of domain resource %s permanently due to: %s";
String INVALID_REPLICAS_VALUE_PATTERN
= "Invalid replicas value in domain resource %s: %s";
}
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ static boolean isInspectionRequired(Packet packet) {
return domainRequiresIntrospectionInCurrentMakeRight(packet) && !wasInspectionRun(packet);
}

boolean isExplicitRecheck();

static boolean isExplicitRecheck(Packet packet) {
return fromPacket(packet).map(MakeRightDomainOperation::isExplicitRecheck).orElse(false);
}

/**
* Returns true if the packet contains info about a domain that requires introspection in a sequences of steps
* before server pods are created or modified.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
import static oracle.kubernetes.operator.EventConstants.DOMAIN_PROCESSING_STARTING_PATTERN;
import static oracle.kubernetes.operator.EventConstants.EVENT_NORMAL;
import static oracle.kubernetes.operator.EventConstants.EVENT_WARNING;
import static oracle.kubernetes.operator.EventConstants.INVALID_REPLICAS_VALUE_EVENT;
import static oracle.kubernetes.operator.EventConstants.INVALID_REPLICAS_VALUE_PATTERN;
import static oracle.kubernetes.operator.EventConstants.WEBLOGIC_OPERATOR_COMPONENT;
import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_ABORTED;
import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_COMPLETED;
Expand Down Expand Up @@ -254,6 +256,28 @@ public String getMessage(DomainPresenceInfo info, EventData eventData) {
}

},
INVALID_REPLICAS_VALUE {
@Override
public String getType() {
return EVENT_WARNING;
}

@Override
public String getReason() {
return INVALID_REPLICAS_VALUE_EVENT;
}

@Override
public String getPattern() {
return INVALID_REPLICAS_VALUE_PATTERN;
}

@Override
public String getMessage(DomainPresenceInfo info, EventData eventData) {
return String.format(INVALID_REPLICAS_VALUE_PATTERN,
info.getDomainUid(), Optional.ofNullable(eventData.message).orElse(""));
}
},
EMPTY {
@Override
protected String getPattern() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,13 @@
import javax.annotation.Nonnull;

import oracle.kubernetes.operator.DomainStatusUpdater;
import oracle.kubernetes.operator.MakeRightDomainOperation;
import oracle.kubernetes.operator.ProcessingConstants;
import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
import oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerShutdownInfo;
import oracle.kubernetes.operator.helpers.DomainPresenceInfo.ServerStartupInfo;
import oracle.kubernetes.operator.helpers.EventHelper.EventData;
import oracle.kubernetes.operator.helpers.EventHelper.EventItem;
import oracle.kubernetes.operator.helpers.PodHelper;
import oracle.kubernetes.operator.logging.LoggingFacade;
import oracle.kubernetes.operator.logging.LoggingFactory;
Expand All @@ -37,6 +40,7 @@
import static java.util.Comparator.comparing;
import static oracle.kubernetes.operator.DomainStatusUpdater.MANAGED_SERVERS_STARTING_PROGRESS_REASON;
import static oracle.kubernetes.operator.DomainStatusUpdater.createProgressingStartedEventStep;
import static oracle.kubernetes.operator.helpers.EventHelper.createEventStep;

public class ManagedServersUpStep extends Step {
static final String SERVERS_UP_MSG =
Expand Down Expand Up @@ -97,9 +101,11 @@ private static void insert(List<Step> steps, Step step) {
public NextAction apply(Packet packet) {
LOGGER.entering();
DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
boolean isExplicitRecheck = MakeRightDomainOperation.isExplicitRecheck(packet);
WlsDomainConfig config = (WlsDomainConfig) packet.get(ProcessingConstants.DOMAIN_TOPOLOGY);

ServersUpStepFactory factory = new ServersUpStepFactory(config, info.getDomain());
ServersUpStepFactory factory = new ServersUpStepFactory(config,
info.getDomain(), info, isExplicitRecheck);

if (LOGGER.isFineEnabled()) {
LOGGER.fine(SERVERS_UP_MSG, factory.domain.getDomainUid(), getRunningServers(info));
Expand Down Expand Up @@ -157,15 +163,21 @@ Step createServerStep(
static class ServersUpStepFactory {
final WlsDomainConfig domainTopology;
final Domain domain;
final DomainPresenceInfo info;
final boolean skipEventCreation;
List<ServerStartupInfo> startupInfos;
List<ServerShutdownInfo> shutdownInfos = new ArrayList<>();
final Collection<String> servers = new ArrayList<>();
final Collection<String> preCreateServers = new ArrayList<>();
final Map<String, Integer> replicas = new HashMap<>();
Step eventStep;

ServersUpStepFactory(WlsDomainConfig domainTopology, Domain domain) {
ServersUpStepFactory(WlsDomainConfig domainTopology, Domain domain,
DomainPresenceInfo info, boolean skipEventCreation) {
this.domainTopology = domainTopology;
this.domain = domain;
this.info = info;
this.skipEventCreation = skipEventCreation;
}

/**
Expand Down Expand Up @@ -223,11 +235,8 @@ boolean exceedsMaxConfiguredClusterSize(WlsClusterConfig clusterConfig) {
}

private Step createNextStep(Step next) {
if (servers.isEmpty()) {
return next;
} else {
return new ManagedServerUpIteratorStep(getStartupInfos(), next);
}
Step nextStep = (servers.isEmpty()) ? next : new ManagedServerUpIteratorStep(getStartupInfos(), next);
return Optional.ofNullable(eventStep).map(s -> Step.chain(s, nextStep)).orElse(nextStep);
}

Collection<ServerStartupInfo> getStartupInfos() {
Expand Down Expand Up @@ -275,6 +284,11 @@ private void logIfReplicasExceedsClusterServersMax(WlsClusterConfig clusterConfi
domain.getReplicaCount(clusterName),
clusterConfig.getMaxDynamicClusterSize(),
clusterName);
String message = LOGGER.formatMessage(MessageKeys.REPLICAS_EXCEEDS_TOTAL_CLUSTER_SERVER_COUNT,
domain.getReplicaCount(clusterName),
clusterConfig.getMaxDynamicClusterSize(),
clusterName);
createInvalidReplicaEventAndStatus(message);
}
}

Expand All @@ -286,13 +300,25 @@ private void logIfReplicasLessThanClusterServersMin(WlsClusterConfig clusterConf
domain.getReplicaCount(clusterName),
clusterConfig.getMinDynamicClusterSize(),
clusterName);
String message = LOGGER.formatMessage(MessageKeys.REPLICAS_LESS_THAN_TOTAL_CLUSTER_SERVER_COUNT,
domain.getReplicaCount(clusterName),
clusterConfig.getMinDynamicClusterSize(),
clusterName);
createInvalidReplicaEventAndStatus(message);

// Reset current replica count so we don't scale down less than minimum
// dynamic cluster size
domain.setReplicaCount(clusterName, clusterConfig.getMinDynamicClusterSize());
}
}

private void createInvalidReplicaEventAndStatus(String message) {
if (!skipEventCreation) {
eventStep = createEventStep(new EventData(EventItem.INVALID_REPLICAS_VALUE, message));
}
info.addValidationWarning(message);
}

private boolean lessThanMinConfiguredClusterSize(WlsClusterConfig clusterConfig) {
if (clusterConfig != null) {
String clusterName = clusterConfig.getClusterName();
Expand Down
2 changes: 1 addition & 1 deletion operator/src/main/resources/Operator.properties
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ WLSKO-0142=Failed to parse results from domain introspector for domain {0} due t
WLSKO-0143=Failed to parse file {0} from domain introspector for domain {1} due to exception: {2}
WLSKO-0144=Unable to start domain with domainUID {0} in namespace {1} after {2} attempts due to exception: {3}
WLSKO-0145=Replacing pod {0} because: {1}
WLSKO-0146=Replica request of {0} exceeds the maximum dynamic server count + server count of {1} configured for cluster {2}
WLSKO-0146=Replica request of {0} exceeds the maximum dynamic server count of {1} configured for cluster {2}
WLSKO-0147=Call {0} has failed with code {1}: message {2}. It has failed {3} times and will be retried up to {4} times.
WLSKO-0148=Current Pod dump [{0}] vs expected pod [{1}].
WLSKO-0150=Creating external channel service for WebLogic domain with UID: {0}.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -313,8 +313,8 @@ public void whenMakeRightCalled_withAbortedEventData_domainProcessingAbortedEven
String.format(DOMAIN_PROCESSING_ABORTED_PATTERN, UID, "Test this failure")), is(true));
}

private V1Event getEventMatchesReason(List<V1Event> events, String reason) {
return (V1Event) Optional.ofNullable(events).get()
private static V1Event getEventMatchesReason(List<V1Event> events, String reason) {
return Optional.ofNullable(events).get()
.stream()
.filter(e -> EventHelperTest.reasonMatches(e, reason)).findFirst().orElse(null);
}
Expand Down Expand Up @@ -347,7 +347,18 @@ private Object containsEventWithInstance(List<V1Event> events, String reason, St
.orElse(""));
}

private Object containsEventWithInvolvedObject(List<V1Event> events, String reason, String name, String namespace) {
/**
* Returns true if an event in the provided list matches the given reason, name, and
* namespace.
*
* @param events A List of events to be checked
* @param reason Reason of the event to be matched
* @param name name of the involved object in the event to be matched
* @param namespace namespace of the involved object in the event to be matched
* @return true if an event in the provided list matches the given conditions.
*/
public static boolean containsEventWithInvolvedObject(List<V1Event> events, String reason,
String name, String namespace) {
return referenceMatches(Optional.ofNullable(getEventMatchesReason(events, reason))
.map(V1Event::getInvolvedObject)
.orElse(null), name, namespace);
Expand All @@ -359,7 +370,7 @@ private Object containsEventWithAction(List<V1Event> events, String reason, Stri
.orElse(null));
}

private Object referenceMatches(V1ObjectReference reference, String name, String namespace) {
private static boolean referenceMatches(V1ObjectReference reference, String name, String namespace) {
return reference != null && name.equals(reference.getName()) && namespace.equals(reference.getNamespace());
}

Expand Down
Loading