Skip to content

Commit dd4ecaa

Browse files
doxiaodoxiao
authored
Owls88781 continue retry start a namespace in non-fullcheck case after hitting 403 (#2315)
* clear isNamespaceStarting flag when failed to create NamespaceWatchingStarted event due to 403 * set isNamespaceStarting flag on success and add a new event
1 parent ce7ec72 commit dd4ecaa

File tree

9 files changed

+234
-15
lines changed

9 files changed

+234
-15
lines changed

‎documentation/staging/content/userguide/managing-domains/domain-events.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ The operator also generates these event types in the operator's namespace, which
3838

3939
* `StartManagingNamespace`: The operator has started managing domains in a namespace.
4040
* `StopManagingNamespace`: The operator has stopped managing domains in a namespace.
41+
* `StartManagingNamespaceFailed`: The operator failed to start managing domains in a namespace because it does not have the required privileges.
4142

4243
#### Operator-generated event details
4344

‎operator/src/main/java/oracle/kubernetes/operator/DomainNamespaces.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,10 +179,14 @@ Step readExistingResources(String ns, DomainProcessor processor) {
179179
return Step.chain(ConfigMapHelper.createScriptConfigMapStep(ns), resources.createListSteps());
180180
}
181181

182-
boolean shouldStartNamespace(String ns) {
182+
public boolean shouldStartNamespace(String ns) {
183183
return getNamespaceStatus(ns).shouldStartNamespace();
184184
}
185185

186+
public void clearNamespaceStartingFlag(String ns) {
187+
getNamespaceStatus(ns).clearNamespaceStartingFlag();
188+
}
189+
186190
interface WatcherFactory<T, W extends Watcher<T>> {
187191
W create(
188192
ThreadFactory threadFactory,

‎operator/src/main/java/oracle/kubernetes/operator/DomainRecheck.java

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636

3737
import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.NAMESPACE_WATCHING_STARTED;
3838
import static oracle.kubernetes.operator.helpers.NamespaceHelper.getOperatorNamespace;
39-
import static oracle.kubernetes.operator.logging.MessageKeys.BEGIN_MANAGING_NAMESPACE;
4039

4140
class DomainRecheck {
4241
private static final LoggingFacade LOGGER = LoggingFactory.getLogger("Operator", "Operator");
@@ -218,6 +217,11 @@ private Step startNamespaceSteps(String ns) {
218217
domainNamespaces.readExistingResources(ns, domainProcessor));
219218
}
220219

220+
// for testing
221+
public Step createStartNamespaceBeforeStep(String ns) {
222+
return new StartNamespaceBeforeStep(ns);
223+
}
224+
221225
private class StartNamespaceBeforeStep extends Step {
222226

223227
private final String ns;
@@ -228,11 +232,11 @@ private class StartNamespaceBeforeStep extends Step {
228232

229233
@Override
230234
public NextAction apply(Packet packet) {
235+
if (domainNamespaces.shouldStartNamespace(ns)) {
236+
return doNext(addNSWatchingStartingEventsStep(), packet);
237+
}
231238
if (fullRecheck) {
232239
return doNext(packet);
233-
} else if (domainNamespaces.shouldStartNamespace(ns)) {
234-
LOGGER.info(BEGIN_MANAGING_NAMESPACE, ns);
235-
return doNext(addNSWatchingStartingEventsStep(), packet);
236240
} else {
237241
return doEnd(packet);
238242
}
@@ -241,7 +245,7 @@ public NextAction apply(Packet packet) {
241245
private Step addNSWatchingStartingEventsStep() {
242246
return Step.chain(
243247
EventHelper.createEventStep(
244-
new EventData(NAMESPACE_WATCHING_STARTED).namespace(ns).resourceName(ns)),
248+
domainNamespaces, new EventData(NAMESPACE_WATCHING_STARTED).namespace(ns).resourceName(ns), null),
245249
EventHelper.createEventStep(
246250
new EventData(EventHelper.EventItem.START_MANAGING_NAMESPACE)
247251
.namespace(getOperatorNamespace()).resourceName(ns)),

‎operator/src/main/java/oracle/kubernetes/operator/EventConstants.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,6 @@ public interface EventConstants {
4545
String START_MANAGING_NAMESPACE_PATTERN = "Start managing namespace %s";
4646
String STOP_MANAGING_NAMESPACE_EVENT = "StopManagingNamespace";
4747
String STOP_MANAGING_NAMESPACE_PATTERN = "Stop managing namespace %s";
48+
String START_MANAGING_NAMESPACE_FAILED_EVENT = "StartManagingNamespaceFailed";
49+
String START_MANAGING_NAMESPACE_FAILED_PATTERN = "Start managing namespace %s failed due to an authorization error";
4850
}

‎operator/src/main/java/oracle/kubernetes/operator/NamespaceStatus.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,8 @@ public AtomicReference<V1SubjectRulesReviewStatus> getRulesReviewStatus() {
3333
boolean shouldStartNamespace() {
3434
return !isNamespaceStarting.getAndSet(true);
3535
}
36+
37+
public void clearNamespaceStartingFlag() {
38+
isNamespaceStarting.set(false);
39+
}
3640
}

‎operator/src/main/java/oracle/kubernetes/operator/helpers/EventHelper.java

Lines changed: 78 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import io.kubernetes.client.openapi.models.V1ObjectMeta;
1212
import io.kubernetes.client.openapi.models.V1ObjectReference;
1313
import jakarta.validation.constraints.NotNull;
14+
import oracle.kubernetes.operator.DomainNamespaces;
1415
import oracle.kubernetes.operator.DomainProcessorImpl;
1516
import oracle.kubernetes.operator.EventConstants;
1617
import oracle.kubernetes.operator.KubernetesConstants;
@@ -52,10 +53,12 @@
5253
import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_ABORTED;
5354
import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_COMPLETED;
5455
import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_PROCESSING_STARTING;
56+
import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.NAMESPACE_WATCHING_STARTED;
5557
import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.NAMESPACE_WATCHING_STOPPED;
5658
import static oracle.kubernetes.operator.helpers.NamespaceHelper.getOperatorNamespace;
5759
import static oracle.kubernetes.operator.helpers.NamespaceHelper.getOperatorPodName;
5860
import static oracle.kubernetes.operator.helpers.NamespaceHelper.getOperatorPodUID;
61+
import static oracle.kubernetes.operator.logging.MessageKeys.BEGIN_MANAGING_NAMESPACE;
5962

6063
/** A Helper Class for the operator to create Kubernetes Events at the key points in the operator's workflow. */
6164
public class EventHelper {
@@ -81,6 +84,18 @@ public static Step createEventStep(EventData eventData) {
8184
return new CreateEventStep(eventData);
8285
}
8386

87+
/**
88+
* Factory for {@link Step} that asynchronously create an event.
89+
*
90+
* @param domainNamespaces DomainSpaces instance
91+
* @param eventData event item
92+
* @param next next step
93+
* @return Step for creating an event
94+
*/
95+
public static Step createEventStep(DomainNamespaces domainNamespaces, EventData eventData, Step next) {
96+
return new CreateEventStep(domainNamespaces, eventData, next);
97+
}
98+
8499
/**
85100
* Factory for {@link Step} that asynchronously create an event.
86101
*
@@ -89,19 +104,21 @@ public static Step createEventStep(EventData eventData) {
89104
* @return Step for creating an event
90105
*/
91106
public static Step createEventStep(EventData eventData, Step next) {
92-
return new CreateEventStep(eventData, next);
107+
return new CreateEventStep(null, eventData, next);
93108
}
94109

95110
public static class CreateEventStep extends Step {
96111
private final EventData eventData;
112+
private final DomainNamespaces domainNamespaces;
97113

98114
CreateEventStep(EventData eventData) {
99-
this(eventData, null);
115+
this(null, eventData, null);
100116
}
101117

102-
CreateEventStep(EventData eventData, Step next) {
118+
CreateEventStep(DomainNamespaces domainNamespaces, EventData eventData, Step next) {
103119
super(next);
104120
this.eventData = eventData;
121+
this.domainNamespaces = domainNamespaces;
105122
}
106123

107124
@Override
@@ -166,6 +183,11 @@ private class CreateEventResponseStep extends ResponseStep<CoreV1Event> {
166183

167184
@Override
168185
public NextAction onSuccess(Packet packet, CallResponse<CoreV1Event> callResponse) {
186+
if (NAMESPACE_WATCHING_STARTED == eventData.eventItem) {
187+
LOGGER.info(BEGIN_MANAGING_NAMESPACE, eventData.getNamespace());
188+
domainNamespaces.shouldStartNamespace(eventData.getNamespace());
189+
}
190+
169191
Optional.ofNullable(packet.getSpi(DomainPresenceInfo.class))
170192
.ifPresent(dpi -> dpi.setLastEventItem(eventData.eventItem));
171193
return doNext(packet);
@@ -178,9 +200,31 @@ public NextAction onFailure(Packet packet, CallResponse<CoreV1Event> callRespons
178200
eventData.eventItem.getReason(), eventData.getNamespace());
179201
return doNext(packet);
180202
}
203+
204+
if (NAMESPACE_WATCHING_STARTED == eventData.eventItem) {
205+
clearNamespaceStartingFlag();
206+
if (isForbidden(callResponse)) {
207+
LOGGER.warning(MessageKeys.CREATING_EVENT_FORBIDDEN,
208+
eventData.eventItem.getReason(), eventData.getNamespace());
209+
return doNext(createStartManagingNSFailedEventStep(), packet);
210+
}
211+
}
212+
181213
return super.onFailure(packet, callResponse);
182214
}
183215

216+
private Step createStartManagingNSFailedEventStep() {
217+
return createEventStep(
218+
new EventData(EventItem.START_MANAGING_NAMESPACE_FAILED)
219+
.namespace(getOperatorNamespace()).resourceName(eventData.getNamespace()));
220+
}
221+
222+
private void clearNamespaceStartingFlag() {
223+
if (domainNamespaces != null) {
224+
domainNamespaces.clearNamespaceStartingFlag(eventData.getNamespace());
225+
}
226+
}
227+
184228
private boolean isForbiddenForNamespaceWatchingStoppedEvent(CallResponse<CoreV1Event> callResponse) {
185229
return isForbidden(callResponse) && NAMESPACE_WATCHING_STOPPED == eventData.eventItem;
186230
}
@@ -483,6 +527,37 @@ protected String generateEventName(EventData eventData) {
483527
return generateOperatorNSEventName(eventData);
484528
}
485529
},
530+
START_MANAGING_NAMESPACE_FAILED {
531+
@Override
532+
protected String getType() {
533+
return EVENT_WARNING;
534+
}
535+
536+
@Override
537+
public String getReason() {
538+
return EventConstants.START_MANAGING_NAMESPACE_FAILED_EVENT;
539+
}
540+
541+
@Override
542+
public String getPattern() {
543+
return EventConstants.START_MANAGING_NAMESPACE_FAILED_PATTERN;
544+
}
545+
546+
@Override
547+
public void addLabels(V1ObjectMeta metadata, EventData eventData) {
548+
addCreatedByOperatorLabel(metadata);
549+
}
550+
551+
@Override
552+
public V1ObjectReference createInvolvedObject(EventData eventData) {
553+
return createOperatorEventInvolvedObject();
554+
}
555+
556+
@Override
557+
protected String generateEventName(EventData eventData) {
558+
return generateOperatorNSEventName(eventData);
559+
}
560+
},
486561
STOP_MANAGING_NAMESPACE {
487562
@Override
488563
public String getReason() {

‎operator/src/test/java/oracle/kubernetes/operator/NamespaceTest.java

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,32 +5,44 @@
55

66
import java.util.ArrayList;
77
import java.util.Arrays;
8+
import java.util.Collection;
89
import java.util.HashSet;
910
import java.util.List;
1011
import java.util.Set;
1112
import java.util.concurrent.atomic.AtomicBoolean;
13+
import java.util.logging.Level;
14+
import java.util.logging.LogRecord;
1215

1316
import com.meterware.simplestub.Memento;
1417
import com.meterware.simplestub.Stub;
1518
import io.kubernetes.client.openapi.models.V1Namespace;
1619
import io.kubernetes.client.openapi.models.V1ObjectMeta;
1720
import oracle.kubernetes.operator.builders.StubWatchFactory;
21+
import oracle.kubernetes.operator.helpers.EventHelper;
22+
import oracle.kubernetes.operator.helpers.EventRetryStrategyStub;
1823
import oracle.kubernetes.operator.helpers.HelmAccessStub;
1924
import oracle.kubernetes.operator.helpers.KubernetesTestSupport;
2025
import oracle.kubernetes.operator.helpers.KubernetesVersion;
2126
import oracle.kubernetes.operator.helpers.SemanticVersion;
2227
import oracle.kubernetes.operator.helpers.TuningParametersStub;
2328
import oracle.kubernetes.utils.TestUtils;
2429
import oracle.kubernetes.weblogic.domain.model.Domain;
30+
import org.hamcrest.MatcherAssert;
2531
import org.jetbrains.annotations.NotNull;
2632
import org.junit.jupiter.api.AfterEach;
2733
import org.junit.jupiter.api.BeforeEach;
2834
import org.junit.jupiter.api.Test;
2935

3036
import static com.meterware.simplestub.Stub.createStrictStub;
37+
import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
3138
import static oracle.kubernetes.operator.DomainProcessorTestSetup.NS;
39+
import static oracle.kubernetes.operator.EventConstants.NAMESPACE_WATCHING_STARTED_EVENT;
40+
import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.NAMESPACE_WATCHING_STARTED;
41+
import static oracle.kubernetes.operator.helpers.EventHelper.createEventStep;
3242
import static oracle.kubernetes.operator.helpers.HelmAccess.OPERATOR_DOMAIN_NAMESPACES;
3343
import static oracle.kubernetes.operator.helpers.KubernetesTestSupport.DOMAIN;
44+
import static oracle.kubernetes.operator.logging.MessageKeys.CREATING_EVENT_FORBIDDEN;
45+
import static oracle.kubernetes.utils.LogMatcher.containsWarning;
3446
import static org.hamcrest.Matchers.contains;
3547
import static org.hamcrest.Matchers.is;
3648
import static org.hamcrest.Matchers.not;
@@ -49,6 +61,9 @@ public class NamespaceTest {
4961
private final DomainNamespaces domainNamespaces = new DomainNamespaces();
5062
private final DomainProcessorStub dp = Stub.createNiceStub(DomainProcessorStub.class);
5163
private final MainDelegateStub delegate = createStrictStub(MainDelegateStub.class, dp, domainNamespaces);
64+
private final TestUtils.ConsoleHandlerMemento loggerControl = TestUtils.silenceOperatorLogger();
65+
private final Collection<LogRecord> logRecords = new ArrayList<>();
66+
private final EventRetryStrategyStub retryStrategy = createStrictStub(EventRetryStrategyStub.class);
5267

5368
@BeforeEach
5469
public void setUp() throws Exception {
@@ -174,6 +189,57 @@ public void whenDomainNamespaceDeleted_isNoLongerInManagedNamespaces() {
174189
assertThat(domainNamespaces.getNamespaces(), not(contains(NS)));
175190
}
176191

192+
@Test
193+
public void whenStartNamespaceBeforeStepRunHit403OnEventCreation_namespaceStartingFlagCleared() {
194+
String namespace = "TEST_NAMESPACE_1";
195+
defineNamespaces(namespace);
196+
specifyDomainNamespaces(namespace);
197+
198+
loggerControl.withLogLevel(Level.INFO).collectLogMessages(logRecords, CREATING_EVENT_FORBIDDEN);
199+
testSupport.failOnCreate(KubernetesTestSupport.EVENT, null, namespace, HTTP_FORBIDDEN);
200+
testSupport.runSteps(new DomainRecheck(delegate).createStartNamespaceBeforeStep(namespace));
201+
202+
MatcherAssert.assertThat(logRecords,
203+
containsWarning(getMessage(CREATING_EVENT_FORBIDDEN, NAMESPACE_WATCHING_STARTED_EVENT, namespace)));
204+
assertThat(domainNamespaces.isStarting(namespace), is(false));
205+
}
206+
207+
@Test
208+
public void whenStartNamespaceBeforeStepRunSucceeds_namespaceStartingFlagIsNotCleared() {
209+
String namespace = "TEST_NAMESPACE_2";
210+
defineNamespaces(namespace);
211+
specifyDomainNamespaces(namespace);
212+
213+
loggerControl.withLogLevel(Level.INFO).collectLogMessages(logRecords, CREATING_EVENT_FORBIDDEN);
214+
testSupport.runSteps(new DomainRecheck(delegate).createStartNamespaceBeforeStep(namespace));
215+
assertThat(logRecords.isEmpty(), is(true));
216+
assertThat(domainNamespaces.isStarting(namespace), is(true));
217+
}
218+
219+
@Test
220+
public void whenStartNamespaceBeforeStepRun403OnEventCreation_thenSucceed_namespaceStartingFlagSet() {
221+
String namespace = "TEST_NAMESPACE_3";
222+
testSupport.addRetryStrategy(retryStrategy);
223+
defineNamespaces(namespace);
224+
specifyDomainNamespaces(namespace);
225+
226+
loggerControl.collectLogMessages(logRecords, CREATING_EVENT_FORBIDDEN);
227+
testSupport.failOnCreate(KubernetesTestSupport.EVENT, null, namespace, HTTP_FORBIDDEN);
228+
testSupport.runSteps(new DomainRecheck(delegate).createStartNamespaceBeforeStep(namespace));
229+
testSupport.cancelFailures();
230+
testSupport.runSteps(createEventStep(delegate.domainNamespaces,
231+
new EventHelper.EventData(NAMESPACE_WATCHING_STARTED)
232+
.namespace(namespace)
233+
.resourceName(namespace), null));
234+
MatcherAssert.assertThat(logRecords,
235+
containsWarning(getMessage(CREATING_EVENT_FORBIDDEN, NAMESPACE_WATCHING_STARTED_EVENT, namespace)));
236+
assertThat(domainNamespaces.isStarting(namespace), is(true));
237+
}
238+
239+
private String getMessage(String pattern, String event, String ns) {
240+
return String.format(pattern, event, ns);
241+
}
242+
177243
private void addDomainNamespace(String namespace) {
178244
currentNamespaces.add(namespace);
179245
HelmAccessStub.defineVariable(OPERATOR_DOMAIN_NAMESPACES, String.join(",", currentNamespaces));

0 commit comments

Comments
 (0)