Easing debugging for security not initialized error

CHANGELOG.md

@@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Introduced new, performance-optimized implementation for tenant privileges ([#5339](https://github.com/opensearch-project/security/pull/5339))
 - Performance improvements: Immutable user object ([#5212](https://github.com/opensearch-project/security/pull/5212))
 - Include mapped roles when setting userInfo in ThreadContext ([#5369](https://github.com/opensearch-project/security/pull/5369))
+- Adds details for debugging Security not initialized error([#5370](https://github.com/opensearch-project/security/pull/5370))
 
 ### Dependencies
 - Bump `guava_version` from 33.4.6-jre to 33.4.8-jre ([#5284](https://github.com/opensearch-project/security/pull/5284))

src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java

@@ -1143,7 +1143,7 @@ public Collection<Object> createComponents(
         userService = new UserService(cs, cr, passwordHasher, settings, localClient);
 
         final XFFResolver xffResolver = new XFFResolver(threadPool);
-        backendRegistry = new BackendRegistry(settings, adminDns, xffResolver, auditLog, threadPool);
+        backendRegistry = new BackendRegistry(settings, adminDns, xffResolver, auditLog, threadPool, cih);
         backendRegistry.registerClusterSettingsChangeListener(clusterService.getClusterSettings());
         tokenManager = new SecurityTokenManager(cs, threadPool, userService);
 
@@ -1181,7 +1181,7 @@ public Collection<Object> createComponents(
             cr.subscribeOnChange(configMap -> { ((DlsFlsValveImpl) dlsFlsValve).updateConfiguration(cr.getConfiguration(CType.ROLES)); });
         }
 
-        sf = new SecurityFilter(settings, evaluator, adminDns, dlsFlsValve, auditLog, threadPool, cs, compatConfig, irr, xffResolver);
+        sf = new SecurityFilter(settings, evaluator, adminDns, dlsFlsValve, auditLog, threadPool, cs, cih, compatConfig, irr, xffResolver);
 
         final String principalExtractorClass = settings.get(SSLConfigConstants.SECURITY_SSL_TRANSPORT_PRINCIPAL_EXTRACTOR_CLASS, null);
 

src/main/java/org/opensearch/security/auth/BackendRegistry.java

@@ -61,6 +61,7 @@
 import org.opensearch.security.auth.blocking.ClientBlockRegistry;
 import org.opensearch.security.auth.internal.NoOpAuthenticationBackend;
 import org.opensearch.security.configuration.AdminDNs;
+import org.opensearch.security.configuration.ClusterInfoHolder;
 import org.opensearch.security.filter.SecurityRequest;
 import org.opensearch.security.filter.SecurityRequestChannel;
 import org.opensearch.security.filter.SecurityResponse;
@@ -101,6 +102,7 @@
     private final AuditLog auditLog;
     private final ThreadPool threadPool;
     private final UserInjector userInjector;
+    private final ClusterInfoHolder clusterInfoHolder;
     private int ttlInMin;
     private Cache<AuthCredentials, User> userCache; // rest standard
     private Cache<String, User> restImpersonationCache; // used for rest impersonation
@@ -152,13 +154,15 @@
         final AdminDNs adminDns,
         final XFFResolver xffResolver,
         final AuditLog auditLog,
-        final ThreadPool threadPool
+        final ThreadPool threadPool,
+        final ClusterInfoHolder clusterInfoHolder
     ) {
         this.adminDns = adminDns;
         this.opensearchSettings = settings;
         this.xffResolver = xffResolver;
         this.auditLog = auditLog;
         this.threadPool = threadPool;
+        this.clusterInfoHolder = clusterInfoHolder;
         this.userInjector = new UserInjector(settings, threadPool, auditLog, xffResolver);
         this.restAuthDomains = Collections.emptySortedSet();
         this.ipAuthFailureListeners = Collections.emptyList();
@@ -275,8 +279,12 @@
         }
 
         if (!isInitialized()) {
-            log.error("Not yet initialized (you may need to run securityadmin)");
-            request.queueForSending(new SecurityResponse(SC_SERVICE_UNAVAILABLE, "OpenSearch Security not initialized."));
+            StringBuilder error = new StringBuilder("OpenSearch Security not initialized.");
+            if (!clusterInfoHolder.hasClusterManager()) {
+                error.append(String.format(" %s", ClusterInfoHolder.CLUSTER_MANAGER_NOT_PRESENT));
+            }
+            log.error("{} (you may need to run securityadmin)", error.toString());
+            request.queueForSending(new SecurityResponse(SC_SERVICE_UNAVAILABLE, error.toString()));
             return false;
         }
 

src/main/java/org/opensearch/security/configuration/ClusterInfoHolder.java

@@ -36,6 +36,7 @@
 import org.opensearch.cluster.node.DiscoveryNodes;
 
 public class ClusterInfoHolder implements ClusterStateListener {
+    public static final String CLUSTER_MANAGER_NOT_PRESENT = "Cluster manager not present";
 
     protected final Logger log = LogManager.getLogger(this.getClass());
     private volatile DiscoveryNodes nodes = null;
@@ -93,4 +94,11 @@ public Boolean hasNode(DiscoveryNode node) {
     public String getClusterName() {
         return this.clusterName;
     }
+
+    public Boolean hasClusterManager() {
+        if (nodes != null) {
+            return nodes.getClusterManagerNode() != null;
+        }
+        return false;
+    }
 }

src/main/java/org/opensearch/security/filter/SecurityFilter.java

@@ -79,6 +79,7 @@
 import org.opensearch.security.auth.UserInjector;
 import org.opensearch.security.compliance.ComplianceConfig;
 import org.opensearch.security.configuration.AdminDNs;
+import org.opensearch.security.configuration.ClusterInfoHolder;
 import org.opensearch.security.configuration.CompatConfig;
 import org.opensearch.security.configuration.DlsFlsRequestValve;
 import org.opensearch.security.http.XFFResolver;
@@ -107,6 +108,7 @@
     private final AuditLog auditLog;
     private final ThreadContext threadContext;
     private final ClusterService cs;
+    private final ClusterInfoHolder clusterInfoHolder;
     private final CompatConfig compatConfig;
     private final IndexResolverReplacer indexResolverReplacer;
     private final XFFResolver xffResolver;
@@ -122,6 +124,7 @@
         AuditLog auditLog,
         ThreadPool threadPool,
         ClusterService cs,
+        final ClusterInfoHolder clusterInfoHolder,
         final CompatConfig compatConfig,
         final IndexResolverReplacer indexResolverReplacer,
         final XFFResolver xffResolver
@@ -132,6 +135,7 @@
         this.auditLog = auditLog;
         this.threadContext = threadPool.getThreadContext();
         this.cs = cs;
+        this.clusterInfoHolder = clusterInfoHolder;
         this.compatConfig = compatConfig;
         this.indexResolverReplacer = indexResolverReplacer;
         this.xffResolver = xffResolver;
@@ -362,10 +366,14 @@
             final PrivilegesEvaluator eval = evalp;
 
             if (!eval.isInitialized()) {
-                log.error("OpenSearch Security not initialized for {}", action);
-                listener.onFailure(
-                    new OpenSearchSecurityException("OpenSearch Security not initialized for " + action, RestStatus.SERVICE_UNAVAILABLE)
-                );
+                StringBuilder error = new StringBuilder("OpenSearch Security not initialized for ");
+                error.append(action);
+                if (!clusterInfoHolder.hasClusterManager()) {
+                    error.append(String.format(". %s", ClusterInfoHolder.CLUSTER_MANAGER_NOT_PRESENT));
+                }
+
+                log.error(error.toString());
+                listener.onFailure(new OpenSearchSecurityException(error.toString(), RestStatus.SERVICE_UNAVAILABLE));
                 return;
             }
 

src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java

@@ -307,7 +307,11 @@ public PrivilegesEvaluationContext createContext(
         Set<String> injectedRoles
     ) {
         if (!isInitialized()) {
-            throw new OpenSearchSecurityException("OpenSearch Security is not initialized.");
+            StringBuilder error = new StringBuilder("OpenSearch Security is not initialized.");
+            if (!clusterInfoHolder.hasClusterManager()) {
+                error.append(String.format(" %s", ClusterInfoHolder.CLUSTER_MANAGER_NOT_PRESENT));
+            }
+            throw new OpenSearchSecurityException(error.toString());
         }
 
         TransportAddress caller = threadContext.getTransient(ConfigConstants.OPENDISTRO_SECURITY_REMOTE_ADDRESS);
@@ -319,7 +323,11 @@ public PrivilegesEvaluationContext createContext(
     public PrivilegesEvaluatorResponse evaluate(PrivilegesEvaluationContext context) {
 
         if (!isInitialized()) {
-            throw new OpenSearchSecurityException("OpenSearch Security is not initialized.");
+            StringBuilder error = new StringBuilder("OpenSearch Security is not initialized.");
+            if (!clusterInfoHolder.hasClusterManager()) {
+                error.append(String.format(" %s", ClusterInfoHolder.CLUSTER_MANAGER_NOT_PRESENT));
+            }
+            throw new OpenSearchSecurityException(error.toString());
         }
 
         String action0 = context.getAction();

src/main/java/org/opensearch/security/securityconf/DynamicConfigFactory.java

@@ -273,6 +273,8 @@ public void onChange(ConfigurationMap typeToConfig) {
             eventBus.post(audit == null ? defaultAuditConfig : audit);
         }
 
+        log.debug("Dispatched config update notification to different subscribers");
+
         initialized.set(true);
     }
 

src/test/java/org/opensearch/security/configuration/ClusterInfoHolderTest.java

@@ -0,0 +1,188 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.security.configuration;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.opensearch.Version;
+import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
+
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ClusterInfoHolderTest {
+
+    private ClusterInfoHolder clusterInfoHolder;
+
+    @Mock
+    private ClusterChangedEvent mockEvent;
+
+    @Mock
+    private ClusterState mockClusterState;
+
+    @Mock
+    private DiscoveryNodes mockNodes;
+
+    @Mock
+    private DiscoveryNode mockNode;
+
+    private static final String TEST_CLUSTER_NAME = "test-cluster";
+
+    @Before
+    public void setUp() {
+        clusterInfoHolder = new ClusterInfoHolder(TEST_CLUSTER_NAME);
+        when(mockEvent.state()).thenReturn(mockClusterState);
+        when(mockClusterState.nodes()).thenReturn(mockNodes);
+    }
+
+    @Test
+    public void testInitialState() {
+        assertFalse("Should not be initialized initially", clusterInfoHolder.isInitialized());
+        assertNull("Should not have cluster manager status initially", clusterInfoHolder.isLocalNodeElectedClusterManager());
+        assertEquals("Should have correct cluster name", TEST_CLUSTER_NAME, clusterInfoHolder.getClusterName());
+    }
+
+    @Test
+    public void testClusterChanged_NodesChanged() {
+        // Execute
+        clusterInfoHolder.clusterChanged(mockEvent);
+
+        // Verify
+        assertTrue("Should be initialized after nodes change", clusterInfoHolder.isInitialized());
+    }
+
+    @Test
+    public void testClusterChanged_LocalNodeClusterManager() {
+        // Setup
+        when(mockEvent.localNodeClusterManager()).thenReturn(Boolean.TRUE);
+
+        // Execute
+        clusterInfoHolder.clusterChanged(mockEvent);
+
+        // Verify
+        assertTrue("Should be cluster manager", clusterInfoHolder.isLocalNodeElectedClusterManager());
+    }
+
+    @Test
+    public void testClusterChanged_NotLocalNodeClusterManager() {
+        // Setup
+        when(mockEvent.localNodeClusterManager()).thenReturn(false);
+
+        // Execute
+        clusterInfoHolder.clusterChanged(mockEvent);
+
+        // Verify
+        assertFalse("Should not be cluster manager", clusterInfoHolder.isLocalNodeElectedClusterManager());
+    }
+
+    @Test
+    public void testGetMinNodeVersion_WhenNotInitialized() {
+        assertNull("Should return null when not initialized", clusterInfoHolder.getMinNodeVersion());
+    }
+
+    @Test
+    public void testGetMinNodeVersion_WhenInitialized() {
+        // Setup
+        Version expectedVersion = Version.CURRENT;
+        when(mockNodes.getMinNodeVersion()).thenReturn(expectedVersion);
+        clusterInfoHolder.clusterChanged(mockEvent);
+
+        // Execute & Verify
+        assertEquals("Should return correct min version", expectedVersion, clusterInfoHolder.getMinNodeVersion());
+    }
+
+    @Test
+    public void testHasNode_WhenNotInitialized() {
+        assertNull("Should return null when not initialized", clusterInfoHolder.hasNode(mockNode));
+    }
+
+    @Test
+    public void testHasNode_WhenNodeExists() {
+        // Setup
+        when(mockNodes.nodeExists(mockNode)).thenReturn(true);
+        clusterInfoHolder.clusterChanged(mockEvent);
+
+        // Execute & Verify
+        assertTrue("Should return true when node exists", clusterInfoHolder.hasNode(mockNode));
+    }
+
+    @Test
+    public void testHasNode_WhenNodeDoesNotExist() {
+        // Setup
+        when(mockNodes.nodeExists(mockNode)).thenReturn(false);
+        clusterInfoHolder.clusterChanged(mockEvent);
+
+        // Execute & Verify
+        assertFalse("Should return false when node doesn't exist", clusterInfoHolder.hasNode(mockNode));
+    }
+
+    @Test
+    public void testHasClusterManager_WhenNotInitialized() {
+        assertFalse("Should return false when not initialized", clusterInfoHolder.hasClusterManager());
+    }
+
+    @Test
+    public void testHasClusterManager_WhenClusterManagerExists() {
+        // Setup
+        when(mockNodes.getClusterManagerNode()).thenReturn(mockNode);
+        clusterInfoHolder.clusterChanged(mockEvent);
+
+        // Execute & Verify
+        assertTrue("Should return true when cluster manager exists", clusterInfoHolder.hasClusterManager());
+    }
+
+    @Test
+    public void testHasClusterManager_WhenNoClusterManager() {
+        // Setup
+        when(mockNodes.getClusterManagerNode()).thenReturn(null);
+        clusterInfoHolder.clusterChanged(mockEvent);
+
+        // Execute & Verify
+        assertFalse("Should return false when no cluster manager", clusterInfoHolder.hasClusterManager());
+    }
+
+    @Test
+    public void testGetClusterManagerNotPresentStatus() {
+        assertEquals("Should return correct status message", "Cluster manager not present", ClusterInfoHolder.CLUSTER_MANAGER_NOT_PRESENT);
+    }
+
+    @Test
+    public void testMultipleClusterChanges() {
+        // First change
+        when(mockEvent.nodesChanged()).thenReturn(true);
+        when(mockEvent.localNodeClusterManager()).thenReturn(true);
+        clusterInfoHolder.clusterChanged(mockEvent);
+
+        assertTrue("Should be initialized", clusterInfoHolder.isInitialized());
+        assertTrue("Should be cluster manager", clusterInfoHolder.isLocalNodeElectedClusterManager());
+
+        // Second change
+        when(mockEvent.nodesChanged()).thenReturn(false);
+        when(mockEvent.localNodeClusterManager()).thenReturn(false);
+        clusterInfoHolder.clusterChanged(mockEvent);
+
+        assertTrue("Should still be initialized", clusterInfoHolder.isInitialized());
+        assertFalse("Should not be cluster manager anymore", clusterInfoHolder.isLocalNodeElectedClusterManager());
+    }
+}

src/test/java/org/opensearch/security/filter/SecurityFilterTests.java

@@ -26,6 +26,7 @@
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.security.auditlog.AuditLog;
 import org.opensearch.security.configuration.AdminDNs;
+import org.opensearch.security.configuration.ClusterInfoHolder;
 import org.opensearch.security.configuration.CompatConfig;
 import org.opensearch.security.configuration.DlsFlsRequestValve;
 import org.opensearch.security.http.XFFResolver;
@@ -84,6 +85,7 @@ public void testImmutableIndicesWildcardMatcher() {
             mock(AuditLog.class),
             mock(ThreadPool.class),
             mock(ClusterService.class),
+            mock(ClusterInfoHolder.class),
             mock(CompatConfig.class),
             mock(IndexResolverReplacer.class),
             mock(XFFResolver.class)
@@ -107,6 +109,7 @@ public void testUnexepectedCausesAreNotSendToCallers() {
             auditLog,
             new ThreadPool(Settings.builder().put("node.name", "mock").build()),
             mock(ClusterService.class),
+            mock(ClusterInfoHolder.class),
             mock(CompatConfig.class),
             mock(IndexResolverReplacer.class),
             mock(XFFResolver.class)