Introduction of AuthToken Authenticator and AuthenticationBackend

pom.xml

@@ -158,6 +158,12 @@
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>commons-validator</groupId>
+            <artifactId>commons-validator</artifactId>
+            <version>1.7</version>
+        </dependency>
+
         <!-- Guava -->
         <dependency>
             <groupId>com.google.guava</groupId>

securityconfig/config.yml

@@ -81,7 +81,24 @@ config:
         ###### more information about XFF https://en.wikipedia.org/wiki/X-Forwarded-For
         ###### and here https://tools.ietf.org/html/rfc7239
         ###### and https://tomcat.apache.org/tomcat-8.0-doc/config/valve.html#Remote_IP_Valve
+    #auth_token_provider:
+    #  enabled: true
+    #  jwt_signing_key_hs512: "test_abc"
+    #  max_validity: "1y"
+    #  max_tokens_per_user: 100
     authc:
+      opendistro_issued_jwt_auth_domain:
+        description: "Authenticate via Json Web Tokens issued by Opendistro Security"
+        http_enabled: false
+        transport_enabled: false
+        # This auth domain is only available for HTTP
+        order: 1
+        http_authenticator:
+          type: auth_token
+          challenge: false
+          # This auth domain automatically pulls configuration from the auth_token_provider config above
+        authentication_backend:
+          type: auth_token
       kerberos_auth_domain:
         http_enabled: false
         transport_enabled: false

src/main/java/com/amazon/dlic/auth/http/jwt/AbstractHTTPJwtAuthenticator.java

@@ -52,10 +52,10 @@ public abstract class AbstractHTTPJwtAuthenticator implements HTTPAuthenticator
 
     private KeyProvider keyProvider;
     private JwtVerifier jwtVerifier;
-    private final String jwtHeaderName;
-    private final boolean isDefaultAuthHeader;
-    private final String jwtUrlParameter;
-    private final String subjectKey;
+    private String jwtHeaderName;
+    private boolean isDefaultAuthHeader;
+    private String jwtUrlParameter;
+    private String subjectKey;
     private final String rolesKey;
 
     public AbstractHTTPJwtAuthenticator(Settings settings, Path configPath) {
@@ -232,4 +232,11 @@ public boolean reRequestAuthentication(RestChannel channel, AuthCredentials auth
         return true;
     }
 
+    public void setAuthenticatorSettings(String jwtHeaderName, String subjectKey) {
+        this.jwtHeaderName = jwtHeaderName;
+        this.isDefaultAuthHeader = HttpHeaders.AUTHORIZATION.equalsIgnoreCase(this.jwtHeaderName);
+        this.jwtUrlParameter = null;
+        this.subjectKey = subjectKey;
+    }
+
 }

src/main/java/com/amazon/dlic/auth/ldap/backend/LDAPAuthenticationBackend.java

@@ -26,6 +26,7 @@
 import java.util.Set;
 import java.util.UUID;
 
+
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -42,12 +43,12 @@
 import com.amazon.dlic.auth.ldap.util.ConfigConstants;
 import com.amazon.dlic.auth.ldap.util.LdapHelper;
 import com.amazon.dlic.auth.ldap.util.Utils;
-import com.amazon.opendistroforelasticsearch.security.auth.AuthenticationBackend;
+import com.amazon.opendistroforelasticsearch.security.auth.SyncAuthenticationBackend;
 import com.amazon.opendistroforelasticsearch.security.user.AuthCredentials;
 import com.amazon.opendistroforelasticsearch.security.user.User;
 import com.amazon.opendistroforelasticsearch.security.support.WildcardMatcher;
 
-public class LDAPAuthenticationBackend implements AuthenticationBackend {
+public class LDAPAuthenticationBackend implements SyncAuthenticationBackend {
 
     static final int ZERO_PLACEHOLDER = 0;
     static final String DEFAULT_USERBASE = "";

src/main/java/com/amazon/dlic/auth/ldap2/LDAPAuthenticationBackend2.java

@@ -43,13 +43,13 @@
 import com.amazon.dlic.auth.ldap.util.ConfigConstants;
 import com.amazon.dlic.auth.ldap.util.Utils;
 import com.amazon.dlic.util.SettingsBasedSSLConfigurator.SSLConfigException;
-import com.amazon.opendistroforelasticsearch.security.auth.AuthenticationBackend;
+import com.amazon.opendistroforelasticsearch.security.auth.SyncAuthenticationBackend;
 import com.amazon.opendistroforelasticsearch.security.auth.Destroyable;
 import com.amazon.opendistroforelasticsearch.security.user.AuthCredentials;
 import com.amazon.opendistroforelasticsearch.security.user.User;
 import com.amazon.opendistroforelasticsearch.security.support.WildcardMatcher;
 
-public class LDAPAuthenticationBackend2 implements AuthenticationBackend, Destroyable {
+public class LDAPAuthenticationBackend2 implements SyncAuthenticationBackend, Destroyable {
 
     protected static final Logger log = LogManager.getLogger(LDAPAuthenticationBackend2.class);
 

src/main/java/com/amazon/opendistroforelasticsearch/security/OpenDistroSecurityPlugin.java

@@ -51,6 +51,7 @@
 import com.amazon.opendistroforelasticsearch.security.auditlog.impl.AuditLogImpl;
 import com.amazon.opendistroforelasticsearch.security.compliance.ComplianceIndexingOperationListenerImpl;
 import com.amazon.opendistroforelasticsearch.security.configuration.DlsFlsValveImpl;
+import com.amazon.opendistroforelasticsearch.security.authtoken.AuthTokenService;
 import com.amazon.opendistroforelasticsearch.security.configuration.OpenDistroSecurityFlsDlsIndexSearcherWrapper;
 import com.amazon.opendistroforelasticsearch.security.configuration.PrivilegesInterceptorImpl;
 import com.amazon.opendistroforelasticsearch.security.configuration.Salt;
@@ -798,7 +799,10 @@ public Collection<Object> createComponents(Client localClient, ClusterService cl
         securityRestHandler = new OpenDistroSecurityRestFilter(backendRegistry, auditLog, threadPool,
                 principalExtractor, settings, configPath, compatConfig);
 
-        final DynamicConfigFactory dcf = new DynamicConfigFactory(cr, settings, configPath, localClient, threadPool, cih);
+
+        AuthTokenService authTokenService = new AuthTokenService();
+        final DynamicConfigFactory dcf = new DynamicConfigFactory(cr, settings, configPath, localClient, threadPool, cih, authTokenService);
+
         dcf.registerDCFListener(backendRegistry);
         dcf.registerDCFListener(compatConfig);
         dcf.registerDCFListener(irr);

src/main/java/com/amazon/opendistroforelasticsearch/security/auth/AuthenticationBackend.java

@@ -35,6 +35,8 @@
 import com.amazon.opendistroforelasticsearch.security.user.AuthCredentials;
 import com.amazon.opendistroforelasticsearch.security.user.User;
 
+import java.util.function.Consumer;
+
 /**
  * Open Distro Security custom authentication backends need to implement this interface.
  * <p/>
@@ -69,7 +71,7 @@ public interface AuthenticationBackend {
      * @throws ElasticsearchSecurityException in case an authentication failure 
      * (when credentials are incorrect, the user does not exist or the backend is not reachable)
      */
-    User authenticate(AuthCredentials credentials) throws ElasticsearchSecurityException;
+    void authenticate(AuthCredentials credentials, Consumer<User> onSuccess, Consumer<Exception> onFailure);
 
     /**
      * 
@@ -81,4 +83,13 @@ public interface AuthenticationBackend {
      */
     boolean exists(User user);
 
+    default UserCachingPolicy userCachingPolicy() {
+        return UserCachingPolicy.ALWAYS;
+    }
+
+    enum UserCachingPolicy {
+        ALWAYS,
+        ONLY_IF_AUTHZ_SEPARATE,
+        NEVER
+    }
 }

src/main/java/com/amazon/opendistroforelasticsearch/security/auth/BackendRegistry.java

@@ -40,12 +40,14 @@
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 
 import javax.naming.InvalidNameException;
 import javax.naming.ldap.LdapName;
 import javax.naming.ldap.Rdn;
 
+import com.amazon.opendistroforelasticsearch.security.authtoken.authenticator.AuthTokenHttpJwtAuthenticator;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchSecurityException;
@@ -296,7 +298,7 @@ public User authenticate(final TransportRequest request, final String sslPrincip
             } else {
                  //auth credentials submitted
                 //impersonation not possible, if requested it will be ignored
-                authenticatedUser = authcz(authenticatedUserCacheTransport, transportRoleCache, creds, authDomain.getBackend(), transportAuthorizers);
+                authenticatedUser = authcz(authenticatedUserCacheTransport, transportRoleCache, creds, (SyncAuthenticationBackend) authDomain.getBackend(), transportAuthorizers);
             }
 
             if (authenticatedUser == null) {
@@ -467,7 +469,11 @@ public boolean authenticate(final RestRequest request, final RestChannel channel
             }
 
             //http completed       
-            authenticatedUser = authcz(userCache, restRoleCache, ac, authDomain.getBackend(), restAuthorizers);
+            if (httpAuthenticator instanceof AuthTokenHttpJwtAuthenticator) {
+                authenticatedUser = authcz(restRoleCache, ac, authDomain.getBackend(), restAuthorizers);
+            } else {
+                authenticatedUser = authcz(userCache, restRoleCache, ac, (SyncAuthenticationBackend) authDomain.getBackend(), restAuthorizers);
+            }
 
             if(authenticatedUser == null) {
                 if(log.isDebugEnabled()) {
@@ -648,7 +654,7 @@ private void authz(User authenticatedUser, Cache<User, Set<String>> roleCache, f
      * @return null if user cannot b authenticated
      */
     private User authcz(final Cache<AuthCredentials, User> cache, Cache<User, Set<String>> roleCache, final AuthCredentials ac,
-                        final AuthenticationBackend authBackend, final Set<AuthorizationBackend> authorizers) {
+                        final SyncAuthenticationBackend authBackend, final Set<AuthorizationBackend> authorizers) {
         if(ac == null) {
             return null;
         }
@@ -683,6 +689,44 @@ public User call() throws Exception {
         }
     }
 
+    private User authcz(Cache<User, Set<String>> roleCache, final AuthCredentials ac,
+                        final AuthenticationBackend authBackend, final Set<AuthorizationBackend> authorizers) {
+
+        AuthenticationBackend.UserCachingPolicy cachingPolicy = authBackend.userCachingPolicy();
+
+        CompletableFuture<User> completableFuture = new CompletableFuture<>();
+        authBackend.authenticate(ac, completableFuture::complete, completableFuture::completeExceptionally);
+
+        User authenticatedUser;
+
+        if (cachingPolicy == AuthenticationBackend.UserCachingPolicy.NEVER) {
+            try {
+                authenticatedUser = completableFuture.get();
+
+                if (!ac.isAuthzComplete() && !authenticatedUser.isAuthzComplete()) {
+                    authz(authenticatedUser, roleCache, authorizers);
+                }
+                return authenticatedUser;
+
+            } catch (Exception e ) {
+                e.printStackTrace();
+            }
+
+        } else if (cachingPolicy == AuthenticationBackend.UserCachingPolicy.ONLY_IF_AUTHZ_SEPARATE && authorizers.isEmpty()) {
+            // noop backend
+            // that means authc and authz was completely done via HTTP (like JWT or PKI)
+
+            try{
+                return completableFuture.get();
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+
+        }
+
+        return null;
+    }
+
     private User impersonate(final TransportRequest tr, final User origPKIuser) throws ElasticsearchSecurityException {
 
         final String impersonatedUser = threadPool.getThreadContext().getHeader("opendistro_security_impersonate_as");

src/main/java/com/amazon/opendistroforelasticsearch/security/auth/SyncAuthenticationBackend.java

@@ -0,0 +1,41 @@
+package com.amazon.opendistroforelasticsearch.security.auth;
+
+import java.util.function.Consumer;
+
+import com.amazon.opendistroforelasticsearch.security.user.AuthCredentials;
+import com.amazon.opendistroforelasticsearch.security.user.User;
+import org.elasticsearch.ElasticsearchSecurityException;
+
+public interface SyncAuthenticationBackend extends AuthenticationBackend {
+
+    /**
+     * Validate credentials and return an authenticated user (or throw an ElasticsearchSecurityException)
+     * <p/>
+     * Results of this method are normally cached so that we not need to query the backend for every authentication attempt.
+     * <p/>
+     * @param The credentials to be validated, never null
+     * @return the authenticated User, never null
+     * @throws ElasticsearchSecurityException in case an authentication failure
+     * (when credentials are incorrect, the user does not exist or the backend is not reachable)
+     */
+    User authenticate(AuthCredentials credentials) throws ElasticsearchSecurityException;
+
+    /**
+     * Validate credentials and return an authenticated user (or throw an ElasticsearchSecurityException)
+     * <p/>
+     * Results of this method are normally cached so that we not need to query the backend for every authentication attempt.
+     * <p/>
+     * @param The credentials to be validated, never null
+     * @return the authenticated User, never null
+     * @throws ElasticsearchSecurityException in case an authentication failure
+     * (when credentials are incorrect, the user does not exist or the backend is not reachable)
+     */
+    default void authenticate(AuthCredentials credentials, Consumer<User> onSuccess, Consumer<Exception> onFailure) {
+        try {
+            User user = this.authenticate(credentials);
+            onSuccess.accept(user);
+        } catch (Exception e) {
+            onFailure.accept(e);
+        }
+    }
+}

src/main/java/com/amazon/opendistroforelasticsearch/security/auth/SyncAuthorizationBackend.java

@@ -0,0 +1,36 @@
+package com.amazon.opendistroforelasticsearch.security.auth;
+
+import com.amazon.opendistroforelasticsearch.security.user.AuthCredentials;
+import com.amazon.opendistroforelasticsearch.security.user.User;
+import org.elasticsearch.ElasticsearchSecurityException;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.function.Consumer;
+
+public interface SyncAuthorizationBackend extends AuthorizationBackend {
+    /**
+     * Populate a {@link User} with backend roles. This method will not be called for cached users.
+     * <p/>
+     * Add them by calling either {@code user.addRole()} or {@code user.addRoles()}
+     * </P>
+     * @param user The authenticated user to populate with backend roles, never null
+     * @param credentials Credentials to authenticate to the authorization backend, maybe null.
+     * <em>This parameter is for future usage, currently always empty credentials are passed!</em>
+     * @throws ElasticsearchSecurityException in case when the authorization backend cannot be reached
+     * or the {@code credentials} are insufficient to authenticate to the authorization backend.
+     */
+    void fillRoles(User user, AuthCredentials credentials) throws ElasticsearchSecurityException;
+
+    default void retrieveRoles(User user, AuthCredentials credentials, Consumer<Collection<String>> onSuccess, Consumer<Exception> onFailure) {
+        try {
+            fillRoles(user, credentials);
+            // TODO notsonice
+
+            onSuccess.accept(Collections.emptyList());
+        } catch (Exception e) {
+            onFailure.accept(e);
+        }
+    }
+
+}

src/main/java/com/amazon/opendistroforelasticsearch/security/auth/internal/InternalAuthenticationBackend.java

@@ -39,17 +39,18 @@
 import java.util.Map;
 import java.util.Map.Entry;
 
+
 import org.bouncycastle.crypto.generators.OpenBSDBCrypt;
 import org.elasticsearch.ElasticsearchSecurityException;
 
-import com.amazon.opendistroforelasticsearch.security.auth.AuthenticationBackend;
-import com.amazon.opendistroforelasticsearch.security.auth.AuthorizationBackend;
+import com.amazon.opendistroforelasticsearch.security.auth.SyncAuthenticationBackend;
+import com.amazon.opendistroforelasticsearch.security.auth.SyncAuthorizationBackend;
 import com.amazon.opendistroforelasticsearch.security.securityconf.InternalUsersModel;
 import com.amazon.opendistroforelasticsearch.security.user.AuthCredentials;
 import com.amazon.opendistroforelasticsearch.security.user.User;
 import org.greenrobot.eventbus.Subscribe;
 
-public class InternalAuthenticationBackend implements AuthenticationBackend, AuthorizationBackend {
+public class InternalAuthenticationBackend implements SyncAuthenticationBackend, SyncAuthorizationBackend {
 
     private InternalUsersModel internalUsersModel;
 

src/main/java/com/amazon/opendistroforelasticsearch/security/auth/internal/NoOpAuthenticationBackend.java

@@ -32,13 +32,14 @@
 
 import java.nio.file.Path;
 
+
 import org.elasticsearch.common.settings.Settings;
 
-import com.amazon.opendistroforelasticsearch.security.auth.AuthenticationBackend;
+import com.amazon.opendistroforelasticsearch.security.auth.SyncAuthenticationBackend;
 import com.amazon.opendistroforelasticsearch.security.user.AuthCredentials;
 import com.amazon.opendistroforelasticsearch.security.user.User;
 
-public class NoOpAuthenticationBackend implements AuthenticationBackend {
+public class NoOpAuthenticationBackend implements SyncAuthenticationBackend {
 
     public NoOpAuthenticationBackend(final Settings settings, final Path configPath) {
         super();