Implement lookup of permissions for API keys

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java

@@ -18,6 +18,8 @@
 
 import java.io.IOException;
 import java.util.Base64;
+import java.util.Collections;
+import java.util.Map;
 import java.util.Objects;
 
 // TODO(hub-cap) Clean this up after moving User over - This class can re-inherit its field AUTHENTICATION_KEY in AuthenticationField.
@@ -28,16 +30,25 @@ public class Authentication implements ToXContentObject {
     private final RealmRef authenticatedBy;
     private final RealmRef lookedUpBy;
     private final Version version;
+    private final AuthenticationType type;
+    private final Map<String, Object> metadata;
 
     public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy) {
         this(user, authenticatedBy, lookedUpBy, Version.CURRENT);
     }
 
     public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy, Version version) {
+        this(user, authenticatedBy, lookedUpBy, version, AuthenticationType.REALM, Collections.emptyMap());
+    }
+
+    public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy, Version version,
+                          AuthenticationType type, Map<String, Object> metadata) {
         this.user = Objects.requireNonNull(user);
         this.authenticatedBy = Objects.requireNonNull(authenticatedBy);
         this.lookedUpBy = lookedUpBy;
         this.version = version;
+        this.type = type;
+        this.metadata = metadata;
     }
 
     public Authentication(StreamInput in) throws IOException {
@@ -49,6 +60,13 @@ public Authentication(StreamInput in) throws IOException {
             this.lookedUpBy = null;
         }
         this.version = in.getVersion();
+        if (in.getVersion().onOrAfter(Version.V_7_0_0)) { // TODO change to V6_6 after backport
+            type = AuthenticationType.values()[in.readVInt()];
+            metadata = in.readMap();
+        } else {
+            type = AuthenticationType.REALM;
+            metadata = Collections.emptyMap();
+        }
     }
 
     public User getUser() {
@@ -67,8 +85,15 @@ public Version getVersion() {
         return version;
     }
 
-    public static Authentication readFromContext(ThreadContext ctx)
-            throws IOException, IllegalArgumentException {
+    public AuthenticationType getAuthenticationType() {
+        return type;
+    }
+
+    public Map<String, Object> getMetadata() {
+        return metadata;
+    }
+
+    public static Authentication readFromContext(ThreadContext ctx) throws IOException, IllegalArgumentException {
         Authentication authentication = ctx.getTransient(AuthenticationField.AUTHENTICATION_KEY);
         if (authentication != null) {
             assert ctx.getHeader(AuthenticationField.AUTHENTICATION_KEY) != null;
@@ -107,8 +132,7 @@ public static Authentication decode(String header) throws IOException {
      * Writes the authentication to the context. There must not be an existing authentication in the context and if there is an
      * {@link IllegalStateException} will be thrown
      */
-    public void writeToContext(ThreadContext ctx)
-            throws IOException, IllegalArgumentException {
+    public void writeToContext(ThreadContext ctx) throws IOException, IllegalArgumentException {
         ensureContextDoesNotContainAuthentication(ctx);
         String header = encode();
         ctx.putTransient(AuthenticationField.AUTHENTICATION_KEY, this);
@@ -141,28 +165,28 @@ public void writeTo(StreamOutput out) throws IOException {
         } else {
             out.writeBoolean(false);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_0_0)) { // TODO change to V6_6 after backport
+            out.writeVInt(type.ordinal());
+            out.writeMap(metadata);
+        }
     }
 
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-
         Authentication that = (Authentication) o;
-
-        if (!user.equals(that.user)) return false;
-        if (!authenticatedBy.equals(that.authenticatedBy)) return false;
-        if (lookedUpBy != null ? !lookedUpBy.equals(that.lookedUpBy) : that.lookedUpBy != null) return false;
-        return version.equals(that.version);
+        return user.equals(that.user) &&
+            authenticatedBy.equals(that.authenticatedBy) &&
+            Objects.equals(lookedUpBy, that.lookedUpBy) &&
+            version.equals(that.version) &&
+            type == that.type &&
+            metadata.equals(that.metadata);
     }
 
     @Override
     public int hashCode() {
-        int result = user.hashCode();
-        result = 31 * result + authenticatedBy.hashCode();
-        result = 31 * result + (lookedUpBy != null ? lookedUpBy.hashCode() : 0);
-        result = 31 * result + version.hashCode();
-        return result;
+        return Objects.hash(user, authenticatedBy, lookedUpBy, version, type, metadata);
     }
 
     @Override
@@ -246,5 +270,11 @@ public int hashCode() {
             return result;
         }
     }
+
+    public enum AuthenticationType {
+        REALM,
+        API_KEY,
+        TOKEN
+    }
 }
 

x-pack/plugin/core/src/main/resources/security-index-template.json

@@ -156,7 +156,7 @@
         },
         "role_descriptors" : {
           "type" : "object",
-          "dynamic" : true
+          "enabled": false
         },
         "version" : {
           "type" : "integer"

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -479,7 +479,7 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
         // minimal
         getLicenseState().addListener(allRolesStore::invalidateAll);
         final AuthorizationService authzService = new AuthorizationService(settings, allRolesStore, clusterService,
-            auditTrailService, failureHandler, threadPool, anonymousUser);
+            auditTrailService, failureHandler, threadPool, anonymousUser, apiKeyService);
         components.add(nativeRolesStore); // used by roles actions
         components.add(reservedRolesStore); // used by roles actions
         components.add(allRolesStore); // for SecurityFeatureSet and clear roles cache

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUserPrivilegesAction.java

@@ -60,13 +60,14 @@ public TransportGetUserPrivilegesAction(ThreadPool threadPool, TransportService
     protected void doExecute(Task task, GetUserPrivilegesRequest request, ActionListener<GetUserPrivilegesResponse> listener) {
         final String username = request.username();
 
-        final User user = Authentication.getAuthentication(threadPool.getThreadContext()).getUser();
+        final Authentication authentication = Authentication.getAuthentication(threadPool.getThreadContext());
+        final User user = authentication.getUser();
         if (user.principal().equals(username) == false) {
             listener.onFailure(new IllegalArgumentException("users may only list the privileges of their own account"));
             return;
         }
 
-        authorizationService.roles(user, ActionListener.wrap(
+        authorizationService.roles(user, authentication, ActionListener.wrap(
             role -> listener.onResponse(buildResponseObject(role)),
             listener::onFailure));
     }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java

@@ -68,13 +68,14 @@ public TransportHasPrivilegesAction(ThreadPool threadPool, TransportService tran
     protected void doExecute(Task task, HasPrivilegesRequest request, ActionListener<HasPrivilegesResponse> listener) {
         final String username = request.username();
 
-        final User user = Authentication.getAuthentication(threadPool.getThreadContext()).getUser();
+        final Authentication authentication = Authentication.getAuthentication(threadPool.getThreadContext());
+        final User user = authentication.getUser();
         if (user.principal().equals(username) == false) {
             listener.onFailure(new IllegalArgumentException("users may only check the privileges of their own account"));
             return;
         }
 
-        authorizationService.roles(user, ActionListener.wrap(
+        authorizationService.roles(user, authentication, ActionListener.wrap(
             role -> resolveApplicationPrivileges(request, ActionListener.wrap(
                 applicationPrivilegeLookup -> checkPrivileges(request, role, applicationPrivilegeLookup, listener),
                 listener::onFailure)),

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java

@@ -19,29 +19,41 @@
 import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
+import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import javax.crypto.SecretKeyFactory;
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.security.NoSuchAlgorithmException;
 import java.time.Clock;
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Base64;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -55,7 +67,12 @@
 public class ApiKeyService {
 
     private static final Logger logger = LogManager.getLogger(ApiKeyService.class);
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
     private static final String TYPE = "doc";
+    static final String API_KEY_ID_KEY = "_security_api_key_id";
+    static final String API_KEY_ROLE_DESCRIPTORS_KEY = "_security_api_key_role_descriptors";
+    static final String API_KEY_ROLE_KEY = "_security_api_key_role";
+
     public static final Setting<String> PASSWORD_HASHING_ALGORITHM = new Setting<>(
         "xpack.security.authc.api_key_hashing.algorithm", "pbkdf2", Function.identity(), (v, s) -> {
         if (Hasher.getAvailableAlgoStoredHash().contains(v.toLowerCase(Locale.ROOT)) == false) {
@@ -126,8 +143,12 @@ public void createApiKey(Authentication authentication, CreateApiKeyRequest requ
                     }
                 }
 
-                builder.array("role_descriptors", request.getRoleDescriptors())
-                    .field("name", request.getName())
+                builder.startObject("role_descriptors");
+                for (RoleDescriptor descriptor : request.getRoleDescriptors()) {
+                    builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true));
+                }
+                builder.endObject();
+                builder.field("name", request.getName())
                     .field("version", version.id)
                     .startObject("creator")
                     .field("principal", authentication.getUser().principal())
@@ -174,7 +195,8 @@ void authenticateWithApiKeyIfPresent(ThreadContext ctx, ActionListener<Authentic
                 executeAsyncWithOrigin(ctx, SECURITY_ORIGIN, getRequest, ActionListener.<GetResponse>wrap(response -> {
                     if (response.isExists()) {
                         try (ApiKeyCredentials ignore = credentials) {
-                            validateApiKeyCredentials(response.getSource(), credentials, clock, listener);
+                            final Map<String, Object> source = response.getSource();
+                            validateApiKeyCredentials(source, credentials, clock, listener);
                         }
                     } else {
                         credentials.close();
@@ -194,6 +216,56 @@ void authenticateWithApiKeyIfPresent(ThreadContext ctx, ActionListener<Authentic
         }
     }
 
+    /**
+     * The current request has been authenticated by an API key and this method enables the
+     * retrieval of role descriptors that are associated with the api key and triggers the building
+     * of the {@link Role} to authorize the request.
+     */
+    public void getRoleForApiKey(Authentication authentication, ThreadContext threadContext, CompositeRolesStore rolesStore,
+                                 FieldPermissionsCache fieldPermissionsCache, ActionListener<Role> listener) {
+        if (authentication.getAuthenticationType() != Authentication.AuthenticationType.API_KEY) {
+            throw new IllegalStateException("authentication type must be api key but is " + authentication.getAuthenticationType());
+        }
+
+        final Map<String, Object> metadata = authentication.getMetadata();
+        final String apiKeyId = (String) metadata.get(API_KEY_ID_KEY);
+        final String contextKeyId = threadContext.getTransient(API_KEY_ID_KEY);
+        if (apiKeyId.equals(contextKeyId)) {
+            final Role preBuiltRole = threadContext.getTransient(API_KEY_ROLE_KEY);
+            if (preBuiltRole != null) {
+                listener.onResponse(preBuiltRole);
+                return;
+            }
+        } else if (contextKeyId != null) {
+            throw new IllegalStateException("authentication api key id [" + apiKeyId + "] does not match context value [" +
+                contextKeyId + "]");
+        }
+
+        final Map<String, Object> roleDescriptors = (Map<String, Object>) metadata.get(API_KEY_ROLE_DESCRIPTORS_KEY);
+        final List<RoleDescriptor> roleDescriptorList = roleDescriptors.entrySet().stream()
+            .map(entry -> {
+                final String name = entry.getKey();
+                final Map<String, Object> rdMap = (Map<String, Object>) entry.getValue();
+                try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+                    builder.map(rdMap);
+                    try (XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY,
+                        new ApiKeyLoggingDeprecationHandler(deprecationLogger, apiKeyId),
+                        BytesReference.bytes(builder).streamInput())) {
+                        return RoleDescriptor.parse(name, parser, false);
+                    }
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }).collect(Collectors.toList());
+
+        rolesStore.buildRoleFromDescriptors(roleDescriptorList, fieldPermissionsCache, ActionListener.wrap(role -> {
+            threadContext.putTransient(API_KEY_ID_KEY, apiKeyId);
+            threadContext.putTransient(API_KEY_ROLE_KEY, role);
+            listener.onResponse(role);
+        }, listener::onFailure));
+
+    }
+
     /**
      * Validates the ApiKey using the source map
      * @param source the source map from a get of the ApiKey document
@@ -214,13 +286,13 @@ static void validateApiKeyCredentials(Map<String, Object> source, ApiKeyCredenti
                 final Map<String, Object> creator = Objects.requireNonNull((Map<String, Object>) source.get("creator"));
                 final String principal = Objects.requireNonNull((String) creator.get("principal"));
                 final Map<String, Object> metadata = (Map<String, Object>) creator.get("metadata");
-                final List<Map<String, Object>> roleDescriptors = (List<Map<String, Object>>) source.get("role_descriptors");
-                final String[] roleNames = roleDescriptors.stream()
-                    .map(rdSource -> (String) rdSource.get("name"))
-                    .collect(Collectors.toList())
-                    .toArray(Strings.EMPTY_ARRAY);
+                final Map<String, Object> roleDescriptors = (Map<String, Object>) source.get("role_descriptors");
+                final String[] roleNames = roleDescriptors.keySet().toArray(Strings.EMPTY_ARRAY);
                 final User apiKeyUser = new User(principal, roleNames, null, null, metadata, true);
-                listener.onResponse(AuthenticationResult.success(apiKeyUser));
+                final Map<String, Object> authResultMetadata = new HashMap<>();
+                authResultMetadata.put(API_KEY_ROLE_DESCRIPTORS_KEY, roleDescriptors);
+                authResultMetadata.put(API_KEY_ID_KEY, credentials.getId());
+                listener.onResponse(AuthenticationResult.success(apiKeyUser, authResultMetadata));
             } else {
                 listener.onResponse(AuthenticationResult.terminate("api key is expired", null));
             }
@@ -310,4 +382,27 @@ public void close() {
             key.close();
         }
     }
+
+    private static class ApiKeyLoggingDeprecationHandler implements DeprecationHandler {
+
+        private final DeprecationLogger deprecationLogger;
+        private final String apiKeyId;
+
+        private ApiKeyLoggingDeprecationHandler(DeprecationLogger logger, String apiKeyId) {
+            this.deprecationLogger = logger;
+            this.apiKeyId = apiKeyId;
+        }
+
+        @Override
+        public void usedDeprecatedName(String usedName, String modernName) {
+            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], expected [{}] instead",
+                usedName, apiKeyId, modernName);
+        }
+
+        @Override
+        public void usedDeprecatedField(String usedName, String replacedWith) {
+            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], replaced by [{}]",
+                usedName, apiKeyId, replacedWith);
+        }
+    }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java

@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.common.Nullable;
@@ -205,7 +206,8 @@ private void checkForApiKey() {
                     if (authResult.isAuthenticated()) {
                         final User user = authResult.getUser();
                         authenticatedBy = new RealmRef("_es_api_key", "_es_api_key", nodeName);
-                        writeAuthToContext(new Authentication(user, authenticatedBy, null));
+                        writeAuthToContext(new Authentication(user, authenticatedBy, null, Version.CURRENT,
+                            Authentication.AuthenticationType.API_KEY, authResult.getMetadata()));
                     } else if (authResult.getStatus() == AuthenticationResult.Status.TERMINATE) {
                         Exception e = (authResult.getException() != null) ? authResult.getException()
                             : Exceptions.authenticationError(authResult.getMessage());