Permission for restricted indices (#37577)

This grants the capability to grant privileges over certain restricted
indices (.security and .security-6 at the moment).
It also removes the special status of the superuser role.

IndicesPermission.Group is extended by adding the `allow_restricted_indices`
boolean flag. By default the flag is false. When it is toggled, you acknowledge
that the indices under the scope of the permission group can cover the
restricted indices as well. Otherwise, by default, restricted indices are ignored
when granting privileges, thus rendering them hidden for authorization purposes.
This effectively adds a confirmation "check-box" for roles that might grant
privileges to restricted indices.

The "special status" of the superuser role has been removed and coded as
any other role:
```
new RoleDescriptor("superuser",
    new String[] { "all" },
    new RoleDescriptor.IndicesPrivileges[] {
        RoleDescriptor.IndicesPrivileges.builder()
            .indices("*")
            .privileges("all")
            .allowRestrictedIndices(true)
// this ----^
            .build() },
            new RoleDescriptor.ApplicationResourcePrivileges[] {
                RoleDescriptor.ApplicationResourcePrivileges.builder()
                    .application("*")
                    .privileges("*")
                    .resources("*")
                    .build()
            },
            null, new String[] { "*" },
    MetadataUtils.DEFAULT_RESERVED_METADATA,
    Collections.emptyMap());
```
In the context of the Backup .security work, this allows the creation of a
"curator role" that would permit listing (get settings) for all indices
(including the restricted ones). That way the curator role would be able to 
ist and snapshot all indices, but not read or restore any of them.

Supersedes #36765
Relates #34454

Author: albertzaharovits

client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/AbstractIndicesPrivileges.java

@@ -39,14 +39,16 @@
 
 public abstract class AbstractIndicesPrivileges {
     static final ParseField NAMES = new ParseField("names");
+    static final ParseField ALLOW_RESTRICTED_INDICES = new ParseField("allow_restricted_indices");
     static final ParseField PRIVILEGES = new ParseField("privileges");
     static final ParseField FIELD_PERMISSIONS = new ParseField("field_security");
     static final ParseField QUERY = new ParseField("query");
 
     protected final Set<String> indices;
     protected final Set<String> privileges;
+    protected final boolean allowRestrictedIndices;
 
-    AbstractIndicesPrivileges(Collection<String> indices, Collection<String> privileges) {
+    AbstractIndicesPrivileges(Collection<String> indices, Collection<String> privileges, boolean allowRestrictedIndices) {
         if (null == indices || indices.isEmpty()) {
             throw new IllegalArgumentException("indices privileges must refer to at least one index name or index name pattern");
         }
@@ -55,6 +57,7 @@ public abstract class AbstractIndicesPrivileges {
         }
         this.indices = Collections.unmodifiableSet(new HashSet<>(indices));
         this.privileges = Collections.unmodifiableSet(new HashSet<>(privileges));
+        this.allowRestrictedIndices = allowRestrictedIndices;
     }
 
     /**
@@ -73,6 +76,15 @@ public Set<String> getPrivileges() {
         return this.privileges;
     }
 
+    /**
+     * True if the privileges cover restricted internal indices too. Certain indices are reserved for internal services and should be
+     * transparent to ordinary users. For that matter, when granting privileges, you also have to toggle this flag to confirm that all
+     * indices, including restricted ones, are in the scope of this permission. By default this is false.
+     */
+    public boolean allowRestrictedIndices() {
+        return this.allowRestrictedIndices;
+    }
+
     /**
      * If {@code true} some documents might not be visible. Only the documents
      * matching {@code query} will be readable.

client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/IndicesPrivileges.java

@@ -50,14 +50,16 @@ public final class IndicesPrivileges extends AbstractIndicesPrivileges implement
             int i = 0;
             final Collection<String> indices = (Collection<String>) constructorObjects[i++];
             final Collection<String> privileges = (Collection<String>) constructorObjects[i++];
+            final boolean allowRestrictedIndices = (Boolean) constructorObjects[i++];
             final FieldSecurity fields = (FieldSecurity) constructorObjects[i++];
             final String query = (String) constructorObjects[i];
-            return new IndicesPrivileges(indices, privileges, fields, query);
+            return new IndicesPrivileges(indices, privileges, allowRestrictedIndices, fields, query);
         });
 
     static {
         PARSER.declareStringArray(constructorArg(), NAMES);
         PARSER.declareStringArray(constructorArg(), PRIVILEGES);
+        PARSER.declareBoolean(constructorArg(), ALLOW_RESTRICTED_INDICES);
         PARSER.declareObject(optionalConstructorArg(), FieldSecurity::parse, FIELD_PERMISSIONS);
         PARSER.declareStringOrNull(optionalConstructorArg(), QUERY);
     }
@@ -66,9 +68,9 @@ public final class IndicesPrivileges extends AbstractIndicesPrivileges implement
     // missing query means all documents, i.e. no restrictions
     private final @Nullable String query;
 
-    private IndicesPrivileges(Collection<String> indices, Collection<String> privileges, @Nullable FieldSecurity fieldSecurity,
-                              @Nullable String query) {
-        super(indices, privileges);
+    private IndicesPrivileges(Collection<String> indices, Collection<String> privileges, boolean allowRestrictedIndices,
+                              @Nullable FieldSecurity fieldSecurity, @Nullable String query) {
+        super(indices, privileges, allowRestrictedIndices);
         this.fieldSecurity = fieldSecurity;
         this.query = query;
     }
@@ -118,13 +120,14 @@ public boolean equals(Object o) {
         IndicesPrivileges that = (IndicesPrivileges) o;
         return indices.equals(that.indices)
             && privileges.equals(that.privileges)
+            && allowRestrictedIndices == that.allowRestrictedIndices
             && Objects.equals(this.fieldSecurity, that.fieldSecurity)
             && Objects.equals(query, that.query);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(indices, privileges, fieldSecurity, query);
+        return Objects.hash(indices, privileges, allowRestrictedIndices, fieldSecurity, query);
     }
 
     @Override
@@ -141,6 +144,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field(NAMES.getPreferredName(), indices);
         builder.field(PRIVILEGES.getPreferredName(), privileges);
+        builder.field(ALLOW_RESTRICTED_INDICES.getPreferredName(), allowRestrictedIndices);
         if (fieldSecurity != null) {
             builder.field(FIELD_PERMISSIONS.getPreferredName(), fieldSecurity, params);
         }
@@ -170,6 +174,7 @@ public static final class Builder {
         Collection<String> deniedFields = null;
         private @Nullable
         String query = null;
+        boolean allowRestrictedIndices = false;
 
         public Builder() {
         }
@@ -223,14 +228,19 @@ public Builder query(@Nullable String query) {
             return this;
         }
 
+        public Builder allowRestrictedIndices(boolean allow) {
+            this.allowRestrictedIndices = allow;
+            return this;
+        }
+
         public IndicesPrivileges build() {
             final FieldSecurity fieldSecurity;
             if (grantedFields == null && deniedFields == null) {
                 fieldSecurity = null;
             } else {
                 fieldSecurity = new FieldSecurity(grantedFields, deniedFields);
             }
-            return new IndicesPrivileges(indices, privileges, fieldSecurity, query);
+            return new IndicesPrivileges(indices, privileges, allowRestrictedIndices, fieldSecurity, query);
         }
     }
 

client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/UserIndicesPrivileges.java

@@ -52,6 +52,7 @@ public class UserIndicesPrivileges extends AbstractIndicesPrivileges {
     static {
         PARSER.declareStringArray(constructorArg(), IndicesPrivileges.NAMES);
         PARSER.declareStringArray(constructorArg(), IndicesPrivileges.PRIVILEGES);
+        PARSER.declareBoolean(constructorArg(), IndicesPrivileges.ALLOW_RESTRICTED_INDICES);
         PARSER.declareObjectArray(optionalConstructorArg(), IndicesPrivileges.FieldSecurity::parse, IndicesPrivileges.FIELD_PERMISSIONS);
         PARSER.declareStringArray(optionalConstructorArg(), IndicesPrivileges.QUERY);
     }
@@ -61,30 +62,23 @@ private static UserIndicesPrivileges buildObjectFromParserArgs(Object[] args) {
         return new UserIndicesPrivileges(
             (List<String>) args[0],
             (List<String>) args[1],
-            (List<IndicesPrivileges.FieldSecurity>) args[2],
-            (List<String>) args[3]
+            (Boolean) args[2],
+            (List<IndicesPrivileges.FieldSecurity>) args[3],
+            (List<String>) args[4]
         );
     }
 
     public static UserIndicesPrivileges fromXContent(XContentParser parser) throws IOException {
         return PARSER.parse(parser, null);
     }
 
-    public UserIndicesPrivileges(Collection<String> indices, Collection<String> privileges,
+    public UserIndicesPrivileges(Collection<String> indices, Collection<String> privileges, boolean allowRestrictedIndices,
                                  Collection<IndicesPrivileges.FieldSecurity> fieldSecurity, Collection<String> query) {
-        super(indices, privileges);
+        super(indices, privileges, allowRestrictedIndices);
         this.fieldSecurity = fieldSecurity == null ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(fieldSecurity));
         this.query = query == null ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(query));
     }
 
-    public Set<String> getIndices() {
-        return indices;
-    }
-
-    public Set<String> getPrivileges() {
-        return privileges;
-    }
-
     public Set<IndicesPrivileges.FieldSecurity> getFieldSecurity() {
         return fieldSecurity;
     }
@@ -114,20 +108,22 @@ public boolean equals(Object o) {
         final UserIndicesPrivileges that = (UserIndicesPrivileges) o;
         return Objects.equals(indices, that.indices) &&
             Objects.equals(privileges, that.privileges) &&
+            allowRestrictedIndices == that.allowRestrictedIndices &&
             Objects.equals(fieldSecurity, that.fieldSecurity) &&
             Objects.equals(query, that.query);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(indices, privileges, fieldSecurity, query);
+        return Objects.hash(indices, privileges, allowRestrictedIndices, fieldSecurity, query);
     }
 
     @Override
     public String toString() {
         return "UserIndexPrivilege{" +
             "indices=" + indices +
             ", privileges=" + privileges +
+            ", allow_restricted_indices=" + allowRestrictedIndices +
             ", fieldSecurity=" + fieldSecurity +
             ", query=" + query +
             '}';

client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java

@@ -393,7 +393,8 @@ public void testPutRole() throws IOException {
         final List<String> indicesPrivilegeDeniedName = Arrays.asList(randomArray(3, String[]::new, () -> randomAlphaOfLength(5)));
         final String indicesPrivilegeQuery = randomAlphaOfLengthBetween(0, 7);
         final IndicesPrivileges indicesPrivilege = IndicesPrivileges.builder().indices(indicesName).privileges(indicesPrivilegeName)
-                .grantedFields(indicesPrivilegeGrantedName).deniedFields(indicesPrivilegeDeniedName).query(indicesPrivilegeQuery).build();
+                .allowRestrictedIndices(randomBoolean()).grantedFields(indicesPrivilegeGrantedName).deniedFields(indicesPrivilegeDeniedName)
+                .query(indicesPrivilegeQuery).build();
         final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values());
         final Map<String, String> expectedParams;
         if (refreshPolicy != RefreshPolicy.NONE) {

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java

@@ -742,8 +742,10 @@ public void testHasPrivileges() throws Exception {
             HasPrivilegesRequest request = new HasPrivilegesRequest(
                 Sets.newHashSet("monitor", "manage"),
                 Sets.newHashSet(
-                    IndicesPrivileges.builder().indices("logstash-2018-10-05").privileges("read", "write").build(),
-                    IndicesPrivileges.builder().indices("logstash-2018-*").privileges("read").build()
+                    IndicesPrivileges.builder().indices("logstash-2018-10-05").privileges("read", "write")
+                        .allowRestrictedIndices(false).build(),
+                    IndicesPrivileges.builder().indices("logstash-2018-*").privileges("read")
+                        .allowRestrictedIndices(true).build()
                 ),
                 null
             );

client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetRolesResponseTests.java

@@ -48,6 +48,7 @@ public void testFromXContent() throws IOException {
                 "      {\n" +
                 "        \"names\" : [ \"index1\", \"index2\" ],\n" +
                 "        \"privileges\" : [ \"all\" ],\n" +
+                "        \"allow_restricted_indices\" : true,\n" +
                 "        \"field_security\" : {\n" +
                 "          \"grant\" : [ \"title\", \"body\" ]}\n" +
                 "      }\n" +
@@ -81,6 +82,7 @@ public void usedDeprecatedField(String usedName, String replacedWith) {
             .indices("index1", "index2")
             .privileges("all")
             .grantedFields("title", "body")
+            .allowRestrictedIndices(true)
             .build();
         assertThat(role.getIndicesPrivileges().contains(expectedIndicesPrivileges), equalTo(true));
         final Map<String, Object> expectedMetadata = new HashMap<>();
@@ -106,6 +108,7 @@ public void testEqualsHashCode() {
             .privileges("write", "monitor", "delete")
             .grantedFields("field1", "field2")
             .deniedFields("field3", "field4")
+            .allowRestrictedIndices(true)
             .build();
         Map<String, Object> metadata = new HashMap<>();
         metadata.put("key", "value");
@@ -125,9 +128,10 @@ public void testEqualsHashCode() {
             .privileges("write", "monitor", "delete")
             .grantedFields("other_field1", "other_field2")
             .deniedFields("other_field3", "other_field4")
+            .allowRestrictedIndices(false)
             .build();
         Map<String, Object> metadata2 = new HashMap<>();
-        metadata.put("other_key", "other_value");
+        metadata2.put("other_key", "other_value");
         final Role role2 = Role.builder()
             .name("role2_name")
             .clusterPrivileges("monitor", "manage", "manage_saml")
@@ -158,6 +162,7 @@ private static GetRolesResponse mutateTestItem(GetRolesResponse original) {
                 .privileges("write", "monitor", "delete")
                 .grantedFields("field1", "field2")
                 .deniedFields("field3", "field4")
+                .allowRestrictedIndices(true)
                 .build();
             Map<String, Object> metadata = new HashMap<String, Object>();
             metadata.put("key", "value");
@@ -179,6 +184,7 @@ private static GetRolesResponse mutateTestItem(GetRolesResponse original) {
                 .privileges("write", "monitor", "delete")
                 .grantedFields("field1", "field2")
                 .deniedFields("field3", "field4")
+                .allowRestrictedIndices(false)
                 .build();
             Map<String, Object> metadata = new HashMap<String, Object>();
             metadata.put("key", "value");

client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetUserPrivilegesResponseTests.java

@@ -48,17 +48,18 @@ public void testParse() throws Exception {
             " {\"application\":{\"manage\":{\"applications\":[\"apps-*\"]}}}" +
             "]," +
             "\"indices\":[" +
-            " {\"names\":[\"test-1-*\"],\"privileges\":[\"read\"]}," +
-            " {\"names\":[\"test-4-*\"],\"privileges\":[\"read\"],\"field_security\":[{\"grant\":[\"*\"],\"except\":[\"private-*\"]}]}," +
-            " {\"names\":[\"test-6-*\",\"test-7-*\"],\"privileges\":[\"read\"]," +
+            " {\"names\":[\"test-1-*\"],\"privileges\":[\"read\"],\"allow_restricted_indices\": false}," +
+            " {\"names\":[\"test-4-*\"],\"privileges\":[\"read\"],\"allow_restricted_indices\": true," +
+            "  \"field_security\":[{\"grant\":[\"*\"],\"except\":[\"private-*\"]}]}," +
+            " {\"names\":[\"test-6-*\",\"test-7-*\"],\"privileges\":[\"read\"],\"allow_restricted_indices\": true," +
             "  \"query\":[\"{\\\"term\\\":{\\\"test\\\":true}}\"]}," +
-            " {\"names\":[\"test-2-*\"],\"privileges\":[\"read\"]," +
+            " {\"names\":[\"test-2-*\"],\"privileges\":[\"read\"],\"allow_restricted_indices\": false," +
             "  \"field_security\":[{\"grant\":[\"*\"],\"except\":[\"secret-*\",\"private-*\"]},{\"grant\":[\"apps-*\"]}]," +
             "  \"query\":[\"{\\\"term\\\":{\\\"test\\\":true}}\",\"{\\\"term\\\":{\\\"apps\\\":true}}\"]}," +
-            " {\"names\":[\"test-3-*\",\"test-6-*\"],\"privileges\":[\"read\",\"write\"]}," +
-            " {\"names\":[\"test-3-*\",\"test-4-*\",\"test-5-*\"],\"privileges\":[\"read\"]," +
+            " {\"names\":[\"test-3-*\",\"test-6-*\"],\"privileges\":[\"read\",\"write\"],\"allow_restricted_indices\": true}," +
+            " {\"names\":[\"test-3-*\",\"test-4-*\",\"test-5-*\"],\"privileges\":[\"read\"],\"allow_restricted_indices\": false," +
             "  \"field_security\":[{\"grant\":[\"test-*\"]}]}," +
-            " {\"names\":[\"test-1-*\",\"test-9-*\"],\"privileges\":[\"all\"]}" +
+            " {\"names\":[\"test-1-*\",\"test-9-*\"],\"privileges\":[\"all\"],\"allow_restricted_indices\": true}" +
             "]," +
             "\"applications\":[" +
             " {\"application\":\"app-dne\",\"privileges\":[\"all\"],\"resources\":[\"*\"]}," +
@@ -80,12 +81,14 @@ public void testParse() throws Exception {
         assertThat(response.getIndicesPrivileges().size(), equalTo(7));
         assertThat(Iterables.get(response.getIndicesPrivileges(), 0).getIndices(), contains("test-1-*"));
         assertThat(Iterables.get(response.getIndicesPrivileges(), 0).getPrivileges(), contains("read"));
+        assertThat(Iterables.get(response.getIndicesPrivileges(), 0).allowRestrictedIndices(), equalTo(false));
         assertThat(Iterables.get(response.getIndicesPrivileges(), 0).getFieldSecurity(), emptyIterable());
         assertThat(Iterables.get(response.getIndicesPrivileges(), 0).getQueries(), emptyIterable());
 
         final UserIndicesPrivileges test4Privilege = Iterables.get(response.getIndicesPrivileges(), 1);
         assertThat(test4Privilege.getIndices(), contains("test-4-*"));
         assertThat(test4Privilege.getPrivileges(), contains("read"));
+        assertThat(test4Privilege.allowRestrictedIndices(), equalTo(true));
         assertThat(test4Privilege.getFieldSecurity(), iterableWithSize(1));
         final IndicesPrivileges.FieldSecurity test4FLS = test4Privilege.getFieldSecurity().iterator().next();
         assertThat(test4FLS.getGrantedFields(), contains("*"));
@@ -95,6 +98,7 @@ public void testParse() throws Exception {
         final UserIndicesPrivileges test2Privilege = Iterables.get(response.getIndicesPrivileges(), 3);
         assertThat(test2Privilege.getIndices(), contains("test-2-*"));
         assertThat(test2Privilege.getPrivileges(), contains("read"));
+        assertThat(test2Privilege.allowRestrictedIndices(), equalTo(false));
         assertThat(test2Privilege.getFieldSecurity(), iterableWithSize(2));
         final Iterator<IndicesPrivileges.FieldSecurity> test2FLSIter = test2Privilege.getFieldSecurity().iterator();
         final IndicesPrivileges.FieldSecurity test2FLS1 = test2FLSIter.next();
@@ -110,6 +114,7 @@ public void testParse() throws Exception {
 
         assertThat(Iterables.get(response.getIndicesPrivileges(), 6).getIndices(), contains("test-1-*", "test-9-*"));
         assertThat(Iterables.get(response.getIndicesPrivileges(), 6).getPrivileges(), contains("all"));
+        assertThat(Iterables.get(response.getIndicesPrivileges(), 6).allowRestrictedIndices(), equalTo(true));
         assertThat(Iterables.get(response.getIndicesPrivileges(), 6).getFieldSecurity(), emptyIterable());
         assertThat(Iterables.get(response.getIndicesPrivileges(), 6).getQueries(), emptyIterable());