diff --git a/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java b/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java index fa99a2fdc7..cbb5ec11f0 100644 --- a/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java +++ b/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java @@ -13,6 +13,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import com.google.common.base.Stopwatch; @@ -23,11 +25,12 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest; import org.opensearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; @@ -83,6 +86,7 @@ import org.opensearch.client.indices.PutMappingRequest; import org.opensearch.client.indices.ResizeRequest; import org.opensearch.client.indices.ResizeResponse; +import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexTemplateMetadata; import org.opensearch.common.settings.Settings; @@ -94,6 +98,7 @@ import org.opensearch.index.reindex.ReindexRequest; import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.security.auditlog.AuditLog; import org.opensearch.test.framework.AuditCompliance; import org.opensearch.test.framework.AuditConfiguration; import org.opensearch.test.framework.AuditFilters; @@ -119,6 +124,7 @@ import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.opensearch.client.RequestOptions.DEFAULT; import static org.opensearch.core.rest.RestStatus.ACCEPTED; +import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; import static org.opensearch.core.rest.RestStatus.FORBIDDEN; import static org.opensearch.core.rest.RestStatus.INTERNAL_SERVER_ERROR; import static org.opensearch.rest.RestRequest.Method.DELETE; @@ -335,25 +341,27 @@ public class SearchOperationTest { * indices with names prefixed by the {@link #INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX} */ static final User USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES = new User("index-operation-tester").roles( - new Role("index-manager").indexPermissions( - "indices:admin/create", - "indices:admin/get", - "indices:admin/delete", - "indices:admin/close", - "indices:admin/close*", - "indices:admin/open", - "indices:admin/resize", - "indices:monitor/stats", - "indices:monitor/settings/get", - "indices:admin/settings/update", - "indices:admin/mapping/put", - "indices:admin/mappings/get", - "indices:admin/cache/clear", - "indices:admin/aliases" - ).on(INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("*")) + new Role("index-manager").clusterPermissions("cluster:monitor/health") + .indexPermissions( + "indices:admin/create", + "indices:admin/get", + "indices:admin/delete", + "indices:admin/close", + "indices:admin/close*", + "indices:admin/open", + "indices:admin/resize", + "indices:monitor/stats", + "indices:monitor/settings/get", + "indices:admin/settings/update", + "indices:admin/mapping/put", + "indices:admin/mappings/get", + "indices:admin/cache/clear", + "indices:admin/aliases" + ) + .on(INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("*")) ); - private static User USER_ALLOWED_TO_CREATE_INDEX = new User("user-allowed-to-create-index").roles( + private static final User USER_ALLOWED_TO_CREATE_INDEX = new User("user-allowed-to-create-index").roles( new Role("create-index-role").indexPermissions("indices:admin/create").on("*") ); @@ -456,7 +464,7 @@ public void cleanData() throws ExecutionException, InterruptedException { if (indicesExistsResponse.isExists()) { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexToBeDeleted); indices.delete(deleteIndexRequest).actionGet(); - Awaitility.await().ignoreExceptions().until(() -> indices.exists(indicesExistsRequest).get().isExists() == false); + Awaitility.await().ignoreExceptions().until(() -> !indices.exists(indicesExistsRequest).get().isExists()); } } @@ -970,12 +978,11 @@ public void shouldIndexDocumentInBulkRequest_positive() throws IOException { } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertAtLeast(4, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER));// sometimes 4 or 6 - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest"));// sometimes 2 or 4 + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } - @Ignore("Audit log verification is shown to be flaky in this test") @Test public void shouldIndexDocumentInBulkRequest_partiallyPositive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -998,10 +1005,10 @@ public void shouldIndexDocumentInBulkRequest_partiallyPositive() throws IOExcept } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "BulkShardRequest").withIndex(SONG_INDEX_NAME)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1030,7 +1037,6 @@ public void shouldIndexDocumentInBulkRequest_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "BulkShardRequest").withIndex(SONG_INDEX_NAME)); } - @Ignore("Audit log verification is shown to be flaky in this test") @Test public void shouldUpdateDocumentsInBulk_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1052,13 +1058,10 @@ public void shouldUpdateDocumentsInBulk_positive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); - + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); } - @Ignore("Audit log verification is shown to be flaky in this test") @Test public void shouldUpdateDocumentsInBulk_partiallyPositive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1081,10 +1084,10 @@ public void shouldUpdateDocumentsInBulk_partiallyPositive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "BulkShardRequest").withIndex(SONG_INDEX_NAME)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1114,6 +1117,10 @@ public void shouldUpdateDocumentsInBulk_negative() throws IOException { @Test public void shouldDeleteDocumentInBulk_positive() throws IOException { + // create index + Settings sourceIndexSettings = Settings.builder().put("index.number_of_replicas", 2).put("index.number_of_shards", 2).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, sourceIndexSettings); + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(IMMEDIATE); bulkRequest.add(new IndexRequest(WRITE_SONG_INDEX_NAME).id("one").source(SONGS[0].asMap())); @@ -1138,14 +1145,16 @@ public void shouldDeleteDocumentInBulk_positive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactly(2, auditPredicate(null).withLayer(AuditLog.Origin.TRANSPORT)); + auditLogsRule.assertAtLeastTransportMessages(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); + auditLogsRule.assertAtLeastTransportMessages(4, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); } - @Ignore("Audit log verification is shown to be flaky in this test") @Test public void shouldDeleteDocumentInBulk_partiallyPositive() throws IOException { + Settings indexSettings = Settings.builder().put("index.number_of_replicas", 0).put("index.number_of_shards", 1).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, indexSettings); + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(IMMEDIATE); bulkRequest.add(new IndexRequest(WRITE_SONG_INDEX_NAME).id("one").source(SONGS[0].asMap())); @@ -1170,10 +1179,8 @@ public void shouldDeleteDocumentInBulk_partiallyPositive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "BulkShardRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1202,7 +1209,6 @@ public void shouldDeleteDocumentInBulk_negative() throws IOException { } - @Ignore("Seems like reindixing isn't completing before the test proceeds") @Test public void shouldReindexDocuments_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(REINDEXING_USER)) { @@ -1221,14 +1227,13 @@ public void shouldReindexDocuments_positive() throws IOException { auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "ReindexRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "SearchRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(REINDEXING_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(REINDEXING_USER, "PutMappingRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "CreateIndexRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "SearchScrollRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(REINDEXING_USER)); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(REINDEXING_USER)); auditLogsRule.assertExactlyOne(missingPrivilege(REINDEXING_USER, "ClearScrollRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(REINDEXING_USER, "PutMappingRequest")); } - @Ignore("Seems like reindixing isn't completing before the test proceeds") @Test public void shouldReindexDocuments_negativeSource() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(REINDEXING_USER)) { @@ -1243,7 +1248,6 @@ public void shouldReindexDocuments_negativeSource() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(REINDEXING_USER, "SearchRequest")); } - @Ignore("Seems like reindixing isn't completing before the test proceeds") @Test public void shouldReindexDocuments_negativeDestination() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(REINDEXING_USER)) { @@ -1262,7 +1266,6 @@ public void shouldReindexDocuments_negativeDestination() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(REINDEXING_USER, "ClearScrollRequest")); } - @Ignore("Seems like reindixing isn't completing before the test proceeds") @Test public void shouldReindexDocuments_negativeSourceAndDestination() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(REINDEXING_USER)) { @@ -1335,7 +1338,6 @@ public void shouldDeleteDocument_negative() throws IOException { } } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldCreateAlias_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { @@ -1349,11 +1351,10 @@ public void shouldCreateAlias_positive() throws IOException { assertThat(internalClient, clusterContainsDocument(TEMPORARY_ALIAS_NAME, ID_S1)); } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_READ_USER).withRestRequest(POST, "/_aliases")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); - auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_READ_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_READ_USER)); } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldCreateAlias_negative() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { @@ -1371,7 +1372,6 @@ public void shouldCreateAlias_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldDeleteAlias_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { @@ -1388,8 +1388,8 @@ public void shouldDeleteAlias_positive() throws IOException { assertThat(internalClient, not(clusterContainsDocument(TEMPORARY_ALIAS_NAME, ID_S1))); } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_READ_USER).withRestRequest(POST, "/_aliases")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); - auditLogsRule.assertExactly(4, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_READ_USER)); + auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); + auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_READ_USER)); } @Test @@ -1409,7 +1409,6 @@ public void shouldDeleteAlias_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldCreateIndexTemplate_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1433,12 +1432,12 @@ public void shouldCreateIndexTemplate_positive() throws IOException { } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_template/musical-index-template")); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/song-transcription-jazz/_doc/0001")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "IndexRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(8, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1471,9 +1470,9 @@ public void shouldDeleteTemplate_positive() throws IOException { } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_template/musical-index-template")); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(DELETE, "/_template/musical-index-template")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "DeleteIndexTemplateRequest")); - auditLogsRule.assertExactly(4, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "DeleteIndexTemplateRequest")); + auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); } @Test @@ -1491,7 +1490,6 @@ public void shouldDeleteTemplate_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "DeleteIndexTemplateRequest")); } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldUpdateTemplate_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1519,12 +1517,12 @@ public void shouldUpdateTemplate_positive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_template/musical-index-template")); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/song-transcription-jazz/_doc/000one")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); + auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "IndexRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(10, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactly(3, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1614,7 +1612,7 @@ public void shouldCreateSnapshotRepository_positive() throws IOException { assertThat(internalClient, clusterContainsSnapshotRepository(TEST_SNAPSHOT_REPOSITORY_NAME)); } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_snapshot/test-snapshot-repository")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); } @Test @@ -1650,8 +1648,8 @@ public void shouldDeleteSnapshotRepository_positive() throws IOException { auditLogsRule.assertExactlyOne( userAuthenticated(LIMITED_WRITE_USER).withRestRequest(DELETE, "/_snapshot/test-snapshot-repository") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "DeleteRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "DeleteRepositoryRequest")); } @Test @@ -1668,9 +1666,10 @@ public void shouldDeleteSnapshotRepository_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "DeleteRepositoryRequest")); } - @Test // Bug which can be reproduced with the below test: https://github.com/opensearch-project/security/issues/2169 + @Test public void shouldCreateSnapshot_positive() throws IOException { final String snapshotName = "snapshot-positive-test"; + long snapshotGetCount; try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); steps.createSnapshotRepository(TEST_SNAPSHOT_REPOSITORY_NAME, cluster.getSnapshotDirPath(), "fs"); @@ -1679,20 +1678,21 @@ public void shouldCreateSnapshot_positive() throws IOException { assertThat(response, notNullValue()); assertThat(response.status(), equalTo(RestStatus.ACCEPTED)); - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); assertThat(internalClient, clusterContainSuccessSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName)); } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_snapshot/test-snapshot-repository")); auditLogsRule.assertExactlyOne( userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_snapshot/test-snapshot-repository/snapshot-positive-test") ); - auditLogsRule.assertAtLeast( - 1, - userAuthenticated(LIMITED_WRITE_USER).withRestRequest(GET, "/_snapshot/test-snapshot-repository/snapshot-positive-test") + auditLogsRule.assertExactly( + snapshotGetCount, + userAuthenticated(LIMITED_WRITE_USER).withEffectiveUser(LIMITED_WRITE_USER) + .withRestRequest(GET, "/_snapshot/test-snapshot-repository/snapshot-positive-test") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertAtLeast(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); } @Test @@ -1717,12 +1717,13 @@ public void shouldCreateSnapshot_negative() throws IOException { @Test public void shouldDeleteSnapshot_positive() throws IOException { String snapshotName = "delete-snapshot-positive"; + long snapshotGetCount; try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); restHighLevelClient.snapshot(); steps.createSnapshotRepository(TEST_SNAPSHOT_REPOSITORY_NAME, cluster.getSnapshotDirPath(), "fs"); steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, SONG_INDEX_NAME); - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); var response = steps.deleteSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); @@ -1736,24 +1737,25 @@ public void shouldDeleteSnapshot_positive() throws IOException { auditLogsRule.assertExactlyOne( userAuthenticated(LIMITED_WRITE_USER).withRestRequest(DELETE, "/_snapshot/test-snapshot-repository/delete-snapshot-positive") ); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(GET, "/_snapshot/test-snapshot-repository/delete-snapshot-positive") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "DeleteSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "DeleteSnapshotRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); } @Test public void shouldDeleteSnapshot_negative() throws IOException { String snapshotName = "delete-snapshot-negative"; + long snapshotGetCount; try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); steps.createSnapshotRepository(TEST_SNAPSHOT_REPOSITORY_NAME, cluster.getSnapshotDirPath(), "fs"); steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, SONG_INDEX_NAME); - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); } try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); @@ -1768,23 +1770,27 @@ public void shouldDeleteSnapshot_negative() throws IOException { auditLogsRule.assertExactlyOne( userAuthenticated(LIMITED_READ_USER).withRestRequest(DELETE, "/_snapshot/test-snapshot-repository/delete-snapshot-negative") ); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(GET, "/_snapshot/test-snapshot-repository/delete-snapshot-negative") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); - auditLogsRule.assertExactly(1, missingPrivilege(LIMITED_READ_USER, "DeleteSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "DeleteSnapshotRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); } - @Ignore("Audit log entries verifcation isn't always consistant") @Test public void shouldRestoreSnapshot_positive() throws IOException { final String snapshotName = "restore-snapshot-positive"; + long snapshotGetCount; + AtomicInteger restoredCount = new AtomicInteger(); try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); // 1. create some documents + Settings indexSettings = Settings.builder().put("index.number_of_replicas", 0).put("index.number_of_shards", 1).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, indexSettings); + BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(new IndexRequest(WRITE_SONG_INDEX_NAME).id("Eins").source(SONGS[0].asMap())); bulkRequest.add(new IndexRequest(WRITE_SONG_INDEX_NAME).id("Zwei").source(SONGS[1].asMap())); @@ -1798,7 +1804,7 @@ public void shouldRestoreSnapshot_positive() throws IOException { steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, WRITE_SONG_INDEX_NAME); // 4. wait till snapshot is ready - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); // 5. introduce some changes bulkRequest = new BulkRequest(); @@ -1818,8 +1824,12 @@ public void shouldRestoreSnapshot_positive() throws IOException { CountRequest countRequest = new CountRequest(RESTORED_SONG_INDEX_NAME); Awaitility.await() .ignoreExceptions() + .pollInterval(100, TimeUnit.MILLISECONDS) .alias("Index contains proper number of documents restored from snapshot.") - .until(() -> restHighLevelClient.count(countRequest, DEFAULT).getCount() == 2); + .until(() -> { + restoredCount.incrementAndGet(); + return restHighLevelClient.count(countRequest, DEFAULT).getCount() == 2; + }); // 8. verify that document are present in restored index assertThat( @@ -1843,30 +1853,33 @@ public void shouldRestoreSnapshot_positive() throws IOException { "/_snapshot/test-snapshot-repository/restore-snapshot-positive/_restore" ) ); + auditLogsRule.assertExactly( + restoredCount.get(), + userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/restored_write_song_index/_count") + ); auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(GET, "/_snapshot/test-snapshot-repository/restore-snapshot-positive") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); - - // Audit events generated in step 7 above - auditLogsRule.assertAtLeast(1, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/restored_write_song_index/_count")); - auditLogsRule.assertAtLeast(1, grantedPrivilege(LIMITED_WRITE_USER, "SearchRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); + auditLogsRule.assertExactly(restoredCount.get(), grantedPrivilege(LIMITED_WRITE_USER, "SearchRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test public void shouldRestoreSnapshot_failureForbiddenIndex() throws IOException { final String snapshotName = "restore-snapshot-negative-forbidden-index"; String restoreToIndex = "forbidden_index"; + long snapshotGetCount; + Settings indexSettings = Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, indexSettings); try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { + SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); // 1. create some documents BulkRequest bulkRequest = new BulkRequest(); @@ -1882,7 +1895,7 @@ public void shouldRestoreSnapshot_failureForbiddenIndex() throws IOException { steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, WRITE_SONG_INDEX_NAME); // 4. wait till snapshot is ready - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); // 5. restore the snapshot assertThatThrownBy( @@ -1908,27 +1921,28 @@ public void shouldRestoreSnapshot_failureForbiddenIndex() throws IOException { ) ); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest( GET, "/_snapshot/test-snapshot-repository/restore-snapshot-negative-forbidden-index" ) ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); - auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertAtLeastTransportMessages(1, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyScanAll(1, missingPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test public void shouldRestoreSnapshot_failureOperationForbidden() throws IOException { String snapshotName = "restore-snapshot-negative-forbidden-operation"; + long snapshotGetCount; + Settings indexSettings = Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, indexSettings); try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); // 1. create some documents @@ -1945,7 +1959,7 @@ public void shouldRestoreSnapshot_failureOperationForbidden() throws IOException steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, WRITE_SONG_INDEX_NAME); // 4. wait till snapshot is ready - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); } // 5. restore the snapshot try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { @@ -1973,21 +1987,19 @@ public void shouldRestoreSnapshot_failureOperationForbidden() throws IOException ) ); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest( GET, "/_snapshot/test-snapshot-repository/restore-snapshot-negative-forbidden-operation" ) ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(1, missingPrivilege(LIMITED_READ_USER, "RestoreSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyScanAll(1, missingPrivilege(LIMITED_READ_USER, "RestoreSnapshotRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -2125,11 +2137,11 @@ public void shouldDeleteIndexByAliasRequest_positive() throws IOException { userAuthenticated(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES).withRestRequest(POST, "/_aliases") ); auditLogsRule.assertExactly( - 2, + 1, grantedPrivilege(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES, "IndicesAliasesRequest") ); auditLogsRule.assertExactly( - 2, + 1, auditPredicate(INDEX_EVENT).withEffectiveUser(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES) ); } @@ -2274,14 +2286,15 @@ public void openIndex_negative() throws IOException { } @Test - @Ignore // required permissions: "indices:admin/resize", "indices:monitor/stats - // todo even when I assign the `indices:admin/resize` and `indices:monitor/stats` permissions to test user, this test fails. - // Issue: https://github.com/opensearch-project/security/issues/2141 public void shrinkIndex_positive() throws IOException { String sourceIndexName = INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("shrink_index_positive_source"); - Settings sourceIndexSettings = Settings.builder().put("index.blocks.write", true).put("index.number_of_shards", 2).build(); String targetIndexName = INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("shrink_index_positive_target"); + Settings sourceIndexSettings = Settings.builder() + .put("index.number_of_replicas", 1) + .put("index.blocks.write", true) + .put("index.number_of_shards", 4) + .build(); IndexOperationsHelper.createIndex(cluster, sourceIndexName, sourceIndexSettings); try ( @@ -2289,6 +2302,17 @@ public void shrinkIndex_positive() throws IOException { USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES ) ) { + ClusterHealthResponse healthResponse = restHighLevelClient.cluster() + .health( + new ClusterHealthRequest(sourceIndexName).waitForNoRelocatingShards(true) + .waitForActiveShards(4) + .waitForNoInitializingShards(true) + .waitForGreenStatus(), + DEFAULT + ); + + assertThat(healthResponse.getStatus(), is(ClusterHealthStatus.GREEN)); + ResizeRequest resizeRequest = new ResizeRequest(targetIndexName, sourceIndexName); ResizeResponse response = restHighLevelClient.indices().shrink(resizeRequest, DEFAULT); @@ -2331,10 +2355,7 @@ public void shrinkIndex_negative() throws IOException { } @Test - @Ignore // required permissions: "indices:admin/resize", "indices:monitor/stats - // todo even when I assign the `indices:admin/resize` and `indices:monitor/stats` permissions to test user, this test fails. - // Issue: https://github.com/opensearch-project/security/issues/2141 public void cloneIndex_positive() throws IOException { String sourceIndexName = INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("clone_index_positive_source"); Settings sourceIndexSettings = Settings.builder().put("index.blocks.write", true).build(); @@ -2351,6 +2372,10 @@ public void cloneIndex_positive() throws IOException { assertThat(response, isSuccessfulResizeResponse(targetIndexName)); assertThat(cluster, indexExists(targetIndexName)); + + // can't clone the same index twice, target already exists + ResizeRequest repeatResizeRequest = new ResizeRequest(targetIndexName, sourceIndexName); + assertThatThrownBy(() -> restHighLevelClient.indices().clone(repeatResizeRequest, DEFAULT), statusException(BAD_REQUEST)); } } @@ -2388,10 +2413,7 @@ public void cloneIndex_negative() throws IOException { } @Test - @Ignore // required permissions: "indices:admin/resize", "indices:monitor/stats - // todo even when I assign the `indices:admin/resize` and `indices:monitor/stats` permissions to test user, this test fails. - // Issue: https://github.com/opensearch-project/security/issues/2141 public void splitIndex_positive() throws IOException { String sourceIndexName = INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("split_index_positive_source"); Settings sourceIndexSettings = Settings.builder().put("index.blocks.write", true).build(); @@ -2706,11 +2728,11 @@ public void shouldCreateIndexWithAlias_positive() throws IOException { ) ); auditLogsRule.assertExactly( - 2, + 1, grantedPrivilege(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES, "CreateIndexRequest") ); auditLogsRule.assertExactly( - 2, + 1, auditPredicate(INDEX_EVENT).withEffectiveUser(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES) ); } diff --git a/src/integrationTest/java/org/opensearch/security/SnapshotSteps.java b/src/integrationTest/java/org/opensearch/security/SnapshotSteps.java index 28aa6abd43..a03891ecca 100644 --- a/src/integrationTest/java/org/opensearch/security/SnapshotSteps.java +++ b/src/integrationTest/java/org/opensearch/security/SnapshotSteps.java @@ -11,6 +11,8 @@ import java.io.IOException; import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.awaitility.Awaitility; @@ -58,13 +60,21 @@ public CreateSnapshotResponse createSnapshot(String repositoryName, String snaps return snapshotClient.create(createSnapshotRequest, DEFAULT); } - public void waitForSnapshotCreation(String repositoryName, String snapshotName) { + public int waitForSnapshotCreation(String repositoryName, String snapshotName) { + AtomicInteger count = new AtomicInteger(); GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest(repositoryName, new String[] { snapshotName }); - Awaitility.await().alias("wait for snapshot creation").ignoreExceptions().until(() -> { - GetSnapshotsResponse snapshotsResponse = snapshotClient.get(getSnapshotsRequest, DEFAULT); - SnapshotInfo snapshotInfo = snapshotsResponse.getSnapshots().get(0); - return SnapshotState.SUCCESS.equals(snapshotInfo.state()); - }); + Awaitility.await() + .pollDelay(250, TimeUnit.MILLISECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .alias("wait for snapshot creation") + .ignoreExceptions() + .until(() -> { + count.incrementAndGet(); + GetSnapshotsResponse snapshotsResponse = snapshotClient.get(getSnapshotsRequest, DEFAULT); + SnapshotInfo snapshotInfo = snapshotsResponse.getSnapshots().get(0); + return SnapshotState.SUCCESS.equals(snapshotInfo.state()); + }); + return count.get(); } // CS-SUPPRESS-SINGLE: RegexpSingleline It is not possible to use phrase "cluster manager" instead of master here diff --git a/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java b/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java index badb938b3a..c41b5f4cda 100644 --- a/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java +++ b/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java @@ -11,21 +11,48 @@ package org.opensearch.security.rest; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.apache.http.HttpStatus; import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; +import org.opensearch.security.auditlog.impl.AuditMessage; +import org.opensearch.test.framework.AuditCompliance; +import org.opensearch.test.framework.AuditConfiguration; +import org.opensearch.test.framework.AuditFilters; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.audit.AuditLogsRule; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import joptsimple.internal.Strings; + import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; +import static org.opensearch.rest.RestRequest.Method.GET; +import static org.opensearch.security.auditlog.impl.AuditCategory.GRANTED_PRIVILEGES; +import static org.opensearch.security.auditlog.impl.AuditCategory.MISSING_PRIVILEGES; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.audit.AuditMessagePredicate.grantedPrivilege; +import static org.opensearch.test.framework.audit.AuditMessagePredicate.privilegePredicateRESTLayer; +import static org.opensearch.test.framework.audit.AuditMessagePredicate.userAuthenticatedPredicate; +import static org.junit.Assert.assertTrue; @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) @@ -34,6 +61,10 @@ public class WhoAmITests { new Role("who_am_i_role").clusterPermissions("security:whoamiprotected") ); + protected final static TestSecurityConfig.User AUDIT_LOG_VERIFIER = new TestSecurityConfig.User("audit_log_verifier").roles( + new Role("audit_log_verifier_role").clusterPermissions("*").indexPermissions("*").on("*") + ); + protected final static TestSecurityConfig.User WHO_AM_I_LEGACY = new TestSecurityConfig.User("who_am_i_user_legacy").roles( new Role("who_am_i_role_legacy").clusterPermissions("cluster:admin/opendistro_security/whoamiprotected") ); @@ -44,50 +75,71 @@ public class WhoAmITests { protected final static TestSecurityConfig.User WHO_AM_I_UNREGISTERED = new TestSecurityConfig.User("who_am_i_user_no_perm"); + protected final String expectedAuthorizedBody = "{\"dn\":null,\"is_admin\":false,\"is_node_certificate_request\":false}"; + protected final String expectedUnuauthorizedBody = + "no permissions for [security:whoamiprotected] and User [name=who_am_i_user_no_perm, backend_roles=[], requestedTenant=null]"; + public static final String WHOAMI_ENDPOINT = "_plugins/_security/whoami"; public static final String WHOAMI_PROTECTED_ENDPOINT = "_plugins/_security/whoamiprotected"; @ClassRule public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) .authc(AUTHC_HTTPBASIC_INTERNAL) - .users(WHO_AM_I, WHO_AM_I_LEGACY, WHO_AM_I_NO_PERM) + .users(WHO_AM_I, WHO_AM_I_LEGACY, WHO_AM_I_NO_PERM, AUDIT_LOG_VERIFIER) + .audit( + new AuditConfiguration(true).compliance(new AuditCompliance().enabled(true)) + .filters(new AuditFilters().enabledRest(true).enabledTransport(true).resolveBulkRequests(true)) + ) .build(); + @Rule + public AuditLogsRule auditLogsRule = new AuditLogsRule(); + @Test - public void testWhoAmIWithGetPermissions() throws Exception { - try (TestRestClient client = cluster.getRestClient(WHO_AM_I)) { - assertThat(client.get(WHOAMI_PROTECTED_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); - } + public void testWhoAmIWithGetPermissions() { try (TestRestClient client = cluster.getRestClient(WHO_AM_I)) { - assertThat(client.get(WHOAMI_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); + assertResponse(client.get(WHOAMI_PROTECTED_ENDPOINT), HttpStatus.SC_OK, expectedAuthorizedBody); + + // audit log, named route + auditLogsRule.assertExactlyOne(userAuthenticatedPredicate(WHO_AM_I, GET, "/" + WHOAMI_PROTECTED_ENDPOINT)); + auditLogsRule.assertExactlyOne(privilegePredicateRESTLayer(GRANTED_PRIVILEGES, WHO_AM_I, GET, "/" + WHOAMI_PROTECTED_ENDPOINT)); + + assertResponse(client.get(WHOAMI_ENDPOINT), HttpStatus.SC_OK, expectedAuthorizedBody); } } @Test - public void testWhoAmIWithGetPermissionsLegacy() throws Exception { + public void testWhoAmIWithGetPermissionsLegacy() { try (TestRestClient client = cluster.getRestClient(WHO_AM_I_LEGACY)) { - assertThat(client.get(WHOAMI_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); - } + assertResponse(client.get(WHOAMI_PROTECTED_ENDPOINT), HttpStatus.SC_OK, expectedAuthorizedBody); - try (TestRestClient client = cluster.getRestClient(WHO_AM_I_LEGACY)) { - assertThat(client.get(WHOAMI_PROTECTED_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); + // audit log, named route + auditLogsRule.assertExactlyOne(userAuthenticatedPredicate(WHO_AM_I_LEGACY, GET, "/" + WHOAMI_PROTECTED_ENDPOINT)); + auditLogsRule.assertExactlyOne( + privilegePredicateRESTLayer(GRANTED_PRIVILEGES, WHO_AM_I_LEGACY, GET, "/" + WHOAMI_PROTECTED_ENDPOINT) + ); + + assertResponse(client.get(WHOAMI_ENDPOINT), HttpStatus.SC_OK, expectedAuthorizedBody); } } @Test - public void testWhoAmIWithoutGetPermissions() throws Exception { + public void testWhoAmIWithoutGetPermissions() { try (TestRestClient client = cluster.getRestClient(WHO_AM_I_NO_PERM)) { - assertThat(client.get(WHOAMI_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); - } + assertResponse(client.get(WHOAMI_PROTECTED_ENDPOINT), HttpStatus.SC_UNAUTHORIZED, expectedUnuauthorizedBody); + // audit log, named route + auditLogsRule.assertExactlyOne(userAuthenticatedPredicate(WHO_AM_I_NO_PERM, GET, "/" + WHOAMI_PROTECTED_ENDPOINT)); + auditLogsRule.assertExactlyOne( + privilegePredicateRESTLayer(MISSING_PRIVILEGES, WHO_AM_I_NO_PERM, GET, "/" + WHOAMI_PROTECTED_ENDPOINT) + ); - try (TestRestClient client = cluster.getRestClient(WHO_AM_I_NO_PERM)) { - assertThat(client.get(WHOAMI_PROTECTED_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_UNAUTHORIZED)); + assertResponse(client.get(WHOAMI_ENDPOINT), HttpStatus.SC_OK, expectedAuthorizedBody); } } @Test - public void testWhoAmIPost() throws Exception { + public void testWhoAmIPost() { try (TestRestClient client = cluster.getRestClient(WHO_AM_I)) { assertThat(client.post(WHOAMI_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); } @@ -104,5 +156,173 @@ public void testWhoAmIPost() throws Exception { assertThat(client.post(WHOAMI_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); } + // No audit logs generated because `/whoami` is passthrough at Transport Layer, and POST route is not a NamedRoute + auditLogsRule.assertAuditLogsCount(0, 0); + } + + @Test + public void testAuditLogSimilarityWithTransportLayer() { + try (TestRestClient client = cluster.getRestClient(AUDIT_LOG_VERIFIER)) { + assertResponse(client.get(WHOAMI_PROTECTED_ENDPOINT), HttpStatus.SC_OK, expectedAuthorizedBody); + auditLogsRule.assertExactlyOne(userAuthenticatedPredicate(AUDIT_LOG_VERIFIER, GET, "/" + WHOAMI_PROTECTED_ENDPOINT)); + auditLogsRule.assertExactlyOne( + privilegePredicateRESTLayer(GRANTED_PRIVILEGES, AUDIT_LOG_VERIFIER, GET, "/" + WHOAMI_PROTECTED_ENDPOINT) + ); + + assertThat(client.get("_cat/indices").getStatusCode(), equalTo(HttpStatus.SC_OK)); + + // transport layer audit messages + auditLogsRule.assertExactly(1, grantedPrivilege(AUDIT_LOG_VERIFIER, "GetSettingsRequest")); + + List grantedPrivilegesMessages = auditLogsRule.getCurrentTestAuditMessages() + .stream() + .filter(msg -> msg.getCategory().equals(GRANTED_PRIVILEGES)) + .collect(Collectors.toList()); + + verifyAuditLogSimilarity(grantedPrivilegesMessages); + } + } + + private void assertResponse(TestRestClient.HttpResponse response, int expectedStatus, String expectedBody) { + assertThat(response.getStatusCode(), equalTo(expectedStatus)); + assertThat(response.getBody(), equalTo(expectedBody)); + } + + private void verifyAuditLogSimilarity(List currentTestAuditMessages) { + List restSet = new ArrayList<>(); + List transportSet = new ArrayList<>(); + + // It is okay to loop through all even though we end up using only 2, as the total number of messages should be around 8 + for (AuditMessage auditMessage : currentTestAuditMessages) { + if ("REST".equals(auditMessage.getAsMap().get(AuditMessage.REQUEST_LAYER).toString())) { + restSet.add(auditMessage); + } else if ("TRANSPORT".equals(auditMessage.getAsMap().get(AuditMessage.REQUEST_LAYER).toString())) { + transportSet.add(auditMessage); + } + } + // We pass 1 message from each layer to check for similarity + checkForStructuralSimilarity(restSet.get(0), transportSet.get(0)); + } + + /** + * Checks for structural similarity between audit message generated at Rest layer vs transport layer + * Example REST audit message for GRANTED_PRIVILEGES: + * { + * "audit_cluster_name":"local_cluster_1", + * "audit_node_name":"data_0", + * "audit_rest_request_method":"GET", + * "audit_category":"GRANTED_PRIVILEGES", + * "audit_request_origin":"REST", + * "audit_node_id":"Dez5cwAAQAC6cdmK_____w", + * "audit_request_layer":"REST", + * "audit_rest_request_path":"/_plugins/_security/whoamiprotected", + * "@timestamp":"2023-08-16T17:35:53.531+00:00", + * "audit_format_version":4, + * "audit_request_remote_address":"127.0.0.1", + * "audit_node_host_address":"127.0.0.1", + * "audit_rest_request_headers":{ + * "Connection":[ + * "keep-alive" + * ], + * "User-Agent":[ + * "Apache-HttpClient/5.2.1 (Java/19.0.1)" + * ], + * "content-length":[ + * "0" + * ], + * "Host":[ + * "127.0.0.1:47210" + * ], + * "Accept-Encoding":[ + * "gzip, x-gzip, deflate" + * ] + * }, + * "audit_request_effective_user":"audit_log_verifier", + * "audit_node_host_name":"127.0.0.1" + * } + * + * + * Example Transport audit message for GRANTED_PRIVILEGES: + * { + * "audit_cluster_name":"local_cluster_1", + * "audit_transport_headers":{ + * "_system_index_access_allowed":"false" + * }, + * "audit_node_name":"data_0", + * "audit_trace_task_id":"Dez5cwAAQAC6cdmK_____w:87", + * "audit_transport_request_type":"GetSettingsRequest", + * "audit_category":"GRANTED_PRIVILEGES", + * "audit_request_origin":"REST", + * "audit_node_id":"Dez5cwAAQAC6cdmK_____w", + * "audit_request_layer":"TRANSPORT", + * "@timestamp":"2023-08-16T17:35:53.621+00:00", + * "audit_format_version":4, + * "audit_request_remote_address":"127.0.0.1", + * "audit_request_privilege":"indices:monitor/settings/get", + * "audit_node_host_address":"127.0.0.1", + * "audit_request_effective_user":"audit_log_verifier", + * "audit_node_host_name":"127.0.0.1" + * } + * + * + * @param restAuditMessage audit message generated at REST layer + * @param transportAuditMessage audit message generated at Transport layer + */ + private void checkForStructuralSimilarity(AuditMessage restAuditMessage, AuditMessage transportAuditMessage) { + + Map restMsgFields = restAuditMessage.getAsMap(); + Map transportMsgFields = transportAuditMessage.getAsMap(); + + Set restAuditSet = restMsgFields.keySet(); + Set transportAuditSet = transportMsgFields.keySet(); + + // Added a magic number here and below, because there are always 15 or more items in each message generated via Audit logs + assertThat(restAuditSet.size(), greaterThan(14)); + assertThat(transportAuditSet.size(), greaterThan(14)); + + // check for values of common fields + Set commonFields = new HashSet<>(restAuditSet); + commonFields.retainAll(transportAuditSet); + + assertCommonFields(commonFields, restMsgFields, transportMsgFields); + + // check for values of uncommon fields + restAuditSet.removeAll(transportAuditMessage.getAsMap().keySet()); + transportAuditSet.removeAll(restAuditMessage.getAsMap().keySet()); + + // We compare two sets and see there were more than 10 items with same keys indicating these logs are similar + // There are a few headers that are generated different for REST vs TRANSPORT layer audit logs, but that is expected + // The end goal of this test is to ensure similarity, not equality. + assertThat(restAuditSet.size(), lessThan(5)); + assertThat(transportAuditSet.size(), lessThan(5)); + + assertThat(restMsgFields.get("audit_rest_request_path"), equalTo("/_plugins/_security/whoamiprotected")); + assertThat(restMsgFields.get("audit_rest_request_method").toString(), equalTo("GET")); + assertThat(restMsgFields.get("audit_rest_request_headers").toString().contains("Connection"), equalTo(true)); + + assertThat(transportMsgFields.get("audit_transport_request_type"), equalTo("GetSettingsRequest")); + assertThat(transportMsgFields.get("audit_request_privilege"), equalTo("indices:monitor/settings/get")); + assertThat(Strings.isNullOrEmpty(transportMsgFields.get("audit_trace_task_id").toString()), equalTo(false)); + } + + private void assertCommonFields(Set commonFields, Map restMsgFields, Map transportMsgFields) { + for (String key : commonFields) { + if (key.equals("@timestamp")) { + String restTimeStamp = restMsgFields.get(key).toString(); + String transportTimeStamp = transportMsgFields.get(key).toString(); + + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + LocalDateTime restDateTime = LocalDateTime.parse(restTimeStamp, formatter); + LocalDateTime transportDateTime = LocalDateTime.parse(transportTimeStamp, formatter); + + // assert that these log messages are generated within 10 seconds of each other + assertTrue(Duration.between(restDateTime, transportDateTime).getSeconds() < 10); + } else if (key.equals("audit_request_layer")) { + assertThat(restMsgFields.get(key).toString(), equalTo("REST")); + assertThat(transportMsgFields.get(key).toString(), equalTo("TRANSPORT")); + } else { + assertThat(restMsgFields.get(key), equalTo(transportMsgFields.get(key))); + } + } } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java b/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java index 3d13d731eb..3f9a0ae466 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java +++ b/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java @@ -39,6 +39,7 @@ public class AuditLogsRule implements TestRule { private static final Logger log = LogManager.getLogger(AuditLogsRule.class); private List currentTestAuditMessages; + private List currentTransportTestAuditMessages; public List getCurrentTestAuditMessages() { return currentTestAuditMessages; @@ -56,6 +57,7 @@ public void waitForAuditLogs() { private void afterWaitingForAuditLogs() { if (log.isDebugEnabled()) { log.debug("Audit records captured during test:\n{}", auditMessagesToString(currentTestAuditMessages)); + log.debug("Audit transport records captured during test:\n{}", auditMessagesToString(currentTransportTestAuditMessages)); } } @@ -63,6 +65,13 @@ public void assertExactlyOne(Predicate predicate) { assertExactly(1, predicate); } + public void assertExactlyScanAll(long expectedNumberOfAuditMessages, Predicate predicate) { + List auditMessages = new ArrayList<>(currentTestAuditMessages); + auditMessages.addAll(currentTransportTestAuditMessages); + assertExactly(exactNumberOfAuditsFulfillPredicate(expectedNumberOfAuditMessages, predicate), auditMessages); + + } + public void assertAuditLogsCount(int from, int to) { int actualCount = currentTestAuditMessages.size(); String message = "Expected audit log count is between " + from + " and " + to + " but was " + actualCount; @@ -70,10 +79,10 @@ public void assertAuditLogsCount(int from, int to) { } public void assertExactly(long expectedNumberOfAuditMessages, Predicate predicate) { - assertExactly(exactNumberOfAuditsFulfillPredicate(expectedNumberOfAuditMessages, predicate)); + assertExactly(exactNumberOfAuditsFulfillPredicate(expectedNumberOfAuditMessages, predicate), currentTestAuditMessages); } - private void assertExactly(Matcher> matcher) { + private void assertExactly(Matcher> matcher, List currentTestAuditMessages) { // pollDelay - initial delay before first evaluation Awaitility.await("Await for audit logs") .atMost(3, TimeUnit.SECONDS) @@ -82,7 +91,11 @@ private void assertExactly(Matcher> matcher) { } public void assertAtLeast(long minCount, Predicate predicate) { - assertExactly(atLeastCertainNumberOfAuditsFulfillPredicate(minCount, predicate)); + assertExactly(atLeastCertainNumberOfAuditsFulfillPredicate(minCount, predicate), currentTestAuditMessages); + } + + public void assertAtLeastTransportMessages(long minCount, Predicate predicate) { + assertExactly(atLeastCertainNumberOfAuditsFulfillPredicate(minCount, predicate), currentTransportTestAuditMessages); } private static String auditMessagesToString(List audits) { @@ -122,16 +135,35 @@ private void whenTimeoutOccurs(String methodName) { private void afterTest() { TestRuleAuditLogSink.unregisterListener(); this.currentTestAuditMessages = null; + this.currentTransportTestAuditMessages = null; } private void beforeTest(String methodName) { log.info("Start collecting audit logs before test {}", methodName); this.currentTestAuditMessages = synchronizedList(new ArrayList<>()); + this.currentTransportTestAuditMessages = synchronizedList(new ArrayList<>()); TestRuleAuditLogSink.registerListener(this); } public void onAuditMessage(AuditMessage auditMessage) { - currentTestAuditMessages.add(auditMessage); - log.debug("New audit message received '{}', total number of audit messages '{}'.", auditMessage, currentTestAuditMessages.size()); + if (auditMessage.getAsMap().keySet().contains("audit_transport_headers")) { + if (log.isDebugEnabled()) { + log.debug( + "New transport audit message received '{}', total number of transport audit messages '{}'.", + auditMessage, + currentTransportTestAuditMessages.size() + ); + } + currentTransportTestAuditMessages.add(auditMessage); + } else { + if (log.isDebugEnabled()) { + log.debug( + "New audit message received '{}', total number of audit messages '{}'.", + auditMessage, + currentTestAuditMessages.size() + ); + } + currentTestAuditMessages.add(auditMessage); + } } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java index 0bf50c7a4d..551a231830 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java @@ -74,6 +74,15 @@ public enum ClusterManager { new NodeSettings(NodeRole.CLUSTER_MANAGER), new NodeSettings(NodeRole.DATA), new NodeSettings(NodeRole.DATA) + ), + + THREE_CLUSTER_MANAGERS_COORDINATOR( + new NodeSettings(NodeRole.CLUSTER_MANAGER), + new NodeSettings(NodeRole.CLUSTER_MANAGER), + new NodeSettings(NodeRole.CLUSTER_MANAGER), + new NodeSettings(NodeRole.DATA), + new NodeSettings(NodeRole.DATA), + new NodeSettings() ); private List nodeSettings = new LinkedList<>();