Provide repository-level stats for searchable snapshots (#55051)

Provides basic repository-level stats that will allow us to get some insight into how many requests are actually being made by the underlying SDK. Currently only tracks GET and LIST calls for S3 repositories. Most of the code is unfortunately boiler plate to add a new endpoint that will help us better understand some of the low-level dynamics of searchable snapshots.
yeohbraddy · Apr 14, 2020 · a610513 · a610513
1 parent d5bb574
commit a610513
Show file tree

Hide file tree

Showing 24 changed files with 837 additions and 19 deletions.
diff --git a/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc b/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc
@@ -1,5 +1,5 @@
 [role="xpack"]
-[testenv="basic"]
+[testenv="platinum"]
 [[searchable-snapshots-api-clear-cache]]
 === Clear cache API
 ++++

diff --git a/docs/reference/searchable-snapshots/apis/get-stats.asciidoc b/docs/reference/searchable-snapshots/apis/get-stats.asciidoc
@@ -1,5 +1,5 @@
 [role="xpack"]
-[testenv="basic"]
+[testenv="platinum"]
 [[searchable-snapshots-api-stats]]
 === Searchable snapshot statistics API
 ++++

diff --git a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc
@@ -1,5 +1,5 @@
 [role="xpack"]
-[testenv="basic"]
+[testenv="platinum"]
 [[searchable-snapshots-api-mount-snapshot]]
 === Mount snapshot API
 ++++

diff --git a/docs/reference/searchable-snapshots/apis/repository-stats.asciidoc b/docs/reference/searchable-snapshots/apis/repository-stats.asciidoc
@@ -0,0 +1,73 @@
+[role="xpack"]
+[testenv="platinum"]
+[[searchable-snapshots-repository-stats]]
+=== Searchable snapshot repository statistics API
+++++
+<titleabbrev>Searchable snapshot repository statistics</titleabbrev>
+++++
+
+experimental[]
+
+Retrieve usage statistics about a snapshot repository.
+
+[[searchable-snapshots-repository-stats-request]]
+==== {api-request-title}
+
+`GET /_snapshot/<repository>/_stats`
+
+[[searchable-snapshots-repository-stats-prereqs]]
+==== {api-prereq-title}
+
+If the {es} {security-features} are enabled, you must have the
+`manage` cluster privilege and the `manage` index privilege
+for any included indices to use this API.
+For more information, see <<security-privileges>>.
+
+[[searchable-snapshots-repository-stats-desc]]
+==== {api-description-title}
+
+
+[[searchable-snapshots-repository-stats-path-params]]
+==== {api-path-parms-title}
+
+`<repository>`::
+(Required, string)
+The repository for which to retrieve stats.
+
+
+[[searchable-snapshots-repository-stats-example]]
+==== {api-examples-title}
+////
+[source,console]
+-----------------------------------
+PUT /docs
+{
+    "settings" : {
+        "index.number_of_shards" : 1,
+        "index.number_of_replicas" : 0
+    }
+}
+
+PUT /_snapshot/my_repository/my_snapshot?wait_for_completion=true
+{
+    "include_global_state": false,
+    "indices": "docs"
+}
+
+DELETE /docs
+
+POST /_snapshot/my_repository/my_snapshot/_mount?wait_for_completion=true
+{
+  "index": "docs"
+}
+-----------------------------------
+// TEST[setup:setup-repository]
+////
+
+Retrieves the statistics of the repository `my_repository`:
+
+[source,console]
+--------------------------------------------------
+GET /_snapshot/my_repository/_stats
+--------------------------------------------------
+// TEST[continued]
diff --git a/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc b/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc
@@ -1,5 +1,5 @@
 [role="xpack"]
-[testenv="basic"]
+[testenv="platinum"]
 [[searchable-snapshots-apis]]
 == Searchable snapshots APIs
 
@@ -10,7 +10,9 @@ You can use the following APIs to perform searchable snapshots operations.
 * <<searchable-snapshots-api-mount-snapshot,Mount snapshot>>
 * <<searchable-snapshots-api-clear-cache,Clear cache>>
 * <<searchable-snapshots-api-stats,Get stats>>
+* <<searchable-snapshots-repository-stats,Repository stats>>
 
 include::mount-snapshot.asciidoc[]
 include::clear-cache.asciidoc[]
 include::get-stats.asciidoc[]
+include::repository-stats.asciidoc[]
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
@@ -151,6 +151,7 @@ public DeleteResult delete() throws IOException {
                     final ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
                     listObjectsRequest.setBucketName(blobStore.bucket());
                     listObjectsRequest.setPrefix(keyPath);
+                    listObjectsRequest.setRequestMetricCollector(blobStore.listMetricCollector);
                     list = SocketAccess.doPrivileged(() -> clientReference.client().listObjects(listObjectsRequest));
                 }
                 final List<String> blobsToDelete = new ArrayList<>();
@@ -307,7 +308,8 @@ private static List<ObjectListing> executeListing(AmazonS3Reference clientRefere
     }
 
     private ListObjectsRequest listObjectsRequest(String keyPath) {
-        return new ListObjectsRequest().withBucketName(blobStore.bucket()).withPrefix(keyPath).withDelimiter("/");
+        return new ListObjectsRequest().withBucketName(blobStore.bucket()).withPrefix(keyPath).withDelimiter("/")
+            .withRequestMetricCollector(blobStore.listMetricCollector);
     }
 
     private String buildKey(String blobName) {

diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java
@@ -19,8 +19,12 @@
 
 package org.elasticsearch.repositories.s3;
 
+import com.amazonaws.Request;
+import com.amazonaws.Response;
+import com.amazonaws.metrics.RequestMetricCollector;
 import com.amazonaws.services.s3.model.CannedAccessControlList;
 import com.amazonaws.services.s3.model.StorageClass;
+import com.amazonaws.util.AWSRequestMetrics;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -29,7 +33,10 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
 
 class S3BlobStore implements BlobStore {
 
@@ -47,6 +54,12 @@ class S3BlobStore implements BlobStore {
 
     private final RepositoryMetadata repositoryMetadata;
 
+    private final Stats stats = new Stats();
+
+    final RequestMetricCollector getMetricCollector;
+    final RequestMetricCollector listMetricCollector;
+
+
     S3BlobStore(S3Service service, String bucket, boolean serverSideEncryption,
                 ByteSizeValue bufferSize, String cannedACL, String storageClass,
                 RepositoryMetadata repositoryMetadata) {
@@ -57,6 +70,26 @@ class S3BlobStore implements BlobStore {
         this.cannedACL = initCannedACL(cannedACL);
         this.storageClass = initStorageClass(storageClass);
         this.repositoryMetadata = repositoryMetadata;
+        this.getMetricCollector = new RequestMetricCollector() {
+            @Override
+            public void collectMetrics(Request<?> request, Response<?> response) {
+                assert request.getHttpMethod().name().equals("GET");
+                final Number requestCount = request.getAWSRequestMetrics().getTimingInfo()
+                    .getCounter(AWSRequestMetrics.Field.RequestCount.name());
+                assert requestCount != null;
+                stats.getCount.addAndGet(requestCount.longValue());
+            }
+        };
+        this.listMetricCollector = new RequestMetricCollector() {
+            @Override
+            public void collectMetrics(Request<?> request, Response<?> response) {
+                assert request.getHttpMethod().name().equals("GET");
+                final Number requestCount = request.getAWSRequestMetrics().getTimingInfo()
+                    .getCounter(AWSRequestMetrics.Field.RequestCount.name());
+                assert requestCount != null;
+                stats.listCount.addAndGet(requestCount.longValue());
+            }
+        };
     }
 
     @Override
@@ -94,6 +127,11 @@ public void close() throws IOException {
         this.service.close();
     }
 
+    @Override
+    public Map<String, Long> stats() {
+        return stats.toMap();
+    }
+
     public CannedAccessControlList getCannedACL() {
         return cannedACL;
     }
@@ -135,4 +173,18 @@ public static CannedAccessControlList initCannedACL(String cannedACL) {
 
         throw new BlobStoreException("cannedACL is not valid: [" + cannedACL + "]");
     }
+
+    static class Stats {
+
+        final AtomicLong listCount = new AtomicLong();
+
+        final AtomicLong getCount = new AtomicLong();
+
+        Map<String, Long> toMap() {
+            final Map<String, Long> results = new HashMap<>();
+            results.put("GET", getCount.get());
+            results.put("LIST", listCount.get());
+            return results;
+        }
+    }
 }
diff --git a/.../repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java b/.../repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java
@@ -25,9 +25,9 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.core.internal.io.IOUtils;
-import org.elasticsearch.Version;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -83,6 +83,7 @@ class S3RetryingInputStream extends InputStream {
     private InputStream openStream() throws IOException {
         try (AmazonS3Reference clientReference = blobStore.clientReference()) {
             final GetObjectRequest getObjectRequest = new GetObjectRequest(blobStore.bucket(), blobKey);
+            getObjectRequest.setRequestMetricCollector(blobStore.getMetricCollector);
             if (currentOffset > 0 || start > 0 || end < Long.MAX_VALUE - 1) {
                 assert start + currentOffset <= end :
                     "requesting beyond end, start = " + start + " offset=" + currentOffset + " end=" + end;