Skip to content

Commit 23f12e4

Browse files
jasontedormartijnvg
jasontedor
and
martijnvg
authored
Expose CCR stats to monitoring (#33617)
This commit exposes the CCR stats endpoint to monitoring collection. Co-authored-by: Martijn van Groningen <martijn.v.groningen@gmail.com>
1 parent 0b567c0 commit 23f12e4

File tree

19 files changed

+728
-104
lines changed

19 files changed

+728
-104
lines changed

‎x-pack/plugin/ccr/qa/multi-cluster-with-security/build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ followClusterTestCluster {
4747
setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
4848
setting 'xpack.license.self_generated.type', 'trial'
4949
setting 'xpack.security.enabled', 'true'
50-
setting 'xpack.monitoring.enabled', 'false'
50+
setting 'xpack.monitoring.collection.enabled', 'true'
5151
extraConfigFile 'roles.yml', 'roles.yml'
5252
setupCommand 'setupTestAdmin',
5353
'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"

‎x-pack/plugin/ccr/qa/multi-cluster-with-security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
3131
import static org.hamcrest.Matchers.containsString;
3232
import static org.hamcrest.Matchers.equalTo;
33+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
3334
import static org.hamcrest.Matchers.is;
3435

3536
public class FollowIndexSecurityIT extends ESRestTestCase {
@@ -80,6 +81,7 @@ public void testFollowIndex() throws Exception {
8081
createAndFollowIndex("leader_cluster:" + allowedIndex, allowedIndex);
8182
assertBusy(() -> verifyDocuments(client(), allowedIndex, numDocs));
8283
assertThat(countCcrNodeTasks(), equalTo(1));
84+
assertBusy(() -> verifyCcrMonitoring(allowedIndex));
8385
assertOK(client().performRequest(new Request("POST", "/" + allowedIndex + "/_ccr/unfollow")));
8486
// Make sure that there are no other ccr relates operations running:
8587
assertBusy(() -> {
@@ -203,4 +205,46 @@ private static boolean indexExists(RestClient client, String index) throws IOExc
203205
return RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode();
204206
}
205207

208+
private static void verifyCcrMonitoring(String expectedLeaderIndex) throws IOException {
209+
ensureYellow(".monitoring-*");
210+
211+
Request request = new Request("GET", "/.monitoring-*/_search");
212+
request.setJsonEntity("{\"query\": {\"term\": {\"type\": \"ccr_stats\"}}}");
213+
Map<String, ?> response = toMap(adminClient().performRequest(request));
214+
215+
int numDocs = (int) XContentMapValues.extractValue("hits.total", response);
216+
assertThat(numDocs, greaterThanOrEqualTo(1));
217+
218+
int numberOfOperationsReceived = 0;
219+
int numberOfOperationsIndexed = 0;
220+
221+
List<?> hits = (List<?>) XContentMapValues.extractValue("hits.hits", response);
222+
for (int i = 0; i < numDocs; i++) {
223+
Map<?, ?> hit = (Map<?, ?>) hits.get(i);
224+
String leaderIndex = (String) XContentMapValues.extractValue("_source.ccr_stats.leader_index", hit);
225+
if (leaderIndex.endsWith(expectedLeaderIndex) == false) {
226+
continue;
227+
}
228+
229+
int foundNumberOfOperationsReceived =
230+
(int) XContentMapValues.extractValue("_source.ccr_stats.operations_received", hit);
231+
numberOfOperationsReceived = Math.max(numberOfOperationsReceived, foundNumberOfOperationsReceived);
232+
int foundNumberOfOperationsIndexed =
233+
(int) XContentMapValues.extractValue("_source.ccr_stats.number_of_operations_indexed", hit);
234+
numberOfOperationsIndexed = Math.max(numberOfOperationsIndexed, foundNumberOfOperationsIndexed);
235+
}
236+
237+
assertThat(numberOfOperationsReceived, greaterThanOrEqualTo(1));
238+
assertThat(numberOfOperationsIndexed, greaterThanOrEqualTo(1));
239+
}
240+
241+
private static void ensureYellow(String index) throws IOException {
242+
Request request = new Request("GET", "/_cluster/health/" + index);
243+
request.addParameter("wait_for_status", "yellow");
244+
request.addParameter("wait_for_no_relocating_shards", "true");
245+
request.addParameter("timeout", "70s");
246+
request.addParameter("level", "shards");
247+
adminClient().performRequest(request);
248+
}
249+
206250
}

‎x-pack/plugin/ccr/qa/multi-cluster/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ followClusterTestCluster {
2727
dependsOn leaderClusterTestRunner
2828
numNodes = 1
2929
clusterName = 'follow-cluster'
30+
setting 'xpack.monitoring.collection.enabled', 'true'
3031
setting 'xpack.license.self_generated.type', 'trial'
3132
setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
3233
}

‎x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
2727
import static org.hamcrest.Matchers.equalTo;
28+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
2829

2930
public class FollowIndexIT extends ESRestTestCase {
3031

@@ -75,6 +76,7 @@ public void testFollowIndex() throws Exception {
7576
index(leaderClient, leaderIndexName, Integer.toString(id + 2), "field", id + 2, "filtered_field", "true");
7677
}
7778
assertBusy(() -> verifyDocuments(followIndexName, numDocs + 3));
79+
assertBusy(() -> verifyCcrMonitoring(leaderIndexName));
7880
}
7981
}
8082

@@ -104,6 +106,7 @@ public void testAutoFollowPatterns() throws Exception {
104106
ensureYellow("logs-20190101");
105107
verifyDocuments("logs-20190101", 5);
106108
});
109+
assertBusy(() -> verifyCcrMonitoring("logs-20190101"));
107110
}
108111

109112
private static void index(RestClient client, String index, String id, Object... fields) throws IOException {
@@ -155,6 +158,39 @@ private static void verifyDocuments(String index, int expectedNumDocs) throws IO
155158
}
156159
}
157160

161+
private static void verifyCcrMonitoring(String expectedLeaderIndex) throws IOException {
162+
ensureYellow(".monitoring-*");
163+
164+
Request request = new Request("GET", "/.monitoring-*/_search");
165+
request.setJsonEntity("{\"query\": {\"term\": {\"type\": \"ccr_stats\"}}}");
166+
Map<String, ?> response = toMap(client().performRequest(request));
167+
168+
int numDocs = (int) XContentMapValues.extractValue("hits.total", response);
169+
assertThat(numDocs, greaterThanOrEqualTo(1));
170+
171+
int numberOfOperationsReceived = 0;
172+
int numberOfOperationsIndexed = 0;
173+
174+
List<?> hits = (List<?>) XContentMapValues.extractValue("hits.hits", response);
175+
for (int i = 0; i < numDocs; i++) {
176+
Map<?, ?> hit = (Map<?, ?>) hits.get(i);
177+
String leaderIndex = (String) XContentMapValues.extractValue("_source.ccr_stats.leader_index", hit);
178+
if (leaderIndex.endsWith(expectedLeaderIndex) == false) {
179+
continue;
180+
}
181+
182+
int foundNumberOfOperationsReceived =
183+
(int) XContentMapValues.extractValue("_source.ccr_stats.operations_received", hit);
184+
numberOfOperationsReceived = Math.max(numberOfOperationsReceived, foundNumberOfOperationsReceived);
185+
int foundNumberOfOperationsIndexed =
186+
(int) XContentMapValues.extractValue("_source.ccr_stats.number_of_operations_indexed", hit);
187+
numberOfOperationsIndexed = Math.max(numberOfOperationsIndexed, foundNumberOfOperationsIndexed);
188+
}
189+
190+
assertThat(numberOfOperationsReceived, greaterThanOrEqualTo(1));
191+
assertThat(numberOfOperationsIndexed, greaterThanOrEqualTo(1));
192+
}
193+
158194
private static Map<String, Object> toMap(Response response) throws IOException {
159195
return toMap(EntityUtils.toString(response.getEntity()));
160196
}

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,21 +40,17 @@
4040
import org.elasticsearch.threadpool.ThreadPool;
4141
import org.elasticsearch.watcher.ResourceWatcherService;
4242
import org.elasticsearch.xpack.ccr.action.AutoFollowCoordinator;
43-
import org.elasticsearch.xpack.ccr.action.TransportUnfollowIndexAction;
44-
import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
45-
import org.elasticsearch.xpack.ccr.action.TransportCreateAndFollowIndexAction;
46-
import org.elasticsearch.xpack.ccr.action.TransportFollowIndexAction;
47-
import org.elasticsearch.xpack.core.ccr.action.CreateAndFollowIndexAction;
4843
import org.elasticsearch.xpack.ccr.action.DeleteAutoFollowPatternAction;
49-
import org.elasticsearch.xpack.core.ccr.action.FollowIndexAction;
5044
import org.elasticsearch.xpack.ccr.action.PutAutoFollowPatternAction;
5145
import org.elasticsearch.xpack.ccr.action.ShardChangesAction;
5246
import org.elasticsearch.xpack.ccr.action.ShardFollowTask;
5347
import org.elasticsearch.xpack.ccr.action.ShardFollowTasksExecutor;
5448
import org.elasticsearch.xpack.ccr.action.TransportCcrStatsAction;
49+
import org.elasticsearch.xpack.ccr.action.TransportCreateAndFollowIndexAction;
5550
import org.elasticsearch.xpack.ccr.action.TransportDeleteAutoFollowPatternAction;
51+
import org.elasticsearch.xpack.ccr.action.TransportFollowIndexAction;
5652
import org.elasticsearch.xpack.ccr.action.TransportPutAutoFollowPatternAction;
57-
import org.elasticsearch.xpack.core.ccr.action.UnfollowIndexAction;
53+
import org.elasticsearch.xpack.ccr.action.TransportUnfollowIndexAction;
5854
import org.elasticsearch.xpack.ccr.action.bulk.BulkShardOperationsAction;
5955
import org.elasticsearch.xpack.ccr.action.bulk.TransportBulkShardOperationsAction;
6056
import org.elasticsearch.xpack.ccr.index.engine.FollowingEngineFactory;
@@ -66,6 +62,10 @@
6662
import org.elasticsearch.xpack.ccr.rest.RestUnfollowIndexAction;
6763
import org.elasticsearch.xpack.core.XPackPlugin;
6864
import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus;
65+
import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
66+
import org.elasticsearch.xpack.core.ccr.action.CreateAndFollowIndexAction;
67+
import org.elasticsearch.xpack.core.ccr.action.FollowIndexAction;
68+
import org.elasticsearch.xpack.core.ccr.action.UnfollowIndexAction;
6969

7070
import java.util.Arrays;
7171
import java.util.Collection;
@@ -76,8 +76,8 @@
7676
import java.util.function.Supplier;
7777

7878
import static java.util.Collections.emptyList;
79-
import static org.elasticsearch.xpack.ccr.CcrSettings.CCR_ENABLED_SETTING;
8079
import static org.elasticsearch.xpack.ccr.CcrSettings.CCR_FOLLOWING_INDEX_SETTING;
80+
import static org.elasticsearch.xpack.core.XPackSettings.CCR_ENABLED_SETTING;
8181

8282
/**
8383
* Container class for CCR functionality.

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import org.elasticsearch.common.settings.Setting;
99
import org.elasticsearch.common.settings.Setting.Property;
1010
import org.elasticsearch.common.unit.TimeValue;
11+
import org.elasticsearch.xpack.core.XPackSettings;
1112

1213
import java.util.Arrays;
1314
import java.util.List;
@@ -22,11 +23,6 @@ private CcrSettings() {
2223

2324
}
2425

25-
/**
26-
* Setting for controlling whether or not CCR is enabled.
27-
*/
28-
static final Setting<Boolean> CCR_ENABLED_SETTING = Setting.boolSetting("xpack.ccr.enabled", true, Property.NodeScope);
29-
3026
/**
3127
* Index setting for a following index.
3228
*/
@@ -46,7 +42,7 @@ private CcrSettings() {
4642
*/
4743
static List<Setting<?>> getSettings() {
4844
return Arrays.asList(
49-
CCR_ENABLED_SETTING,
45+
XPackSettings.CCR_ENABLED_SETTING,
5046
CCR_FOLLOWING_INDEX_SETTING,
5147
CCR_AUTO_FOLLOW_POLL_INTERVAL);
5248
}

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.elasticsearch.tasks.TaskId;
2121
import org.elasticsearch.xpack.ccr.action.bulk.BulkShardOperationsResponse;
2222
import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus;
23+
import org.elasticsearch.xpack.core.ccr.action.FollowIndexAction;
2324

2425
import java.util.ArrayList;
2526
import java.util.Arrays;

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@
3434

3535
public class TransportCcrStatsAction extends TransportTasksAction<
3636
ShardFollowNodeTask,
37-
CcrStatsAction.TasksRequest,
38-
CcrStatsAction.TasksResponse, CcrStatsAction.TaskResponse> {
37+
CcrStatsAction.StatsRequest,
38+
CcrStatsAction.StatsResponses, CcrStatsAction.StatsResponse> {
3939

4040
private final IndexNameExpressionResolver resolver;
4141
private final CcrLicenseChecker ccrLicenseChecker;
@@ -54,8 +54,8 @@ public TransportCcrStatsAction(
5454
clusterService,
5555
transportService,
5656
actionFilters,
57-
CcrStatsAction.TasksRequest::new,
58-
CcrStatsAction.TasksResponse::new,
57+
CcrStatsAction.StatsRequest::new,
58+
CcrStatsAction.StatsResponses::new,
5959
Ccr.CCR_THREAD_POOL_NAME);
6060
this.resolver = Objects.requireNonNull(resolver);
6161
this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker);
@@ -64,8 +64,8 @@ public TransportCcrStatsAction(
6464
@Override
6565
protected void doExecute(
6666
final Task task,
67-
final CcrStatsAction.TasksRequest request,
68-
final ActionListener<CcrStatsAction.TasksResponse> listener) {
67+
final CcrStatsAction.StatsRequest request,
68+
final ActionListener<CcrStatsAction.StatsResponses> listener) {
6969
if (ccrLicenseChecker.isCcrAllowed() == false) {
7070
listener.onFailure(LicenseUtils.newComplianceException("ccr"));
7171
return;
@@ -74,21 +74,21 @@ protected void doExecute(
7474
}
7575

7676
@Override
77-
protected CcrStatsAction.TasksResponse newResponse(
78-
final CcrStatsAction.TasksRequest request,
79-
final List<CcrStatsAction.TaskResponse> taskResponses,
77+
protected CcrStatsAction.StatsResponses newResponse(
78+
final CcrStatsAction.StatsRequest request,
79+
final List<CcrStatsAction.StatsResponse> statsRespons,
8080
final List<TaskOperationFailure> taskOperationFailures,
8181
final List<FailedNodeException> failedNodeExceptions) {
82-
return new CcrStatsAction.TasksResponse(taskOperationFailures, failedNodeExceptions, taskResponses);
82+
return new CcrStatsAction.StatsResponses(taskOperationFailures, failedNodeExceptions, statsRespons);
8383
}
8484

8585
@Override
86-
protected CcrStatsAction.TaskResponse readTaskResponse(final StreamInput in) throws IOException {
87-
return new CcrStatsAction.TaskResponse(in);
86+
protected CcrStatsAction.StatsResponse readTaskResponse(final StreamInput in) throws IOException {
87+
return new CcrStatsAction.StatsResponse(in);
8888
}
8989

9090
@Override
91-
protected void processTasks(final CcrStatsAction.TasksRequest request, final Consumer<ShardFollowNodeTask> operation) {
91+
protected void processTasks(final CcrStatsAction.StatsRequest request, final Consumer<ShardFollowNodeTask> operation) {
9292
final ClusterState state = clusterService.state();
9393
final Set<String> concreteIndices = new HashSet<>(Arrays.asList(resolver.concreteIndexNames(state, request)));
9494
for (final Task task : taskManager.getTasks().values()) {
@@ -103,10 +103,10 @@ protected void processTasks(final CcrStatsAction.TasksRequest request, final Con
103103

104104
@Override
105105
protected void taskOperation(
106-
final CcrStatsAction.TasksRequest request,
106+
final CcrStatsAction.StatsRequest request,
107107
final ShardFollowNodeTask task,
108-
final ActionListener<CcrStatsAction.TaskResponse> listener) {
109-
listener.onResponse(new CcrStatsAction.TaskResponse(task.getFollowShardId(), task.getStatus()));
108+
final ActionListener<CcrStatsAction.StatsResponse> listener) {
109+
listener.onResponse(new CcrStatsAction.StatsResponse(task.getFollowShardId(), task.getStatus()));
110110
}
111111

112112
}

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestCcrStatsAction.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ public String getName() {
3333

3434
@Override
3535
protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) throws IOException {
36-
final CcrStatsAction.TasksRequest request = new CcrStatsAction.TasksRequest();
36+
final CcrStatsAction.StatsRequest request = new CcrStatsAction.StatsRequest();
3737
request.setIndices(Strings.splitStringByCommaToArray(restRequest.param("index")));
3838
request.setIndicesOptions(IndicesOptions.fromRequest(restRequest, request.indicesOptions()));
3939
return channel -> client.execute(CcrStatsAction.INSTANCE, request, new RestToXContentListener<>(channel));

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,9 @@ public void onFailure(final Exception e) {
9090

9191
public void testThatCcrStatsAreUnavailableWithNonCompliantLicense() throws InterruptedException {
9292
final CountDownLatch latch = new CountDownLatch(1);
93-
client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.TasksRequest(), new ActionListener<CcrStatsAction.TasksResponse>() {
93+
client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.StatsRequest(), new ActionListener<CcrStatsAction.StatsResponses>() {
9494
@Override
95-
public void onResponse(final CcrStatsAction.TasksResponse tasksResponse) {
95+
public void onResponse(final CcrStatsAction.StatsResponses statsResponses) {
9696
latch.countDown();
9797
fail();
9898
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,12 @@ private XPackSettings() {
3535
throw new IllegalStateException("Utility class should not be instantiated");
3636
}
3737

38+
39+
/**
40+
* Setting for controlling whether or not CCR is enabled.
41+
*/
42+
public static final Setting<Boolean> CCR_ENABLED_SETTING = Setting.boolSetting("xpack.ccr.enabled", true, Property.NodeScope);
43+
3844
/** Setting for enabling or disabling security. Defaults to true. */
3945
public static final Setting<Boolean> SECURITY_ENABLED = Setting.boolSetting("xpack.security.enabled", true, Setting.Property.NodeScope);
4046

0 commit comments

Comments
 (0)