[Transform] Add yml test suite for testing remote clusters (CCS)

x-pack/qa/multi-cluster-tests-with-security/build.gradle

@@ -0,0 +1,59 @@
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+apply plugin: 'elasticsearch.testclusters'
+apply plugin: 'elasticsearch.standalone-test'
+
+dependencies {
+  testCompile project(':x-pack:qa')
+  testCompile project(':client:rest-high-level')
+}
+
+task 'remote-cluster'(type: RestIntegTestTask) {
+  mustRunAfter(precommit)
+  runner {
+    systemProperty 'tests.rest.suite', 'remote_cluster'
+  }
+}
+
+testClusters.'remote-cluster' {
+  testDistribution = 'DEFAULT'
+  numberOfNodes = 2
+  setting 'cluster.remote.connect', "false"
+  setting 'xpack.ilm.enabled', 'false'
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.watcher.enabled', 'false'
+  setting 'xpack.monitoring.enabled', 'false'
+  setting 'xpack.license.self_generated.type', 'trial'
+
+  user username: "test_user", password: "x-pack-test-password"
+}
+
+task 'mixed-cluster'(type: RestIntegTestTask) {
+  dependsOn 'remote-cluster'
+  runner {
+    useCluster testClusters.'remote-cluster'
+    systemProperty 'tests.rest.suite', 'multi_cluster'
+  }
+}
+
+testClusters.'mixed-cluster' {
+  testDistribution = 'DEFAULT'
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.watcher.enabled', 'false'
+  setting 'xpack.monitoring.enabled', 'false'
+  setting 'xpack.license.self_generated.type', 'trial'
+  setting 'cluster.remote.my_remote_cluster.seeds', {
+    testClusters.'remote-cluster'.getAllTransportPortURI().collect { "\"$it\"" }.toString()
+  }
+  setting 'cluster.remote.connections_per_cluster', "1"
+  setting 'cluster.remote.connect', "true"
+
+  user username: "test_user", password: "x-pack-test-password"
+}
+
+task integTest {
+  dependsOn 'mixed-cluster'
+}
+
+test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test
+check.dependsOn(integTest)

x-pack/qa/multi-cluster-tests-with-security/src/test/java/org/elasticsearch/multi_cluster/MultiClusterYamlTestSuiteIT.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.multi_cluster;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
+import org.apache.lucene.util.TimeUnits;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue;
+
+@TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
+public class MultiClusterYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    private static final String USER = "test_user";
+    private static final String PASS = "x-pack-test-password";
+
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    public MultiClusterYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()));
+        return Settings.builder()
+                .put(super.restClientSettings())
+                .put(ThreadContext.PREFIX + ".Authorization", token)
+                .build();
+    }
+}

x-pack/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml

@@ -0,0 +1,261 @@
+---
+setup:
+  - skip:
+      features: headers
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+  - do:
+      security.put_user:
+        username: "joe"
+        body:  >
+            {
+              "password": "transform",
+              "roles" : [  "transform_admin", "x_cluster_role" ]
+            }
+  - do:
+      security.put_role:
+        name: "x_cluster_role"
+        body:  >
+            {
+              "cluster": [],
+              "indices": [
+                {
+                  "names": ["test_index", "my_remote_cluster:test_i*", "my_remote_cluster:aliased_test_index"],
+                  "privileges": ["all", "view_index_metadata"]
+                },
+                {
+                  "names": ["simple-remote-transform", "simple-local-remote-transform"],
+                  "privileges": ["all"]
+                }
+              ]
+            }
+---
+teardown:
+  - do:
+      security.delete_user:
+        username: "joe"
+        ignore: 404
+
+---
+"Search remote cluster":
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      search:
+        rest_total_hits_as_int: true
+        index: my_remote_cluster:test_index
+        body:
+          aggs:
+            user:
+              terms:
+                field: user
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 9 }
+  - length: { aggregations.user.buckets: 3 }
+  - match: { aggregations.user.buckets.0.key: "a" }
+  - match: { aggregations.user.buckets.0.doc_count: 5 }
+
+---
+"Batch transform from remote cluster":
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      transform.put_transform:
+        transform_id: "simple-remote-transform"
+        body: >
+          {
+            "source": { "index": "my_remote_cluster:test_index" },
+            "dest": { "index": "simple-remote-transform" },
+            "pivot": {
+              "group_by": { "user": {"terms": {"field": "user"}}},
+              "aggs": {"avg_stars": {"avg": {"field": "stars"}}}
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      transform.start_transform:
+        transform_id: "simple-remote-transform"
+  - match: { acknowledged: true }
+
+  - do:
+      transform.get_transform_stats:
+        transform_id: "simple-remote-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "simple-remote-transform" }
+  - match: { transforms.0.state: "/started|indexing|stopping|stopped/" }
+
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      transform.stop_transform:
+        transform_id: "simple-remote-transform"
+        wait_for_completion: true
+        wait_for_checkpoint: true
+  - match: { acknowledged: true }
+
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      transform.get_transform_stats:
+        transform_id: "simple-remote-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "simple-remote-transform" }
+  - match: { transforms.0.state: "stopped" }
+  - match: { transforms.0.checkpointing.last.checkpoint: 1 }
+
+  # workaround: refresh dest index, to be removed, see gh #51154
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      indices.refresh:
+        index: simple-remote-transform
+
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      search:
+        rest_total_hits_as_int: true
+        index: simple-remote-transform
+        sort: user
+
+  - match: { hits.total: 3 }
+  - match: { hits.hits.0._index: simple-remote-transform }
+  - match: { hits.hits.0._source.avg_stars: 3.6 }
+  - match: { hits.hits.0._source.user: a }
+  - match: { hits.hits.1._source.avg_stars: 2.0 }
+  - match: { hits.hits.1._source.user: b }
+
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      transform.update_transform:
+        transform_id: "simple-remote-transform"
+        body: >
+          {
+            "source": { "index": ["my_remote_cluster:test_index", "my_remote_cluster:test_index_2"] }
+          }
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      transform.get_transform_stats:
+        transform_id: "simple-remote-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "simple-remote-transform" }
+  - match: { transforms.0.state: "stopped" }
+  # we added test_index_2, which has 2 more docs:
+  - match: { transforms.0.checkpointing.operations_behind: 2 }
+
+---
+"Batch transform from local and remote cluster":
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_shards: 3
+              number_of_replicas: 0
+          aliases:
+            test_alias: {}
+          mappings:
+            properties:
+              time:
+                type: date
+              user:
+                type: keyword
+              stars:
+                type: integer
+              coolness:
+                type: integer
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+            - '{"index": {"_index": "test_index"}}'
+            - '{"user": "a", "stars": 3, "date" : "2018-11-29T12:12:12.123456789Z"}'
+            - '{"index": {"_index": "test_index"}}'
+            - '{"user": "c", "stars": 5, "date" : "2018-11-29T12:14:12.123456789Z"}'
+            - '{"index": {"_index": "test_index"}}'
+            - '{"user": "d", "stars": 5, "date" : "2018-11-29T12:16:12.123456789Z"}'
+            - '{"index": {"_index": "test_index"}}'
+            - '{"user": "e", "stars": 2, "date" : "2018-11-29T12:17:12.123456789Z"}'
+            - '{"index": {"_index": "test_index"}}'
+            - '{"user": "b", "stars": 3, "date" : "2018-11-29T12:22:12.123456789Z"}'
+            - '{"index": {"_index": "test_index"}}'
+            - '{"user": "c", "stars": 5, "date" : "2018-11-29T12:23:12.123456789Z"}'
+            - '{"index": {"_index": "test_index"}}'
+            - '{"user": "d", "stars": 1, "date" : "2018-11-29T12:32:12.123456789Z"}'
+            - '{"index": {"_index": "test_index"}}'
+            - '{"user": "e", "stars": 3, "date" : "2018-11-29T12:34:12.123456789Z"}'
+            - '{"index": {"_index": "test_index"}}'
+            - '{"user": "c", "stars": 4, "date" : "2018-11-29T12:35:12.123456789Z"}'
+
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      transform.put_transform:
+        transform_id: "simple-local-remote-transform"
+        body: >
+          {
+            "source": { "index": ["test_index", "my_remote_cluster:test_index"] },
+            "dest": { "index": "simple-local-remote-transform" },
+            "pivot": {
+              "group_by": { "user": {"terms": {"field": "user"}}},
+              "aggs": {
+                "avg_stars": {"avg": {"field": "stars"}},
+                "count": {"value_count": {"field": "user"}}
+              }
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      transform.start_transform:
+        transform_id: "simple-local-remote-transform"
+  - match: { acknowledged: true }
+
+  - do:
+      transform.get_transform_stats:
+        transform_id: "simple-local-remote-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "simple-local-remote-transform" }
+  - match: { transforms.0.state: "/started|indexing|stopping|stopped/" }
+
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      transform.stop_transform:
+        transform_id: "simple-local-remote-transform"
+        wait_for_completion: true
+        wait_for_checkpoint: true
+  - match: { acknowledged: true }
+
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      transform.get_transform_stats:
+        transform_id: "simple-local-remote-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "simple-local-remote-transform" }
+  - match: { transforms.0.state: "stopped" }
+  - match: { transforms.0.checkpointing.last.checkpoint: 1 }
+
+  # workaround: refresh dest index, to be removed, see gh #51154
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      indices.refresh:
+        index: simple-local-remote-transform
+
+  - do:
+      headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
+      search:
+        rest_total_hits_as_int: true
+        index: simple-local-remote-transform
+        sort: user
+
+  - match: { hits.total: 5 }
+  - match: { hits.hits.0._index: simple-local-remote-transform }
+  - match: { hits.hits.0._source.avg_stars: 3.5 }
+  - match: { hits.hits.0._source.count: 6 }
+  - match: { hits.hits.0._source.user: a }
+  - match: { hits.hits.1._source.avg_stars: 2.25 }
+  - match: { hits.hits.1._source.user: b }
+  - match: { hits.hits.3._source.count: 2 }
+  - match: { hits.hits.3._source.user: d }