Merge branch '6.x' into ccr-6.x

* 6.x:
  Upgrade to Lucene-7.4.0-snapshot-518d303506 (#31360)
  [ML] Implement new rules design (#31110) (#31294)
  Remove RestGetAllAliasesAction (#31308)
  CCS: don't proxy requests for already connected node (#31273)
  Rankeval: Fold template test project into main module (#31203)
  [Docs] Remove reference to repository-s3 plugin creating an S3 bucket (#31359)
  More detailed tracing when writing metadata (#31319)
  Add details section for dcg ranking metric (#31177)

Author: dnhatn

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 6.4.0
-lucene            = 7.4.0-snapshot-0a7c3f462f
+lucene            = 7.4.0-snapshot-518d303506
 
 # optional dependencies
 spatial4j         = 0.7

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.client;
 
 import com.fasterxml.jackson.core.JsonParseException;
+
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
@@ -608,7 +609,7 @@ public void testDefaultNamedXContents() {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(7, namedXContents.size());
+        assertEquals(8, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -626,9 +627,10 @@ public void testProvidedNamedXContents() {
         assertTrue(names.contains(PrecisionAtK.NAME));
         assertTrue(names.contains(DiscountedCumulativeGain.NAME));
         assertTrue(names.contains(MeanReciprocalRank.NAME));
-        assertEquals(Integer.valueOf(2), categories.get(MetricDetail.class));
+        assertEquals(Integer.valueOf(3), categories.get(MetricDetail.class));
         assertTrue(names.contains(PrecisionAtK.NAME));
         assertTrue(names.contains(MeanReciprocalRank.NAME));
+        assertTrue(names.contains(DiscountedCumulativeGain.NAME));
     }
 
     private static class TrackingActionListener implements ActionListener<Integer> {

docs/plugins/repository-s3.asciidoc

@@ -284,22 +284,7 @@ You may further restrict the permissions by specifying a prefix within the bucke
 // NOTCONSOLE
 
 The bucket needs to exist to register a repository for snapshots. If you did not create the bucket then the repository
-registration will fail. If you want Elasticsearch to create the bucket instead, you can add the permission to create a
-specific bucket like this:
-
-[source,js]
-----
-{
-   "Action": [
-      "s3:CreateBucket"
-   ],
-   "Effect": "Allow",
-   "Resource": [
-      "arn:aws:s3:::snaps.example.com"
-   ]
-}
-----
-// NOTCONSOLE
+registration will fail.
 
 [[repository-s3-aws-vpc]]
 [float]

modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+a57659a275921d8ab3f7ec580e9bf713ce6143b1

modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-518d303506.jar.sha1

@@ -23,3 +23,12 @@ esplugin {
   hasClientJar = true
 }
 
+integTestCluster {
+  // Modules who's integration is explicitly tested in integration tests
+  module project(':modules:lang-mustache')
+}
+
+run {
+  // Modules who's integration is explicitly tested in integration tests
+  module project(':modules:lang-mustache')
+}

modules/rank-eval/build.gradle

@@ -36,6 +36,7 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRatings;
 
@@ -129,26 +130,31 @@ public EvalQueryQuality evaluate(String taskId, SearchHit[] hits,
                 .collect(Collectors.toList());
         List<RatedSearchHit> ratedHits = joinHitsWithRatings(hits, ratedDocs);
         List<Integer> ratingsInSearchHits = new ArrayList<>(ratedHits.size());
+        int unratedResults = 0;
         for (RatedSearchHit hit : ratedHits) {
-            // unknownDocRating might be null, which means it will be unrated docs are
-            // ignored in the dcg calculation
-            // we still need to add them as a placeholder so the rank of the subsequent
-            // ratings is correct
+            // unknownDocRating might be null, in which case unrated docs will be ignored in the dcg calculation.
+            // we still need to add them as a placeholder so the rank of the subsequent ratings is correct
             ratingsInSearchHits.add(hit.getRating().orElse(unknownDocRating));
+            if (hit.getRating().isPresent() == false) {
+                unratedResults++;
+            }
         }
-        double dcg = computeDCG(ratingsInSearchHits);
+        final double dcg = computeDCG(ratingsInSearchHits);
+        double result = dcg;
+        double idcg = 0;
 
         if (normalize) {
             Collections.sort(allRatings, Comparator.nullsLast(Collections.reverseOrder()));
-            double idcg = computeDCG(allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size())));
-            if (idcg > 0) {
-                dcg = dcg / idcg;
+            idcg = computeDCG(allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size())));
+            if (idcg != 0) {
+                result = dcg / idcg;
             } else {
-                dcg = 0;
+                result = 0;
             }
         }
-        EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, dcg);
+        EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, result);
         evalQueryQuality.addHitsAndRatings(ratedHits);
+        evalQueryQuality.setMetricDetails(new Detail(dcg, idcg, unratedResults));
         return evalQueryQuality;
     }
 
@@ -167,7 +173,7 @@ private static double computeDCG(List<Integer> ratings) {
     private static final ParseField K_FIELD = new ParseField("k");
     private static final ParseField NORMALIZE_FIELD = new ParseField("normalize");
     private static final ParseField UNKNOWN_DOC_RATING_FIELD = new ParseField("unknown_doc_rating");
-    private static final ConstructingObjectParser<DiscountedCumulativeGain, Void> PARSER = new ConstructingObjectParser<>("dcg_at", false,
+    private static final ConstructingObjectParser<DiscountedCumulativeGain, Void> PARSER = new ConstructingObjectParser<>("dcg", false,
             args -> {
                 Boolean normalized = (Boolean) args[0];
                 Integer optK = (Integer) args[2];
@@ -217,4 +223,118 @@ public final boolean equals(Object obj) {
     public final int hashCode() {
         return Objects.hash(normalize, unknownDocRating, k);
     }
+
+    public static final class Detail implements MetricDetail {
+
+        private static ParseField DCG_FIELD = new ParseField("dcg");
+        private static ParseField IDCG_FIELD = new ParseField("ideal_dcg");
+        private static ParseField NDCG_FIELD = new ParseField("normalized_dcg");
+        private static ParseField UNRATED_FIELD = new ParseField("unrated_docs");
+        private final double dcg;
+        private final double idcg;
+        private final int unratedDocs;
+
+        Detail(double dcg, double idcg, int unratedDocs) {
+            this.dcg = dcg;
+            this.idcg = idcg;
+            this.unratedDocs = unratedDocs;
+        }
+
+        Detail(StreamInput in) throws IOException {
+            this.dcg = in.readDouble();
+            this.idcg = in.readDouble();
+            this.unratedDocs = in.readVInt();
+        }
+
+        @Override
+        public
+        String getMetricName() {
+            return NAME;
+        }
+
+        @Override
+        public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field(DCG_FIELD.getPreferredName(), this.dcg);
+            if (this.idcg != 0) {
+                builder.field(IDCG_FIELD.getPreferredName(), this.idcg);
+                builder.field(NDCG_FIELD.getPreferredName(), this.dcg / this.idcg);
+            }
+            builder.field(UNRATED_FIELD.getPreferredName(), this.unratedDocs);
+            return builder;
+        }
+
+        private static final ConstructingObjectParser<Detail, Void> PARSER = new ConstructingObjectParser<>(NAME, true, args -> {
+            return new Detail((Double) args[0], (Double) args[1] != null ? (Double) args[1] : 0.0d, (Integer) args[2]);
+        });
+
+        static {
+            PARSER.declareDouble(constructorArg(), DCG_FIELD);
+            PARSER.declareDouble(optionalConstructorArg(), IDCG_FIELD);
+            PARSER.declareInt(constructorArg(), UNRATED_FIELD);
+        }
+
+        public static Detail fromXContent(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeDouble(this.dcg);
+            out.writeDouble(this.idcg);
+            out.writeVInt(this.unratedDocs);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME;
+        }
+
+        /**
+         * @return the discounted cumulative gain
+         */
+        public double getDCG() {
+            return this.dcg;
+        }
+
+        /**
+         * @return the ideal discounted cumulative gain, can be 0 if nothing was computed, e.g. because no normalization was required
+         */
+        public double getIDCG() {
+            return this.idcg;
+        }
+
+        /**
+         * @return the normalized discounted cumulative gain, can be 0 if nothing was computed, e.g. because no normalization was required
+         */
+        public double getNDCG() {
+            return (this.idcg != 0) ? this.dcg / this.idcg : 0;
+        }
+
+        /**
+         * @return the number of unrated documents in the search results
+         */
+        public Object getUnratedDocs() {
+            return this.unratedDocs;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj == null || getClass() != obj.getClass()) {
+                return false;
+            }
+            DiscountedCumulativeGain.Detail other = (DiscountedCumulativeGain.Detail) obj;
+            return (this.dcg == other.dcg &&
+                    this.idcg == other.idcg &&
+                    this.unratedDocs == other.unratedDocs);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(this.dcg, this.idcg, this.unratedDocs);
+        }
+    }
 }
+

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java

@@ -41,6 +41,8 @@ public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
                 PrecisionAtK.Detail::fromXContent));
         namedXContent.add(new NamedXContentRegistry.Entry(MetricDetail.class, new ParseField(MeanReciprocalRank.NAME),
                 MeanReciprocalRank.Detail::fromXContent));
+        namedXContent.add(new NamedXContentRegistry.Entry(MetricDetail.class, new ParseField(DiscountedCumulativeGain.NAME),
+                DiscountedCumulativeGain.Detail::fromXContent));
         return namedXContent;
     }
 }

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java

@@ -61,8 +61,9 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         namedWriteables.add(
                 new NamedWriteableRegistry.Entry(EvaluationMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetail.class, PrecisionAtK.NAME, PrecisionAtK.Detail::new));
-        namedWriteables
-                .add(new NamedWriteableRegistry.Entry(MetricDetail.class, MeanReciprocalRank.NAME, MeanReciprocalRank.Detail::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetail.class, MeanReciprocalRank.NAME, MeanReciprocalRank.Detail::new));
+        namedWriteables.add(
+                new NamedWriteableRegistry.Entry(MetricDetail.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain.Detail::new));
         return namedWriteables;
     }
 

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.rankeval;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.text.Text;
@@ -254,9 +255,8 @@ private void assertParsedCorrect(String xContent, Integer expectedUnknownDocRati
 
     public static DiscountedCumulativeGain createTestItem() {
         boolean normalize = randomBoolean();
-        Integer unknownDocRating = Integer.valueOf(randomIntBetween(0, 1000));
-
-        return new DiscountedCumulativeGain(normalize, unknownDocRating, 10);
+        Integer unknownDocRating = frequently() ? Integer.valueOf(randomIntBetween(0, 1000)) : null;
+        return new DiscountedCumulativeGain(normalize, unknownDocRating, randomIntBetween(1, 10));
     }
 
     public void testXContentRoundtrip() throws IOException {
@@ -283,7 +283,25 @@ public void testXContentParsingIsNotLenient() throws IOException {
             parser.nextToken();
             XContentParseException exception = expectThrows(XContentParseException.class,
                     () -> DiscountedCumulativeGain.fromXContent(parser));
-            assertThat(exception.getMessage(), containsString("[dcg_at] unknown field"));
+            assertThat(exception.getMessage(), containsString("[dcg] unknown field"));
+        }
+    }
+
+    public void testMetricDetails() {
+        double dcg = randomDoubleBetween(0, 1, true);
+        double idcg = randomBoolean() ? 0.0 : randomDoubleBetween(0, 1, true);
+        double expectedNdcg = idcg != 0 ? dcg / idcg : 0.0;
+        int unratedDocs = randomIntBetween(0, 100);
+        DiscountedCumulativeGain.Detail detail = new DiscountedCumulativeGain.Detail(dcg, idcg, unratedDocs);
+        assertEquals(dcg, detail.getDCG(), 0.0);
+        assertEquals(idcg, detail.getIDCG(), 0.0);
+        assertEquals(expectedNdcg, detail.getNDCG(), 0.0);
+        assertEquals(unratedDocs, detail.getUnratedDocs());
+        if (idcg != 0) {
+            assertEquals("{\"dcg\":{\"dcg\":" + dcg + ",\"ideal_dcg\":" + idcg + ",\"normalized_dcg\":" + expectedNdcg
+                    + ",\"unrated_docs\":" + unratedDocs + "}}", Strings.toString(detail));
+        } else {
+            assertEquals("{\"dcg\":{\"dcg\":" + dcg + ",\"unrated_docs\":" + unratedDocs + "}}", Strings.toString(detail));
         }
     }
 

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java

@@ -68,10 +68,20 @@ public static EvalQueryQuality randomEvalQueryQuality() {
         EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAlphaOfLength(10),
                 randomDoubleBetween(0.0, 1.0, true));
         if (randomBoolean()) {
-            if (randomBoolean()) {
+            int metricDetail = randomIntBetween(0, 2);
+            switch (metricDetail) {
+            case 0:
                 evalQueryQuality.setMetricDetails(new PrecisionAtK.Detail(randomIntBetween(0, 1000), randomIntBetween(0, 1000)));
-            } else {
+                break;
+            case 1:
                 evalQueryQuality.setMetricDetails(new MeanReciprocalRank.Detail(randomIntBetween(0, 1000)));
+                break;
+            case 2:
+                evalQueryQuality.setMetricDetails(new DiscountedCumulativeGain.Detail(randomDoubleBetween(0, 1, true),
+                        randomBoolean() ? randomDoubleBetween(0, 1, true) : 0, randomInt()));
+                break;
+            default:
+                throw new IllegalArgumentException("illegal randomized value in test");
             }
         }
         evalQueryQuality.addHitsAndRatings(ratedHits);

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java

@@ -0,0 +1 @@
+b91a260d8d12ee4b3302a63059c73a34de0ce146

qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/10_rank_eval_templated.yml renamed to modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml

@@ -0,0 +1 @@
+cc1ca9bd9e2c162dd1da8c2e7111913fd8033e48

plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+2fa3662a10a9e085b1c7b87293d727422cbe6224

plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-518d303506.jar.sha1

@@ -0,0 +1 @@
+60aa50c11857e6739e68936cb45102562b2c46b4

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+4586368007785a3be26db4b9ce404ffb8c76f350

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-518d303506.jar.sha1

@@ -0,0 +1 @@
+9c6d030ab2c148df7a6ba73a774ef4b8c720a6cb

plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+8275bf8df2644d5fcec2963cf237d14b6e00fefe