codelibs · marevol · Mar 12, 2022 · Mar 1, 2022 · Mar 11, 2022
diff --git a/README.md b/README.md
@@ -21,7 +21,7 @@ Please file an [issue](https://github.com/codelibs/elasticsearch-dynarank/issues
 
 ## Installation
 
-    $ $ES_HOME/bin/elasticsearch-plugin install org.codelibs:elasticsearch-dynarank:7.6.0
+    $ $ES_HOME/bin/elasticsearch-plugin install org.codelibs:elasticsearch-dynarank:7.16.0
 
 ## Getting Started
 
@@ -100,6 +100,7 @@ The configuration is below:
           "script_sort":{
             "lang":"dynarank_diversity_sort",
             "params":{
+              "bucket_factory": "standard",
               "diversity_fields":["filedname1", "filedname2"],
               "diversity_thresholds":[0.95, 1]
             }
@@ -109,5 +110,6 @@ The configuration is below:
       }
     }'
 
-diversity\_fields is fields for a diversity.
+bucket\_factory is bucket type. use minhash type field for sort, specify "minhash".(default: standard)  
+diversity\_fields is fields for a diversity.  
 diversity\_thresholds is a threshold for a similarity of each document.
diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.codelibs</groupId>
 	<artifactId>elasticsearch-dynarank</artifactId>
-	<version>7.9.0-SNAPSHOT</version>
+	<version>7.16.0-SNAPSHOT</version>
 	<packaging>jar</packaging>
 	<description>This plugin re-orders top N documents in a search results.</description>
 	<inceptionYear>2011</inceptionYear>
@@ -27,7 +27,7 @@
 		<version>9</version>
 	</parent>
 	<properties>
-		<elasticsearch.version>7.9.3</elasticsearch.version>
+		<elasticsearch.version>7.16.3</elasticsearch.version>
 		<elasticsearch.plugin.classname>org.codelibs.elasticsearch.dynarank.DynamicRankingPlugin</elasticsearch.plugin.classname>
 		<maven.compiler.source>1.8</maven.compiler.source>
 		<maven.compiler.target>1.8</maven.compiler.target>
@@ -95,6 +95,23 @@
 			</plugin>
 		</plugins>
 	</build>
+	<repositories>
+		<repository>
+			<id>central</id>
+			<url>https://repo1.maven.org/maven2/</url>
+			<releases>
+				<enabled>true</enabled>
+			</releases>
+			<snapshots>
+				<enabled>true</enabled>
+			</snapshots>
+		</repository>
+		<repository>
+			<id>codelibs.org</id>
+			<name>CodeLibs Repository</name>
+			<url>https://maven.codelibs.org</url>
+		</repository>
+	</repositories>
 	<dependencies>
 		<dependency>
 			<groupId>org.elasticsearch</groupId>
@@ -139,7 +156,7 @@
 		<dependency>
 			<groupId>org.codelibs</groupId>
 			<artifactId>elasticsearch-minhash</artifactId>
-			<version>7.9.0</version>
+			<version>7.16.0</version>
 			<scope>test</scope>
 		</dependency>
 		<dependency>

diff --git a/src/main/java/org/codelibs/elasticsearch/dynarank/ranker/DynamicRanker.java b/src/main/java/org/codelibs/elasticsearch/dynarank/ranker/DynamicRanker.java
@@ -9,6 +9,7 @@
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
+import java.util.Objects;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -28,6 +29,7 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -39,8 +41,8 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptType;
@@ -49,7 +51,7 @@
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.profile.SearchProfileShardResults;
+import org.elasticsearch.search.profile.SearchProfileResults;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -254,11 +256,15 @@ public ScriptInfo getScriptInfo(final String index) {
                     return ScriptInfo.NO_SCRIPT_INFO;
                 }
 
-                final ScriptInfo[] scriptInfos = indexAbstraction.getIndices().stream().map(md -> md.getSettings())
-                        .filter(s -> SETTING_INDEX_DYNARANK_LANG.get(s).length() > 0)
-                        .map(settings -> new ScriptInfo(SETTING_INDEX_DYNARANK_SCRIPT.get(settings),
-                                SETTING_INDEX_DYNARANK_LANG.get(settings), SETTING_INDEX_DYNARANK_TYPE.get(settings),
-                                SETTING_INDEX_DYNARANK_PARAMS.get(settings), SETTING_INDEX_DYNARANK_REORDER_SIZE.get(settings), SETTING_INDEX_DYNARANK_KEEP_TOPN.get(settings)))
+                final ScriptInfo[] scriptInfos = indexAbstraction.getIndices().stream()
+                        .map(metaData::index)
+                        .filter(idx -> SETTING_INDEX_DYNARANK_LANG.get(idx.getSettings()).length() > 0)
+                        .map(idx ->
+                            new ScriptInfo(SETTING_INDEX_DYNARANK_SCRIPT.get(idx.getSettings()), SETTING_INDEX_DYNARANK_LANG.get(idx.getSettings()),
+                                    SETTING_INDEX_DYNARANK_TYPE.get(idx.getSettings()), SETTING_INDEX_DYNARANK_PARAMS.get(idx.getSettings()),
+                                    SETTING_INDEX_DYNARANK_REORDER_SIZE.get(idx.getSettings()), SETTING_INDEX_DYNARANK_KEEP_TOPN.get(idx.getSettings()),
+                                    idx.mapping())
+                        )
                         .toArray(n -> new ScriptInfo[n]);
 
                 if (scriptInfos.length == 0) {
@@ -337,7 +343,7 @@ public void onResponse(final Response response) {
                     final Suggest suggest = in.readBoolean() ? new Suggest(in) : null;
                     final boolean timedOut = in.readBoolean();
                     final Boolean terminatedEarly = in.readOptionalBoolean();
-                    final SearchProfileShardResults profileResults = in.readOptionalWriteable(SearchProfileShardResults::new);
+                    final SearchProfileResults profileResults = in.readOptionalWriteable(SearchProfileResults::new);
                     final int numReducePhases =  in.readVInt();
 
                     final SearchResponseSections internalResponse = new InternalSearchResponse(newHits, aggregations, suggest,
@@ -490,7 +496,7 @@ public static class ScriptInfo {
             // nothing
         }
 
-        ScriptInfo(final String script, final String lang, final String scriptType, final Settings settings, final int reorderSize,final int keepTopN) {
+        ScriptInfo(final String script, final String lang, final String scriptType, final Settings settings, final int reorderSize, final int keepTopN, final MappingMetadata mappingMetadata) {
             this.script = script;
             this.lang = lang;
             this.reorderSize = reorderSize;
@@ -500,6 +506,7 @@ public static class ScriptInfo {
                 final List<String> list = settings.getAsList(name);
                 this.settings.put(name, list.toArray(new String[list.size()]));
             }
+            this.settings.put("source_as_map", mappingMetadata.getSourceAsMap());
             if ("STORED".equalsIgnoreCase(scriptType)) {
                 this.scriptType = ScriptType.STORED;
             } else {
@@ -577,7 +584,8 @@ public void run() {
 
                     final ScriptInfo scriptInfo = new ScriptInfo(script, SETTING_INDEX_DYNARANK_LANG.get(indexSettings),
                             SETTING_INDEX_DYNARANK_TYPE.get(indexSettings), SETTING_INDEX_DYNARANK_PARAMS.get(indexSettings),
-                            SETTING_INDEX_DYNARANK_REORDER_SIZE.get(indexSettings),   SETTING_INDEX_DYNARANK_KEEP_TOPN.get(indexSettings));
+                            SETTING_INDEX_DYNARANK_REORDER_SIZE.get(indexSettings),   SETTING_INDEX_DYNARANK_KEEP_TOPN.get(indexSettings),
+                            indexMD.mapping());
                     if (logger.isDebugEnabled()) {
                         logger.debug("Reload cache for {} => {}", index, scriptInfo);
                     }

diff --git a/src/main/java/org/codelibs/elasticsearch/dynarank/script/DiversitySortScriptEngine.java b/src/main/java/org/codelibs/elasticsearch/dynarank/script/DiversitySortScriptEngine.java
@@ -13,6 +13,7 @@
 import org.apache.logging.log4j.Logger;
 import org.codelibs.elasticsearch.dynarank.script.bucket.BucketFactory;
 import org.codelibs.elasticsearch.dynarank.script.bucket.Buckets;
+import org.codelibs.elasticsearch.dynarank.script.bucket.impl.MinhashBucketFactory;
 import org.codelibs.elasticsearch.dynarank.script.bucket.impl.StandardBucketFactory;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.settings.Setting;
@@ -29,6 +30,8 @@ public class DiversitySortScriptEngine implements ScriptEngine {
 
     private static final String STANDARD = "standard";
 
+    private static final String MINHASH = "minhash";
+
     public static final Setting<Settings> SETTING_SCRIPT_DYNARANK_BUCKET =
             Setting.groupSetting("script.dynarank.bucket.", Property.NodeScope);
 
@@ -40,6 +43,7 @@ public DiversitySortScriptEngine(final Settings settings) {
 
         bucketFactories = new HashMap<>();
         bucketFactories.put(STANDARD, new StandardBucketFactory(settings));
+        bucketFactories.put(MINHASH, new MinhashBucketFactory(settings));
 
         for (final String name : bucketSettings.names()) {
             try {
@@ -91,9 +95,9 @@ public SearchHit[] execute(SearchHit[] searchHit) {
             if (logger.isDebugEnabled()) {
                 logger.debug("Starting DiversitySortScript...");
             }
-            Object bucketFactoryName = params.get("bucket_factory");
-            if (bucketFactoryName == null) {
-                bucketFactoryName = STANDARD;
+            Object bucketFactoryName = STANDARD;
+            if (params.get("bucket_factory") != null) {
+                bucketFactoryName = ((String[]) params.get("bucket_factory"))[0];
             }
             final BucketFactory bucketFactory = bucketFactories.get(bucketFactoryName);
             if (bucketFactory == null) {

diff --git a/src/main/java/org/codelibs/elasticsearch/dynarank/script/bucket/impl/MinhashBucket.java b/src/main/java/org/codelibs/elasticsearch/dynarank/script/bucket/impl/MinhashBucket.java
@@ -0,0 +1,78 @@
+package org.codelibs.elasticsearch.dynarank.script.bucket.impl;
+
+import org.codelibs.elasticsearch.dynarank.script.bucket.Bucket;
+import org.codelibs.minhash.MinHash;
+import org.elasticsearch.search.SearchHit;
+
+import java.util.LinkedList;
+import java.util.Queue;
+
+public class MinhashBucket implements Bucket {
+    protected Queue<SearchHit> queue = new LinkedList<>();
+
+    protected Object hash;
+
+    private final float threshold;
+
+    private final boolean isMinhash;
+
+    public MinhashBucket(final SearchHit hit, final Object hash, final float threshold, final boolean isMinhash) {
+        this.hash = hash;
+        this.threshold = threshold;
+        this.isMinhash = isMinhash;
+        queue.add(hit);
+    }
+
+    @Override
+    public void consume() {
+        queue.poll();
+    }
+
+    @Override
+    public SearchHit get() {
+        return queue.peek();
+    }
+
+    @Override
+    public boolean contains(final Object value) {
+        if (hash == null) {
+            return value == null;
+        }
+
+        if (value == null) {
+            return false;
+        }
+
+        if (!hash.getClass().equals(value.getClass())) {
+            return false;
+        }
+
+        if (value instanceof String) {
+            if (isMinhash) {
+                return MinHash.compare(hash.toString(), value.toString()) >= threshold;
+            }
+            return value.toString().equals(hash);
+        } else if (value instanceof Number) {
+            return Math.abs(((Number) value).doubleValue() - ((Number) hash).doubleValue()) < threshold;
+        } else if (value instanceof byte[]) {
+            final byte[] target = (byte[]) value;
+            return MinHash.compare((byte[]) hash, target) >= threshold;
+        }
+        return false;
+    }
+
+    @Override
+    public void add(final Object... args) {
+        queue.add((SearchHit) args[0]);
+    }
+
+    @Override
+    public int size() {
+        return queue.size();
+    }
+
+    @Override
+    public String toString() {
+        return "MinhashBucket [queue=" + queue + ", hash=" + hash + ", threshold=" + threshold + ", isMinhash=" + isMinhash + "]";
+    }
+}
diff --git a/...ain/java/org/codelibs/elasticsearch/dynarank/script/bucket/impl/MinhashBucketFactory.java b/...ain/java/org/codelibs/elasticsearch/dynarank/script/bucket/impl/MinhashBucketFactory.java
@@ -0,0 +1,28 @@
+package org.codelibs.elasticsearch.dynarank.script.bucket.impl;
+
+import org.codelibs.elasticsearch.dynarank.script.bucket.Bucket;
+import org.codelibs.elasticsearch.dynarank.script.bucket.BucketFactory;
+import org.codelibs.elasticsearch.dynarank.script.bucket.Buckets;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.SearchHit;
+
+import java.util.Map;
+
+public class MinhashBucketFactory implements BucketFactory {
+
+    protected Settings settings;
+
+    public MinhashBucketFactory(final Settings settings) {
+        this.settings = settings;
+    }
+
+    @Override
+    public Buckets createBucketList(final Map<String, Object> params) {
+        return new MinhashBuckets(this, params);
+    }
+
+    @Override
+    public Bucket createBucket(final Object... args) {
+        return new MinhashBucket((SearchHit) args[0], args[1], (float) args[2], (boolean) args[3]);
+    }
+}