Added as yet untested auto colour correlogram (fixed at 4 distance sets) distance calculator.

Author: ndtreviv

README.md

@@ -24,6 +24,19 @@ The result is subtracted from Double.MAX_VALUE to create a descending scoring va
 
 The first hash is sent in as a parameter: `param_hash`, the other is expected to be stored in a field also identified by a parameter: `param_field`.
 
+## AutoColourCorrelogram
+This script will calculate a distance between a given 64x4 correlogram (4 distance sets) in the format:
+```
+X0Y0,X0Y1,X0Y2,X0Y4;X1Y0,X1Y1,X1Y2,X1Y3;...;X64Y0,X64Y1,X64Y2,X64Y3
+```
+Because in distance terms, the smaller the distance the more likely the match, an in elastic terms, the greater the score the more likely
+the match, we need to turn the score on its head. So the result returned is actually `Integer.MAX_VALUE - distance`.
+
+If the parameter is incorrect, or the document does not have all the expected fields for the correlogram, the result will be `Integer.MAX_VALUE`.
+
+The script expects that the correlogram is stored in a field called `correlogram` in the same format as the parameter is provided (see above).
+
+
 **See Usage Examples below for clarity.**
 
 # Setup
@@ -34,7 +47,7 @@ The first hash is sent in as a parameter: `param_hash`, the other is expected to
 Include the following in your elasticsearch.yml config file:
 
     script.native:
-        hamming_distance.type: com.cameraforensics.elasticsearch.plugins.HammingDistanceScriptFactory
+        hamming_distance.type: com.cameraforensics.elasticsearch.plugins.AutoColourCorrelogramDistanceScriptFactory
         euclidean_distance.type: com.cameraforensics.elasticsearch.plugins.EuclideanDistanceScriptFactory
 
 **Note:** If you don’t do this, they still show up on the plugins list (see later) but you’ll get errors when you try to use either of them saying that elasticsearch can’t find the plugin.
@@ -124,4 +137,28 @@ produces something like:
           ]
         }
       }
+    }'
+    
+### Auto Colour Correlogram
+
+    curl -XPOST 'http://localhost:9200/twitter/_search?pretty' -d '{
+      "query": {
+        "function_score": {     
+          "min_score": IDEAL MIN SCORE HERE,
+          "query":{
+           "match_all":{}
+          },
+          "functions": [
+            {
+              "script_score": {
+                "script": "autocolourcorrelogram",
+                "lang" : "native",
+                "params": {
+                  "param_acc": "CORRELOGRAM TO COMPARE WITH"
+                }
+              }
+            }
+          ]
+        }
+      }
     }'

autocolourcorrelogram/build.gradle

@@ -0,0 +1,36 @@
+group 'com.cameraforensics'
+version '2.4.1.0'
+
+sourceCompatibility = JavaVersion.VERSION_1_7
+targetCompatibility = JavaVersion.VERSION_1_7
+
+configurations {
+    provided
+}
+
+apply plugin: 'idea'
+
+idea {
+    module {
+        scopes.PROVIDED.plus += [configurations.provided]
+    }
+}
+
+sourceSets {
+    main.compileClasspath += configurations.provided
+    test.compileClasspath += configurations.provided
+    test.runtimeClasspath += configurations.provided
+}
+
+jar {
+    baseName = 'autocolourcorrelogram'
+    version =  '2.4.1.0'
+    from {
+        configurations.compile.collect { it.isDirectory() ? it : zipTree(it) }
+    }
+}
+
+dependencies {
+    provided 'org.elasticsearch:elasticsearch:2.4.1'
+    compile "commons-codec:commons-codec:1.10"
+}

autocolourcorrelogram/src/main/java/com/cameraforensics/elasticsearch/plugins/AutoColourCorrelogramDistancePlugin.java

@@ -0,0 +1,23 @@
+package com.cameraforensics.elasticsearch.plugins;
+
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptModule;
+
+public class AutoColourCorrelogramDistancePlugin extends Plugin {
+
+    private static final String PLUGIN_NAME = "autocolourcorrelogram";
+
+    @Override
+    public String name() {
+        return PLUGIN_NAME;
+    }
+
+    @Override
+    public String description() {
+        return "A scoring function to calculate the distance between two Auto Colour Correlograms.";
+    }
+
+    public void onModule(ScriptModule scriptModule) {
+        scriptModule.registerScript(PLUGIN_NAME, AutoColourCorrelogramDistanceScriptFactory.class);
+    }
+}

autocolourcorrelogram/src/main/java/com/cameraforensics/elasticsearch/plugins/AutoColourCorrelogramDistanceScript.java

@@ -0,0 +1,84 @@
+package com.cameraforensics.elasticsearch.plugins;
+
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.script.AbstractFloatSearchScript;
+
+import java.util.Map;
+
+public class AutoColourCorrelogramDistanceScript extends AbstractFloatSearchScript {
+
+    private static final int SIZE_X = 64;
+    private static final int SIZE_Y = 4;
+
+    private float[][] queryCorrelogram = new float[SIZE_X][SIZE_Y];
+
+    public AutoColourCorrelogramDistanceScript(Map<String, Object> params) {
+        super();
+        queryCorrelogram = parseCorrelogram((String) params.get("param_acc"));
+    }
+
+    /**
+     * EG: This:
+     * ```
+     * 11.0,9.0,7.0,7.0;13.0,12.0,12.0,11.0;8.0,5.0,4.0,3.0;10.0,7.0,6.0,6.0;5.0,3.0,2.0,1.0;3.0,0.0,0.0,0.0;2.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;10.0,8.0,6.0,5.0;8.0,4.0,3.0,2.0;12.0,10.0,9.0,8.0;10.0,7.0,6.0,5.0;8.0,5.0,4.0,3.0;10.0,6.0,3.0,1.0;1.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;6.0,3.0,1.0,1.0;3.0,0.0,0.0,0.0;3.0,1.0,0.0,0.0;0.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;7.0,3.0,2.0,2.0;4.0,1.0,0.0,0.0;3.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;1.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;1.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;7.0,3.0,1.0,1.0;15.0,14.0,13.0,13.0;10.0,7.0,6.0,4.0;15.0,15.0,14.0,14.0;13.0,11.0,9.0,8.0;15.0,15.0,15.0,15.0;1.0,0.0,0.0,0.0;10.0,7.0,5.0,3.0;11.0,8.0,6.0,5.0;12.0,10.0,9.0,9.0;13.0,11.0,10.0,9.0;4.0,1.0,0.0,0.0;5.0,2.0,1.0,0.0;0.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;8.0,5.0,3.0,2.0;9.0,7.0,5.0,5.0;5.0,2.0,1.0,0.0;0.0,0.0,0.0,0.0;1.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;11.0,8.0,7.0,6.0;11.0,8.0,7.0,7.0;6.0,2.0,1.0,0.0;2.0,0.0,0.0,0.0;2.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;0.0,0.0,0.0,0.0;
+     * ```
+     * will produce:
+     * [[11.0, 9.0, 7.0, 7.0], [13.0, 12.0, 12.0, 11.0] ... ]
+     * @param correlogramStr
+     * @return
+     */
+    private float[][] parseCorrelogram(String correlogramStr) {
+        if (correlogramStr == null) {
+            return null;
+        }
+
+        float[][] correlogram = new float[SIZE_X][SIZE_Y];
+        String[] rows = correlogramStr.split(";");
+        if (rows.length != SIZE_X) {
+            return null;
+        }
+
+        for (int i = 0; i < rows.length; i++) {
+            String[] cols = rows[i].split(",");
+            if (cols.length != SIZE_Y) {
+                return null;
+            }
+
+            for (int j = 0; j < cols.length; j++) {
+                try {
+                    correlogram[i][j] = Integer.parseInt(cols[j]);
+                } catch (NumberFormatException nfe) {
+                    return null;
+                }
+            }
+        }
+        return correlogram;
+    }
+
+    private float getDistance(float[][] correlogram, float[][] queryCorrelogram) {
+        float result = 0;
+        for (int i = 0; i < correlogram.length; i++) {
+            float[] ints = correlogram[i];
+            for (int j = 0; j < ints.length; j++) {
+                result += (correlogram[i][j] > 0 ? (correlogram[i][j] / 2f) * Math.log((2f * correlogram[i][j]) / (correlogram[i][j] + queryCorrelogram[i][j])) : 0) +
+                        (queryCorrelogram[i][j] > 0 ? (queryCorrelogram[i][j] / 2f) * Math.log((2f * queryCorrelogram[i][j]) / (correlogram[i][j] + queryCorrelogram[i][j])) : 0);
+            }
+        }
+
+        return (float) Integer.MAX_VALUE - result;
+    }
+
+    @Override
+    public float runAsFloat() {
+        if (queryCorrelogram == null) {
+            return Integer.MAX_VALUE;
+        }
+
+        float[][] correlogram = parseCorrelogram(((ScriptDocValues.Strings) doc().get("correlogram")).getValue());
+        if (correlogram == null) {
+            return Integer.MAX_VALUE;
+        }
+
+        return getDistance(correlogram, queryCorrelogram);
+    }
+}

autocolourcorrelogram/src/main/java/com/cameraforensics/elasticsearch/plugins/AutoColourCorrelogramDistanceScriptFactory.java

@@ -0,0 +1,19 @@
+package com.cameraforensics.elasticsearch.plugins;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.NativeScriptFactory;
+
+import java.util.Map;
+
+public class AutoColourCorrelogramDistanceScriptFactory implements NativeScriptFactory {
+    @Override
+    public ExecutableScript newScript(@Nullable Map<String, Object> params) {
+        return new AutoColourCorrelogramDistanceScript(params);
+    }
+
+    @Override
+    public boolean needsScores() {
+        return false;
+    }
+}

autocolourcorrelogram/src/main/resources/plugin-descriptor.properties

@@ -0,0 +1,7 @@
+name=autocolourcorrelogram
+description=A scoring function to calculate the distance between two Auto Colour Correlograms.
+version=2.4.1.0
+jvm=true
+classname=com.cameraforensics.elasticsearch.plugins.AutoColourCorrelogramDistancePlugin
+java.version=1.7
+elasticsearch.version=2.4.1

settings.gradle

@@ -1,4 +1,5 @@
 rootProject.name = 'elasticsearch-plugins'
 include 'hammingdistance'
 include 'euclideandistance'
+include 'autocolourcorrelogram'
 

travis-build.gradle

@@ -1,7 +1,6 @@
 group 'com.cameraforensics'
 version '2.4.1'
 
-apply plugin: 'java'
 apply plugin: 'java'
 
 sourceCompatibility = JavaVersion.VERSION_1_7