refactor and improve testing infrastructure

externalize hadoop settings to allow testing against remote clusters
add provisioning of wrap tests + libs as jars for remote execution
add special handling of Win for remote execution (w/o using Cygwin)

Author: costin

build.gradle

@@ -224,6 +224,12 @@ task javadocJar(type: Jar) {
     from javadoc
 }
 
+// jar used for testing Hadoop remotely (es-hadoop + tests)
+task hadoopTestingJar(type: Jar) {
+	classifier = 'testing'
+	from sourceSets.test.output
+	from sourceSets.main.output
+}
 
 jar {
     manifest.attributes["Created-By"] = "${System.getProperty("java.version")} (${System.getProperty("java.specification.vendor")})"
@@ -266,5 +272,6 @@ task wrapper(type: Wrapper) {
     gradleVersion = '1.4'
 }
 
-assemble.dependsOn = ['jar', 'sourcesJar']
+// skip creation of javadoc jars 
+assemble.dependsOn = ['jar', 'hadoopTestingJar']
 defaultTasks 'build'

src/main/java/org/elasticsearch/hadoop/cfg/PropertiesSettings.java

@@ -24,7 +24,7 @@
  */
 public class PropertiesSettings extends Settings {
 
-    private final Properties props;
+    protected final Properties props;
 
     public PropertiesSettings(Properties props) {
         Assert.notNull(props, "Non-null properties expected");

src/test/java/org/elasticsearch/hadoop/ProvisionerTest.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2013 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.elasticsearch.hadoop;
+
+import org.elasticsearch.hadoop.integration.HdpBootstrap;
+import org.elasticsearch.hadoop.integration.Provisioner;
+import org.junit.Test;
+
+public class ProvisionerTest {
+
+    @Test
+    public void testBasicProvisier() {
+        Provisioner.provision(HdpBootstrap.hadoopConfig());
+    }
+}

src/test/java/org/elasticsearch/hadoop/integration/TestSettings.java renamed to src/test/java/org/elasticsearch/hadoop/integration/HdfsUtils.java

@@ -15,29 +15,24 @@
  */
 package org.elasticsearch.hadoop.integration;
 
-import java.util.Properties;
+import java.io.IOException;
 
-import org.elasticsearch.hadoop.cfg.PropertiesSettings;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
 
 /**
- * Tweaked settings for testing.
+ *
  */
-public class TestSettings extends PropertiesSettings {
-
-    public final static Properties TESTING_PROPS = new Properties();
-
-    static {
-        TESTING_PROPS.put(ES_BATCH_SIZE_BYTES, "8kb");
-        // see TestSettings
-        TESTING_PROPS.put(ES_PORT, "9500");
-    }
+public class HdfsUtils {
 
-    public TestSettings() {
-        super(new Properties(TESTING_PROPS));
+    public static void copyFromLocal(String localPath) throws IOException {
+        copyFromLocal(localPath, localPath);
     }
 
-    public TestSettings(String uri) {
-        this();
-        setProperty(ES_RESOURCE, uri);
+    public static void copyFromLocal(String localPath, String destination) throws IOException {
+        JobConf hadoopConfig = HdpBootstrap.hadoopConfig();
+        FileSystem fs = FileSystem.get(hadoopConfig);
+        fs.copyFromLocalFile(false, true, new Path(localPath), new Path(destination));
     }
 }

src/test/java/org/elasticsearch/hadoop/util/TestUtils.java renamed to src/test/java/org/elasticsearch/hadoop/integration/HdpBootstrap.java

@@ -13,26 +13,30 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.elasticsearch.hadoop.util;
+package org.elasticsearch.hadoop.integration;
 
-import java.io.File;
+import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
 
 import org.apache.commons.lang.reflect.FieldUtils;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.JobSubmissionFiles;
+import org.elasticsearch.hadoop.util.TestSettings;
 
-public class TestUtils {
+public class HdpBootstrap {
+
+    private static boolean hackVerified = false;
 
     /**
      * Hack to allow Hadoop client to run on windows (which otherwise fails due to some permission problem).
      */
-    public static void hackHadoopStagingOnWin() {
+    private static void hackHadoopStagingOnWin() {
         // do the assignment only on Windows systems
-        if (isWindows()) {
+        if (TestUtils.isWindows()) {
             // 0655 = -rwxr-xr-x
             JobSubmissionFiles.JOB_DIR_PERMISSION.fromShort((short) 0650);
             JobSubmissionFiles.JOB_FILE_PERMISSION.fromShort((short) 0650);
@@ -51,29 +55,40 @@ public static void hackHadoopStagingOnWin() {
         }
     }
 
-    public static boolean isWindows() {
-        return System.getProperty("os.name").toLowerCase().startsWith("win");
-    }
-
-    public static boolean delete(File file) {
-        if (file == null || !file.exists()) {
-            return false;
+    public static JobConf addProperties(JobConf conf, Properties props) {
+        for (Entry<Object, Object> entry : props.entrySet()) {
+            conf.set(entry.getKey().toString(), entry.getValue().toString());
         }
+        return conf;
+    }
 
-        boolean result = true;
-        if (file.isDirectory()) {
-            String[] children = file.list();
-            for (int i = 0; i < children.length; i++) {
-                result &= delete(new File(file, children[i]));
+    public static JobConf hadoopConfig() {
+        if (!hackVerified) {
+            hackVerified = true;
+            // check local execution
+            if ("local".equals(TestSettings.TESTING_PROPS.get("mapred.job.tracker"))) {
+                hackHadoopStagingOnWin();
             }
+            // damn HADOOP-9123
+            System.setProperty("path.separator", ":");
         }
-        return file.delete() & result;
+
+        JobConf conf = addProperties(new JobConf(), TestSettings.TESTING_PROPS);
+        // provision by default
+        Provisioner.provision(conf);
+
+        return conf;
     }
 
-    public static Configuration addProperties(Configuration conf, Properties props) {
-        for (Entry<Object, Object> entry : props.entrySet()) {
-            conf.set(entry.getKey().toString(), entry.getValue().toString());
+    public static Properties asProperties(Configuration cfg) {
+        Properties props = new Properties();
+
+        if (cfg != null) {
+            for (Map.Entry<String, String> entry : cfg) {
+                props.setProperty(entry.getKey(), entry.getValue());
+            }
         }
-        return conf;
+
+        return props;
     }
 }

src/test/java/org/elasticsearch/hadoop/integration/LocalES.java

@@ -17,7 +17,6 @@
 
 import java.io.File;
 
-import org.elasticsearch.hadoop.util.TestUtils;
 import org.junit.rules.ExternalResource;
 
 public class LocalES extends ExternalResource {
@@ -36,7 +35,7 @@ public class LocalES extends ExternalResource {
 
     @Override
     protected void before() throws Throwable {
-        TestUtils.hackHadoopStagingOnWin();
+        //TestUtils.hackHadoopStagingOnWin();
 
         if (master == null) {
             System.out.println("Starting Elasticsearch Master...");

src/test/java/org/elasticsearch/hadoop/integration/Provisioner.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2013 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.elasticsearch.hadoop.integration;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.elasticsearch.hadoop.util.Assert;
+
+/**
+ * Builder class for adding jars to the Hadoop environment.
+ */
+public abstract class Provisioner {
+
+    private static final String ESHADOOP_TESTING_JAR;
+    private static final String[] CASCADING_JARS;
+
+    static {
+        // init ES-Hadoop JAR
+        // expect the jar under build\libs
+        try {
+            File folder = new File("." + File.separator + "build" + File.separator + "libs" + File.separator).getCanonicalFile();
+            // find proper jar
+            File[] files = folder.listFiles(new FileFilter() {
+
+                @Override
+                public boolean accept(File pathname) {
+                    return pathname.getName().contains("-testing");
+                }
+            });
+            Assert.isTrue(files != null && files.length == 1,
+                    String.format("Cannot find elasticsearch hadoop jar;%s", Arrays.toString(files)));
+            ESHADOOP_TESTING_JAR = files[0].getAbsoluteFile().toURI().toString();
+
+        } catch (IOException ex) {
+            throw new RuntimeException("Cannot find required files", ex);
+        }
+
+        // initialize cascading jars
+        // read them from the classpath
+
+        List<String> jars = new ArrayList<String>();
+        // cascading-core
+        jars.add(findContainingJar("cascading/cascade/Cascade.class"));
+        // cascading-hadoop
+        jars.add(findContainingJar("cascading/flow/hadoop/HadoopFlow.class"));
+        // jgrapht
+        jars.add(findContainingJar("org/jgrapht/Graph.class"));
+        jars.add(findContainingJar("riffle/process/Process.class"));
+        // riffle
+        jars.add(findContainingJar("org/codehaus/janino/Java.class"));
+        // janino commons-compiler
+        jars.add(findContainingJar("org/codehaus/commons/compiler/CompileException.class"));
+
+        CASCADING_JARS = jars.toArray(new String[jars.size()]);
+    }
+
+
+    public static JobConf provision(JobConf conf) {
+        // set job jar
+        conf.set("mapred.jar", ESHADOOP_TESTING_JAR);
+        return conf;
+    }
+
+    public static JobConf cascading(JobConf conf) {
+        addLibs(conf, CASCADING_JARS);
+        return conf;
+    }
+
+    protected static void addLibs(Configuration configuration, String... libs) {
+        addResource(configuration, libs, "-libjars");
+    }
+
+    private static void addResource(Configuration cfg, String[] locations, String param) {
+        Assert.notNull(cfg, "a non-null configuration is required");
+
+        List<String> list = new ArrayList<String>();
+
+        try {
+            if (locations != null) {
+                int count = locations.length;
+                list.add(param);
+
+                StringBuilder sb = new StringBuilder();
+                for (String location : locations) {
+                    if (location != null) {
+                        sb.append(location);
+                        if (--count > 0) {
+                            sb.append(",");
+                        }
+                    }
+                }
+                list.add(sb.toString());
+            }
+
+            new GenericOptionsParser(cfg, list.toArray(new String[list.size()]));
+        } catch (IOException ex) {
+            throw new IllegalStateException(ex);
+        }
+    }
+
+    private static String findContainingJar(String binaryName) {
+        ClassLoader loader = Provisioner.class.getClassLoader();
+
+        try {
+            for (Enumeration<URL> urls = loader.getResources(binaryName); urls.hasMoreElements();) {
+                URL url = urls.nextElement();
+                // remove jar:
+                if ("jar".equals(url.getProtocol())) {
+                    return url.getPath().replaceAll("!.*$", "");
+                }
+            }
+        } catch (IOException ex) {
+            throw new IllegalArgumentException("Cannot find jar for class " + binaryName, ex);
+        }
+        throw new IllegalArgumentException("Cannot find class " + binaryName);
+    }
+}

src/test/java/org/elasticsearch/hadoop/integration/TestUtils.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2013 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.elasticsearch.hadoop.integration;
+
+import java.io.File;
+
+public class TestUtils {
+
+    public static boolean delete(File file) {
+        if (file == null || !file.exists()) {
+            return false;
+        }
+
+        boolean result = true;
+        if (file.isDirectory()) {
+            String[] children = file.list();
+            for (int i = 0; i < children.length; i++) {
+                result &= delete(new File(file, children[i]));
+            }
+        }
+        return file.delete() & result;
+    }
+
+    public static boolean isWindows() {
+        return System.getProperty("os.name").toLowerCase().startsWith("win");
+    }
+}