Merge branch 'titan05' into log

Conflicts: titan-hbase/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java
twilmes · Feb 16, 2014 · dc4d0ec · dc4d0ec
2 parents 7484589 + 0d7afbd
commit dc4d0ec
Show file tree

Hide file tree

Showing 224 changed files with 9,905 additions and 321 deletions.
diff --git a/README.textile b/README.textile
@@ -11,6 +11,7 @@ h2. Features
 * Support for various "storage backends":https://github.com/thinkaurelius/titan/wiki/Storage-Backend-Overview:
 ** "Apache Cassandra":http://cassandra.apache.org/ (distributed)
 ** "Apache HBase":http://hbase.apache.org/ (distributed)
+** "MapR Tables":http://www.mapr.com/products/mapr-editions/m7-edition (distributed)
 ** "Oracle BerkeleyDB":http://www.oracle.com/technetwork/database/berkeleydb/overview/index-093405.html (local)
 ** "Persistit":https://github.com/pbeaman/persistit (local)
 * Support for various "indexing backends":https://github.com/thinkaurelius/titan/wiki/Indexing-Backend-Overview:

diff --git a/pom.xml b/pom.xml
@@ -73,6 +73,7 @@
         <jersey.version>1.17</jersey.version>
         <jna.version>3.2.7</jna.version>
         <kuali.s3.wagon.version>1.1.20</kuali.s3.wagon.version>
+        <jasper.version>5.5.23</jasper.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <titan.testdir>${project.build.directory}/titan-test</titan.testdir>
@@ -279,6 +280,11 @@
                     <artifactId>wagon-maven-plugin</artifactId>
                     <version>1.0-beta-4</version>
                 </plugin>
+                <plugin>
+                    <groupId>org.codehaus.mojo</groupId>
+                    <artifactId>build-helper-maven-plugin</artifactId>
+                    <version>1.8</version>
+                </plugin>
                 <plugin>
                     <groupId>org.codehaus.mojo</groupId>
                     <artifactId>exec-maven-plugin</artifactId>
@@ -442,6 +448,16 @@
                 <artifactId>jettison</artifactId>
                 <version>1.3.3</version>
             </dependency>
+            <dependency>
+                <groupId>tomcat</groupId>
+                <artifactId>jasper-runtime</artifactId>
+                <version>${jasper.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>tomcat</groupId>
+                <artifactId>jasper-compiler</artifactId>
+                <version>${jasper.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.mortbay.jetty</groupId>
                 <artifactId>jetty</artifactId>
@@ -561,6 +577,11 @@
                 <artifactId>netty</artifactId>
                 <version>3.2.7.Final</version>
             </dependency>
+            <dependency>
+                <groupId>io.netty</groupId>
+                <artifactId>netty</artifactId>
+                <version>3.6.6.Final</version>
+            </dependency>
             <!-- Package prefix is org.apache.commons.httpclient -->
             <dependency>
                 <groupId>commons-httpclient</groupId>

diff --git a/titan-all/pom.xml b/titan-all/pom.xml
@@ -40,7 +40,7 @@
         </dependency>
         <dependency>
             <groupId>com.thinkaurelius.titan</groupId>
-            <artifactId>titan-hbase</artifactId>
+            <artifactId>titan-hbase-096</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>

diff --git a/titan-cassandra/pom.xml b/titan-cassandra/pom.xml
@@ -150,29 +150,6 @@
         </dependency>
     </dependencies>
 
-    <profiles>
-        <profile>
-            <id>aurelius-release</id>
-
-            <build>
-                <plugins>
-                    <plugin>
-                        <artifactId>maven-jar-plugin</artifactId>
-                        <executions>
-                            <execution>
-                                <id>pack-test-jar</id>
-                                <phase>package</phase>
-                                <goals>
-                                    <goal>test-jar</goal>
-                                </goals>
-                            </execution>
-                        </executions>
-                    </plugin>
-                </plugins>
-            </build>
-        </profile>
-    </profiles>
-
     <build>
 
         <resources>
@@ -351,6 +328,18 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <artifactId>maven-jar-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>pack-test-jar</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 </project>

diff --git a/...ain/java/com/thinkaurelius/titan/diskstorage/cassandra/AbstractCassandraStoreManager.java b/...ain/java/com/thinkaurelius/titan/diskstorage/cassandra/AbstractCassandraStoreManager.java
@@ -96,7 +96,7 @@ public static Partitioner getPartitioner(String className) {
      */
     public static final ConfigOption<String> CASSANDRA_COMPRESSION_TYPE = new ConfigOption<String>(STORAGE_NS,"compression-type",
             "The particular compression type to use for Cassandra sstable compression",
-            ConfigOption.Type.FIXED, "SnappyCompressor");
+            ConfigOption.Type.FIXED, "LZ4Compressor");
 
 //    public static final String COMPRESSION_KEY = "compression.sstable_compression";
 //    public static final String DEFAULT_COMPRESSION = "SnappyCompressor";

diff --git a/titan-cassandra/src/test/java/com/thinkaurelius/titan/CassandraStorageSetup.java b/titan-cassandra/src/test/java/com/thinkaurelius/titan/CassandraStorageSetup.java
@@ -1,13 +1,15 @@
 package com.thinkaurelius.titan;
 
 import static com.thinkaurelius.titan.diskstorage.cassandra.AbstractCassandraStoreManager.CASSANDRA_KEYSPACE;
+import static com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration.IDS_PARTITION;
 
 import java.io.File;
 import java.io.IOException;
 
 import com.thinkaurelius.titan.diskstorage.cassandra.utils.CassandraDaemonWrapper;
 import com.thinkaurelius.titan.diskstorage.configuration.ModifiableConfiguration;
 import com.thinkaurelius.titan.diskstorage.configuration.WriteConfiguration;
+import com.thinkaurelius.titan.graphdb.database.idassigner.IDPartitionMode;
 
 import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
@@ -50,7 +52,7 @@ public static ModifiableConfiguration getEmbeddedConfiguration(String ks) {
 
     public static ModifiableConfiguration getEmbeddedCassandraPartitionConfiguration(String ks) {
         ModifiableConfiguration config = getEmbeddedConfiguration(ks);
-        config.set(IDS_PARTITION,true);
+        config.set(IDS_PARTITION, IDPartitionMode.ENABLED);
         config.set(IDS_FLUSH,false);
 //        config.subset(GraphDatabaseConfiguration.METRICS_NAMESPACE).addProperty(GraphDatabaseConfiguration.METRICS_CONSOLE_INTERVAL, 3000L);
         return config;

diff --git a/...m/thinkaurelius/titan/diskstorage/cassandra/AbstractCassandraKeyColumnValueStoreTest.java b/...m/thinkaurelius/titan/diskstorage/cassandra/AbstractCassandraKeyColumnValueStoreTest.java
@@ -78,7 +78,7 @@ public void testDefaultCFCompressor() throws StorageException {
 
         Map<String, String> defaultCfCompressionOps =
                 new ImmutableMap.Builder<String, String>()
-                .put("sstable_compression", DEFAULT_COMPRESSOR_PACKAGE + ".SnappyCompressor")
+                .put("sstable_compression", DEFAULT_COMPRESSOR_PACKAGE + "." + AbstractCassandraStoreManager.CASSANDRA_COMPRESSION_TYPE.getDefaultValue())
                 .put("chunk_length_kb", "64")
                 .build();
 

diff --git a/titan-cassandra/src/test/resources/log4j.properties b/titan-cassandra/src/test/resources/log4j.properties
@@ -22,3 +22,7 @@ log4j.rootLogger=INFO, A1, A2
 log4j.logger.org.apache.cassandra=INFO
 log4j.logger.org.apache.hadoop=INFO
 log4j.logger.org.apache.zookeeper=INFO
+# Disable all messages from ExpectedValueCheckingTransaction.  The point is to
+# suppress scary-looking ERROR messages that are deliberately induced by
+# LockKeyColumnValueStoreTest.
+log4j.logger.com.thinkaurelius.titan.diskstorage.locking.consistentkey.ExpectedValueCheckingTransaction=OFF
diff --git a/...c/main/java/com/thinkaurelius/titan/graphdb/configuration/GraphDatabaseConfiguration.java b/...c/main/java/com/thinkaurelius/titan/graphdb/configuration/GraphDatabaseConfiguration.java
@@ -37,6 +37,7 @@
 import com.thinkaurelius.titan.core.DefaultTypeMaker;
 import com.thinkaurelius.titan.diskstorage.Backend;
 import com.thinkaurelius.titan.graphdb.blueprints.BlueprintsDefaultTypeMaker;
+import com.thinkaurelius.titan.graphdb.database.idassigner.IDPartitionMode;
 import com.thinkaurelius.titan.graphdb.database.idassigner.VertexIDAssigner;
 import com.thinkaurelius.titan.graphdb.database.serialize.Serializer;
 import com.thinkaurelius.titan.graphdb.transaction.StandardTransactionBuilder;
@@ -611,9 +612,9 @@ public boolean apply(@Nullable Integer uniqueIdBitWidth) {
      * enabled to ensure an even distribution of data. If the keyspace is random/hashed, then enabling this only has the benefit
      * of de-congesting a single id pool in the database.
      */
-    public static final ConfigOption<Boolean> IDS_PARTITION = new ConfigOption<Boolean>(IDS_NS,"partition",
+    public static final ConfigOption<IDPartitionMode> IDS_PARTITION = new ConfigOption<IDPartitionMode>(IDS_NS,"partition",
             "Whether the id space should be partitioned for equal distribution of keys",
-            ConfigOption.Type.FIXED, false);
+            ConfigOption.Type.FIXED, IDPartitionMode.DEFAULT);
 //    public static final String IDS_PARTITION_KEY = "partition";
 //    public static final boolean IDS_PARTITION_DEFAULT = false;
 

diff --git a/...re/src/main/java/com/thinkaurelius/titan/graphdb/database/idassigner/IDPartitionMode.java b/...re/src/main/java/com/thinkaurelius/titan/graphdb/database/idassigner/IDPartitionMode.java
@@ -0,0 +1,22 @@
+package com.thinkaurelius.titan.graphdb.database.idassigner;
+
+import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreFeatures;
+
+public enum IDPartitionMode {
+    /**
+     * Automatically enable partitioning when the storage backend reports true
+     * for both {@link StoreFeatures#isDistributed()} and
+     * {@link StoreFeatures#isKeyOrdered()}.
+     */
+    DEFAULT,
+
+    /**
+     * Force partitioning on.
+     */
+    ENABLED,
+
+    /**
+     * Force partitioning off.
+     */
+    DISABLED;
+}
diff --git a/...e/src/main/java/com/thinkaurelius/titan/graphdb/database/idassigner/VertexIDAssigner.java b/...e/src/main/java/com/thinkaurelius/titan/graphdb/database/idassigner/VertexIDAssigner.java
@@ -70,23 +70,24 @@ public class VertexIDAssigner {
     private final int maxPartitionID;
     private final boolean hasLocalPartitions;
 
-
     public VertexIDAssigner(Configuration config, IDAuthority idAuthority, StoreFeatures idAuthFeatures) {
         Preconditions.checkNotNull(idAuthority);
         this.idAuthority = idAuthority;
 
         long partitionBits;
-        boolean partitionIDs = config.get(IDS_PARTITION);
-        if (partitionIDs) {
+        IDPartitionMode partitionIDs = config.get(IDS_PARTITION);
+        boolean storeWantsPartitioning = idAuthFeatures.isKeyOrdered() && idAuthFeatures.isDistributed();
+        if (partitionIDs.equals(IDPartitionMode.ENABLED) ||
+                (storeWantsPartitioning && partitionIDs.equals(IDPartitionMode.DEFAULT))) {
             //Use a placement strategy that balances partitions
             partitionBits = DEFAULT_PARTITION_BITS;
             hasLocalPartitions = idAuthFeatures.hasLocalKeyPartition();
 
             placementStrategy = Backend.getImplementationClass(config, config.get(PLACEMENT_STRATEGY),
                     REGISTERED_PLACEMENT_STRATEGIES);
         } else {
-            if (idAuthFeatures.isKeyOrdered() && idAuthFeatures.isDistributed())
-                log.warn("ID Partitioning is disabled which will likely cause uneven data distribution");
+            if (storeWantsPartitioning)
+                log.warn("ID Partitioning is disabled, which will likely cause uneven data distribution and sequentially increasing keys");
             //Use the default placement strategy
             partitionBits = 0;
             hasLocalPartitions = false;
@@ -122,8 +123,10 @@ private void setLocalPartitions(long partitionBits) {
             try {
                 List<KeyRange> locals = idAuthority.getLocalIDPartition();
                 if (locals==null || locals.isEmpty()) throw new IllegalStateException("Returned partitions were empty");
+                log.debug("Processing {} local ID partition range(s)", locals.size());
                 for (KeyRange local : locals) {
-                    Preconditions.checkArgument(local.getStart().length() >= 4 && local.getEnd().length() >= 4);
+                    Preconditions.checkArgument(local.getStart().length() >= 4);
+                    Preconditions.checkArgument(local.getEnd().length() >= 4);
                     int[] partition = new int[2];
                     for (int i = 0; i < 2; i++) {
                         partition[i] = local.getAt(i).getInt(0);
@@ -133,6 +136,7 @@ private void setLocalPartitions(long partitionBits) {
                     else partition[0] = (partition[0] >>> 2);
                     //Upper bound needs to be exclusive
                     partition[1] = (partition[1] >>> 2) - 1;
+                    partition[1] &= 0x3FFFFFFF;
                     if (partition[0]==partition[1]) {
                         log.warn("Individual key range is too small for partition block: {}",local);
                         continue;

diff --git a/.../java/com/thinkaurelius/titan/graphdb/database/idassigner/placement/PartitionIDRange.java b/.../java/com/thinkaurelius/titan/graphdb/database/idassigner/placement/PartitionIDRange.java
@@ -24,9 +24,12 @@ public class PartitionIDRange {
 
 
     public PartitionIDRange(int lowerID, int upperID, int idUpperBound) {
-        Preconditions.checkArgument(idUpperBound>0 && idUpperBound<=Integer.MAX_VALUE);
-        Preconditions.checkArgument(lowerID>=0 && lowerID<idUpperBound);
-        Preconditions.checkArgument(upperID>=0 && upperID<=idUpperBound);
+        Preconditions.checkArgument(idUpperBound>0, "Partition limit " + idUpperBound + " must be positive");
+        Preconditions.checkArgument(idUpperBound<=Integer.MAX_VALUE, "Partition limit cannot exceed representable range of an integer");
+        Preconditions.checkArgument(lowerID>=0, "Negative partition lower bound " + lowerID);
+        Preconditions.checkArgument(lowerID< idUpperBound, "Partition lower bound" + lowerID + " equal to or greater than limit " + idUpperBound);
+        Preconditions.checkArgument(upperID>=0, "Negative partition upper bound " + upperID);
+        Preconditions.checkArgument(upperID<=idUpperBound, "Partition upper bound " + upperID + " exceeds limit " + idUpperBound);
         this.lowerID = lowerID;
         this.upperID = upperID;
         this.idUpperBound = idUpperBound;

diff --git a/titan-dist/pom.xml b/titan-dist/pom.xml
@@ -23,9 +23,38 @@
     <dependencies>
         <dependency>
             <groupId>${project.groupId}</groupId>
-            <artifactId>titan-all</artifactId>
+            <artifactId>titan-berkeleyje</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>titan-cassandra</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>titan-es</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>titan-hbase-096</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase-client</artifactId>
+            <version>0.96.1.1-hadoop2</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>titan-lucene</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>titan-persistit</artifactId>
             <version>${project.version}</version>
-            <type>pom</type>
         </dependency>
         <dependency>
             <groupId>${project.groupId}</groupId>

diff --git a/titan-dist/src/assembly/static/conf/cassandra.yaml b/titan-dist/src/assembly/static/conf/cassandra.yaml
@@ -315,7 +315,7 @@ ssl_storage_port: 7001
 # address associated with the hostname (it might not be).
 #
 # Setting this to 0.0.0.0 is always wrong.
-listen_address: localhost
+listen_address: 127.0.0.1
 
 # Address to broadcast to other Cassandra nodes
 # Leaving this blank will set it to the same value as listen_address
@@ -346,7 +346,7 @@ start_rpc: true
 # 
 # Leaving this blank has the same effect it does for ListenAddress,
 # (i.e. it will be based on the configured hostname of the node).
-rpc_address: localhost
+rpc_address: 127.0.0.1
 # port for Thrift to listen for clients on
 rpc_port: 9160
 

diff --git a/titan-hbase/.gitignore b/titan-hbase/.gitignore
@@ -0,0 +1,6 @@
+/titan-hbase-*/.settings/
+/titan-hbase-*/.project
+/titan-hbase-*/.classpath
+/titan-hbase-*/bin/
+/titan-hbase-*/conf/
+/titan-hbase-*/logs/
diff --git a/titan-hbase/conf/.gitignore b/titan-hbase/conf/.gitignore