Merge pull request #3 from EmergentOrder/master

Add support for BIDMach & VW LR + SPPMI feature vectorization

Author: EmergentOrder

build.sbt

@@ -3,9 +3,20 @@ name := "org.template.textclassification"
 
 organization := "io.prediction"
 
+scalaVersion := "2.10.5"
+
 libraryDependencies ++= Seq(
-  "io.prediction"    %% "core"        % pioVersion.value % "provided",
-  "org.apache.spark" %% "spark-core" % "1.3.1" % "provided",
-  "org.apache.spark" %% "spark-mllib" % "1.3.1" % "provided",
+  "io.prediction"    % "core_2.10"        % pioVersion.value % "provided",
+  "org.apache.spark" %% "spark-core" % "1.4.1" % "provided",
+  "org.apache.spark" %% "spark-mllib" % "1.4.1" % "provided",
+  "com.github.fommil.netlib" % "all" % "1.1.2" pomOnly(),
+  "com.github.johnlangford" % "vw-jni" % "8.0.0",
   "org.xerial.snappy" % "snappy-java" % "1.1.1.7"
 )
+
+mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) =>
+  {
+    case y if y.startsWith("doc")     => MergeStrategy.discard
+    case x => old(x)
+  }
+}

data/Twitter140sample.txt

@@ -0,0 +1,46 @@
+"""
+Import sample data for classification engine
+"""
+
+import predictionio
+import argparse
+
+def import_events(client, file):
+  f = open(file, 'r')
+  count = 0
+  print "Importing data..."
+  for line in f:
+    data = line.rstrip('\r\n').split(",")
+    plan = data[0] 
+    #Not strictly CSV, after the first comma, no longer delimiting
+    text = ",".join(data[1:])
+    client.create_event(
+      event="$set",
+      entity_type="user",
+      entity_id=str(count), # use the count num as user ID
+      properties= {
+        "text" : text,
+        "category" : plan,
+        "label" : int(plan)
+      }
+    )
+    count += 1
+  f.close()
+  print "%s events are imported." % count
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(
+    description="Import sample data for classification engine")
+  parser.add_argument('--access_key', default='invald_access_key')
+  parser.add_argument('--url', default="http://localhost:7070")
+  parser.add_argument('--file', default="./data/Twitter140sample.txt")
+
+  args = parser.parse_args()
+  print args
+
+  client = predictionio.EventClient(
+    access_key=args.access_key,
+    url=args.url,
+    threads=5,
+    qsize=500)
+  import_events(client, args.file)

data/import_eventserver.py

@@ -9,15 +9,22 @@
   },
   "preparator": {
     "params": {
-      "nGram": 2,
-      "numFeatures": 15000
+      "nGram": 1,
+      "numFeatures": 500,
+      "SPPMI": false
     }
   },
   "algorithms": [
     {
-      "name": "nb",
+      "name": "lr",
       "params": {
-        "lambda": 0.25
+         "maxIter": 1,
+        "regParam": 0.00000005,
+        "stepSize": 5.0,
+        "bitPrecision": 22,
+        "modelName": "model.vw",
+        "namespace": "n",
+        "ngram": 1 
       }
     }
   ]

engine.json

@@ -0,0 +1,7 @@
+public class getnativepath {
+    public static void main(String [] args) 
+    {
+        String v = System.getProperty("java.library.path");
+        System.out.print(v);
+    }
+}