grounding handler, pipeline completed

.gitignore

@@ -10,6 +10,7 @@ mappings/
 datasets/
 linksets/
 log/
+logs/
 publications-*/
 old/
 tmp/

mandolin.properties

@@ -1,5 +1,5 @@
 # GENERAL CONFIGURATION FOR MANDOLIN
-pgsql_home=/usr/local/Cellar/postgresql/9.4.1/
+pgsql_home=/usr/local/Cellar/postgresql/9.4.1
 pgsql_username=tom
 pgsql_password=
 pgsql_url=localhost

pgsql/sql/load-body.sql

@@ -0,0 +1,18 @@
+
+-- build relationships table with type information
+CREATE TABLE probkb.relationships AS
+SELECT nextval('probkb.relids') AS id, r.rel AS rel,
+       r.ent1 AS ent1, rc.class1 AS class1,
+       r.ent2 AS ent2, rc.class2 AS class2, AVG(weight) AS weight
+FROM probkb.extractions r, probkb.relClasses rc, probkb.entClasses ec1, probkb.entClasses ec2
+WHERE r.rel = rc.rel
+AND r.ent1 = ec1.ent AND ec1.class = rc.class1
+AND r.ent2 = ec2.ent AND ec2.class = rc.class2
+GROUP BY r.rel, r.ent1, rc.class1, r.ent2, rc.class2;
+CREATE INDEX relationships_rel_idx ON probkb.relationships(rel);
+CLUSTER probkb.relationships USING relationships_rel_idx;
+
+DELETE FROM probkb.relationships WHERE ent1 = ent2;
+
+SELECT probkb.qc();
+

pgsql/sql/load-head.sql

@@ -0,0 +1,11 @@
+SET work_mem='4GB';
+SET enable_mergejoin=OFF;
+
+-- generate random types for 0 typed entities
+--INSERT INTO entClasses
+--SELECT tt.ent, trunc(random()*156) AS class
+--FROM (SELECT id AS ent FROM entities
+--        EXCEPT
+--      SELECT ent FROM entClasses) tt;
+
+-- import csv

pgsql/sql/load-tail.sql

@@ -0,0 +1,8 @@
+
+ANALYZE probkb.relationships;  -- gather statistics for better query plan
+ANALYZE probkb.mln1;
+ANALYZE probkb.mln2;
+ANALYZE probkb.mln3;
+ANALYZE probkb.mln4;
+ANALYZE probkb.mln5;
+ANALYZE probkb.mln6;

pgsql/sql/run.sql

@@ -0,0 +1,4 @@
+-- Run grounding phase...
+
+SELECT probkb.ground();
+SELECT probkb.groundFactors();

src/org/aksw/mandolin/MandolinProbKB.java

@@ -4,8 +4,8 @@
 
 import org.aksw.mandolin.amie.RDFToTSV;
 import org.aksw.mandolin.amie.RuleMiner;
+import org.aksw.mandolin.grounding.Grounding;
 import org.aksw.mandolin.inference.ProbKBToRockitGibbsSampling;
-import org.aksw.mandolin.inference.RockitGroundingAndGibbsSampling;
 import org.aksw.mandolin.model.PredictionLiteral;
 import org.aksw.mandolin.model.PredictionSet;
 
@@ -84,7 +84,7 @@ private void run() throws Exception {
 		RDFToTSV.run(map, BASE, TEMP_OUTPUT);
 		RuleMiner.run(map, BASE, TEMP_OUTPUT);
 
-		// TODO update SQL file dynamically and launch grounding from PGSQL console
+		Grounding.ground(BASE);
 
 		PredictionSet pset = new ProbKBToRockitGibbsSampling(map).infer();
 //		PredictionSet pset = new RockitGroundingAndGibbsSampling(AIM_RELATION, "eval/11_publi-mln/prog.mln",

src/org/aksw/mandolin/eval/PostgreDB.java

@@ -6,6 +6,7 @@
 import java.sql.SQLException;
 import java.sql.Statement;
 
+import org.aksw.mandolin.util.Bundle;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -25,9 +26,11 @@ public PostgreDB() {
 
 	public void connect() {
 
-		String url = "jdbc:postgresql://localhost/probkb";
-		String user = "tom";
-		String password = "";
+		String host = Bundle.getString("pgsql_url");
+		String db = Bundle.getString("pgsql_database");
+		String url = "jdbc:postgresql://"+host+"/"+db;
+		String user = Bundle.getString("pgsql_username");
+		String password = Bundle.getString("pgsql_password");
 
 		try {
 			con = DriverManager.getConnection(url, user, password);

src/org/aksw/mandolin/grounding/Grounding.java

@@ -0,0 +1,114 @@
+package org.aksw.mandolin.grounding;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.PrintWriter;
+import java.util.Scanner;
+
+import org.aksw.mandolin.util.Bundle;
+import org.aksw.mandolin.util.Shell;
+
+/**
+ * @author Tommaso Soru <tsoru@informatik.uni-leipzig.de>
+ *
+ */
+public class Grounding {
+
+	public static void ground(String base) throws FileNotFoundException {
+		// prepare SQL files
+		prepare(base);
+		// generate tables and procedures
+		generate(base);
+		// run scripts for grounding
+		run();
+	}
+
+	private static void run() {
+		String[] cmd = {
+		// Drop schema
+		Bundle.getString("pgsql_home") + "/bin/psql probkb -f "
+				+ System.getProperty("user.dir") + "/pgsql/sql/run.sql", };
+		for (String c : cmd) {
+			System.out.println("> " + c);
+			Shell.execute(c, true);
+		}
+	}
+
+	private static void generate(String base) {
+		String[] cmd = {
+				// Drop schema
+				Bundle.getString("pgsql_home") + "/bin/psql probkb -f "
+						+ System.getProperty("user.dir")
+						+ "/pgsql/sql/drop.sql",
+				// Create db
+				Bundle.getString("pgsql_home") + "/bin/createdb probkb",
+				// Create the probkb schema and tables.
+				Bundle.getString("pgsql_home") + "/bin/psql probkb -f "
+						+ System.getProperty("user.dir")
+						+ "/pgsql/sql/create.sql",
+				// Create quality control procedures.
+				Bundle.getString("pgsql_home") + "/bin/psql probkb -f "
+						+ System.getProperty("user.dir") + "/pgsql/sql/qc.sql",
+				// Load the files in CSV format.
+				Bundle.getString("pgsql_home") + "/bin/psql probkb -f "
+						+ System.getProperty("user.dir") + "/" + base
+						+ "/load.sql",
+				// Create grounding procedures.
+				Bundle.getString("pgsql_home") + "/bin/psql probkb -f "
+						+ System.getProperty("user.dir")
+						+ "/pgsql/sql/ground.sql" };
+		for (String c : cmd) {
+			System.out.println("> " + c);
+			Shell.execute(c, true);
+		}
+	}
+
+	private static void prepare(String base) throws FileNotFoundException {
+
+		PrintWriter load = new PrintWriter(new File(base + "/load.sql"));
+
+		// write head
+		write("pgsql/sql/load-head.sql", load);
+
+		// write graph tables
+		String[] tables = { "classes", "entities", "relations", "entClasses",
+				"relClasses", "functionals", "extractions", };
+		// due to a stylistic choice from ProbKB, table `extractions`
+		// corresponds to file `relationships.csv`
+		String[] csv = { "classes", "entities", "relations", "entClasses",
+				"relClasses", "functionals", "relationships", };
+		for (int i = 0; i < tables.length; i++)
+			load.write("COPY probkb." + tables[i] + " FROM '"
+					+ System.getProperty("user.dir") + "/" + base + "/"
+					+ csv[i] + ".csv' DELIMITERS ',' CSV;\n");
+
+		// write body
+		write("pgsql/sql/load-body.sql", load);
+
+		// write MLN tables
+		for (int i = 1; i <= 6; i++)
+			load.write("COPY probkb.mln" + i + " FROM '"
+					+ System.getProperty("user.dir") + "/" + base + "/mln" + i
+					+ ".csv' DELIMITERS ',' CSV;\n");
+
+		// write tail
+		write("pgsql/sql/load-tail.sql", load);
+		load.close();
+
+	}
+
+	private static void write(String filename, PrintWriter pw)
+			throws FileNotFoundException {
+		Scanner in = new Scanner(new File(filename));
+		while (in.hasNextLine())
+			pw.write(in.nextLine() + "\n");
+		in.close();
+	}
+
+	public static void main(String[] args) throws FileNotFoundException {
+
+		ground("eval/0001");
+
+	}
+
+}

src/org/aksw/mandolin/util/Shell.java

@@ -50,21 +50,7 @@ public static String execute(String command, boolean show) {
 	 * @return
 	 */
 	public static String execute(String command) {
-		StringBuffer sb = new StringBuffer();
-		Process p;
-		try {
-			p = Runtime.getRuntime().exec(command);
-
-			BufferedReader reader = new BufferedReader(new InputStreamReader(
-					p.getInputStream()));
-
-			String line = "";
-			while ((line = reader.readLine()) != null)
-				sb.append(line);
-		} catch (Exception e) {
-			e.printStackTrace();
-		}
-		return sb.toString();
+		return execute(command, false);
 	}
 
 	/**