Skip to content

Commit cdaaf45

Browse files
pwwpchepwwpche
committed
Have to give up, too much to improve :(
1 parent 2f94c7a commit cdaaf45

24 files changed

+980
-187
lines changed

‎CypherCustomVisitor.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,16 +65,15 @@ void executeQuery() {
6565
System.out.println("Executing query...");
6666
QueryExecution execution = new QueryExecution(new DBUtil(conn), bestPlan);
6767
ResultTable table = execution.execute();
68-
System.out.println("Finished. ");
69-
70-
System.out.println("Query content: ");
7168
System.out.println("========================");
7269
System.out.println(table.toString());
7370
System.out.println("========================\n\n\n");
7471
System.out.println("SQL: ");
7572
System.out.println("========================");
7673
System.out.println(String.join("\n", execution.getUsedSQL()));
7774
System.out.println("========================\n\n\n");
75+
76+
7877
}
7978

8079
@Override

‎DataImport/DBHandler.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,10 +78,13 @@ public boolean checkExist(Map<String, Object> item) throws SQLException {
7878
return true;
7979
}
8080

81-
public String getUniqueGidBy(String prop, String value) throws SQLException {
82-
83-
String sql = "SELECT gid FROM P_" + prop + " WHERE VALUE = \"" + value + "\";\n";
84-
return util.getStringFromSQL(sql);
81+
public String getUniqueGidBy(String prop, String value) {
82+
try {
83+
String sql = "SELECT gid FROM P_" + prop + " WHERE VALUE = \"" + value + "\";\n";
84+
return util.getStringFromSQL(sql);
85+
}catch (SQLException e){
86+
return "";
87+
}
8588
}
8689

8790
public List<Integer> getGidBy(String property, String value) throws SQLException {

‎DataImport/FileParser.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ private void getMetadata() throws IOException {
9191
}
9292

9393
// NOTICE:
94-
// type of "rel_type" is STRING!!!
94+
// type of "rel_type" is String!!!
9595
keyType.put("rel_type", "string");
9696

9797
// Relation among columns
@@ -269,7 +269,7 @@ private List<String> constructMetaSchema() {
269269
return result;
270270
}
271271

272-
public void run() throws IOException, SQLException {
272+
public void run(boolean createSchema) throws IOException, SQLException {
273273
System.out.println("Creating tables...");
274274
getMetadata();
275275
MyBufferedReader br = new MyBufferedReader(fileName);
@@ -279,7 +279,7 @@ public void run() throws IOException, SQLException {
279279
// Distinguish node property from edge property.
280280

281281
String line;
282-
if(true) {
282+
if(createSchema) {
283283
// All relations are of type 0.
284284
// Typeid of nodes starts from 1.
285285
int typeId = 1;
@@ -369,7 +369,7 @@ PRIMARY KEY (gid)
369369

370370
for (String key : typeToProperties.keySet()) {
371371
String baseSql = "INSERT INTO typeProperty(id, name) VALUES (\""
372-
+ key.toString() + "\", ";
372+
+ key + "\", ";
373373
for (String prop : typeToProperties.get(key)) {
374374
String propSql = baseSql + "\"" + prop + "\");\n";
375375
sqlSchema.add(propSql);
@@ -436,7 +436,6 @@ PRIMARY KEY (gid)
436436

437437
if (item.containsKey("id")) {
438438
String id = item.get("id").toString();
439-
440439
if(item.containsKey("text")){
441440
// For tweets, use BloomFilter to find if it check.
442441
if(filter.contains(id)) {
@@ -453,6 +452,10 @@ PRIMARY KEY (gid)
453452
item.put("text", value);
454453

455454
String gid = getUniqueGlobalId().toString();
455+
if(gid.equals("763")){
456+
System.out.println(763);
457+
}
458+
filter.add(id);
456459
handler.insertObject(gid, item);
457460
lineGidType.put(type, gid);
458461
newGid.put(type, Integer.valueOf(gid));

‎Main.java‎

Lines changed: 35 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,11 @@
1010
import com.mysql.jdbc.jdbc2.optional.SuspendableXAConnection;
1111
import jdk.nashorn.internal.parser.JSONParser;
1212
import org.antlr.v4.runtime.ANTLRFileStream;
13+
import org.antlr.v4.runtime.ANTLRInputStream;
1314
import org.antlr.v4.runtime.CommonTokenStream;
1415
import org.json.JSONObject;
1516

16-
import java.io.BufferedReader;
17-
import java.io.File;
18-
import java.io.FileReader;
19-
import java.io.IOException;
17+
import java.io.*;
2018

2119

2220
import java.sql.Connection;
@@ -36,40 +34,27 @@ Sampling, paralleling, condensing, overlapping(DP)
3634
public class Main {
3735

3836
public static void main(String[] args) throws IOException, SQLException {
39-
// MyBufferedReader myBufferedReader = new MyBufferedReader("src/large.csv");
40-
// String line;
41-
// int count = 0;
42-
// while((line = myBufferedReader.readLine()) != null){
43-
// if(count % 10000 == 0){
44-
// System.out.println(count);
45-
// }
46-
// count++;
47-
// JSONObject object = new JSONObject(line);
48-
// Map<String, Object> map = JsonParser.jsonToMap(object);
49-
// if(map.containsKey("relationship") && !"".equals(map.get("relationship").toString())){
50-
// System.out.println(line);
51-
// }
52-
// }
5337

5438
//Parsing this CSV file:
55-
String url = "jdbc:mysql://localhost:3306/graphDB";
39+
String url = "jdbc:mysql://localhost:3306/graphDB_updated";
5640
String fileName = "large.csv";
5741

5842
String username = "root";
5943
String password = "";
6044
System.out.println("Connecting to MySQL...");
6145
Connection connection = DriverManager.getConnection(url, username, password);
46+
connection.setAutoCommit(false);
6247
System.out.println("MySQL connected.");
6348

6449

6550
// System.out.println("Parsing file...");
6651
// FileParser fileParser = new FileParser(fileName, connection);
67-
// fileParser.run();
52+
// fileParser.run(false);
6853
// System.out.println("Parsing complete.");
6954

7055

7156
System.out.println("Creating index...");
72-
QueryIndexer queryIndexer = new QueryIndexer(connection);
57+
QueryIndexer queryIndexer = new QueryIndexer(connection, false );
7358
System.out.println("Index created...");
7459

7560
System.out.println("Parsing ANTLR query...");
@@ -81,11 +66,36 @@ public static void main(String[] args) throws IOException, SQLException {
8166
visitor.setConnection(connection);
8267
visitor.visit(cypher);
8368

84-
// MyBufferedReader bufferedReader = new MyBufferedReader("src/small.csv");
85-
// String line = "";
86-
// while ((line = bufferedReader.readLine()) != null){
87-
// System.out.println(line);
69+
70+
// while(true){
71+
// try {
72+
// BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
73+
// StringBuilder query = new StringBuilder();
74+
// System.out.println("Enter Query : ");
75+
// System.out.println("use wq to finish input");
76+
// while (true) {
77+
//
78+
// String input = br.readLine();
79+
// if ("wq".equals(input)) {
80+
// break;
81+
// }
82+
// query.append(input);
83+
// }
84+
// System.out.println("Query : \n" + query.toString());
85+
// CypherLexer lexer = new CypherLexer(new ANTLRInputStream(query.toString()));
86+
// CypherParser parser = new CypherParser(new CommonTokenStream(lexer));
87+
// CypherParser.CypherContext cypher = parser.cypher();
88+
// CypherCustomVisitor visitor = new CypherCustomVisitor();
89+
// visitor.setIndexer(queryIndexer);
90+
// visitor.setConnection(connection);
91+
// visitor.visit(cypher);
92+
//
93+
// } catch (Exception e) {
94+
// e.printStackTrace();
8895
// }
96+
// }
97+
98+
8999
}
90100

91101
}

‎Query/Engine/QueryExecution.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
public class QueryExecution {
1818
private DBUtil dbUtil;
1919
private PlanTree tree;
20+
private List<RangePath> pathMem = new ArrayList<>();
2021

2122
public List<String> getUsedSQL() {
2223
return usedSQL;
@@ -46,9 +47,9 @@ public ResultTable execute(){
4647
}else if(plan instanceof ExpandAllPlan){
4748
executionList.add(new ExpandAllExec(dbUtil, plan));
4849
}else if(plan instanceof RangeExpandAllPlan){
49-
executionList.add(new RangeExpandAllExec(dbUtil, plan));
50+
executionList.add(new RangeExpandAllExec(dbUtil, plan, pathMem));
5051
}else if(plan instanceof RangeExpandIntoPlan){
51-
executionList.add(new RangeExpandIntoExec(dbUtil, plan));
52+
executionList.add(new RangeExpandIntoExec(dbUtil, plan, pathMem));
5253
}else if(plan instanceof NodeHashJoinPlan){
5354
executionList.add(new NodeHashJoinExec(dbUtil, plan));
5455
}else if(plan instanceof CartesianProductPlan){
@@ -58,7 +59,7 @@ public ResultTable execute(){
5859
}else if(plan instanceof FilterRelationEqualityPlan){
5960
executionList.add(new FilterRelationEqualityExec(dbUtil, plan));
6061
}else if(plan instanceof ProduceResultPlan) {
61-
executionList.add(new ProduceResultExec(dbUtil, plan));
62+
executionList.add(new ProduceResultExec(dbUtil, plan, pathMem));
6263
}
6364

6465
}

‎Query/Engine/QueryIndexer.java‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,9 @@
22

33
import DataImport.JsonParser;
44
import Utility.DBUtil;
5-
import org.json.JSONArray;
65
import org.json.JSONObject;
76

87
import java.io.*;
9-
import java.nio.Buffer;
108
import java.sql.Connection;
119
import java.sql.SQLException;
1210
import java.util.*;
@@ -84,12 +82,12 @@ private void dump() throws IOException {
8482
}
8583

8684

87-
public QueryIndexer(Connection conn) throws IOException, SQLException {
85+
public QueryIndexer(Connection conn, boolean rewrite) throws IOException, SQLException {
8886
this.conn = conn;
8987
this.dbUtil = new DBUtil(conn);
9088

9189
File indexerFile = new File("indexer.dat");
92-
if(indexerFile.exists()){
90+
if(indexerFile.exists() && !rewrite){
9391
load();
9492
}else{
9593
// Get number of nodes and relations
@@ -107,11 +105,11 @@ public QueryIndexer(Connection conn) throws IOException, SQLException {
107105

108106
// Get number of nodes with same label
109107
statement = "select label, COUNT(*) from NodeLabel GROUP BY (label);";
110-
labelNodes = dbUtil.getMapFromSQL(statement);
108+
labelNodes = dbUtil.getMapIntFromSQL(statement);
111109

112110
// Get number of relations with same label
113111
statement = "SELECT rel_type, COUNT(*) from Edge GROUP BY rel_type;";
114-
labelRelation = dbUtil.getMapFromSQL(statement);
112+
labelRelation = dbUtil.getMapIntFromSQL(statement);
115113

116114
// Get number of distinct values of each property in nodes.
117115
statement = "SELECT DISTINCT(name) FROM typeProperty WHERE id > 0";
@@ -126,11 +124,11 @@ public QueryIndexer(Connection conn) throws IOException, SQLException {
126124

127125
// Get number of edges that comes out of nodes with same label.
128126
statement = "SELECT label, COUNT(DISTINCT eid) from (Edge e LEFT JOIN NodeLabel n ON e.node1 = n.gid) GROUP BY (label);";
129-
nodeLabelOutgoing = dbUtil.getMapFromSQL(statement);
127+
nodeLabelOutgoing = dbUtil.getMapIntFromSQL(statement);
130128

131129
// Get number of edges that goes into nodes with same label.
132130
statement = "SELECT label, COUNT(DISTINCT eid) from (Edge e LEFT JOIN NodeLabel n ON e.node2 = n.gid) GROUP BY (label);";
133-
nodeLabelIncoming = dbUtil.getMapFromSQL(statement);
131+
nodeLabelIncoming = dbUtil.getMapIntFromSQL(statement);
134132

135133

136134
for(String label : labelNodes.keySet()){

‎Query/Execution/AllNodeScanExec.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public AllNodeScanExec(DBUtil util, Plan plan) {
1717
public ResultTable execute() {
1818
this.exeUtil.startRecording();
1919
ResultTable table = new ResultTable();
20-
table.putAll(plan.getVariable(), ResultTable.ObjectType.NODE, exeUtil.getAllNodes());
20+
table.putAll(plan.getVariable(), ResultTable.ObjectType.Node_ID, exeUtil.getAllNodes());
2121
this.querySQL = this.exeUtil.getHistory();
2222
return table;
2323
}

‎Query/Execution/Execution.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ public class Execution {
1818

1919
protected List<String> querySQL = new ArrayList<>();
2020

21+
2122
public Execution(DBUtil util, Plan plan){
2223
this.exeUtil = new ExecutionUtility(util);
2324
this.plan = plan;

‎Query/Execution/ExecutionUtility.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,25 @@ public String getPropertyByGid(String field, String gid) {
248248
}
249249
}
250250

251+
public List<String> getPropertyByGids(String field, List<String> gids) {
252+
if(gids.size() < 500){
253+
List<String> result = new ArrayList<>();
254+
gids.forEach(gid -> result.add(getPropertyByGid(field, gid)));
255+
return result;
256+
}
257+
258+
try {
259+
String statement = "SELECT * FROM P_" + field + ";";
260+
Map<String, String> propMap = dbUtil.getMapStringFromSQL(statement);
261+
List<String> result = new ArrayList<>(gids.size());
262+
gids.forEach(gid -> result.add(propMap.get(gid)));
263+
return result;
264+
} catch (SQLException e) {
265+
e.printStackTrace();
266+
return null;
267+
}
268+
}
269+
251270
public Map<String, String> expandObject(String gid) {
252271
try {
253272
String statement = "SELECT type FROM ObjectType WHERE gid = \"" + gid + "\";";

‎Query/Execution/ExpandAllExec.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,10 @@
11
package Query.Execution;
22

3-
import Entity.Constraint;
43
import Entity.QueryConstraints;
54
import Query.Entities.RelationEdge;
65
import Query.Plan.ExpandAllPlan;
7-
import Query.Plan.ExpandIntoPlan;
86
import Query.Plan.Plan;
97
import Utility.DBUtil;
10-
import com.sun.org.apache.regexp.internal.RE;
118

129
import java.util.*;
1310

@@ -99,7 +96,7 @@ public ResultTable execute(ResultTable table) {
9996
}
10097
}
10198
retItems = new ArrayList<>(Arrays.asList(expandedNode, edge.name));
102-
List<ResultTable.ObjectType> retTypes = new ArrayList<>(Arrays.asList(ResultTable.ObjectType.NODE, ResultTable.ObjectType.RELATIONSHIP));
99+
List<ResultTable.ObjectType> retTypes = new ArrayList<>(Arrays.asList(ResultTable.ObjectType.Node_ID, ResultTable.ObjectType.Edge_ID));
103100
table.expand(insideNode, retItems, retTypes, outingEdges);
104101

105102
this.querySQL = exeUtil.getHistory();

0 commit comments

Comments
 (0)