Merge pull request #188 from ldbc/sf1-test

Test SF1 in CI

Author: szarnyasg

.circleci/config.yml

@@ -9,7 +9,7 @@ workflows:
 
 jobs:
   test:
-    resource_class: large
+    resource_class: xlarge
     machine:
       image: ubuntu-2004:202008-01
     steps:
@@ -29,21 +29,21 @@ jobs:
           name: Build project
           command: ./build.sh
       - run:
-          name: Load Cypher/Neo4j database (1st time)
+          name: Tiny data set - Load Cypher/Neo4j database (1st time)
           command: |
             cd cypher
             . scripts/environment-variables-default.sh
             scripts/load-in-one-step.sh
             cd ..
       - run:
-          name: Create validation parameters with Neo4j/Cypher project
+          name: Tiny data set - Create validation parameters with Neo4j/Cypher project
           command: |
             cd cypher
             driver/create-validation-parameters.sh
-            mv validation_params.csv validation_params_cypher.csv
+            mv validation_params.csv validation_params_cypher_tiny.csv
             cd ..
       - run:
-          name: Load PostgreSQL database (1st time)
+          name: Tiny data set - Load PostgreSQL database (1st time)
           command: |
             cd postgres
             export POSTGRES_CSV_DIR=`pwd`/test-data/
@@ -52,29 +52,29 @@ jobs:
             scripts/load.sh
             cd ..
       - run:
-          name: Create validation parameters with the PostgreSQL project
+          name: Tiny data set - Create validation parameters with the PostgreSQL project
           command: |
             cd postgres
             driver/create-validation-parameters.sh
-            mv validation_params.csv validation_params_postgres.csv
+            mv validation_params.csv validation_params_postgres_tiny.csv
             cd ..
       - run:
-          name: Load Cypher/Neo4j database (2nd time)
+          name: Tiny data set - Load Cypher/Neo4j database (2nd time)
           command: |
             cd cypher
             . scripts/environment-variables-default.sh
             scripts/load-in-one-step.sh
             cd ..
       - run:
-          name: Cross-validate the Neo4j/Cypher project based on the results from Postgres
+          name: Tiny data set - Cross-validate the Neo4j/Cypher project based on the results from Postgres
           command: |
-            mv postgres/validation_params_postgres.csv cypher/validation_params.csv
+            mv postgres/validation_params_postgres_tiny.csv cypher/validation_params.csv
             cd cypher
             driver/validate.sh | tee validation-log.txt
             grep 'Validation Result: PASS' validation-log.txt
             cd ..
       - run:
-          name: Load PostgreSQL database (2nd time)
+          name: Tiny data set - Load PostgreSQL database (2nd time)
           command: |
             cd postgres
             export POSTGRES_CSV_DIR=`pwd`/test-data/
@@ -83,9 +83,49 @@ jobs:
             scripts/load.sh
             cd ..
       - run:
-          name: Cross-validate the Postgres project based on the results from Neo4j/Cypher
+          name: Tiny data set - Cross-validate the Postgres project based on the results from Neo4j/Cypher
           command: |
-            mv cypher/validation_params_cypher.csv postgres/validation_params.csv
+            mv cypher/validation_params_cypher_tiny.csv postgres/validation_params.csv
+            cd postgres
+            driver/validate.sh | tee validation-log.txt
+            grep 'Validation Result: PASS' validation-log.txt
+            cd ..
+      - run:
+          name: Download SF1 data set
+          command: |
+            wget https://surfdrive.surf.nl/files/index.php/s/Ar7kacLw3LVzQaf/download --quiet --output-document=sf1.zip
+            unzip sf1.zip
+            # substitution parameters and updateStreams for Neo4j/Cypher
+            cp ./sf1/substitution_parameters/* ./cypher/test-data/substitution_parameters/
+            cp ./sf1/social_network-csv_composite/updateStream* ./cypher/test-data/update_streams/
+      - run:
+          name: SF1 data set - Load Cypher/Neo4j database
+          command: |
+            cd cypher
+            . scripts/environment-variables-default.sh
+            export NEO4J_VANILLA_CSV_DIR=`pwd`/../sf1/social_network-csv_composite
+            scripts/load-in-one-step.sh
+            cd ..
+      - run:
+          name: SF1 data set - Create validation parameters with the Neo4j/Cypher
+          command: |
+            cd cypher
+            driver/create-validation-parameters.sh
+            mv validation_params.csv validation_params_cypher_sf1.csv
+            cd ..
+      - run:
+          name: SF1 data set - Load PostgreSQL database
+          command: |
+            cd postgres
+            export POSTGRES_CSV_DIR=`pwd`/../sf1/social_network-csv_merge_foreign
+            scripts/start.sh
+            scripts/create-db.sh
+            scripts/load.sh
+            cd ..
+      - run:
+          name: SF1 data set - Cross-validate the Postgres project based on the results from Neo4j/Cypher
+          command: |
+            mv cypher/validation_params_cypher_sf1.csv postgres/validation_params.csv
             cd postgres
             driver/validate.sh | tee validation-log.txt
             grep 'Validation Result: PASS' validation-log.txt

common/src/main/java/com/ldbc/impls/workloads/ldbc/snb/converter/Converter.java

@@ -12,6 +12,7 @@
 public class Converter {
 
     final static String DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'+0000'";
+    final static String DATE_FORMAT = "yyyy-MM-dd";
 
     public String convertDateTime(Date date) {
         final SimpleDateFormat sdf = new SimpleDateFormat(DATETIME_FORMAT);
@@ -20,7 +21,9 @@ public String convertDateTime(Date date) {
     }
 
     public String convertDate(Date date) {
-        return convertDateTime(date);
+        final SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT);
+        sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+        return "'" + sdf.format(date) + "'";
     }
 
     /**
@@ -85,10 +88,6 @@ public String convertOrganisations(List<LdbcUpdate1AddPerson.Organization> value
         return res;
     }
 
-    public String convertBlacklist(List<String> words) {
-        return convertStringList(words);
-    }
-
     /**
      * Some implementations, e.g. the (deprecated) SPARQL one, will not work with a simple toString and require some tinkering,
      * e.g. padding the id with '0' characters.

cypher/driver/benchmark.properties

@@ -2,6 +2,7 @@ endpoint=bolt://localhost:7687
 user=neo4j
 password=admin
 queryDir=queries/
+
 printQueryNames=false
 printQueryStrings=false
 printQueryResults=false

cypher/driver/create-validation-parameters-sf10.properties

@@ -2,8 +2,9 @@ endpoint=bolt://localhost:7687
 user=neo4j
 password=admin
 queryDir=queries/
+
 printQueryNames=true
-printQueryStrings=true
+printQueryStrings=false
 printQueryResults=false
 
 status=1

cypher/driver/create-validation-parameters.properties

@@ -2,6 +2,7 @@ endpoint=bolt://localhost:7687
 user=neo4j
 password=admin
 queryDir=queries/
+
 printQueryNames=true
 printQueryStrings=false
 printQueryResults=false

cypher/queries/interactive-update-1.cypher

@@ -4,7 +4,7 @@ CREATE (p:Person {
     firstName: $personFirstName,
     lastName: $personLastName,
     gender: $gender,
-    birthday: datetime($birthday),
+    birthday: date($birthday),
     creationDate: datetime($creationDate),
     locationIP: $locationIP,
     browserUsed: $browserUsed,

postgres/driver/benchmark.properties

@@ -4,6 +4,7 @@ password=mysecretpassword
 databaseName=ldbcsnb
 jdbcDriver=org.postgresql.ds.PGPoolingDataSource
 queryDir=queries
+
 printQueryNames=false
 printQueryStrings=false
 printQueryResults=false

postgres/driver/create-validation-parameters.properties

@@ -4,6 +4,7 @@ password=mysecretpassword
 databaseName=ldbcsnb
 jdbcDriver=org.postgresql.ds.PGPoolingDataSource
 queryDir=queries
+
 printQueryNames=true
 printQueryStrings=false
 printQueryResults=false

postgres/src/main/java/com/ldbc/impls/workloads/ldbc/snb/postgres/converter/PostgresConverter.java

@@ -23,6 +23,10 @@ public String convertDateTime(Date date) {
         return "'" + sdf.format(date) + "'::timestamp";
     }
 
+    public String convertDate(Date date) {
+        return super.convertDate(date) + "::date";
+    }
+
     @Override
     public String convertString(String value) {
         return "'" + value.replace("'", "''") + "'";