Sync project with origin ldbc_snb_interactive main

.circleci/config.yml

@@ -9,105 +9,121 @@ workflows:
 
 jobs:
   test:
-    resource_class: xlarge
+    resource_class: large
     machine:
       image: ubuntu-2004:202008-01
     steps:
       - checkout
       - run:
-          name: Setup
+          name: Install dependencies
           command: |
             export DEBIAN_FRONTEND=noninteractive
-            sudo apt update
-            # install dependencies
-            sudo apt install -y git
-            # clone, build, and install driver
-            git clone --depth 1 --branch 0.3.5 https://github.com/ldbc/ldbc_snb_driver && cd ldbc_snb_driver && mvn install -DskipTests && cd ..
-            # PostgreSQL
-            pip3 install --user --progress-bar off psycopg2-binary
+            sudo apt update && sudo apt install -y zstd
+      ### DuckDB
+      - run:
+          name: Install DuckDB binary
+          command: |
+            duckdb/scripts/get.sh
       - run:
           name: Build project
           command: ./build.sh
       - run:
-          name: Tiny data set - Load Cypher/Neo4j database (1st time)
+          name: Tiny data set - Load DuckDB
+          command: |
+            cd duckdb
+            export DUCKDB_CSV_DIR=`pwd`/../postgres/test-data
+            scripts/load-in-one-step.sh
+            cd ..
+      ### Neo4j
+      - run:
+          name: Tiny data set - Load Neo4j database and create a backup
           command: |
             cd cypher
-            . scripts/environment-variables-default.sh
+            export NEO4J_VANILLA_CSV_DIR=`pwd`/test-data/vanilla/
+            export NEO4J_CONVERTED_CSV_DIR=`pwd`/test-data/converted/
             scripts/load-in-one-step.sh
+            scripts/backup-database.sh
             cd ..
       - run:
-          name: Tiny data set - Create validation parameters with Neo4j/Cypher project
+          name: Tiny data set - Create validation parameters with Neo4j project
           command: |
             cd cypher
             driver/create-validation-parameters.sh
             mv validation_params.csv validation_params_cypher_tiny.csv
             cd ..
+      ### PostgreSQL
       - run:
-          name: Tiny data set - Load PostgreSQL database (1st time)
+          name: Install PostgreSQL dependencies
+          command: |
+            postgres/scripts/install-dependencies.sh
+      - run:
+          name: Tiny data set - Load PostgreSQL database and create a backup
           command: |
             cd postgres
             export POSTGRES_CSV_DIR=`pwd`/test-data/
-            scripts/start.sh
-            scripts/create-db.sh
-            scripts/load.sh
+            scripts/load-in-one-step.sh
+            scripts/backup-database.sh
             cd ..
       - run:
-          name: Tiny data set - Create validation parameters with the PostgreSQL project
+          name: Tiny data set - Create validation parameters with PostgreSQL
           command: |
             cd postgres
             driver/create-validation-parameters.sh
             mv validation_params.csv validation_params_postgres_tiny.csv
             cd ..
+      ### Neo4j
       - run:
-          name: Tiny data set - Load Cypher/Neo4j database (2nd time)
-          command: |
-            cd cypher
-            . scripts/environment-variables-default.sh
-            scripts/load-in-one-step.sh
-            cd ..
-      - run:
-          name: Tiny data set - Cross-validate the Neo4j/Cypher project based on the results from Postgres
+          name: Tiny data set - Cross-validate the Neo4j project based on the results from PostgreSQL
           command: |
             mv postgres/validation_params_postgres_tiny.csv cypher/validation_params.csv
             cd cypher
+            scripts/restore-database.sh
             driver/validate.sh | tee validation-log.txt
             grep 'Validation Result: PASS' validation-log.txt
             cd ..
       - run:
-          name: Tiny data set - Load PostgreSQL database (2nd time)
+          name: Tiny data set - Benchmark Neo4j
           command: |
-            cd postgres
-            export POSTGRES_CSV_DIR=`pwd`/test-data/
-            scripts/start.sh
-            scripts/create-db.sh
-            scripts/load.sh
+            cd cypher
+            scripts/restore-database.sh
+            driver/benchmark.sh
             cd ..
+      ### PostgreSQL
       - run:
-          name: Tiny data set - Cross-validate the Postgres project based on the results from Neo4j/Cypher
+          name: Tiny data set - Cross-validate the PostgreSQL project based on the results from Neo4j
           command: |
             mv cypher/validation_params_cypher_tiny.csv postgres/validation_params.csv
             cd postgres
+            scripts/restore-database.sh
             driver/validate.sh | tee validation-log.txt
             grep 'Validation Result: PASS' validation-log.txt
             cd ..
+      - run:
+          name: Tiny data set - Benchmark PostgreSQL
+          command: |
+            cd postgres
+            scripts/restore-database.sh
+            driver/benchmark.sh
+            cd ..
+      ## SF1 tests
       - run:
           name: Download SF1 data set
           command: |
-            wget https://surfdrive.surf.nl/files/index.php/s/Ar7kacLw3LVzQaf/download --quiet --output-document=sf1.zip
-            unzip sf1.zip
-            # substitution parameters and updateStreams for Neo4j/Cypher
+            curl -L https://surfdrive.surf.nl/files/index.php/s/Ng2eCg6m06ZNKKo/download | tar -xv --use-compress-program=unzstd
+            # substitution parameters and updateStreams for Neo4j
             cp ./sf1/substitution_parameters/* ./cypher/test-data/substitution_parameters/
-            cp ./sf1/social_network-csv_composite/updateStream* ./cypher/test-data/update_streams/
+            cp ./sf1/social_network-csv_composite-longdateformatter/updateStream* ./cypher/test-data/update_streams/
       - run:
-          name: SF1 data set - Load Cypher/Neo4j database
+          name: SF1 data set - Load Neo4j database
           command: |
             cd cypher
-            . scripts/environment-variables-default.sh
-            export NEO4J_VANILLA_CSV_DIR=`pwd`/../sf1/social_network-csv_composite
+            export NEO4J_VANILLA_CSV_DIR=`pwd`/../sf1/social_network-csv_composite-longdateformatter
+            export NEO4J_CONVERTED_CSV_DIR=`pwd`/../sf1/social_network-csv_composite-converted
+            mkdir ${NEO4J_CONVERTED_CSV_DIR}
             scripts/load-in-one-step.sh
             cd ..
       - run:
-          name: SF1 data set - Create validation parameters with the Neo4j/Cypher
+          name: SF1 data set - Create validation parameters with Neo4j
           command: |
             cd cypher
             driver/create-validation-parameters.sh
@@ -117,13 +133,12 @@ jobs:
           name: SF1 data set - Load PostgreSQL database
           command: |
             cd postgres
+            export POSTGRES_CUSTOM_CONFIGURATION=`pwd`/config/postgresql.conf
             export POSTGRES_CSV_DIR=`pwd`/../sf1/social_network-csv_merge_foreign
-            scripts/start.sh
-            scripts/create-db.sh
-            scripts/load.sh
+            scripts/load-in-one-step.sh
             cd ..
       - run:
-          name: SF1 data set - Cross-validate the Postgres project based on the results from Neo4j/Cypher
+          name: SF1 data set - Cross-validate the PostgreSQL project based on the results from Neo4j
           command: |
             mv cypher/validation_params_cypher_sf1.csv postgres/validation_params.csv
             cd postgres

.gitignore

@@ -6,9 +6,14 @@ release.properties
 dependency-reduced-pom.xml
 .idea/
 *.iml
-validation_params.csv
+validation_params*.csv
 *.swp
 *-actual.json
 *-expected.json
 results/
 *.DS_Store
+sf1/
+sf1.zip
+tcr.log
+*.priv
+*.priv.*

NOTICE.txt

@@ -1,4 +1,4 @@
- Copyright [2020-]2021 Linked Data Benchmark Council
+ Copyright [2020-]2022 Linked Data Benchmark Council
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.