Add deletions

.circleci/config.yml

@@ -9,7 +9,7 @@ workflows:
 
 jobs:
   test:
-    resource_class: large
+    resource_class: xlarge
     machine:
       image: ubuntu-2004:202008-01
     steps:
@@ -20,16 +20,54 @@ jobs:
             export DEBIAN_FRONTEND=noninteractive
             sudo apt update && sudo apt install -y zstd
             scripts/install-dependencies.sh
+            paramgen/scripts/install-dependencies.sh
       - run:
           name: Build project
           command: scripts/build.sh
+      - run:
+          name: Download small data set for Cypher
+          command: |
+            # Cypher
+            cd cypher
+            scripts/get-sample-data-set.sh
+            cd ..
+      - run:
+          name: Download small data set for Postgres
+          command: |
+            # Postgres
+            cd postgres
+            scripts/get-sample-data-set.sh
+            cd ..
+      - run:
+          name: Download small data set for updates
+          command: |
+            # regular CSVs for updates
+            wget -q https://ldbcouncil.org/ldbc_snb_datagen_spark/social-network-sf0.003-bi-composite-merged-fk.zip
+            unzip -q social-network-sf0.003-bi-composite-merged-fk.zip
+      - run:
+          name: Convert update streams
+          command: |
+            git clone https://github.com/ldbc/ldbc_snb_interactive_driver
+            cd ldbc_snb_interactive_driver/scripts/
+            ./install-dependencies-for-convert-script.sh
+            export LDBC_SNB_DATA_ROOT_DIRECTORY=../../social-network-sf0.003-bi-composite-merged-fk/
+            ./convert.sh
+            mv inserts ../../update-streams/
+            mv deletes ../../update-streams/
+            cd ../..
+      - run:
+          name: Generate parameters
+          command: |
+            cd paramgen
+            scripts/get-sample-factors.sh
+            scripts/paramgen.sh
+            cd ..
       ### Neo4j
       - run:
           name: Tiny data set - Load Neo4j database and create a backup
           command: |
             cd cypher
-            export NEO4J_VANILLA_CSV_DIR=`pwd`/test-data/vanilla/
-            export NEO4J_CONVERTED_CSV_DIR=`pwd`/test-data/converted/
+            . scripts/use-sample-data-set.sh
             scripts/load-in-one-step.sh
             scripts/backup-database.sh
             cd ..
@@ -38,14 +76,14 @@ jobs:
           command: |
             cd cypher
             driver/create-validation-parameters.sh
-            cp validation_params.csv validation_params_cypher_tiny.csv
+            cp validation_params.json validation_params_cypher_tiny.json
             cd ..
       ### PostgreSQL
       - run:
           name: Tiny data set - Load PostgreSQL database and create a backup
           command: |
             cd postgres
-            export POSTGRES_CSV_DIR=`pwd`/test-data/
+            . scripts/use-sample-data-set.sh
             scripts/load-in-one-step.sh
             scripts/backup-database.sh
             cd ..
@@ -55,13 +93,13 @@ jobs:
             cd postgres
             driver/create-validation-parameters.sh
             scripts/stop.sh
-            cp validation_params.csv validation_params_postgres_tiny.csv
+            cp validation_params.json validation_params_postgres_tiny.json
             cd ..
       ### Neo4j
       - run:
           name: Tiny data set - Cross-validate the Neo4j project based on the results from PostgreSQL
           command: |
-            cp postgres/validation_params_postgres_tiny.csv cypher/validation_params.csv
+            cp postgres/validation_params_postgres_tiny.json cypher/validation_params.json
             cd cypher
             scripts/restore-database.sh
             driver/validate.sh | tee validation-log.txt
@@ -74,24 +112,11 @@ jobs:
             scripts/restore-database.sh
             driver/benchmark.sh
             cd ..
-      ### DuckDB
-      - run:
-          name: Install DuckDB binary
-          command: |
-            duckdb/scripts/get.sh
-      - run:
-          name: Tiny data set - Load DuckDB database
-          command: |
-            cd duckdb
-            export DUCKDB_CSV_DIR=`pwd`/../postgres/test-data/
-            scripts/load-in-one-step.sh
-            scripts/backup-database.sh
-            cd ..
       ### PostgreSQL
       - run:
           name: Tiny data set - Cross-validate the PostgreSQL project based on the results from Neo4j
           command: |
-            cp cypher/validation_params_cypher_tiny.csv postgres/validation_params.csv
+            cp cypher/validation_params_cypher_tiny.json postgres/validation_params.json
             cd postgres
             scripts/restore-database.sh
             driver/validate.sh | tee validation-log.txt
@@ -105,116 +130,104 @@ jobs:
             driver/benchmark.sh
             scripts/stop.sh
             cd ..
-      ### Umbra
-      - run:
-          name: Load Umbra Docker image
-          command: |
-            . umbra/scripts/vars.sh
-            # ${UMBRA_URL_PREFIX} should be set as a sercet variable
-            curl -s ${UMBRA_URL_PREFIX}${UMBRA_VERSION}.tar.gz | docker load
-      - run:
-          name: Tiny data set - Load Umbra database and create a backup
-          command: |
-            cd umbra
-            export UMBRA_CSV_DIR=`pwd`/../postgres/test-data/
-            scripts/load-in-one-step.sh
-            scripts/backup-database.sh
-            cd ..
-      - run:
-          name: Tiny data set - Create validation parameters with Umbra
-          command: |
-            cd umbra
-            driver/create-validation-parameters.sh
-            cp validation_params.csv validation_params_umbra_tiny.csv
-            scripts/stop.sh
-            cd ..
       - run:
-          name: Tiny data set - Cross-validate the Umbra project based on the results from Neo4j
+          name: Get SF1 data set
           command: |
-            cp cypher/validation_params_cypher_tiny.csv umbra/validation_params.csv
-            cd umbra
-            export UMBRA_CSV_DIR=`pwd`/../postgres/test-data/
-            scripts/restore-database.sh
-            driver/validate.sh | tee validation-log.txt
-            grep 'Validation Result: PASS' validation-log.txt
-            cd ..
+            curl -L https://surfdrive.surf.nl/files/index.php/s/IbWgmBVQfdu1BRV/download | tar -xv --use-compress-program=unzstd
       - run:
-          name: Tiny data set - Benchmark Umbra
+          name: Process SF1 data set
           command: |
-            cd umbra
-            export UMBRA_CSV_DIR=`pwd`/../postgres/test-data/
-            scripts/restore-database.sh
-            driver/benchmark.sh
-            scripts/stop.sh
+            export SF=1
+            export LDBC_SNB_DRIVER_DIR=`pwd`/ldbc_snb_interactive_driver
+            export LDBC_SNB_DATA_ROOT_DIRECTORY=`pwd`/out-sf${SF}
+            export LDBC_SNB_IMPLS_DIR=`pwd`
+
+            echo "==================== Generate update streams ===================="
+            cd ${LDBC_SNB_IMPLS_DIR}
+            mkdir update-streams-sf${SF}/
+
+            cd ${LDBC_SNB_DRIVER_DIR}
+            cd scripts
+            ./convert.sh
+            mv inserts ${LDBC_SNB_IMPLS_DIR}/update-streams-sf${SF}/
+            mv deletes ${LDBC_SNB_IMPLS_DIR}/update-streams-sf${SF}/
+
+            echo "==================== Generate parameters ===================="
+            cd ${LDBC_SNB_IMPLS_DIR}
+            mkdir parameters-sf${SF}
+
+            cd paramgen
+            scripts/get-factors.sh
+            scripts/paramgen.sh
             cd ..
-      ## SF1 tests
-      - run:
-          name: Download SF1 data set
-          command: |
-            curl -L https://surfdrive.surf.nl/files/index.php/s/Ng2eCg6m06ZNKKo/download | tar -xv --use-compress-program=unzstd
-            # substitution parameters and updateStreams for Neo4j
-            cp ./sf1/substitution_parameters/* ./cypher/test-data/substitution_parameters/
-            cp ./sf1/substitution_parameters/* ./postgres/test-data/substitution_parameters/
-            cp ./sf1/social_network-csv_composite-longdateformatter/updateStream* ./cypher/test-data/update_streams/
-            cp ./sf1/social_network-csv_composite-longdateformatter/updateStream* ./postgres/test-data/update_streams/
+
+            mv parameters/*.parquet parameters-sf${SF}/
       ### Neo4j
       - run:
-          name: SF1 data set - Load Neo4j database
+          name: SF1 data set - Load Neo4j database and create a backup
           command: |
+            export SF=1
+            export LDBC_SNB_DATAGEN_DIR=`pwd`
             cd cypher
-            export NEO4J_VANILLA_CSV_DIR=`pwd`/../sf1/social_network-csv_composite-longdateformatter
-            export NEO4J_CONVERTED_CSV_DIR=`pwd`/../sf1/social_network-csv_composite-converted
-            mkdir ${NEO4J_CONVERTED_CSV_DIR}
+            . scripts/use-datagen-data-set.sh
             scripts/load-in-one-step.sh
+            scripts/backup-database.sh
             cd ..
       - run:
-          name: SF1 data set - Create validation parameters with Neo4j
+          name: SF1 data set - Create validation parameters with Neo4j project
           command: |
+            export SF=1
+            export LDBC_SNB_DATAGEN_DIR=`pwd`
             cd cypher
-            driver/create-validation-parameters.sh
-            cp validation_params.csv validation_params_cypher_sf1.csv
-            cd ..
-      ### DuckDB
-      - run:
-          name: SF1 data set - Load DuckDB database
-          command: |
-            cd duckdb
-            export DUCKDB_CSV_DIR=`pwd`/../sf1/social_network-csv_merge_foreign
-            scripts/load-in-one-step.sh
+            . scripts/use-datagen-data-set.sh
+            driver/create-validation-parameters.sh driver/create-validation-parameters-sf${SF}.properties
+            cp validation_params.json validation_params_cypher_sf${SF}.json
             cd ..
       ### PostgreSQL
       - run:
-          name: SF1 data set - Load PostgreSQL database
+          name: SF1 data set - Load PostgreSQL database and create a backup
           command: |
+            export SF=1
+            export LDBC_SNB_DATAGEN_DIR=`pwd`
             cd postgres
-            export POSTGRES_CUSTOM_CONFIGURATION=`pwd`/config/postgresql.conf
-            export POSTGRES_CSV_DIR=`pwd`/../sf1/social_network-csv_merge_foreign
+            . scripts/use-datagen-data-set.sh
             scripts/load-in-one-step.sh
+            scripts/backup-database.sh
             cd ..
       - run:
           name: SF1 data set - Cross-validate the PostgreSQL project based on the results from Neo4j
           command: |
-            cp cypher/validation_params_cypher_sf1.csv postgres/validation_params.csv
+            export SF=1
+            export LDBC_SNB_DATAGEN_DIR=`pwd`
+            cp cypher/validation_params_cypher_sf${SF}.json postgres/validation_params.json
             cd postgres
+            . scripts/use-datagen-data-set.sh
+            scripts/restore-database.sh
+            sed -i "s|ldbc.snb.interactive.parameters_dir=../parameters/|ldbc.snb.interactive.parameters_dir=../parameters-sf${SF}/|" driver/validate.properties
             driver/validate.sh | tee validation-log.txt
-            scripts/stop.sh
             grep 'Validation Result: PASS' validation-log.txt
             cd ..
-      ### Umbra
+      ### Neo4j
       - run:
-          name: SF1 data set - Load Umbra database
+          name: SF1 data set - Benchmark Neo4j
           command: |
-            cd umbra
-            export UMBRA_CSV_DIR=`pwd`/../sf1/social_network-csv_merge_foreign
-            scripts/load-in-one-step.sh
+            export SF=1
+            export LDBC_SNB_DATAGEN_DIR=`pwd`
+            cd cypher
+            . scripts/use-datagen-data-set.sh
+            scripts/restore-database.sh
+            driver/benchmark.sh driver/benchmark-sf${SF}.properties
             cd ..
+      ### PostgreSQL
       - run:
-          name: SF1 data set - Cross-validate the Umbra project based on the results from Neo4j
+          name: SF1 data set - Benchmark PostgreSQL
           command: |
-            cp cypher/validation_params_cypher_sf1.csv umbra/validation_params.csv
-            cd umbra
-            driver/validate.sh | tee validation-log.txt
+            export SF=1
+            export LDBC_SNB_DATAGEN_DIR=`pwd`
+            cd postgres
+            . scripts/use-datagen-data-set.sh
+            scripts/restore-database.sh
+            driver/benchmark.sh driver/benchmark-sf${SF}.properties
             scripts/stop.sh
-            grep 'Validation Result: PASS' validation-log.txt
             cd ..
-      - slack/status
+      - slack/status

.gitignore

@@ -7,6 +7,7 @@ dependency-reduced-pom.xml
 .idea/
 *.iml
 validation_params*.csv
+validation_params*.json
 *.swp
 *-actual.json
 *-expected.json
@@ -18,3 +19,12 @@ tcr.log
 *.priv
 *.priv.*
 umbra/.dev.env
+
+social-network-sf*/
+social-network-sf*.*
+
+parameters-sf*/
+parameters-sf*.*
+
+update-streams-sf*/
+update-streams-sf*.*