remove dev stuff before merging to master

data.csv

@@ -1,36 +1,36 @@
 task,data,active
-groupby,G1_1e6_1e2_0_0,TRUE
-groupby,G1_1e6_1e1_0_0,TRUE
-groupby,G1_1e6_2e0_0_0,TRUE
-groupby,G1_1e6_1e2_0_1,TRUE
-groupby,G1_1e7_1e2_0_0,FALSE
-groupby,G1_1e7_1e1_0_0,FALSE
-groupby,G1_1e7_2e0_0_0,FALSE
-groupby,G1_1e7_1e2_0_1,FALSE
-groupby,G1_1e8_1e2_0_0,FALSE
-groupby,G1_1e8_1e1_0_0,FALSE
-groupby,G1_1e8_2e0_0_0,FALSE
-groupby,G1_1e8_1e2_0_1,FALSE
-groupby,G1_1e9_1e2_0_0,FALSE
-groupby,G1_1e9_1e1_0_0,FALSE
-groupby,G1_1e9_2e0_0_0,FALSE
-groupby,G1_1e9_1e2_0_1,FALSE
-groupby,G2_1e6_1e2_0_0,TRUE
-groupby,G2_1e6_1e1_0_0,TRUE
-groupby,G2_1e6_2e0_0_0,TRUE
-groupby,G2_1e6_1e2_0_1,TRUE
-groupby,G2_1e7_1e2_0_0,FALSE
-groupby,G2_1e7_1e1_0_0,FALSE
-groupby,G2_1e7_2e0_0_0,FALSE
-groupby,G2_1e7_1e2_0_1,FALSE
-groupby,G2_1e8_1e2_0_0,FALSE
-groupby,G2_1e8_1e1_0_0,FALSE
-groupby,G2_1e8_2e0_0_0,FALSE
-groupby,G2_1e8_1e2_0_1,FALSE
-groupby,G2_1e9_1e2_0_0,FALSE
-groupby,G2_1e9_1e1_0_0,FALSE
-groupby,G2_1e9_2e0_0_0,FALSE
-groupby,G2_1e9_1e2_0_1,FALSE
+groupby,G1_1e6_1e2_0_0,FALSE
+groupby,G1_1e6_1e1_0_0,FALSE
+groupby,G1_1e6_2e0_0_0,FALSE
+groupby,G1_1e6_1e2_0_1,FALSE
+groupby,G1_1e7_1e2_0_0,TRUE
+groupby,G1_1e7_1e1_0_0,TRUE
+groupby,G1_1e7_2e0_0_0,TRUE
+groupby,G1_1e7_1e2_0_1,TRUE
+groupby,G1_1e8_1e2_0_0,TRUE
+groupby,G1_1e8_1e1_0_0,TRUE
+groupby,G1_1e8_2e0_0_0,TRUE
+groupby,G1_1e8_1e2_0_1,TRUE
+groupby,G1_1e9_1e2_0_0,TRUE
+groupby,G1_1e9_1e1_0_0,TRUE
+groupby,G1_1e9_2e0_0_0,TRUE
+groupby,G1_1e9_1e2_0_1,TRUE
+groupby,G2_1e6_1e2_0_0,FALSE
+groupby,G2_1e6_1e1_0_0,FALSE
+groupby,G2_1e6_2e0_0_0,FALSE
+groupby,G2_1e6_1e2_0_1,FALSE
+groupby,G2_1e7_1e2_0_0,TRUE
+groupby,G2_1e7_1e1_0_0,TRUE
+groupby,G2_1e7_2e0_0_0,TRUE
+groupby,G2_1e7_1e2_0_1,TRUE
+groupby,G2_1e8_1e2_0_0,TRUE
+groupby,G2_1e8_1e1_0_0,TRUE
+groupby,G2_1e8_2e0_0_0,TRUE
+groupby,G2_1e8_1e2_0_1,TRUE
+groupby,G2_1e9_1e2_0_0,TRUE
+groupby,G2_1e9_1e1_0_0,TRUE
+groupby,G2_1e9_2e0_0_0,TRUE
+groupby,G2_1e9_1e2_0_1,TRUE
 join,J1_1e6_NA_0_0,FALSE
 join,J1_1e7_NA_0_0,TRUE
 join,J1_1e8_NA_0_0,FALSE

report.R

@@ -5,8 +5,8 @@ get_report_status_file = function(path=getwd()) {
   file.path(path, "report-done")
 }
 get_report_solutions = function() {
-  "clickhouse"
-  #c("data.table", "dplyr", "pandas", "pydatatable", "spark", "dask", "juliadf")
+  #"clickhouse"
+  c("data.table", "dplyr", "pandas", "pydatatable", "spark", "dask", "juliadf")
 }
 get_excluded_batch = function() {
   c(

run.conf

@@ -2,7 +2,7 @@
 export RUN_TASKS="groupby" # join sort read"
 
 # solution, used in init-setup-iteration.R
-export RUN_SOLUTIONS="clickhouse"  #"data.table dplyr pandas pydatatable spark dask juliadf" # clickhouse"
+export RUN_SOLUTIONS="data.table dplyr pandas pydatatable spark dask juliadf" # clickhouse"
 
 # not run benchmarks but print what would run and what skipped
 export MOCKUP=false
@@ -14,10 +14,10 @@ export CSV_VERBOSE=false
 export DO_UPGRADE=true
 
 # force run, ignore if same version was run already
-export FORCE_RUN=true
+export FORCE_RUN=false
 
 # flag to build reports, used in ruh.sh before publish
 export DO_REPORT=false
 
 # flag to publish, used in ruh.sh before exit
-export DO_PUBLISH=true
+export DO_PUBLISH=false

run.sh

@@ -10,9 +10,9 @@ export BATCH=$(date +%s)
 # confirm stop flag disabled
 if [[ -f ./stop ]]; then echo "# Benchmark run $BATCH aborted. 'stop' file exists, should be removed before calling 'run.sh'" && exit; fi;
 
-# confirm clickhouse is not running - TODO resolve sudo-less start/stop clickhouse first
-#source ./ch.sh
-#ch_active && echo "# Benchmark run $BATCH aborted. clickhouse-server is running, shut it down before calling 'run.sh'" && exit;
+# confirm clickhouse is not running
+source ./ch.sh
+ch_active && echo "# Benchmark run $BATCH aborted. clickhouse-server is running, shut it down before calling 'run.sh'" && exit;
 
 # set lock
 if [[ -f ./run.lock ]]; then echo "# Benchmark run $BATCH aborted. 'run.lock' file exists, this should be checked before calling 'run.sh'. Ouput redirection mismatch might have happened if writing output to same file as currently running $(cat ./run.lock) benchmark run" && exit; else echo $BATCH > run.lock; fi;
@@ -32,10 +32,10 @@ if [[ "$DO_UPGRADE" == true && "$RUN_SOLUTIONS" =~ "modin" ]]; then ./modin/init
 if [[ "$DO_UPGRADE" == true && "$RUN_SOLUTIONS" =~ "pandas" ]]; then ./pandas/init-pandas.sh; fi;
 if [[ "$DO_UPGRADE" == true && "$RUN_SOLUTIONS" =~ "pydatatable" ]]; then ./pydatatable/init-pydatatable.sh; fi;
 if [[ "$DO_UPGRADE" == true && "$RUN_SOLUTIONS" =~ "spark" ]]; then ./spark/init-spark.sh; fi;
-#if [[ "$DO_UPGRADE" == true && "$RUN_SOLUTIONS" =~ "clickhouse" ]]; then ./clickhouse/init-clickhouse.sh; fi; # for now manual as requires sudo
+#if [[ "$DO_UPGRADE" == true && "$RUN_SOLUTIONS" =~ "clickhouse" ]]; then ./clickhouse/init-clickhouse.sh; fi; # manual as requires sudo: apt-get install --only-upgrade clickhouse-server clickhouse-client
 
 # produce VERSION, REVISION files for each solution
-#./versions.sh
+./versions.sh
 
 # run
 Rscript ./launcher.R