Generearate synthetic data and run experiments

.gitignore

@@ -127,3 +127,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+input/gemsec_facebook_dataset/*

TODO.md

@@ -10,7 +10,7 @@
 
 [X] Sanity check. Compare results (not time)
 
-1. graphframe_bs.py
+1. `graphframe_bs.py`
 
 ```bash
 spark-submit --master local[3] --packages graphframes:graphframes:0.8.1-spark3.0-s_2.12 src/graphframe_bs.py 2 cache
@@ -41,7 +41,7 @@ Execution time : 27.456836223602295
 
 Different results triangles on each experiment, but with the same probability!
 
-2. rdd_bs.py
+2. `rdd_bs.py`
 
 ```bash
 spark-submit --master local[3] --packages graphframes:graphframes:0.8.1-spark3.0-s_2.12 src/rdd_bs.py 10
@@ -59,7 +59,7 @@ spark-submit --master local[3] --packages graphframes:graphframes:0.8.1-spark3.0
     Execution time : 8.00922679901123
 ```
 
-3. rdd_fast.py
+3. `rdd_fast.py`
 
 ```bash
 spark-submit --master local[3] --packages graphframes:graphframes:0.8.1-spark3.0-s_2.12 src/rdd_fast.py 10 0.8

experiments.csv

@@ -1,4 +1,19 @@
-script,cores,k,data_efficiency,time
-graphframe_bs.py,3,1,None,20.040249824523926
-graphframe_bs.py,3,1,None,20.405229568481445
-graphframe_bs.py,3,1,None,20.430413007736206
+script,cores,dataset,k,time
+src/graphframe_bs.py,1,artists_uniform,10,177.28104901313782
+src/graphframe_bs.py,1,artists_uniform,100,165.02419257164001
+src/graphframe_bs.py,1,artists_uniform,1000,199.8896152973175
+src/graphframe_bs.py,2,artists_uniform,10,120.59347820281982
+src/graphframe_bs.py,2,artists_uniform,100,109.26687359809875
+src/graphframe_bs.py,2,artists_uniform,1000,102.84408783912659
+src/graphframe_bs.py,8,artists_uniform,10,80.65441060066223
+src/graphframe_bs.py,8,artists_uniform,100,84.33814811706543
+src/graphframe_bs.py,8,artists_uniform,1000,93.380126953125
+src/graphframe_bs.py,1,artists_normal,10,167.7195463180542
+src/graphframe_bs.py,1,artists_normal,100,176.1601779460907
+src/graphframe_bs.py,1,artists_normal,1000,163.45222926139832
+src/graphframe_bs.py,2,artists_normal,10,101.96716809272766
+src/graphframe_bs.py,2,artists_normal,100,108.68491959571838
+src/graphframe_bs.py,2,artists_normal,1000,107.02509760856628
+src/graphframe_bs.py,8,artists_normal,10,90.41069388389587
+src/graphframe_bs.py,8,artists_normal,100,87.18181848526001
+src/graphframe_bs.py,8,artists_normal,1000,93.27506709098816

experiments_collins.csv

@@ -0,0 +1,28 @@
+script,cores,dataset,k,time
+src/graphframe_bs.py,1,collins,10,30.58803701400757
+src/graphframe_bs.py,1,collins,100,31.38023281097412
+src/graphframe_bs.py,1,collins,1000,29.2769193649292
+src/graphframe_bs.py,2,collins,10,24.48428463935852
+src/graphframe_bs.py,2,collins,100,22.833560705184937
+src/graphframe_bs.py,2,collins,1000,19.599233150482178
+src/graphframe_bs.py,8,collins,10,18.67221188545227
+src/graphframe_bs.py,8,collins,100,18.908872604370117
+src/graphframe_bs.py,8,collins,1000,18.513964891433716
+src/rdd_bs.py,1,collins,10,9.219404935836792
+src/rdd_bs.py,1,collins,100,8.947774410247803
+src/rdd_bs.py,1,collins,1000,9.352253675460815
+src/rdd_bs.py,2,collins,10,9.313251495361328
+src/rdd_bs.py,2,collins,100,9.166396856307983
+src/rdd_bs.py,2,collins,1000,9.044026851654053
+src/rdd_bs.py,8,collins,10,9.171149730682373
+src/rdd_bs.py,8,collins,100,9.086631774902344
+src/rdd_bs.py,8,collins,1000,9.210607528686523
+src/rdd_fast.py,1,collins,10,4.090131998062134
+src/rdd_fast.py,1,collins,100,4.0312323570251465
+src/rdd_fast.py,1,collins,1000,4.045019626617432
+src/rdd_fast.py,2,collins,10,4.059403419494629
+src/rdd_fast.py,2,collins,100,3.798642158508301
+src/rdd_fast.py,2,collins,1000,4.034178256988525
+src/rdd_fast.py,8,collins,10,3.867384910583496
+src/rdd_fast.py,8,collins,100,4.296428680419922
+src/rdd_fast.py,8,collins,1000,4.024558782577515