quakers.yml

tag: dev  # Required
#datastore:  # All properties required
#  - config:
#      hostname: !ENV ${MYSQL_HOST}
#      username: !ENV ${MYSQL_USERNAME}
#      password: !ENV ${MYSQL_PASSWORD}
#      db_name: !ENV ${MYSQL_DB_NAME}
#      port: 3306
#    type: mysql
spark:
  - config:
      spark.master: local[*]  # Required
      spark.submit.deployMode: client  # Required
      spark_warehouse_folder: data/spark-warehouse  # Required
      spark.ui.port: 4040
      spark.driver.cores: 5
      spark.driver.memory: 8g
      spark.driver.memoryOverhead: 4096
      spark.driver.maxResultSize: 0
      spark.executor.instances: 2
      spark.executor.cores: 3
      spark.executor.memory: 4g
      spark.executor.memoryOverhead: 4096
      spark.sql.broadcastTimeout: 3600
      spark.sql.autoBroadcastJoinThreshold: -1
      spark.sql.shuffle.partitions: 4
      spark.default.parallelism: 4
      spark.network.timeout: 3600s
    dirs:
      df_data_folder: data/dataframes
      spark_warehouse_folder: data/spark-warehouse
      checkpoints_folder: data/checkpoints
      communities_csv_folder: data/csv_data
input:
  - config:  # All properties required
      name: Quakers
      nodes:
        path: data/input_graphs/Quakers/quakers_nodelist.csv2
        has_header: true
        delimiter: ','
        encoding: ISO-8859-1
        feature_names:
          - id
          - Historical_Significance
          - Gender
          - Birthdate
          - Deathdate
          - internal_id
      edges:
        path: data/input_graphs/Quakers/quakers_edgelist.csv2
        has_header: true
        has_weights: false
        delimiter: ','
    type: local
run_options:  # All properties required
  - config:
      cached_init_step: false
      feature_min_avg: 0.33
      r_lvl1_thres: 0.50
      r_lvl2_thres: 0.85
      max_edge_weight: 0.50
      betweenness_thres: 10
      max_sp_length: 2
      min_comp_size: 2
      max_steps: 30
      features_to_check:
        - id
        - Gender
output:  # All properties required
  - config:
      logs_folder: data/logs
      save_communities_to_csvs: false
      visualizer:
        dimensions: 3
        save_img: true
        folder: data/plots
        steps:
#          - 0
          - -1  # The Last step