ADD: Neuron Coverage DNN

Author: phantran

README.md

@@ -0,0 +1,15 @@
+## Introduction
+
+The main purpose of this source code is to demonstrate the correlation between image transformations and neuron
+coverage. A model is trained on generated BeamNG dataset for the simple task of keeping vehicles on the road. A part of
+the dataset is then transformed (rotate, adjust brightness, etc...) and input to the trained model. Neuron coverage
+and output from the trained model on both original and transformed image data will be recorded and summarize to verify
+the correlation.
+
+## Setup
+
+Clone <strong>config.yml.sample</strong> file and rename it to <strong>config.yml</strong>. Modify the configurations corresponding to your local setup.
+
+## Running
+`python run.py --task TASK_TO_PERFORM` <br>
+Replace TASK_TO_PERFORM with one of the following options: <strong>collecting, training, evaluating</strong>

requirements.txt

@@ -0,0 +1,11 @@
+beamngpy==1.18
+Keras==2.4.3
+tensorflow==2.4.0
+shapely==1.7.1
+pandas==1.2.1
+imageio==2.9.0
+scikit-learn==0.24.1
+opencv-python==4.5.1.48
+PyYAML~=5.3.1
+numpy~=1.19.5
+matplotlib~=3.3.3

run.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Author: Tran Phan
+
+import argparse
+
+from src.modules import logger
+
+
+def get_console_arguments():
+    # parse console arguments
+    parser = argparse.ArgumentParser(description='Neuron Coverage Demonstration')
+    parser.add_argument('--task', dest='task', choices=['collecting', 'data_cleaning', 'training', 'evaluating'],
+                        default="",
+                        required=True, help='Enter one of the following: collecting, training, evaluating')
+    parser.add_argument('--q', dest='number_of_images', type=int, default=1000,
+                        required=False, help='Number of images data to collect')
+    parser.add_argument('--cm', dest='collect_mode', choices=['manual_mode', 'ai_mode'], default="manual_mode",
+                        required=False, help='Data collect mode: manual_mode or ai_mode')
+    parser.add_argument('--model', dest='model', choices=['inception_v3', 'cnn'],
+                        default="inception_v3",
+                        help='Enter one of the following model name for training: inception_v2, cnn')
+    parser.add_argument('--mf', dest='model_file', type=str, default="model_1.h5",
+                        help='Model file name for running evaluator')
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = get_console_arguments()
+
+    if args.task == "collecting":
+        logger.info("Task: Collecting")
+        from src.training import data_collector
+
+        data_collector.run_collector(int(args.number_of_images), args.collect_mode)
+
+    if args.task == "data_cleaning":
+        logger.info("Task: Cleaning training data")
+        from src.training import data_cleaner
+
+        data_cleaner.run_data_cleaner()
+
+    elif args.task == "training":
+        logger.info("Task: Training - " + args.model)
+        from src.training import model_training
+
+        model_training.run_trainer(args.model)
+
+    elif args.task == "evaluating":
+        logger.info("Task: Evaluating")
+        from src.evaluation import evaluate
+
+        evaluate.run_evaluator(args.model_file)

src/__init__.py

@@ -0,0 +1,32 @@
+logging:
+  filename:
+  level: DEBUG
+  encoding: utf-8
+  format: '%(asctime)s | %(levelname)s | %(name)s | %(message)s'
+
+data_collecting:
+  mode: "manual"
+  number_of_images: 5
+  image_width: 299
+  image_height: 299
+  scenario_level: "west_coast_usa"
+  scenario_name: "example"
+  vehicle_model: "etk800"
+  pos: "-717,101,118"
+  rot_quat: "0,0,0.3826834,0.9238795"
+  data_path: "/training/data/images/"  # relative to the root of the project
+  csv_path: "/training/data/training_data.csv"
+  model_path: "/training/models/"
+  sleep: 1  #seconds
+
+training:
+  epochs: 30
+
+evaluation:
+  neuron_cover_threshold: 0.2
+  evaluation_data_path: "/evaluation/data/images/"
+
+beamNG:
+  home: "Directory of BeamNG program"
+  host: "localhost"
+  port: 64256

src/__pycache__/__init__.cpython-38.pyc

@@ -0,0 +1,32 @@
+logging:
+  filename:
+  level: DEBUG
+  encoding: utf-8
+  format: '%(asctime)s | %(levelname)s | %(name)s | %(message)s'
+
+data_collecting:
+  mode: "manual"
+  number_of_images: 5
+  image_width: 500
+  image_height: 500
+  scenario_level: "west_coast_usa"
+  scenario_name: "example"
+  vehicle_model: "etk800"
+  pos: "-717,101,118"
+  rot_quat: "0,0,0.3826834,0.9238795"
+  data_path: "/training/data/images/"  # relative to the root of the project
+  csv_path: "/training/data/training_data.csv"
+  model_path: "/training/models/"
+  sleep: 1  #seconds
+
+training:
+  epochs: 5
+
+evaluation:
+  neuron_cover_threshold: 0.2
+  evaluation_data_path: "/evaluation/data/images/"
+
+beamNG:
+  home: "Directory of BeamNG program"
+  host: "localhost"
+  port: 64256

src/__pycache__/definitions.cpython-38.pyc

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Author: Tran Phan
+
+import os
+
+ROOT_DIR = os.path.dirname(os.path.abspath(__file__))