Merge branch 'develop' into main

Author: lukasgehring

.DS_Store

@@ -133,6 +133,10 @@ dmypy.json
 
 # PyCharm
 .idea/
+.DS_Store
+
+# Database
+*.db
 
 # todo
 todo.md

.gitignore

@@ -8,7 +8,11 @@ results in a database. Errors that occur during execution are also captured by t
 ## Installation and configuration
 In order to install the package via pip use the command
 ```
-pip install py_experimenter
+pip install py-experimenter
+```
+You can then import the PyExperimenter by using
+```
+from py_experimenter import PyExperimenter
 ```
 
 ### Configuration file
@@ -23,12 +27,15 @@ In the following we will go through these sections and go into more detail about
 The database section contains the credentials for the database where the experiment parameters, results, potential errors and meta information of the experiments are written.
 ```
 [DATABASE]
-host=<hostname>
-user=<username>
+provider=<sqlite or mysql>
+host=<hostname>     (only required when using mysql as provider)
+user=<username>     (only required when using mysql as provider)
+password=<password> (only required when using mysql as provider)
 database=<databasename>
-password=<password>
 table=<tablename>
 ```
+You can use `sqlite` as well as `mysql` as provider for the database. As you can see above, for mysql you need to provide a host, user and password for the database,
+while sqlit only requires a database and table name. 
 Note that the table is automatically generated by PyExperimenter.
 In the configuration file you should therefore only enter the desired name.
 
@@ -67,8 +74,6 @@ custom.setting1=<value>
 custom.setting2=<value>
 ```
 
-An example of a configuration file can be found in `config/example_config.cfg`.
-
 ## How should the function to be executed look like?
 To use the PyExperimenter for your experiments, you need to create a function (again, we call this function `own_function`) with 3 parameters.
 This function will be called by the PyExperimenter to process each instance of the experiment.
@@ -97,7 +102,7 @@ file to the experiment. Note that this parameter is needed even if you have not
 ## How to run the PyExperimenter
 To run the PyExperimenter you need to initialize a `PyExperimenter` object:
 ```python
-import PyExperimenter
+from py_experimenter import PyExperimenter
 
 # initialize the PyExperiementer
 experimenter = PyExperimenter()
@@ -186,3 +191,8 @@ so the machine id is identical for each experiment. The status values for the `s
 
 This is followed by the `creation_date`,`start_date` and `end_date` columns, which contain the creation, start and end times. 
 The last column contains the error message of the exercise. If no error occurred, this column contains the entry `NULL`.
+
+
+###Examples
+In the `examples` folder you can find examples for most of the functionality explained here. Note that `example1.py`, `example2.py` and `example4.py` are sqlite examples,
+while `example3.py` is a mysql example, where you need to add your database credentials in the config file.

README.md

@@ -1 +1 @@
-from base_folder.py_experimenter.py_experimenter import PyExperimenter
+from py_experimenter.experimenter import PyExperimenter

base_folder/py_experimenter/__init__.py

@@ -1,98 +1,16 @@
-import logging
-import sys
 from typing import List
 
-import mysql
-import numpy as np
-from mysql.connector import connect, ProgrammingError
-from datetime import datetime
-import pandas as pd
-import base_folder.py_experimenter.utils as utils
-
 
 class DatabaseConnector:
 
-    def __init__(self, config):
-        self.config = config
-        self.table_name, host, user, database, password = utils.extract_db_credentials_and_table_name_from_config(
-            config)
-        self._db_credentials = dict(host=host, user=user, database=database, password=password)
-
-        # try connection to database and exit program if connection not possible
-        try:
-            connection = connect(**self._db_credentials)
-
-        except mysql.connector.Error as err:
-            logging.error(err)
-            sys.exit(1)
-
-        else:
-            connection.close()
-
     def create_table_if_not_exists(self) -> None:
         """
         Check if tables does exist. If not, a new table will be created.
         :param mysql_connection: mysql_connector to the database
         :param table_name: name of the table from the config
         :param experiment_config: experiment section of the config file
         """
-        experiment_config = self.config['PY_EXPERIMENTER']
-
-        try:
-            connection = connect(**self._db_credentials)
-            cursor = connection.cursor()
-
-            cursor.execute(f"SHOW TABLES LIKE '{self.table_name}'")
-
-            # load column names from config
-            fields = experiment_config['keyfields'].split(',') + experiment_config['resultfields'].split(',')
-
-            # remove whitespace
-            clean_fields = [field.replace(' ', '') for field in fields]
-
-            # (name, type) - default type is 'VARCHAR(255)'
-            typed_fields = [tuple(field.split(':')) if len(field.split(':')) == 2 else (field, 'VARCHAR(255)') for
-                            field in clean_fields]
-
-            # exit if table already exist
-            if cursor.fetchall():
-                cursor.execute(f"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '{self.table_name}'")
-                columns = [k[0] for k in cursor.fetchall()][:-6]
-                config_columns = [k[0] for k in typed_fields]
-
-                # check if fields in config match with columns in database
-                if set(columns) == set(config_columns):
-                    return
-
-                logging.error("Keyfields or resultfields from the configuration do not match columns in the existing "
-                              "table. Please change your configuration or delete the table in your database.")
-                sys.exit()
-
-            # extend experiment columns by pyexperimenter columns
-            typed_fields.extend(
-                [('status', 'VARCHAR(255)'), ('machine', 'VARCHAR(255)'), ('creation_date', 'VARCHAR(255)'),
-                 ('start_date', 'VARCHAR(255)'),
-                 ('end_date', 'VARCHAR(255)'), ('error', 'LONGTEXT')])
-
-            # set default value for each column to NULL
-            columns = ['%s %s DEFAULT NULL' % (field, datatype) for field, datatype in typed_fields]
-
-            stmt = f"CREATE TABLE {self.table_name} ({','.join(columns)})"
-
-            try:
-                cursor.execute(stmt)
-            except ProgrammingError:
-                logging.error("An error occurred while creating the table in the database. Please check the "
-                              "configuration file for allowed characters and data types for the keyfields and resultfields "
-                              "as well as the table name.")
-                sys.exit(1)
-
-        except mysql.connector.Error as err:
-            logging.error(err)
-            sys.exit(1)
-
-        else:
-            connection.close()
+        pass
 
     def fill_table(self, individual_parameters=None, parameters=None) -> None:
         """
@@ -102,133 +20,22 @@ def fill_table(self, individual_parameters=None, parameters=None) -> None:
         :param config: config file
 
         """
-
-        # ref: https://www.kite.com/python/answers/how-to-get-all-element-combinations-of-two-numpy-arrays-in-python
-        keyfield_names = utils.get_keyfields(self.config)
-
-        if individual_parameters is None:
-            if parameters is None:
-                keyfield_data = utils.get_keyfield_data(self.config)
-            else:
-                keyfield_data = list(parameters.values())
-
-            combinations = np.array(np.meshgrid(*keyfield_data)).T.reshape(-1, len(keyfield_data))
-            combinations = [dict(zip(keyfield_names, combination)) for combination in combinations]
-        else:
-            combinations = individual_parameters
-
-        # check if combinations exist
-        if len(combinations) == 0:
-            return
-
-        if individual_parameters is not None:
-            _number_of_keys = 0
-            for key in combinations[0].keys():
-
-                if key not in keyfield_names:
-                    logging.error(f"Keyfield '{key}' is not defined in configuration file")
-                    sys.exit()
-
-                _number_of_keys += 1
-
-            if _number_of_keys != len(keyfield_names):
-                logging.error(f"{len(keyfield_names) - _number_of_keys} keyfield(s) missing! Please check passed parameters contain all keyfields defined in the configuration file.")
-                sys.exit()
-
-
-
-        columns_names = np.array2string(np.array(keyfield_names), separator=',') \
-            .replace('[', '') \
-            .replace(']', '') \
-            .replace("'", "")
-
-        try:
-            connection = connect(**self._db_credentials)
-            cursor = connection.cursor()
-            cursor.execute(f"SELECT {columns_names} FROM {self.table_name}")
-            existing_rows = list(map(np.array2string, np.array(cursor.fetchall())))
-
-        except mysql.connector.Error as err:
-            logging.error(err)
-            sys.exit(1)
-
-        else:
-            connection.close()
-
-        columns_names += ",status"
-        columns_names += ",creation_date"
-
-        time = datetime.now()
-        for combination in combinations:
-            if ("['" + "' '".join([str(value) for value in combination.values()]) + "']") in existing_rows:
-                continue
-            values = list(combination.values())
-            values.append("created")
-            values.append("%s" % time.strftime("%m/%d/%Y, %H:%M:%S"))
-
-            self._write_to_database(columns_names.split(', '), values)
+        pass
 
     def get_parameters_to_execute(self) -> List[dict]:
-        experiment_config = self.config['PY_EXPERIMENTER']
-
-        execute_condition = "status='created'"
-
-        keyfields = experiment_config['keyfields'].split(',')
-        keyfield_names = utils.get_field_names(keyfields)
-
-        stmt = f"SELECT {', '.join(keyfield_names)} FROM {self.table_name} WHERE {execute_condition}"
-
-        try:
-            connection = connect(**self._db_credentials)
-            cursor = connection.cursor()
-
-            try:
-                cursor.execute(stmt)
-            except ProgrammingError as e:
-                logging.error(str(e) + "\nPlease check if 'fill_table()' was called correctly.")
-                sys.exit(1)
-
-            parameters = pd.DataFrame(cursor.fetchall())
-            if parameters.empty:
-                return []
-            parameters.columns = [i[0] for i in cursor.description]
-
-        except mysql.connector.Error as err:
-            logging.error(err)
-            sys.exit(1)
-
-        else:
-            connection.close()
-
-        named_parameters = [dict(parameter.to_dict()) for _, parameter in parameters.iterrows()]
-
-        return named_parameters
+        pass
 
     def _write_to_database(self, keys, values) -> None:
         """
         Write new row(s) to database
         :param keys: Column names
         :param values: Values for column names
         """
-        if len(keys[0].split(',')) != len(values):
-            logging.error('Keyfield(s) missing! Please check passed parameters contain all keyfields defined in the configuration file.')
-            sys.exit(1)
-
-        keys = ", ".join(keys)
-        values = "'" + "', '".join([str(value) for value in values]) + "'"
-
-        stmt = f"INSERT INTO {self.table_name} ({keys}) VALUES ({values})"
-
-        try:
-            connection = connect(**self._db_credentials)
-            cursor = connection.cursor()
+        pass
 
-            cursor.execute(stmt)
-            connection.commit()
 
-        except mysql.connector.Error as err:
-            logging.error(err)
-            sys.exit(1)
+    def _update_database(self, keys, values, where):
+        pass
 
-        else:
-            connection.close()
+    def get_not_executed_yet(self, where) -> bool:
+        pass