Merge pull request #209 from tornede/207-get_table-with-condition

207 get table with condition

Author: LukasFehring

.gitignore

@@ -150,6 +150,8 @@ output/
 # codecarbon
 .codecarbon.config
 emissions.csv
+powermetrics_log.txt
 
 # development folder
 development/
+

CHANGELOG.rst

@@ -1,7 +1,14 @@
 =========
 Changelog
 =========
+v1.4.3 (**.**.2025)
+===================
+
+Fix
+===
 
+- Fix bug, where the logtable_name was not overwritten by `table_name` updates in the `PyExperimenter` class.
+- Allow condition for experimenter.get_table() and experimenter.get_logtable()
 
 v1.4.2 (12.06.2024)
 ===================

docs/source/examples/example_logtables.ipynb

@@ -1,15 +1,18 @@
 import logging
 from abc import ABC, abstractclassmethod
 from logging import Logger
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import numpy as np
 import omegaconf
 from attr import dataclass
 from omegaconf import DictConfig, ListConfig, OmegaConf
 
 from py_experimenter import utils
-from py_experimenter.exceptions import InvalidColumnError, InvalidConfigError, InvalidLogtableError
+from py_experimenter.exceptions import (
+    InvalidColumnError,
+    InvalidLogtableError,
+)
 
 
 class Cfg(ABC):
@@ -362,8 +365,10 @@ def __init__(
         self.codecarbon_configuration = codecarbon_configuration
 
     @staticmethod
-    def extract_config(config_path: str, logger: logging.Logger) -> "PyExperimenterCfg":
+    def extract_config(config_path: str, logger: logging.Logger, overwritten_table_name: Optional[str] = None) -> "PyExperimenterCfg":
         config = omegaconf.OmegaConf.load(config_path)
+        if overwritten_table_name is not None:
+            config["PY_EXPERIMENTER"]["Database"]["table"]["name"] = overwritten_table_name
 
         if "n_jobs" not in config["PY_EXPERIMENTER"]:
             config["PY_EXPERIMENTER"]["n_jobs"] = 1

poetry.lock

@@ -390,15 +390,17 @@ def delete_table(self) -> None:
         self.commit(connection)
         self.close_connection(connection)
 
-    def get_logtable(self, logtable_name: str) -> pd.DataFrame:
-        return self.get_table(f"{self.database_configuration.table_name}__{logtable_name}")
+    def get_logtable(self, logtable_name: str, condition = Optional[str]) -> pd.DataFrame:
+        return self.get_table(f"{self.database_configuration.table_name}__{logtable_name}", condition)
 
     def get_codecarbon_table(self) -> pd.DataFrame:
         return self.get_table(f"{self.database_configuration.table_name}_codecarbon")
 
-    def get_table(self, table_name: Optional[str] = None) -> pd.DataFrame:
+    def get_table(self, table_name: Optional[str] = None, condition:Optional[str] = None) -> pd.DataFrame:
         connection = self.connect()
         query = f"SELECT * FROM {self.database_configuration.table_name}" if table_name is None else f"SELECT * FROM {table_name}"
+        if condition:
+            query += f" WHERE {condition}"
         # suppress warning for pandas
         import warnings
 

py_experimenter/config.py

@@ -96,7 +96,7 @@ def __init__(
             handler.setFormatter(formatter)
             self.logger.addHandler(handler)
 
-        self.config = PyExperimenterCfg.extract_config(experiment_configuration_file_path, logger=self.logger)
+        self.config = PyExperimenterCfg.extract_config(experiment_configuration_file_path, logger=self.logger, overwritten_table_name=table_name)
 
         self.use_codecarbon = use_codecarbon
 
@@ -109,8 +109,6 @@ def __init__(
         if use_ssh_tunnel is not None:
             self.config.database_configuration.use_ssh_tunnel = use_ssh_tunnel
 
-        if table_name is not None:
-            self.config.database_configuration.table_name = table_name
         if database_name is not None:
             self.config.database_configuration.database_name = database_name
         self.name = name
@@ -494,16 +492,20 @@ def delete_table(self) -> None:
         """
         self.db_connector.delete_table()
 
-    def get_table(self) -> pd.DataFrame:
+    def get_table(self, condition:Optional[str] = None) -> pd.DataFrame:
         """
         Returns the database table as `Pandas.DataFrame`.
 
+        :param condition: The condition to filter the table in sql syntax. The condition is added as a where clause.
+          If None, the whole table is returned.
+        :type condition: str
+
         :return: The database table as `Pandas.DataFrame`.
         :rtype: pd.DataFrame
         """
         return self.db_connector.get_table()
 
-    def get_logtable(self, logtable_name: str) -> pd.DataFrame:
+    def get_logtable(self, logtable_name: str, condition:Optional[str] = None) -> pd.DataFrame:
         """
         Returns the log table as `Pandas.DataFrame`.
 
@@ -512,7 +514,7 @@ def get_logtable(self, logtable_name: str) -> pd.DataFrame:
         :return: The log table as `Pandas.DataFrame`.
         :rtype: pd.DataFrame
         """
-        return self.db_connector.get_logtable(logtable_name)
+        return self.db_connector.get_logtable(logtable_name, condition)
 
     def get_codecarbon_table(self) -> pd.DataFrame:
         """

py_experimenter/database_connector.py

@@ -37,6 +37,7 @@ codecarbon = ">=2.2.1"
 pymysql = "^1.0.3"
 omegaconf = "^2.3.0"
 sshtunnel = "^0.4.0"
+setuptools = "^69.5.1"
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.0"

py_experimenter/experimenter.py

@@ -10,9 +10,9 @@
 
 @pytest.fixture
 def experimenter():
-    configuration_path = os.path.join("test", "test_codecarbon", "configs", "integration_test_sqlite.yml")
+    configuration_path = os.path.join("test", "test_codecarbon", "configs", "integration_test_mysql.yml")
 
-    return PyExperimenter(configuration_path)
+    return PyExperimenter(configuration_path, use_ssh_tunnel=True)
 
 
 def run_ml(parameters: dict, result_processor: ResultProcessor, custom_config: dict):