From 40f9c9b36a70673a85806977153389a725a38807 Mon Sep 17 00:00:00 2001 From: Mike Doran Date: Mon, 30 Aug 2021 13:29:45 +0100 Subject: [PATCH 1/6] version correction --- MANIFEST | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 MANIFEST diff --git a/MANIFEST b/MANIFEST new file mode 100644 index 0000000..1bf2332 --- /dev/null +++ b/MANIFEST @@ -0,0 +1,6 @@ +# file GENERATED by distutils, do NOT edit +setup.cfg +setup.py +easy_postgres_engine/__init__.py +easy_postgres_engine/postgres_engine.py +easy_postgres_engine/retry_decorator.py diff --git a/setup.py b/setup.py index e579a2d..17f1f3c 100644 --- a/setup.py +++ b/setup.py @@ -20,5 +20,5 @@ "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", ], - python_requires='>=3.9' + python_requires='>=3.8' ) From 6c6e68d32538716eaeda2189cdd85fd2ae80a5fa Mon Sep 17 00:00:00 2001 From: Mike Doran Date: Tue, 14 Sep 2021 16:17:20 +0100 Subject: [PATCH 2/6] handing nan better and config update --- .flake8 | 10 ++++ .pre-commit-config.yaml | 15 ++++++ easy_postgres_engine/postgres_engine.py | 46 +++++++++++-------- easy_postgres_engine/retry_decorator.py | 5 +- .../tests/test_postgres_engine.py | 46 +++++++++---------- easy_postgres_engine/utils/__init__.py | 0 .../utils/dataframe_functions.py | 7 +++ pyproject.toml | 4 ++ requirements.txt | 4 ++ setup.py | 26 +++++------ 10 files changed, 103 insertions(+), 60 deletions(-) create mode 100644 .flake8 create mode 100644 .pre-commit-config.yaml create mode 100644 easy_postgres_engine/utils/__init__.py create mode 100644 easy_postgres_engine/utils/dataframe_functions.py create mode 100644 pyproject.toml diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..a467e27 --- /dev/null +++ b/.flake8 @@ -0,0 +1,10 @@ +[flake8] +ignore = E501, E203, W503 +exclude = + .git, + __pycache__, + build, + dist, + scripts, + tests +max-complexity = 10 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..c124238 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,15 @@ + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: detect-private-key + - id: requirements-txt-fixer + - repo: https://github.com/psf/black + rev: 21.5b2 + hooks: + - id: black diff --git a/easy_postgres_engine/postgres_engine.py b/easy_postgres_engine/postgres_engine.py index c015755..398d7dc 100644 --- a/easy_postgres_engine/postgres_engine.py +++ b/easy_postgres_engine/postgres_engine.py @@ -1,15 +1,16 @@ import logging +from typing import Any, Dict, Optional, Union import pandas as pd import psycopg2 import psycopg2.extras from .retry_decorator import retry +from .utils.dataframe_functions import replace_nan_with_none_in_dataframe class PostgresEngine: - - def __init__(self, databaseName: str, user: str, password: str, host: str = 'localhost', port: int = 5432): + def __init__(self, databaseName: str, user: str, password: str, host: str = "localhost", port: int = 5432): """ Class for accessing Postgres databases more easily. @@ -27,72 +28,79 @@ def __init__(self, databaseName: str, user: str, password: str, host: str = 'loc self.connection = None self.cursor = None - def _get_connection(self): + def _get_connection(self) -> None: try: - self.connection = psycopg2.connect(user=self.user, password=self.password, host=self.host, port=self.port, database=self.databaseName) + self.connection = psycopg2.connect( + user=self.user, password=self.password, host=self.host, port=self.port, database=self.databaseName + ) except Exception as ex: - logging.exception(f'Error connecting to PostgreSQL {ex}') + logging.exception(f"Error connecting to PostgreSQL {ex}") raise ex - def _get_cursor(self, isInsertionQuery: bool): + def _get_cursor(self, isInsertionQuery: bool) -> None: if isInsertionQuery: self.cursor = self.connection.cursor() else: self.cursor = self.connection.cursor(cursor_factory=psycopg2.extras.RealDictCursor) - def _close_connection(self): + def _close_connection(self) -> None: self.connection.close() - def _close_cursor(self): + def _close_cursor(self) -> None: self.cursor.close() - def close(self): + def close(self) -> None: if self.connection is not None: self._close_connection() if self.cursor is not None: self._close_cursor() - def create_table(self, schema: str): + def create_table(self, schema: str) -> None: self._get_connection() self._get_cursor(isInsertionQuery=True) self.cursor.execute(schema) try: self.connection.commit() except Exception as ex: - logging.exception(f'error: {ex} \nschemaQuery: {schema}') + logging.exception(f"error: {ex} \nschemaQuery: {schema}") raise ex finally: self.close() - def create_index(self, tableName: str, column: str): + def create_index(self, tableName: str, column: str) -> None: self._get_connection() self._get_cursor(isInsertionQuery=True) - indexQuery = f'CREATE INDEX IF NOT EXISTS {tableName}_{column} ON {tableName}({column});' + indexQuery = f"CREATE INDEX IF NOT EXISTS {tableName}_{column} ON {tableName}({column});" self.cursor.execute(indexQuery) try: self.connection.commit() except Exception as ex: - logging.exception(f'error: {ex} \nindexQuery: {indexQuery}') + logging.exception(f"error: {ex} \nindexQuery: {indexQuery}") raise ex finally: self.close() @retry(numRetries=5, retryDelaySeconds=3, backoffScalingFactor=2) - def run_select_query(self, query: str, parameters: dict = None): + def run_select_query_with_retry(self, query: str, parameters: Optional[Dict[str, Any]] = None) -> pd.DataFrame: + return self.run_select_query(query=query, parameters=parameters) + + def run_select_query(self, query: str, parameters: Optional[Dict[str, Any]] = None) -> pd.DataFrame: self._get_connection() self._get_cursor(isInsertionQuery=False) self.cursor.execute(query, parameters) outputs = self.cursor.fetchall() self.close() outputDataframe = pd.DataFrame(outputs) - return outputDataframe.where(outputDataframe.notnull(), None).dropna(axis=0, how='all') + return replace_nan_with_none_in_dataframe(dataframe=outputDataframe) @retry(numRetries=5, retryDelaySeconds=3, backoffScalingFactor=2) - def run_update_query(self, query: str, parameters: dict = None, returnId: bool = True): + def run_update_query( + self, query: str, parameters: Optional[Dict[str, Any]] = None, returnId: bool = True + ) -> Union[None, int]: self._get_connection() self._get_cursor(isInsertionQuery=True) if returnId: - query = f'{query}\nRETURNING id' + query = f"{query}\nRETURNING id" self.cursor.execute(query, parameters) if returnId: insertedId = self.cursor.fetchone()[0] @@ -101,7 +109,7 @@ def run_update_query(self, query: str, parameters: dict = None, returnId: bool = try: self.connection.commit() except Exception as ex: - logging.exception(f'error: {ex} \nquery: {query} \nparameters: {parameters}') + logging.exception(f"error: {ex} \nquery: {query} \nparameters: {parameters}") raise ex finally: self.close() diff --git a/easy_postgres_engine/retry_decorator.py b/easy_postgres_engine/retry_decorator.py index f1c552c..27c2797 100644 --- a/easy_postgres_engine/retry_decorator.py +++ b/easy_postgres_engine/retry_decorator.py @@ -4,7 +4,6 @@ def retry(numRetries: int = 5, retryDelaySeconds: int = 3, backoffScalingFactor: int = 2): - def retry_decorator(func): @wraps(func) def retry_function(*args, **kwargs): @@ -13,11 +12,13 @@ def retry_function(*args, **kwargs): try: return func(*args, **kwargs) except Exception as ex: - exceptionMessage = f'{ex}, Retrying in {currentDelay} seconds...' + exceptionMessage = f"{ex}, Retrying in {currentDelay} seconds..." logging.warning(exceptionMessage) sleep(currentDelay) numTries -= 1 currentDelay *= backoffScalingFactor return func(*args, **kwargs) + return retry_function + return retry_decorator diff --git a/easy_postgres_engine/tests/test_postgres_engine.py b/easy_postgres_engine/tests/test_postgres_engine.py index bbc7889..c690a55 100644 --- a/easy_postgres_engine/tests/test_postgres_engine.py +++ b/easy_postgres_engine/tests/test_postgres_engine.py @@ -1,18 +1,18 @@ import testing.postgresql from unittest import TestCase -from easy_postgres_engine.tests.test_table_schema import TEST_TABLE_SCHEMA -from easy_postgres_engine.postgres_engine import PostgresEngine +from .test_table_schema import TEST_TABLE_SCHEMA +from ..postgres_engine import PostgresEngine def create_table(postgresqlConnection): config = postgresqlConnection.dsn() dbEngine = PostgresEngine( - databaseName=config['database'], - user=config['user'], - password=config.get('password'), - port=config['port'], - host=config['host'] + databaseName=config["database"], + user=config["user"], + password=config.get("password"), + port=config["port"], + host=config["host"], ) dbEngine.create_table(schema=TEST_TABLE_SCHEMA) @@ -27,24 +27,23 @@ def tearDownModule(): class TestPostgresEngine(TestCase): - - def __init__(self, methodName='runTest'): + def __init__(self, methodName="runTest"): super(TestPostgresEngine, self).__init__(methodName=methodName) self.firstCustomerId = 10 - self.firstCustomerName = 'Mary' + self.firstCustomerName = "Mary" self.secondCustomerId = 50 - self.secondCustomerName = 'John' + self.secondCustomerName = "John" def setUp(self): super().setUp() self.postgresql = Postgresql() config = self.postgresql.dsn() self.dbEngine = PostgresEngine( - databaseName=config['database'], - user=config['user'], - password=config.get('password'), - port=config['port'], - host=config['host'] + databaseName=config["database"], + user=config["user"], + password=config.get("password"), + port=config["port"], + host=config["host"], ) def tearDown(self): @@ -60,19 +59,18 @@ def test_engine(self): (%(customerId)s, %(customerName)s) """ insertedId1 = self.dbEngine.run_update_query( - query=TEST_INSERTION_QUERY, - parameters={'customerId': self.firstCustomerId, 'customerName': self.firstCustomerName} + query=TEST_INSERTION_QUERY, parameters={"customerId": self.firstCustomerId, "customerName": self.firstCustomerName} ) self.assertEqual(insertedId1, 1) insertedId2 = self.dbEngine.run_update_query( query=TEST_INSERTION_QUERY, - parameters={'customerId': self.secondCustomerId, 'customerName': self.secondCustomerName} + parameters={"customerId": self.secondCustomerId, "customerName": self.secondCustomerName}, ) self.assertEqual(insertedId2, 2) - queryResults = self.dbEngine.run_select_query(query='SELECT * FROM tbl_example') - self.assertSequenceEqual(list(queryResults['customer_id'].values), [self.firstCustomerId, self.secondCustomerId]) - self.assertSequenceEqual(list(queryResults['customer_name'].values), [self.firstCustomerName, self.secondCustomerName]) + queryResults = self.dbEngine.run_select_query(query="SELECT * FROM tbl_example") + self.assertSequenceEqual(list(queryResults["customer_id"].values), [self.firstCustomerId, self.secondCustomerId]) + self.assertSequenceEqual(list(queryResults["customer_name"].values), [self.firstCustomerName, self.secondCustomerName]) specificQueryResults = self.dbEngine.run_select_query( query=""" @@ -83,7 +81,7 @@ def test_engine(self): WHERE customer_id = %(customerId)s """, - parameters={'customerId': self.firstCustomerId} + parameters={"customerId": self.firstCustomerId}, ) self.assertEqual(len(specificQueryResults), 1) - self.assertEqual(specificQueryResults['customer_name'].iloc[0], self.firstCustomerName) + self.assertEqual(specificQueryResults["customer_name"].iloc[0], self.firstCustomerName) diff --git a/easy_postgres_engine/utils/__init__.py b/easy_postgres_engine/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/easy_postgres_engine/utils/dataframe_functions.py b/easy_postgres_engine/utils/dataframe_functions.py new file mode 100644 index 0000000..6295715 --- /dev/null +++ b/easy_postgres_engine/utils/dataframe_functions.py @@ -0,0 +1,7 @@ +import numpy as np +import pandas as pd + + +def replace_nan_with_none_in_dataframe(dataframe: pd.DataFrame) -> pd.DataFrame: + dataframe = dataframe.where(dataframe.notnull(), None).dropna(axis=0, how="all") + return dataframe.replace({np.nan: None}) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b021620 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,4 @@ +[tool.black] +line-length = 127 +target-version = ['py36', 'py37', 'py38'] +include = '\.pyi?$' diff --git a/requirements.txt b/requirements.txt index 2cd4853..e7982e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ +black==21.5b2 +flake8==3.9.2 +nose2[coverage_plugin]==0.10.0 +numpy==1.19.2 pandas==1.3.2 psycopg2-binary==2.9.1 testing.postgresql==1.3.0 diff --git a/setup.py b/setup.py index 17f1f3c..6fa1ace 100644 --- a/setup.py +++ b/setup.py @@ -1,24 +1,20 @@ from distutils.core import setup setup( - name='easy_postgres_engine', - packages=['easy_postgres_engine'], - version='0.2', - description='Engine class for easier connections to postgres databases', - author='Michael Doran', - author_email='mikrdoran@gmail.com', - url='https://github.com/miksyr/easy_postgres_engine', - download_url='https://github.com/miksyr/easy_postgres_engine/archive/v_02.tar.gz', - keywords=['postgreSQL', 'postgres'], - install_requires=[ - 'pandas==1.3.2', - 'psycopg2-binary==2.9.1', - 'testing.postgresql==1.3.0' - ], + name="easy_postgres_engine", + packages=["easy_postgres_engine"], + version="0.2", + description="Engine class for easier connections to postgres databases", + author="Michael Doran", + author_email="mikrdoran@gmail.com", + url="https://github.com/miksyr/easy_postgres_engine", + download_url="https://github.com/miksyr/easy_postgres_engine/archive/v_02.tar.gz", + keywords=["postgreSQL", "postgres"], + install_requires=["pandas==1.3.2", "psycopg2-binary==2.9.1", "testing.postgresql==1.3.0"], classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", ], - python_requires='>=3.8' + python_requires=">=3.8", ) From 34dbc75474e6d23470c035b853f28f98d6b788a2 Mon Sep 17 00:00:00 2001 From: Mike Doran Date: Tue, 14 Sep 2021 16:18:47 +0100 Subject: [PATCH 3/6] requirements update --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6fa1ace..3932bc3 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ url="https://github.com/miksyr/easy_postgres_engine", download_url="https://github.com/miksyr/easy_postgres_engine/archive/v_02.tar.gz", keywords=["postgreSQL", "postgres"], - install_requires=["pandas==1.3.2", "psycopg2-binary==2.9.1", "testing.postgresql==1.3.0"], + install_requires=["pandas==1.3.2", "psycopg2-binary==2.9.1", "testing.postgresql==1.3.0", "numpy==1.19.2"], classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", From 892ec8afb2f54a17c1afe161a6690c807b8fa8fd Mon Sep 17 00:00:00 2001 From: Mike Doran Date: Tue, 14 Sep 2021 16:27:03 +0100 Subject: [PATCH 4/6] bug fix for nan handling --- easy_postgres_engine/postgres_engine.py | 7 ++++++- easy_postgres_engine/utils/__init__.py | 0 easy_postgres_engine/utils/dataframe_functions.py | 7 ------- 3 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 easy_postgres_engine/utils/__init__.py delete mode 100644 easy_postgres_engine/utils/dataframe_functions.py diff --git a/easy_postgres_engine/postgres_engine.py b/easy_postgres_engine/postgres_engine.py index 398d7dc..aa1045d 100644 --- a/easy_postgres_engine/postgres_engine.py +++ b/easy_postgres_engine/postgres_engine.py @@ -1,12 +1,17 @@ import logging from typing import Any, Dict, Optional, Union +import numpy as np import pandas as pd import psycopg2 import psycopg2.extras from .retry_decorator import retry -from .utils.dataframe_functions import replace_nan_with_none_in_dataframe + + +def replace_nan_with_none_in_dataframe(dataframe: pd.DataFrame) -> pd.DataFrame: + dataframe = dataframe.where(dataframe.notnull(), None).dropna(axis=0, how="all") + return dataframe.replace({np.nan: None}) class PostgresEngine: diff --git a/easy_postgres_engine/utils/__init__.py b/easy_postgres_engine/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/easy_postgres_engine/utils/dataframe_functions.py b/easy_postgres_engine/utils/dataframe_functions.py deleted file mode 100644 index 6295715..0000000 --- a/easy_postgres_engine/utils/dataframe_functions.py +++ /dev/null @@ -1,7 +0,0 @@ -import numpy as np -import pandas as pd - - -def replace_nan_with_none_in_dataframe(dataframe: pd.DataFrame) -> pd.DataFrame: - dataframe = dataframe.where(dataframe.notnull(), None).dropna(axis=0, how="all") - return dataframe.replace({np.nan: None}) From ce43197dc0a6366c0272fea80d3151542352d511 Mon Sep 17 00:00:00 2001 From: Mike Doran Date: Wed, 27 Apr 2022 19:07:43 +0100 Subject: [PATCH 5/6] requirements bump --- requirements.txt | 14 +++++++------- setup.py | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index e7982e7..f2b0642 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -black==21.5b2 -flake8==3.9.2 -nose2[coverage_plugin]==0.10.0 -numpy==1.19.2 -pandas==1.3.2 -psycopg2-binary==2.9.1 -testing.postgresql==1.3.0 +black>=21.5b2 +flake8>=3.9.2 +nose2[coverage_plugin]>=0.10.0 +numpy>=1.19.2 +pandas>=1.3.2 +psycopg2-binary>=2.9.1 +testing.postgresql>=1.3.0 diff --git a/setup.py b/setup.py index 3932bc3..b96a931 100644 --- a/setup.py +++ b/setup.py @@ -3,14 +3,14 @@ setup( name="easy_postgres_engine", packages=["easy_postgres_engine"], - version="0.2", + version="0.2.1", description="Engine class for easier connections to postgres databases", author="Michael Doran", author_email="mikrdoran@gmail.com", url="https://github.com/miksyr/easy_postgres_engine", - download_url="https://github.com/miksyr/easy_postgres_engine/archive/v_02.tar.gz", + download_url="https://github.com/miksyr/easy_postgres_engine/archive/v_03.tar.gz", keywords=["postgreSQL", "postgres"], - install_requires=["pandas==1.3.2", "psycopg2-binary==2.9.1", "testing.postgresql==1.3.0", "numpy==1.19.2"], + install_requires=["pandas>=1.3.2", "psycopg2-binary>=2.9.1", "testing.postgresql>=1.3.0", "numpy>=1.19.2"], classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", From eda125d9e9755a3b4124226fe9b08c22921fad01 Mon Sep 17 00:00:00 2001 From: Mike Doran Date: Fri, 15 Jul 2022 13:28:46 +0100 Subject: [PATCH 6/6] updated exception handling --- easy_postgres_engine/postgres_engine.py | 10 +++++----- setup.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/easy_postgres_engine/postgres_engine.py b/easy_postgres_engine/postgres_engine.py index aa1045d..0d1b581 100644 --- a/easy_postgres_engine/postgres_engine.py +++ b/easy_postgres_engine/postgres_engine.py @@ -106,12 +106,12 @@ def run_update_query( self._get_cursor(isInsertionQuery=True) if returnId: query = f"{query}\nRETURNING id" - self.cursor.execute(query, parameters) - if returnId: - insertedId = self.cursor.fetchone()[0] - else: - insertedId = None try: + self.cursor.execute(query, parameters) + if returnId: + insertedId = self.cursor.fetchone()[0] + else: + insertedId = None self.connection.commit() except Exception as ex: logging.exception(f"error: {ex} \nquery: {query} \nparameters: {parameters}") diff --git a/setup.py b/setup.py index b96a931..f1f10a0 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="easy_postgres_engine", packages=["easy_postgres_engine"], - version="0.2.1", + version="0.2.4", description="Engine class for easier connections to postgres databases", author="Michael Doran", author_email="mikrdoran@gmail.com",