From 6e19eddb66f705168eee832ce77bf431f7bcb592 Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Wed, 6 Mar 2024 14:26:48 +0100 Subject: [PATCH 01/16] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Changed=20credential?= =?UTF-8?q?s=20logic=20to=20get=20credentials=20from=20KeyVault?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/sap_rfc_to_adls.py | 8 +++++++ viadot/sources/sap_rfc.py | 38 +++++++++++++++++++-------------- viadot/tasks/sap_rfc.py | 37 ++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 16 deletions(-) diff --git a/viadot/flows/sap_rfc_to_adls.py b/viadot/flows/sap_rfc_to_adls.py index d7a2ac390..e8d917581 100644 --- a/viadot/flows/sap_rfc_to_adls.py +++ b/viadot/flows/sap_rfc_to_adls.py @@ -17,6 +17,8 @@ def __init__( rfc_total_col_width_character_limit: int = 400, rfc_unique_id: List[str] = None, sap_credentials: dict = None, + saprfc_credentials_key: str = "SAP", + env: str = "PROD", output_file_extension: str = ".parquet", local_file_path: str = None, file_sep: str = "\t", @@ -67,6 +69,8 @@ def __init__( ... ) sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. + saprfc_credentials_key (str, optional): Azure KV secret. Defaults to "SAP". + env (str, optional): SAP environment. Defaults to "PROD". output_file_extension (str, optional): Output file extension - to allow selection of .csv for data which is not easy to handle with parquet. Defaults to ".parquet". local_file_path (str, optional): Local destination path. Defaults to None. file_sep(str, optional): The separator to use in the CSV. Defaults to "\t". @@ -91,6 +95,8 @@ def __init__( self.rfc_total_col_width_character_limit = rfc_total_col_width_character_limit self.rfc_unique_id = rfc_unique_id self.sap_credentials = sap_credentials + self.saprfc_credentials_key = saprfc_credentials_key + self.env = env self.output_file_extension = output_file_extension self.local_file_path = local_file_path self.file_sep = file_sep @@ -121,6 +127,8 @@ def gen_flow(self) -> Flow: rfc_unique_id=self.rfc_unique_id, alternative_version=self.alternative_version, credentials=self.sap_credentials, + saprfc_credentials_key=self.saprfc_credentials_key, + env=self.env, flow=self, ) if self.validate_df_dict: diff --git a/viadot/sources/sap_rfc.py b/viadot/sources/sap_rfc.py index a9d109148..7e6f6d7a2 100644 --- a/viadot/sources/sap_rfc.py +++ b/viadot/sources/sap_rfc.py @@ -256,16 +256,17 @@ def __init__( """ self._con = None - DEFAULT_CREDENTIALS = local_config.get("SAP").get("DEV") + saprfc_credentials_key = kwargs.pop("saprfc_credentials_key") + env = kwargs.pop("env") credentials = kwargs.pop("credentials", None) if credentials is None: - credentials = DEFAULT_CREDENTIALS - if credentials is None: - raise CredentialError("Missing credentials.") logger.warning( - "Your credentials will use DEV environment. If you would like to use different one - please specified it." + f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in sap_credentials parameter." ) + credentials = local_config.get(saprfc_credentials_key).get(env) + if credentials is None: + raise CredentialError(f"Missing {env} credentials!") super().__init__(*args, credentials=credentials, **kwargs) @@ -455,9 +456,11 @@ def _get_columns(self, sql: str, aliased: bool = False) -> List[str]: self.aliases_keyed_by_columns = aliases_keyed_by_columns columns = [ - aliases_keyed_by_columns[col] - if col in aliases_keyed_by_columns - else col + ( + aliases_keyed_by_columns[col] + if col in aliases_keyed_by_columns + else col + ) for col in columns ] @@ -699,16 +702,17 @@ def __init__( """ self._con = None - DEFAULT_CREDENTIALS = local_config.get("SAP").get("DEV") + saprfc_credentials_key = kwargs.pop("saprfc_credentials_key") + env = kwargs.pop("env") credentials = kwargs.pop("credentials", None) if credentials is None: - credentials = DEFAULT_CREDENTIALS - if credentials is None: - raise CredentialError("Missing credentials.") logger.warning( - "Your credentials will use DEV environment. If you would like to use different one - please specified it in 'sap_credentials' variable inside the flow." + f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in sap_credentials parameter." ) + credentials = local_config.get(saprfc_credentials_key).get(env) + if credentials is None: + raise CredentialError(f"Missing {env} credentials!") super().__init__(*args, credentials=credentials, **kwargs) @@ -904,9 +908,11 @@ def _get_columns(self, sql: str, aliased: bool = False) -> List[str]: self.aliases_keyed_by_columns = aliases_keyed_by_columns columns = [ - aliases_keyed_by_columns[col] - if col in aliases_keyed_by_columns - else col + ( + aliases_keyed_by_columns[col] + if col in aliases_keyed_by_columns + else col + ) for col in columns ] diff --git a/viadot/tasks/sap_rfc.py b/viadot/tasks/sap_rfc.py index 19d55353d..c70becf2d 100644 --- a/viadot/tasks/sap_rfc.py +++ b/viadot/tasks/sap_rfc.py @@ -2,13 +2,18 @@ from typing import List import pandas as pd +import json from prefect import Task from prefect.utilities.tasks import defaults_from_attrs +from viadot.tasks import AzureKeyVaultSecret try: from viadot.sources import SAPRFC, SAPRFCV2 except ImportError: raise +from prefect.utilities import logging + +logger = logging.get_logger() class SAPRFCToDF(Task): @@ -20,6 +25,8 @@ def __init__( func: str = None, rfc_total_col_width_character_limit: int = 400, credentials: dict = None, + saprfc_credentials_key: str = "SAP", + env: str = "PROD", max_retries: int = 3, retry_delay: timedelta = timedelta(seconds=10), timeout: int = 3600, @@ -52,12 +59,16 @@ def __init__( 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. credentials (dict, optional): The credentials to use to authenticate with SAP. + saprfc_credentials_key (str, optional): Azure KV secret. Defaults to "SAP". + env (str, optional): SAP environment. Defaults to "PROD". By default, they're taken from the local viadot config. """ self.query = query self.sep = sep self.replacement = replacement self.credentials = credentials + self.saprfc_credentials_key = saprfc_credentials_key + self.env = env self.func = func self.rfc_total_col_width_character_limit = rfc_total_col_width_character_limit @@ -84,6 +95,8 @@ def run( sep: str = None, replacement: str = "-", credentials: dict = None, + saprfc_credentials_key: str = "SAP", + env: str = "PROD", func: str = None, rfc_total_col_width_character_limit: int = None, rfc_unique_id: List[str] = None, @@ -97,6 +110,9 @@ def run( multiple options are automatically tried. Defaults to None. replacement (str, optional): In case of sep is on a columns, set up a new character to replace inside the string to avoid flow breakdowns. Defaults to "-". + credentials (dict, optional): The credentials to use to authenticate with SAP. + saprfc_credentials_key (str, optional): Azure KV secret. Defaults to "SAP". + env (str, optional): SAP environment. Defaults to "PROD". func (str, optional): SAP RFC function to use. Defaults to None. rfc_total_col_width_character_limit (int, optional): Number of characters by which query will be split in chunks in case of too many columns for RFC function. According to SAP documentation, the limit is @@ -116,6 +132,23 @@ def run( Returns: pd.DataFrame: DataFrame with SAP data. """ + + if isinstance(credentials, dict): + credentials_keys = list(credentials.keys()) + required_credentials_params = ["sysnr", "user", "passwd", "ashost"] + for key in required_credentials_params: + if key not in credentials_keys: + self.logger.warning( + f"Required key '{key}' not found in your 'sap_credentials' dictionary!" + ) + credentials = None + + if credentials is None: + credentials_str = AzureKeyVaultSecret( + secret=saprfc_credentials_key, + ).run() + credentials = json.loads(credentials_str).get(env) + if query is None: raise ValueError("Please provide the query.") @@ -128,6 +161,8 @@ def run( sep=sep, replacement=replacement, credentials=credentials, + saprfc_credentials_key=saprfc_credentials_key, + env=env, func=func, rfc_total_col_width_character_limit=rfc_total_col_width_character_limit, rfc_unique_id=rfc_unique_id, @@ -136,6 +171,8 @@ def run( sap = SAPRFC( sep=sep, credentials=credentials, + saprfc_credentials_key=saprfc_credentials_key, + env=env, func=func, rfc_total_col_width_character_limit=rfc_total_col_width_character_limit, ) From 58ec8fcb67a08ea9e0bbb070cb25f3fce361eb2e Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Wed, 6 Mar 2024 14:34:21 +0100 Subject: [PATCH 02/16] =?UTF-8?q?=F0=9F=9A=80=20Added=20changes=20to=20cha?= =?UTF-8?q?ngelog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c49b99fd7..c022d5dc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Added option for sap_rfc connector to get credentials from Azure KeyVault or directly passing dictionary inside flow. -### Fixed +### Fixed ### Changed From db63d913192a114c179c5db7d185289e4584d000 Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Thu, 14 Mar 2024 13:55:48 +0100 Subject: [PATCH 03/16] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Change=20'query'=20p?= =?UTF-8?q?arameter=20as=20required=20on=20the=20=5F=5Finit=5F=5F=20level?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/sap_rfc_to_adls.py | 2 +- viadot/tasks/sap_rfc.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/viadot/flows/sap_rfc_to_adls.py b/viadot/flows/sap_rfc_to_adls.py index e8d917581..c95b6e181 100644 --- a/viadot/flows/sap_rfc_to_adls.py +++ b/viadot/flows/sap_rfc_to_adls.py @@ -10,7 +10,7 @@ class SAPRFCToADLS(Flow): def __init__( self, name: str, - query: str = None, + query: str, rfc_sep: str = None, rfc_replacement: str = "-", func: str = "RFC_READ_TABLE", diff --git a/viadot/tasks/sap_rfc.py b/viadot/tasks/sap_rfc.py index c70becf2d..58f7008a9 100644 --- a/viadot/tasks/sap_rfc.py +++ b/viadot/tasks/sap_rfc.py @@ -19,7 +19,7 @@ class SAPRFCToDF(Task): def __init__( self, - query: str = None, + query: str, sep: str = None, replacement: str = "-", func: str = None, @@ -91,7 +91,7 @@ def __init__( ) def run( self, - query: str = None, + query: str, sep: str = None, replacement: str = "-", credentials: dict = None, @@ -149,9 +149,6 @@ def run( ).run() credentials = json.loads(credentials_str).get(env) - if query is None: - raise ValueError("Please provide the query.") - if alternative_version is True: if rfc_unique_id: self.logger.warning( From bec218bf2b9f2b579393942315662dc35b38d765 Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Thu, 14 Mar 2024 14:13:49 +0100 Subject: [PATCH 04/16] =?UTF-8?q?=F0=9F=8E=A8=20Changed=20docstring=20for?= =?UTF-8?q?=20parameter=20'saprfc=5Fcredentials=5Fkey'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/sap_rfc_to_adls.py | 2 +- viadot/tasks/sap_rfc.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/viadot/flows/sap_rfc_to_adls.py b/viadot/flows/sap_rfc_to_adls.py index c95b6e181..d60eb2e1a 100644 --- a/viadot/flows/sap_rfc_to_adls.py +++ b/viadot/flows/sap_rfc_to_adls.py @@ -69,7 +69,7 @@ def __init__( ... ) sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - saprfc_credentials_key (str, optional): Azure KV secret. Defaults to "SAP". + saprfc_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". env (str, optional): SAP environment. Defaults to "PROD". output_file_extension (str, optional): Output file extension - to allow selection of .csv for data which is not easy to handle with parquet. Defaults to ".parquet". local_file_path (str, optional): Local destination path. Defaults to None. diff --git a/viadot/tasks/sap_rfc.py b/viadot/tasks/sap_rfc.py index 58f7008a9..56c541d7f 100644 --- a/viadot/tasks/sap_rfc.py +++ b/viadot/tasks/sap_rfc.py @@ -59,7 +59,7 @@ def __init__( 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. credentials (dict, optional): The credentials to use to authenticate with SAP. - saprfc_credentials_key (str, optional): Azure KV secret. Defaults to "SAP". + saprfc_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". env (str, optional): SAP environment. Defaults to "PROD". By default, they're taken from the local viadot config. """ @@ -111,7 +111,7 @@ def run( replacement (str, optional): In case of sep is on a columns, set up a new character to replace inside the string to avoid flow breakdowns. Defaults to "-". credentials (dict, optional): The credentials to use to authenticate with SAP. - saprfc_credentials_key (str, optional): Azure KV secret. Defaults to "SAP". + saprfc_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". env (str, optional): SAP environment. Defaults to "PROD". func (str, optional): SAP RFC function to use. Defaults to None. rfc_total_col_width_character_limit (int, optional): Number of characters by which query will be split in chunks From bdec339732d9b4deb3fdb1a840f5acc3bb9773bc Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Thu, 14 Mar 2024 14:45:53 +0100 Subject: [PATCH 05/16] =?UTF-8?q?=F0=9F=8E=A8=20Change=20'sap=5Fcredential?= =?UTF-8?q?s'=20varable=20name=20and=20docstring?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/sap_rfc_to_adls.py | 2 +- viadot/sources/sap_rfc.py | 20 ++++++++++---------- viadot/tasks/sap_rfc.py | 24 ++++++++++++------------ 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/viadot/flows/sap_rfc_to_adls.py b/viadot/flows/sap_rfc_to_adls.py index d60eb2e1a..38114707c 100644 --- a/viadot/flows/sap_rfc_to_adls.py +++ b/viadot/flows/sap_rfc_to_adls.py @@ -126,7 +126,7 @@ def gen_flow(self) -> Flow: rfc_total_col_width_character_limit=self.rfc_total_col_width_character_limit, rfc_unique_id=self.rfc_unique_id, alternative_version=self.alternative_version, - credentials=self.sap_credentials, + sap_credentials=self.sap_credentials, saprfc_credentials_key=self.saprfc_credentials_key, env=self.env, flow=self, diff --git a/viadot/sources/sap_rfc.py b/viadot/sources/sap_rfc.py index 7e6f6d7a2..4873d568a 100644 --- a/viadot/sources/sap_rfc.py +++ b/viadot/sources/sap_rfc.py @@ -259,16 +259,16 @@ def __init__( saprfc_credentials_key = kwargs.pop("saprfc_credentials_key") env = kwargs.pop("env") - credentials = kwargs.pop("credentials", None) - if credentials is None: + sap_credentials = kwargs.pop("sap_credentials", None) + if sap_credentials is None: logger.warning( f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in sap_credentials parameter." ) - credentials = local_config.get(saprfc_credentials_key).get(env) - if credentials is None: + sap_credentials = local_config.get(saprfc_credentials_key).get(env) + if sap_credentials is None: raise CredentialError(f"Missing {env} credentials!") - super().__init__(*args, credentials=credentials, **kwargs) + super().__init__(*args, credentials=sap_credentials, **kwargs) self.sep = sep self.client_side_filters = None @@ -705,16 +705,16 @@ def __init__( saprfc_credentials_key = kwargs.pop("saprfc_credentials_key") env = kwargs.pop("env") - credentials = kwargs.pop("credentials", None) - if credentials is None: + sap_credentials = kwargs.pop("sap_credentials", None) + if sap_credentials is None: logger.warning( f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in sap_credentials parameter." ) - credentials = local_config.get(saprfc_credentials_key).get(env) - if credentials is None: + sap_credentials = local_config.get(saprfc_credentials_key).get(env) + if sap_credentials is None: raise CredentialError(f"Missing {env} credentials!") - super().__init__(*args, credentials=credentials, **kwargs) + super().__init__(*args, credentials=sap_credentials, **kwargs) self.sep = sep self.replacement = replacement diff --git a/viadot/tasks/sap_rfc.py b/viadot/tasks/sap_rfc.py index 56c541d7f..f79e2e2aa 100644 --- a/viadot/tasks/sap_rfc.py +++ b/viadot/tasks/sap_rfc.py @@ -24,7 +24,7 @@ def __init__( replacement: str = "-", func: str = None, rfc_total_col_width_character_limit: int = 400, - credentials: dict = None, + sap_credentials: dict = None, saprfc_credentials_key: str = "SAP", env: str = "PROD", max_retries: int = 3, @@ -58,7 +58,7 @@ def __init__( in case of too many columns for RFC function. According to SAP documentation, the limit is 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. - credentials (dict, optional): The credentials to use to authenticate with SAP. + sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. saprfc_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". env (str, optional): SAP environment. Defaults to "PROD". By default, they're taken from the local viadot config. @@ -66,7 +66,7 @@ def __init__( self.query = query self.sep = sep self.replacement = replacement - self.credentials = credentials + self.sap_credentials = sap_credentials self.saprfc_credentials_key = saprfc_credentials_key self.env = env self.func = func @@ -94,7 +94,7 @@ def run( query: str, sep: str = None, replacement: str = "-", - credentials: dict = None, + sap_credentials: dict = None, saprfc_credentials_key: str = "SAP", env: str = "PROD", func: str = None, @@ -110,7 +110,7 @@ def run( multiple options are automatically tried. Defaults to None. replacement (str, optional): In case of sep is on a columns, set up a new character to replace inside the string to avoid flow breakdowns. Defaults to "-". - credentials (dict, optional): The credentials to use to authenticate with SAP. + sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. saprfc_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". env (str, optional): SAP environment. Defaults to "PROD". func (str, optional): SAP RFC function to use. Defaults to None. @@ -133,21 +133,21 @@ def run( pd.DataFrame: DataFrame with SAP data. """ - if isinstance(credentials, dict): - credentials_keys = list(credentials.keys()) + if isinstance(sap_credentials, dict): + credentials_keys = list(sap_credentials.keys()) required_credentials_params = ["sysnr", "user", "passwd", "ashost"] for key in required_credentials_params: if key not in credentials_keys: self.logger.warning( f"Required key '{key}' not found in your 'sap_credentials' dictionary!" ) - credentials = None + sap_credentials = None - if credentials is None: + if sap_credentials is None: credentials_str = AzureKeyVaultSecret( secret=saprfc_credentials_key, ).run() - credentials = json.loads(credentials_str).get(env) + sap_credentials = json.loads(credentials_str).get(env) if alternative_version is True: if rfc_unique_id: @@ -157,7 +157,7 @@ def run( sap = SAPRFCV2( sep=sep, replacement=replacement, - credentials=credentials, + sap_credentials=sap_credentials, saprfc_credentials_key=saprfc_credentials_key, env=env, func=func, @@ -167,7 +167,7 @@ def run( else: sap = SAPRFC( sep=sep, - credentials=credentials, + sap_credentials=sap_credentials, saprfc_credentials_key=saprfc_credentials_key, env=env, func=func, From 505196a7a5b15853b4dff301ceaf3c7a12a4ae80 Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Thu, 14 Mar 2024 15:03:47 +0100 Subject: [PATCH 06/16] =?UTF-8?q?=F0=9F=8E=A8=20changed=20variable=20name?= =?UTF-8?q?=20from=20'saprfc=5Fcredential=5Fkey'=20to=20'sap=5Fcredential?= =?UTF-8?q?=5Fkey'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/sap_rfc_to_adls.py | 10 +++++----- viadot/sources/sap_rfc.py | 8 ++++---- viadot/tasks/sap_rfc.py | 16 ++++++++-------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/viadot/flows/sap_rfc_to_adls.py b/viadot/flows/sap_rfc_to_adls.py index 38114707c..a4f65eb3c 100644 --- a/viadot/flows/sap_rfc_to_adls.py +++ b/viadot/flows/sap_rfc_to_adls.py @@ -17,7 +17,7 @@ def __init__( rfc_total_col_width_character_limit: int = 400, rfc_unique_id: List[str] = None, sap_credentials: dict = None, - saprfc_credentials_key: str = "SAP", + sap_credentials_key: str = "SAP", env: str = "PROD", output_file_extension: str = ".parquet", local_file_path: str = None, @@ -69,7 +69,7 @@ def __init__( ... ) sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - saprfc_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". + sap_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". env (str, optional): SAP environment. Defaults to "PROD". output_file_extension (str, optional): Output file extension - to allow selection of .csv for data which is not easy to handle with parquet. Defaults to ".parquet". local_file_path (str, optional): Local destination path. Defaults to None. @@ -95,7 +95,7 @@ def __init__( self.rfc_total_col_width_character_limit = rfc_total_col_width_character_limit self.rfc_unique_id = rfc_unique_id self.sap_credentials = sap_credentials - self.saprfc_credentials_key = saprfc_credentials_key + self.sap_credentials_key = sap_credentials_key self.env = env self.output_file_extension = output_file_extension self.local_file_path = local_file_path @@ -126,8 +126,8 @@ def gen_flow(self) -> Flow: rfc_total_col_width_character_limit=self.rfc_total_col_width_character_limit, rfc_unique_id=self.rfc_unique_id, alternative_version=self.alternative_version, - sap_credentials=self.sap_credentials, - saprfc_credentials_key=self.saprfc_credentials_key, + credentials=self.sap_credentials, + sap_credentials_key=self.sap_credentials_key, env=self.env, flow=self, ) diff --git a/viadot/sources/sap_rfc.py b/viadot/sources/sap_rfc.py index 4873d568a..5a9cf5b2d 100644 --- a/viadot/sources/sap_rfc.py +++ b/viadot/sources/sap_rfc.py @@ -256,7 +256,7 @@ def __init__( """ self._con = None - saprfc_credentials_key = kwargs.pop("saprfc_credentials_key") + sap_credentials_key = kwargs.pop("sap_credentials_key") env = kwargs.pop("env") sap_credentials = kwargs.pop("sap_credentials", None) @@ -264,7 +264,7 @@ def __init__( logger.warning( f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in sap_credentials parameter." ) - sap_credentials = local_config.get(saprfc_credentials_key).get(env) + sap_credentials = local_config.get(sap_credentials_key).get(env) if sap_credentials is None: raise CredentialError(f"Missing {env} credentials!") @@ -702,7 +702,7 @@ def __init__( """ self._con = None - saprfc_credentials_key = kwargs.pop("saprfc_credentials_key") + sap_credentials_key = kwargs.pop("sap_credentials_key") env = kwargs.pop("env") sap_credentials = kwargs.pop("sap_credentials", None) @@ -710,7 +710,7 @@ def __init__( logger.warning( f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in sap_credentials parameter." ) - sap_credentials = local_config.get(saprfc_credentials_key).get(env) + sap_credentials = local_config.get(sap_credentials_key).get(env) if sap_credentials is None: raise CredentialError(f"Missing {env} credentials!") diff --git a/viadot/tasks/sap_rfc.py b/viadot/tasks/sap_rfc.py index f79e2e2aa..1925439df 100644 --- a/viadot/tasks/sap_rfc.py +++ b/viadot/tasks/sap_rfc.py @@ -25,7 +25,7 @@ def __init__( func: str = None, rfc_total_col_width_character_limit: int = 400, sap_credentials: dict = None, - saprfc_credentials_key: str = "SAP", + sap_credentials_key: str = "SAP", env: str = "PROD", max_retries: int = 3, retry_delay: timedelta = timedelta(seconds=10), @@ -59,7 +59,7 @@ def __init__( 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - saprfc_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". + sap_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". env (str, optional): SAP environment. Defaults to "PROD". By default, they're taken from the local viadot config. """ @@ -67,7 +67,7 @@ def __init__( self.sep = sep self.replacement = replacement self.sap_credentials = sap_credentials - self.saprfc_credentials_key = saprfc_credentials_key + self.sap_credentials_key = sap_credentials_key self.env = env self.func = func self.rfc_total_col_width_character_limit = rfc_total_col_width_character_limit @@ -95,7 +95,7 @@ def run( sep: str = None, replacement: str = "-", sap_credentials: dict = None, - saprfc_credentials_key: str = "SAP", + sap_credentials_key: str = "SAP", env: str = "PROD", func: str = None, rfc_total_col_width_character_limit: int = None, @@ -111,7 +111,7 @@ def run( replacement (str, optional): In case of sep is on a columns, set up a new character to replace inside the string to avoid flow breakdowns. Defaults to "-". sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - saprfc_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". + sap_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". env (str, optional): SAP environment. Defaults to "PROD". func (str, optional): SAP RFC function to use. Defaults to None. rfc_total_col_width_character_limit (int, optional): Number of characters by which query will be split in chunks @@ -145,7 +145,7 @@ def run( if sap_credentials is None: credentials_str = AzureKeyVaultSecret( - secret=saprfc_credentials_key, + secret=sap_credentials_key, ).run() sap_credentials = json.loads(credentials_str).get(env) @@ -158,7 +158,7 @@ def run( sep=sep, replacement=replacement, sap_credentials=sap_credentials, - saprfc_credentials_key=saprfc_credentials_key, + sap_credentials_key=sap_credentials_key, env=env, func=func, rfc_total_col_width_character_limit=rfc_total_col_width_character_limit, @@ -168,7 +168,7 @@ def run( sap = SAPRFC( sep=sep, sap_credentials=sap_credentials, - saprfc_credentials_key=saprfc_credentials_key, + sap_credentials_key=sap_credentials_key, env=env, func=func, rfc_total_col_width_character_limit=rfc_total_col_width_character_limit, From 2895eb8f6933e46f78e4e79f859c9b5f842e8d1d Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Thu, 14 Mar 2024 15:08:01 +0100 Subject: [PATCH 07/16] =?UTF-8?q?=F0=9F=8E=A8=20Changed=20env=20default=20?= =?UTF-8?q?value=20'PROD'=20to=20'DEV'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/sap_rfc_to_adls.py | 4 ++-- viadot/tasks/sap_rfc.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/viadot/flows/sap_rfc_to_adls.py b/viadot/flows/sap_rfc_to_adls.py index a4f65eb3c..009296d44 100644 --- a/viadot/flows/sap_rfc_to_adls.py +++ b/viadot/flows/sap_rfc_to_adls.py @@ -18,7 +18,7 @@ def __init__( rfc_unique_id: List[str] = None, sap_credentials: dict = None, sap_credentials_key: str = "SAP", - env: str = "PROD", + env: str = "DEV", output_file_extension: str = ".parquet", local_file_path: str = None, file_sep: str = "\t", @@ -70,7 +70,7 @@ def __init__( ) sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. sap_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". - env (str, optional): SAP environment. Defaults to "PROD". + env (str, optional): SAP environment. Defaults to "DEV". output_file_extension (str, optional): Output file extension - to allow selection of .csv for data which is not easy to handle with parquet. Defaults to ".parquet". local_file_path (str, optional): Local destination path. Defaults to None. file_sep(str, optional): The separator to use in the CSV. Defaults to "\t". diff --git a/viadot/tasks/sap_rfc.py b/viadot/tasks/sap_rfc.py index 1925439df..b2ef7552b 100644 --- a/viadot/tasks/sap_rfc.py +++ b/viadot/tasks/sap_rfc.py @@ -26,7 +26,7 @@ def __init__( rfc_total_col_width_character_limit: int = 400, sap_credentials: dict = None, sap_credentials_key: str = "SAP", - env: str = "PROD", + env: str = "DEV", max_retries: int = 3, retry_delay: timedelta = timedelta(seconds=10), timeout: int = 3600, @@ -60,7 +60,7 @@ def __init__( of characters, so we add a safety margin. Defaults to 400. sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. sap_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". - env (str, optional): SAP environment. Defaults to "PROD". + env (str, optional): SAP environment. Defaults to "DEV". By default, they're taken from the local viadot config. """ self.query = query @@ -96,7 +96,7 @@ def run( replacement: str = "-", sap_credentials: dict = None, sap_credentials_key: str = "SAP", - env: str = "PROD", + env: str = "DEV", func: str = None, rfc_total_col_width_character_limit: int = None, rfc_unique_id: List[str] = None, @@ -112,7 +112,7 @@ def run( inside the string to avoid flow breakdowns. Defaults to "-". sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. sap_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". - env (str, optional): SAP environment. Defaults to "PROD". + env (str, optional): SAP environment. Defaults to "DEV". func (str, optional): SAP RFC function to use. Defaults to None. rfc_total_col_width_character_limit (int, optional): Number of characters by which query will be split in chunks in case of too many columns for RFC function. According to SAP documentation, the limit is From f456ed295f7c2bfada5e011cd8cdce7d0f80118f Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Fri, 15 Mar 2024 08:49:17 +0100 Subject: [PATCH 08/16] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Change=20the=20logic?= =?UTF-8?q?=20of=20passing=20'env'=20,=20'sap=5Fcredentials'=20and=20'sap?= =?UTF-8?q?=5Fcredentias=5Fkey'=20variables=20into=20source=20class?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/sources/sap_rfc.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/viadot/sources/sap_rfc.py b/viadot/sources/sap_rfc.py index 5a9cf5b2d..365fa72f6 100644 --- a/viadot/sources/sap_rfc.py +++ b/viadot/sources/sap_rfc.py @@ -237,6 +237,9 @@ def __init__( sep: str = None, func: str = "RFC_READ_TABLE", rfc_total_col_width_character_limit: int = 400, + sap_credentials: dict = None, + sap_credentials_key: str = "SAP", + env: str = "DEV", *args, **kwargs, ): @@ -250,16 +253,19 @@ def __init__( in case of too many columns for RFC function. According to SAP documentation, the limit is 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. + sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. + sap_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". + env (str, optional): SAP environment. Defaults to "DEV". Raises: CredentialError: If provided credentials are incorrect. """ self._con = None - sap_credentials_key = kwargs.pop("sap_credentials_key") - env = kwargs.pop("env") + self.sap_credentials = sap_credentials + self.sap_credentials_key = sap_credentials_key + self.env = env - sap_credentials = kwargs.pop("sap_credentials", None) if sap_credentials is None: logger.warning( f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in sap_credentials parameter." @@ -680,6 +686,9 @@ def __init__( func: str = "RFC_READ_TABLE", rfc_total_col_width_character_limit: int = 400, rfc_unique_id: List[str] = None, + sap_credentials: dict = None, + sap_credentials_key: str = "SAP", + env: str = "DEV", *args, **kwargs, ): @@ -696,16 +705,19 @@ def __init__( 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. rfc_unique_id (List[str], optional): Reference columns to merge chunks Data Frames. These columns must to be unique. Defaults to None. + sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. + sap_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". + env (str, optional): SAP environment. Defaults to "DEV". Raises: CredentialError: If provided credentials are incorrect. """ self._con = None - sap_credentials_key = kwargs.pop("sap_credentials_key") - env = kwargs.pop("env") + self.sap_credentials = sap_credentials + self.sap_credentials_key = sap_credentials_key + self.env = env - sap_credentials = kwargs.pop("sap_credentials", None) if sap_credentials is None: logger.warning( f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in sap_credentials parameter." From e6a98107aee79bdb629967c1f649a3e01e454bce Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Fri, 15 Mar 2024 09:04:48 +0100 Subject: [PATCH 09/16] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Change=20logic=20of?= =?UTF-8?q?=20checking=20credentials=5Fkeys=20dictionary=20from=20task=20t?= =?UTF-8?q?o=20the=20source=20level?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/sources/sap_rfc.py | 20 ++++++++++++++++++++ viadot/tasks/sap_rfc.py | 10 ---------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/viadot/sources/sap_rfc.py b/viadot/sources/sap_rfc.py index 365fa72f6..261a673bb 100644 --- a/viadot/sources/sap_rfc.py +++ b/viadot/sources/sap_rfc.py @@ -266,6 +266,16 @@ def __init__( self.sap_credentials_key = sap_credentials_key self.env = env + if isinstance(sap_credentials, dict): + credentials_keys = list(sap_credentials.keys()) + required_credentials_params = ["sysnr", "user", "passwd", "ashost"] + for key in required_credentials_params: + if key not in credentials_keys: + self.logger.warning( + f"Required key '{key}' not found in your 'sap_credentials' dictionary!" + ) + sap_credentials = None + if sap_credentials is None: logger.warning( f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in sap_credentials parameter." @@ -718,6 +728,16 @@ def __init__( self.sap_credentials_key = sap_credentials_key self.env = env + if isinstance(sap_credentials, dict): + credentials_keys = list(sap_credentials.keys()) + required_credentials_params = ["sysnr", "user", "passwd", "ashost"] + for key in required_credentials_params: + if key not in credentials_keys: + self.logger.warning( + f"Required key '{key}' not found in your 'sap_credentials' dictionary!" + ) + sap_credentials = None + if sap_credentials is None: logger.warning( f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in sap_credentials parameter." diff --git a/viadot/tasks/sap_rfc.py b/viadot/tasks/sap_rfc.py index b2ef7552b..469b0b1df 100644 --- a/viadot/tasks/sap_rfc.py +++ b/viadot/tasks/sap_rfc.py @@ -133,16 +133,6 @@ def run( pd.DataFrame: DataFrame with SAP data. """ - if isinstance(sap_credentials, dict): - credentials_keys = list(sap_credentials.keys()) - required_credentials_params = ["sysnr", "user", "passwd", "ashost"] - for key in required_credentials_params: - if key not in credentials_keys: - self.logger.warning( - f"Required key '{key}' not found in your 'sap_credentials' dictionary!" - ) - sap_credentials = None - if sap_credentials is None: credentials_str = AzureKeyVaultSecret( secret=sap_credentials_key, From 92c2b7043d80207c8deb3f6c1484b214c86630aa Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Fri, 15 Mar 2024 13:05:59 +0100 Subject: [PATCH 10/16] =?UTF-8?q?=F0=9F=90=9B=20fix=20parameter=20name=20b?= =?UTF-8?q?ug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/sap_rfc_to_adls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viadot/flows/sap_rfc_to_adls.py b/viadot/flows/sap_rfc_to_adls.py index 009296d44..5bc4153fd 100644 --- a/viadot/flows/sap_rfc_to_adls.py +++ b/viadot/flows/sap_rfc_to_adls.py @@ -126,7 +126,7 @@ def gen_flow(self) -> Flow: rfc_total_col_width_character_limit=self.rfc_total_col_width_character_limit, rfc_unique_id=self.rfc_unique_id, alternative_version=self.alternative_version, - credentials=self.sap_credentials, + sap_credentials=self.sap_credentials, sap_credentials_key=self.sap_credentials_key, env=self.env, flow=self, From 2d3ee8d89d6422595a2eda335f50cc251d33afe1 Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Fri, 15 Mar 2024 13:12:33 +0100 Subject: [PATCH 11/16] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactor=20saprfc=20?= =?UTF-8?q?task=20in=20order=20to=20meet=20new=20logic=20in=20source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/tasks/sap_rfc.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/viadot/tasks/sap_rfc.py b/viadot/tasks/sap_rfc.py index 469b0b1df..fb059a467 100644 --- a/viadot/tasks/sap_rfc.py +++ b/viadot/tasks/sap_rfc.py @@ -19,7 +19,7 @@ class SAPRFCToDF(Task): def __init__( self, - query: str, + query: str = None, sep: str = None, replacement: str = "-", func: str = None, @@ -87,7 +87,7 @@ def __init__( "replacement", "func", "rfc_total_col_width_character_limit", - "credentials", + "sap_credentials", ) def run( self, @@ -134,10 +134,15 @@ def run( """ if sap_credentials is None: - credentials_str = AzureKeyVaultSecret( - secret=sap_credentials_key, - ).run() - sap_credentials = json.loads(credentials_str).get(env) + try: + credentials_str = AzureKeyVaultSecret( + secret=sap_credentials_key, + ).run() + sap_credentials = json.loads(credentials_str).get(env) + except: + logger.warning( + f"Getting credentials from Azure Key Vault was not possible. Either there is no key: {sap_credentials_key} or env: {env} or there is not Key Vault in your environment." + ) if alternative_version is True: if rfc_unique_id: From e5ee5b735299c3daf6200e2289a4f1383fb9d197 Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Fri, 15 Mar 2024 13:22:25 +0100 Subject: [PATCH 12/16] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor=20logic=20i?= =?UTF-8?q?n=20source=20regarding=20credentials=20exceptions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/sources/sap_rfc.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/viadot/sources/sap_rfc.py b/viadot/sources/sap_rfc.py index 261a673bb..4a63b44e9 100644 --- a/viadot/sources/sap_rfc.py +++ b/viadot/sources/sap_rfc.py @@ -262,7 +262,6 @@ def __init__( """ self._con = None - self.sap_credentials = sap_credentials self.sap_credentials_key = sap_credentials_key self.env = env @@ -271,7 +270,7 @@ def __init__( required_credentials_params = ["sysnr", "user", "passwd", "ashost"] for key in required_credentials_params: if key not in credentials_keys: - self.logger.warning( + logger.warning( f"Required key '{key}' not found in your 'sap_credentials' dictionary!" ) sap_credentials = None @@ -280,12 +279,21 @@ def __init__( logger.warning( f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in sap_credentials parameter." ) - sap_credentials = local_config.get(sap_credentials_key).get(env) + try: + sap_credentials = local_config.get(sap_credentials_key).get(env) + except AttributeError: + raise CredentialError( + f"Sap_credentials_key: {sap_credentials_key} is not stored neither in KeyVault or Local Config!" + ) if sap_credentials is None: raise CredentialError(f"Missing {env} credentials!") - super().__init__(*args, credentials=sap_credentials, **kwargs) + super().__init__( + *args, + **kwargs, + ) + self.sap_credentials = sap_credentials self.sep = sep self.client_side_filters = None self.func = func @@ -295,7 +303,7 @@ def __init__( def con(self) -> pyrfc.Connection: if self._con is not None: return self._con - con = pyrfc.Connection(**self.credentials) + con = pyrfc.Connection(**self.sap_credentials) self._con = con return con @@ -724,7 +732,6 @@ def __init__( """ self._con = None - self.sap_credentials = sap_credentials self.sap_credentials_key = sap_credentials_key self.env = env @@ -733,7 +740,7 @@ def __init__( required_credentials_params = ["sysnr", "user", "passwd", "ashost"] for key in required_credentials_params: if key not in credentials_keys: - self.logger.warning( + logger.warning( f"Required key '{key}' not found in your 'sap_credentials' dictionary!" ) sap_credentials = None @@ -742,12 +749,18 @@ def __init__( logger.warning( f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in sap_credentials parameter." ) - sap_credentials = local_config.get(sap_credentials_key).get(env) + try: + sap_credentials = local_config.get(sap_credentials_key).get(env) + except AttributeError: + raise CredentialError( + f"Sap_credentials_key: {sap_credentials_key} is not stored neither in KeyVault or Local Config!" + ) if sap_credentials is None: raise CredentialError(f"Missing {env} credentials!") - super().__init__(*args, credentials=sap_credentials, **kwargs) + super().__init__(*args, **kwargs) + self.sap_credentials = sap_credentials self.sep = sep self.replacement = replacement self.client_side_filters = None From 371ddbd684c25dbdb55c6b764eedc44d97a66707 Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Thu, 21 Mar 2024 15:50:30 +0100 Subject: [PATCH 13/16] =?UTF-8?q?=F0=9F=9A=80=20Add=20tests=20for=20new=20?= =?UTF-8?q?logic=20for=20source=20and=20task=20level?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/integration/tasks/test_sap_rfc_to_df.py | 47 ++++++++++- tests/integration/test_sap_rfc.py | 83 ++++++++++++++++++- 2 files changed, 123 insertions(+), 7 deletions(-) diff --git a/tests/integration/tasks/test_sap_rfc_to_df.py b/tests/integration/tasks/test_sap_rfc_to_df.py index 4151adfdc..7007a978c 100644 --- a/tests/integration/tasks/test_sap_rfc_to_df.py +++ b/tests/integration/tasks/test_sap_rfc_to_df.py @@ -1,13 +1,52 @@ +import pytest +import logging + +from viadot.exceptions import CredentialError from viadot.config import local_config from viadot.tasks import SAPRFCToDF def test_sap_rfc_to_df_bbp(): - sap_test_creds = local_config.get("SAP").get("QA") task = SAPRFCToDF( - credentials=sap_test_creds, - query="SELECT MATNR, MATKL, MTART, LAEDA FROM MARA WHERE LAEDA LIKE '2022%'", + query="SELECT MATNR, MATKL, MTART, LAEDA FROM MARA WHERE LAEDA LIKE '20220110%'", func="BBP_RFC_READ_TABLE", ) - df = task.run() + df = task.run(sap_credentials_key="SAP", env="QA") assert len(df.columns) == 4 and not df.empty + + +def test_sap_rfc_to_df_wrong_sap_credential_key_bbp(caplog): + task = SAPRFCToDF( + query="SELECT MATNR, MATKL, MTART, LAEDA FROM MARA WHERE LAEDA LIKE '20220110%'", + func="BBP_RFC_READ_TABLE", + ) + with pytest.raises( + CredentialError, + match="Sap_credentials_key: SAP_test is not stored neither in KeyVault or Local Config!", + ): + task.run( + sap_credentials_key="SAP_test", + ) + assert ( + f"Getting credentials from Azure Key Vault was not possible. Either there is no key: SAP_test or env: DEV or there is not Key Vault in your environment." + in caplog.text + ) + + +def test_sap_rfc_to_df_wrong_env_bbp(caplog): + task = SAPRFCToDF( + query="SELECT MATNR, MATKL, MTART, LAEDA FROM MARA WHERE LAEDA LIKE '20220110%'", + func="BBP_RFC_READ_TABLE", + ) + with pytest.raises( + CredentialError, + match="Missing PROD_test credentials!", + ): + task.run( + sap_credentials_key="SAP", + env="PROD_test", + ) + assert ( + f"Getting credentials from Azure Key Vault was not possible. Either there is no key: SAP or env: PROD_test or there is not Key Vault in your environment." + in caplog.text + ) diff --git a/tests/integration/test_sap_rfc.py b/tests/integration/test_sap_rfc.py index 28ab044a2..2667d4d81 100644 --- a/tests/integration/test_sap_rfc.py +++ b/tests/integration/test_sap_rfc.py @@ -1,6 +1,9 @@ -from collections import OrderedDict +import pytest +import logging +from collections import OrderedDict from viadot.sources import SAPRFC, SAPRFCV2 +from viadot.exceptions import CredentialError sap = SAPRFC() sap2 = SAPRFCV2() @@ -192,7 +195,7 @@ def test___build_pandas_filter_query_v2(): def test_default_credentials_warning_SAPRFC(caplog): _ = SAPRFC() assert ( - "Your credentials will use DEV environment. If you would like to use different one - please specified it." + f"Your credentials will use DEV environment from local config. If you would like to use different one - please specified it in sap_credentials parameter" in caplog.text ) @@ -200,6 +203,80 @@ def test_default_credentials_warning_SAPRFC(caplog): def test_default_credentials_warning_SAPRFCV2(caplog): _ = SAPRFCV2() assert ( - "Your credentials will use DEV environment. If you would like to use different one - please specified it." + f"Your credentials will use DEV environment from local config. If you would like to use different one - please specified it in sap_credentials parameter" + in caplog.text + ) + + +def test_credentials_dictionary_wrong_key_warning_SAPRFC(caplog): + _ = SAPRFC( + sap_credentials={ + "sysnr_test": "test", + "user": "test", + "passwd": "test", + "ashost": "test", + } + ) + assert ( + f"Required key 'sysnr' not found in your 'sap_credentials' dictionary!" + in caplog.text + ) + assert ( + f"Your credentials will use DEV environment from local config. If you would like to use different one - please specified it in sap_credentials parameter" in caplog.text ) + + +def test_credentials_dictionary_wrong_key_warning_SAPRFCV2(caplog): + _ = SAPRFCV2( + sap_credentials={ + "sysnr_test": "test", + "user": "test", + "passwd": "test", + "ashost": "test", + } + ) + assert ( + f"Required key 'sysnr' not found in your 'sap_credentials' dictionary!" + in caplog.text + ) + assert ( + f"Your credentials will use DEV environment from local config. If you would like to use different one - please specified it in sap_credentials parameter" + in caplog.text + ) + + +def test_sap_credentials_key_wrong_value_error_SAPRFC(caplog): + with pytest.raises( + CredentialError, + match="Sap_credentials_key: SAP_test is not stored neither in KeyVault or Local Config!", + ): + with caplog.at_level(logging.ERROR): + _ = SAPRFC(sap_credentials_key="SAP_test") + + +def test_sap_credentials_key_wrong_value_error_SAPRFCV2(caplog): + with pytest.raises( + CredentialError, + match="Sap_credentials_key: SAP_test is not stored neither in KeyVault or Local Config!", + ): + with caplog.at_level(logging.ERROR): + _ = SAPRFC(sap_credentials_key="SAP_test") + + +def test_env_wrong_value_error_SAPRFC(caplog): + with pytest.raises( + CredentialError, + match="Missing PROD_test credentials!", + ): + with caplog.at_level(logging.ERROR): + _ = SAPRFC(env="PROD_test") + + +def test_env_wrong_value_error_SAPRFCV2(caplog): + with pytest.raises( + CredentialError, + match="Missing PROD_test credentials!", + ): + with caplog.at_level(logging.ERROR): + _ = SAPRFC(env="PROD_test") From ad17a3f3c4ad2480b2351d0ce7d98970598ad82e Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Tue, 2 Apr 2024 08:07:47 +0200 Subject: [PATCH 14/16] =?UTF-8?q?=F0=9F=8E=A8=20Changed=20docstrings=20in?= =?UTF-8?q?=20source,=20task=20nad=20flow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/sap_rfc_to_adls.py | 4 ++-- viadot/sources/sap_rfc.py | 8 ++++---- viadot/tasks/sap_rfc.py | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/viadot/flows/sap_rfc_to_adls.py b/viadot/flows/sap_rfc_to_adls.py index 5bc4153fd..d5f171ada 100644 --- a/viadot/flows/sap_rfc_to_adls.py +++ b/viadot/flows/sap_rfc_to_adls.py @@ -69,8 +69,8 @@ def __init__( ... ) sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - sap_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". - env (str, optional): SAP environment. Defaults to "DEV". + sap_credentials_key (str, optional): The key for sap credentials located in the local config for Azure Key Vault. Defaults to "SAP". + env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV" output_file_extension (str, optional): Output file extension - to allow selection of .csv for data which is not easy to handle with parquet. Defaults to ".parquet". local_file_path (str, optional): Local destination path. Defaults to None. file_sep(str, optional): The separator to use in the CSV. Defaults to "\t". diff --git a/viadot/sources/sap_rfc.py b/viadot/sources/sap_rfc.py index 4a63b44e9..df9404c06 100644 --- a/viadot/sources/sap_rfc.py +++ b/viadot/sources/sap_rfc.py @@ -254,8 +254,8 @@ def __init__( 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - sap_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". - env (str, optional): SAP environment. Defaults to "DEV". + sap_credentials_key (str, optional): The key for sap credentials located in the local config for Azure Key Vault. Defaults to "SAP". + env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV". Raises: CredentialError: If provided credentials are incorrect. @@ -724,8 +724,8 @@ def __init__( of characters, so we add a safety margin. Defaults to 400. rfc_unique_id (List[str], optional): Reference columns to merge chunks Data Frames. These columns must to be unique. Defaults to None. sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - sap_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". - env (str, optional): SAP environment. Defaults to "DEV". + sap_credentials_key (str, optional): The key for sap credentials located in the local config for Azure Key Vault. Defaults to "SAP". + env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV". Raises: CredentialError: If provided credentials are incorrect. diff --git a/viadot/tasks/sap_rfc.py b/viadot/tasks/sap_rfc.py index fb059a467..91c754678 100644 --- a/viadot/tasks/sap_rfc.py +++ b/viadot/tasks/sap_rfc.py @@ -59,8 +59,8 @@ def __init__( 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - sap_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". - env (str, optional): SAP environment. Defaults to "DEV". + sap_credentials_key (str, optional): The key for sap credentials located in the local config for Azure Key Vault. Defaults to "SAP". + env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV". By default, they're taken from the local viadot config. """ self.query = query @@ -111,8 +111,8 @@ def run( replacement (str, optional): In case of sep is on a columns, set up a new character to replace inside the string to avoid flow breakdowns. Defaults to "-". sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - sap_credentials_key (str, optional): Local config or Azure KV secret. Defaults to "SAP". - env (str, optional): SAP environment. Defaults to "DEV". + sap_credentials_key (str, optional): The key for sap credentials located in the local config for Azure Key Vault. Defaults to "SAP". + env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV". func (str, optional): SAP RFC function to use. Defaults to None. rfc_total_col_width_character_limit (int, optional): Number of characters by which query will be split in chunks in case of too many columns for RFC function. According to SAP documentation, the limit is From f0d218acec88f58309a482fb7620c8ac42050272 Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Tue, 2 Apr 2024 08:10:12 +0200 Subject: [PATCH 15/16] =?UTF-8?q?=F0=9F=8E=A8=20Changed=20logger.warning?= =?UTF-8?q?=20string=20in=20source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/sources/sap_rfc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/viadot/sources/sap_rfc.py b/viadot/sources/sap_rfc.py index df9404c06..0ebad30f4 100644 --- a/viadot/sources/sap_rfc.py +++ b/viadot/sources/sap_rfc.py @@ -283,7 +283,7 @@ def __init__( sap_credentials = local_config.get(sap_credentials_key).get(env) except AttributeError: raise CredentialError( - f"Sap_credentials_key: {sap_credentials_key} is not stored neither in KeyVault or Local Config!" + f"sap_credentials_key: {sap_credentials_key} is not stored neither in KeyVault or Local Config!" ) if sap_credentials is None: raise CredentialError(f"Missing {env} credentials!") @@ -753,7 +753,7 @@ def __init__( sap_credentials = local_config.get(sap_credentials_key).get(env) except AttributeError: raise CredentialError( - f"Sap_credentials_key: {sap_credentials_key} is not stored neither in KeyVault or Local Config!" + f"sap_credentials_key: {sap_credentials_key} is not stored neither in KeyVault or Local Config!" ) if sap_credentials is None: raise CredentialError(f"Missing {env} credentials!") From 5108fec38ae51f6f63900b14c914a1b8e0f8827d Mon Sep 17 00:00:00 2001 From: adrian-wojcik Date: Thu, 4 Apr 2024 10:02:35 +0200 Subject: [PATCH 16/16] =?UTF-8?q?=F0=9F=8E=A8=20Correct=20docstrings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/sap_rfc_to_adls.py | 4 ++-- viadot/sources/sap_rfc.py | 8 ++++---- viadot/tasks/sap_rfc.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/viadot/flows/sap_rfc_to_adls.py b/viadot/flows/sap_rfc_to_adls.py index d5f171ada..5a380bbaa 100644 --- a/viadot/flows/sap_rfc_to_adls.py +++ b/viadot/flows/sap_rfc_to_adls.py @@ -68,8 +68,8 @@ def __init__( rfc_unique_id=["VBELN", "LPRIO"], ... ) - sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - sap_credentials_key (str, optional): The key for sap credentials located in the local config for Azure Key Vault. Defaults to "SAP". + sap_credentials (dict, optional): The credentials to use to authenticate with SAP. Defaults to None. + sap_credentials_key (str, optional): The key for sap credentials located in the local config or Azure Key Vault. Defaults to "SAP". env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV" output_file_extension (str, optional): Output file extension - to allow selection of .csv for data which is not easy to handle with parquet. Defaults to ".parquet". local_file_path (str, optional): Local destination path. Defaults to None. diff --git a/viadot/sources/sap_rfc.py b/viadot/sources/sap_rfc.py index 0ebad30f4..8d53bd230 100644 --- a/viadot/sources/sap_rfc.py +++ b/viadot/sources/sap_rfc.py @@ -253,8 +253,8 @@ def __init__( in case of too many columns for RFC function. According to SAP documentation, the limit is 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. - sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - sap_credentials_key (str, optional): The key for sap credentials located in the local config for Azure Key Vault. Defaults to "SAP". + sap_credentials (dict, optional): The credentials to use to authenticate with SAP. Defaults to None. + sap_credentials_key (str, optional): The key for sap credentials located in the local config or Azure Key Vault. Defaults to "SAP". env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV". Raises: @@ -723,8 +723,8 @@ def __init__( 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. rfc_unique_id (List[str], optional): Reference columns to merge chunks Data Frames. These columns must to be unique. Defaults to None. - sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - sap_credentials_key (str, optional): The key for sap credentials located in the local config for Azure Key Vault. Defaults to "SAP". + sap_credentials (dict, optional): The credentials to use to authenticate with SAP. Defaults to None. + sap_credentials_key (str, optional): The key for sap credentials located in the local config or Azure Key Vault. Defaults to "SAP". env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV". Raises: diff --git a/viadot/tasks/sap_rfc.py b/viadot/tasks/sap_rfc.py index 91c754678..2d47e573a 100644 --- a/viadot/tasks/sap_rfc.py +++ b/viadot/tasks/sap_rfc.py @@ -59,7 +59,7 @@ def __init__( 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - sap_credentials_key (str, optional): The key for sap credentials located in the local config for Azure Key Vault. Defaults to "SAP". + sap_credentials_key (str, optional): The key for sap credentials located in the local config or Azure Key Vault. Defaults to "SAP". env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV". By default, they're taken from the local viadot config. """ @@ -110,8 +110,8 @@ def run( multiple options are automatically tried. Defaults to None. replacement (str, optional): In case of sep is on a columns, set up a new character to replace inside the string to avoid flow breakdowns. Defaults to "-". - sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. - sap_credentials_key (str, optional): The key for sap credentials located in the local config for Azure Key Vault. Defaults to "SAP". + sap_credentials (dict, optional): The credentials to use to authenticate with SAP. Defaults to None. + sap_credentials_key (str, optional): The key for sap credentials located in the local config or Azure Key Vault. Defaults to "SAP". env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV". func (str, optional): SAP RFC function to use. Defaults to None. rfc_total_col_width_character_limit (int, optional): Number of characters by which query will be split in chunks