Merge pull request #810 from marcinpurtak/feature/sharepoint_list_ext…

…ension_KPI5 Feature/sharepoint list extension kpi5
dyvenia · Nov 15, 2023 · 3389cb1 · 3389cb1
2 parents 8025068 + 5dc7401
commit 3389cb1
Show file tree

Hide file tree

Showing 5 changed files with 514 additions and 183 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,21 +5,42 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
 ### Added
 - Added new view type `agent_interaction_view_type` in `Genesys`source.
 - Added `TM1` source class.
 - Added `TM1ToDF` task class.
 - Added `set_prefect_kv` parameter to `BigQueryToADLS` with `False` as a default. If there is a need to create new pair in KV Store the parameter can be changed to `True`.
+- Added `_rename_duplicated_fields` method to `SharepointListToDF` task class for finding and rename duplicated columns
+- Added new view type `agent_interaction_view_type` in `Genesys`source.
 - Added libraries `nltk` and `sklearn` to `requirements`.
 
-### Fixed
-
 ### Changed
+- Splitted test for Eurostat on source tests and task tests
+- Modified `SharepointList` source class: 
+  -> docstrings update
+- Modified `SharepointToADLS` flow class:
+  -> docstrings update
+  -> changed set_prefect_kv: bool = False to prevent forced KV store append
+- Modified `SharepointListToADLS` flow class:
+  -> changed set_prefect_kv: bool = False to prevent forced KV store append
+- Modified `SharepointList` source class:
+  -> docstrings update
+  -> Changed `_unpack_fields` method to handle Sharepoint MultiChoiceField type + small improvements
+  -> Changed `get_fields` method to handle special characters - different approach to call get() and execute_query()
+  -> Renamed method from `select_expandable_user_fields` to `select_fields` + update for MultiChoiceField type
+  -> Changed `check_filters` method errors messages and more checks added
+  -> Changed `operators_mapping` method errors messages
+  -> Changed `make_filter_for_df` method errors messages
+- Modified `SharepointListToDF` task class:
+  -> docstrings update
 - Splitted test for Eurostat on source tests and task tests.
 - Modified `CustomerGauge` source class with simplified logic to return json structure.
 - Expanded `CustomerGaugeToDF` task class with separate cleaning functions and handling nested json structure flattening with two new methods `_field_reference_unpacker` and `_nested_dict_transformer`.
 - Changed `CustomerGaugeToADLS` to containing new arguments.
 
+### Fixed
+
 
 ## [0.4.21] - 2023-10-26
 ### Added
@@ -641,4 +662,4 @@ specified in the `SUPERMETRICS_DEFAULT_USER` secret
 - Moved from poetry to pip
 
 ### Fixed
-- Fix `AzureBlobStorage`'s `to_storage()` method is missing the final upload blob part
+- Fix `AzureBlobStorage`'s `to_storage()` method is missing the final upload blob part
diff --git a/tests/integration/test_sharepoint.py b/tests/integration/test_sharepoint.py
@@ -167,10 +167,11 @@ def test_get_data_types(file_name):
     assert "String" in dtypes
 
 
+### SECTION FOR TESTING SHAREPOINT LIST CONNECTOR ###
 @pytest.fixture(scope="session")
 def sharepoint_list():
     """
-    Fixture for creating a Sharepoint class instance.
+    Fixture for creating a SharepointList class instance.
     The class instance can be used within a test functions to interact with Sharepoint.
     """
     spl = SharepointList()
@@ -186,39 +187,60 @@ def test_valid_filters(sharepoint_list):
     assert result is True
 
 
-def test_invalid_dtype(sharepoint_list):
+def test_filters_missing_dtype(sharepoint_list):
+    filters = {
+        "filter1": {"operator1": ">", "value1": 10},
+    }
+    with pytest.raises(
+        ValueError,
+        match=re.escape("dtype for filter1 is missing!"),
+    ):
+        sharepoint_list.check_filters(filters)
+
+
+def test_filters_invalid_dtype(sharepoint_list):
     filters = {
         "filter1": {"dtype": "list", "operator1": ">", "value1": 10},
     }
-    with pytest.raises(ValueError, match="dtype not allowed!"):
+    with pytest.raises(
+        ValueError,
+        match=re.escape(
+            "dtype not allowed! Expected: ['datetime', 'date', 'bool', 'int', 'float', 'complex', 'str'] got: list ."
+        ),
+    ):
         sharepoint_list.check_filters(filters)
 
 
-def test_missing_operator1(sharepoint_list):
+def test_filters_missing_operator1(sharepoint_list):
     filters = {
         "filter1": {"dtype": "int", "value1": 10},
     }
     with pytest.raises(ValueError, match="Operator1 is missing!"):
         sharepoint_list.check_filters(filters)
 
 
-def test_invalid_operator1(sharepoint_list):
+def test_filters_invalid_operator1(sharepoint_list):
     filters = {
         "filter1": {"dtype": "int", "operator1": "*", "value1": 10},
     }
-    with pytest.raises(ValueError, match="Operator type not allowed!"):
+    with pytest.raises(
+        ValueError,
+        match=re.escape(
+            "Operator1 type not allowed! Expected: ['<', '>', '<=', '>=', '==', '!='] got: * ."
+        ),
+    ):
         sharepoint_list.check_filters(filters)
 
 
-def test_missing_value1(sharepoint_list):
+def test_filters_missing_value1(sharepoint_list):
     filters = {
         "filter1": {"dtype": "int", "operator1": ">", "value1": None},
     }
-    with pytest.raises(ValueError, match="Value for operator1 is missing!"):
+    with pytest.raises(ValueError, match="Value1 for operator1 is missing!"):
         sharepoint_list.check_filters(filters)
 
 
-def test_missing_operators_conjuction(sharepoint_list):
+def test_filters_missing_operators_conjuction(sharepoint_list):
     filters = {
         "filter1": {
             "dtype": "int",
@@ -228,11 +250,16 @@ def test_missing_operators_conjuction(sharepoint_list):
             "value2": 20,
         },
     }
-    with pytest.raises(ValueError, match="Operators for conjuction is missing!"):
+    with pytest.raises(
+        ValueError,
+        match=re.escape(
+            "Operator for conjuction is missing! Expected: ['&', '|'] got empty."
+        ),
+    ):
         sharepoint_list.check_filters(filters)
 
 
-def test_invalid_operators_conjuction(sharepoint_list):
+def test_filters_invalid_operators_conjuction(sharepoint_list):
     filters = {
         "filter1": {
             "dtype": "int",
@@ -243,11 +270,16 @@ def test_invalid_operators_conjuction(sharepoint_list):
             "operators_conjuction": "!",
         },
     }
-    with pytest.raises(ValueError, match="Operators for conjuction not allowed!"):
+    with pytest.raises(
+        ValueError,
+        match=re.escape(
+            "Operator for conjuction not allowed! Expected: ['&', '|'] got ! ."
+        ),
+    ):
         sharepoint_list.check_filters(filters)
 
 
-def test_invalid_filters_conjuction(sharepoint_list):
+def test_filters_conjuction_not_allowed(sharepoint_list):
     filters = {
         "filter1": {
             "dtype": "int",
@@ -257,35 +289,80 @@ def test_invalid_filters_conjuction(sharepoint_list):
         },
     }
     with pytest.raises(
-        ValueError, match="Filters operators for conjuction not allowed!"
+        ValueError,
+        match=re.escape(
+            "Filters conjuction allowed only when more then one filter provided!"
+        ),
+    ):
+        sharepoint_list.check_filters(filters)
+
+
+def test_filters_invalid_conjuction(sharepoint_list):
+    filters = {
+        "filter1": {
+            "dtype": "int",
+            "value1": 10,
+            "operator1": ">",
+            "filters_conjuction": "!",
+        },
+        "filter2": {
+            "dtype": "int",
+            "operator1": "==",
+        },
+    }
+    with pytest.raises(
+        ValueError,
+        match=re.escape(
+            "Filter operator for conjuction not allowed! Expected: ['&', '|'] got ! ."
+        ),
     ):
         sharepoint_list.check_filters(filters)
 
 
 def test_valid_mapping(sharepoint_list):
     filters = {
         "filter1": {
+            "dtype": "int",
+            "value1": 10,
+            "value2": 20,
             "operator1": ">",
             "operator2": "<=",
             "operators_conjuction": "&",
             "filters_conjuction": "|",
         },
-        "filter2": {"operator1": "==", "operator2": "!=", "operators_conjuction": "|"},
+        "filter2": {
+            "dtype": "int",
+            "value1": 30,
+            "value2": 0,
+            "operator1": "==",
+            "operator2": "!=",
+            "operators_conjuction": "|",
+        },
     }
     expected_result = {
         "filter1": {
+            "dtype": "int",
+            "value1": 10,
+            "value2": 20,
             "operator1": "gt",
             "operator2": "le",
             "operators_conjuction": "and",
             "filters_conjuction": "or",
         },
-        "filter2": {"operator1": "eq", "operator2": "ne", "operators_conjuction": "or"},
+        "filter2": {
+            "dtype": "int",
+            "value1": 30,
+            "value2": 0,
+            "operator1": "eq",
+            "operator2": "ne",
+            "operators_conjuction": "or",
+        },
     }
-    result = sharepoint_list.operators_mapping(deepcopy(filters))
+    result = sharepoint_list.operators_mapping(filters)
     assert result == expected_result
 
 
-def test_invalid_comparison_operator(sharepoint_list):
+def test_operators_mapping_invalid_comparison_operator(sharepoint_list):
     filters = {
         "filter1": {
             "operator1": "*",
@@ -296,10 +373,10 @@ def test_invalid_comparison_operator(sharepoint_list):
     }
     error_message = "This comparison operator: * is not allowed. Please read the function documentation for details!"
     with pytest.raises(ValueError, match=re.escape(error_message)):
-        sharepoint_list.operators_mapping(deepcopy(filters))
+        sharepoint_list.operators_mapping(filters)
 
 
-def test_invalid_logical_operator(sharepoint_list):
+def test_operators_mapping_invalid_logical_operator(sharepoint_list):
     filters = {
         "filter1": {
             "operator1": ">",
@@ -308,9 +385,23 @@ def test_invalid_logical_operator(sharepoint_list):
             "filters_conjuction": "|",
         },
     }
-    error_message = "This conjuction(logical) operator: ! is not allowed. Please read the function documentation for details!"
+    error_message = "This conjuction (logical) operator: ! is not allowed. Please read the function documentation for details!"
+    with pytest.raises(ValueError, match=re.escape(error_message)):
+        sharepoint_list.operators_mapping(filters)
+
+
+def test_operators_mapping_invalid_filters_logical_operator(sharepoint_list):
+    filters = {
+        "filter1": {
+            "operator1": ">",
+            "operator2": "<=",
+            "operators_conjuction": "&",
+            "filters_conjuction": "!",
+        },
+    }
+    error_message = "This filters conjuction (logical) operator: ! is not allowed. Please read the function documentation for details!"
     with pytest.raises(ValueError, match=re.escape(error_message)):
-        sharepoint_list.operators_mapping(deepcopy(filters))
+        sharepoint_list.operators_mapping(filters)
 
 
 def test_single_filter_datetime_api(sharepoint_list):