feat: improve expression language object accessing (#53)

* fix: improve expression language object accessing and refactor expression transformer

* chore: bump version

* refactor: dry grammar and transformer

* refactor: dry grammar and transformer

* refactor: dry grammar and transformer

* chore: bump cicd package version

* fix: apply PR feedback

Author: arjendev

.github/workflows/ci-cd.yml

@@ -51,7 +51,7 @@ jobs:
         run: poetry run pytest .
       - name: Set build version and build package
         run: |
-          poetry version 0.0.0.alpha${{ github.run_number}}
+          poetry version 0.0.1.alpha${{ github.run_number}}
           poetry build
       - name: Publish package distributions to PyPI
         if: github.ref == 'refs/heads/main'

examples/data_factory/batch_job/test_data_factory_batchjob_functional.py

@@ -37,7 +37,7 @@ def test_batch_job_pipeline(request: pytest.FixtureRequest) -> None:
                 "WorkloadUserAssignedIdentityClientId",
                 "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name",
             ),
-            RunParameter(RunParameterType.Pipeline, "JobAdditionalEnvironmentSettings", "[]"),
+            RunParameter(RunParameterType.Pipeline, "JobAdditionalEnvironmentSettings", []),
             RunParameter(
                 RunParameterType.Pipeline,
                 "OutputStorageAccountName",
@@ -82,7 +82,7 @@ def test_batch_job_pipeline(request: pytest.FixtureRequest) -> None:
     assert activity.name == "Set CommonEnvironmentSettings"
     assert activity.type_properties["variableName"] == "CommonEnvironmentSettings"
 
-    common_environment_settings = activity.type_properties["value"].get_json_value()
+    common_environment_settings = activity.type_properties["value"].value
     assert len(common_environment_settings) == 8
     assert common_environment_settings[0]["name"] == "WORKLOAD_APP_PACKAGE"
     assert common_environment_settings[0]["value"] == "test-application"

examples/data_factory/batch_job/test_data_factory_batchjob_unit.py

@@ -155,7 +155,7 @@ def test_set_common_environment_settings(test_framework: TestFramework, pipeline
     activity.evaluate(state)
 
     # Assert
-    env_settings = activity.type_properties["value"].get_json_value()
+    env_settings = activity.type_properties["value"].value
     assert env_settings[0]["name"] == "WORKLOAD_APP_PACKAGE"
     assert env_settings[0]["value"] == "workload"
     assert env_settings[1]["name"] == "WORKLOAD_APP_PACKAGE_VERSION"
@@ -260,7 +260,7 @@ def test_start_job_pipeline(test_framework: TestFramework, pipeline: Pipeline) -
             RunParameter[str](
                 RunParameterType.Pipeline,
                 "JobAdditionalEnvironmentSettings",
-                '[{"name": "STORAGE_ACCOUNT_NAME", "value": "teststorage"}]',
+                [{"name": "STORAGE_ACCOUNT_NAME", "value": "teststorage"}],
             ),
         ],
         variables=[
@@ -281,7 +281,7 @@ def test_start_job_pipeline(test_framework: TestFramework, pipeline: Pipeline) -
                 default_value="$AZ_BATCH_APP_PACKAGE_workload_0_13_2/workload.tar.gz",
             ),
             PipelineRunVariable(
-                name="CommonEnvironmentSettings", default_value='[{"name":"COMMON_ENV_SETTING","value":"dummy"}]'
+                name="CommonEnvironmentSettings", default_value=[{"name": "COMMON_ENV_SETTING", "value": "dummy"}]
             ),
         ],
     )

examples/fabric/batch_job/test_fabric_batchjob_functional.py

@@ -35,7 +35,7 @@ def test_batch_job_pipeline(request: pytest.FixtureRequest) -> None:
                 "WorkloadUserAssignedIdentityClientId",
                 "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name",
             ),
-            RunParameter(RunParameterType.Pipeline, "JobAdditionalEnvironmentSettings", "[]"),
+            RunParameter(RunParameterType.Pipeline, "JobAdditionalEnvironmentSettings", []),
             RunParameter(
                 RunParameterType.Pipeline,
                 "OutputStorageAccountName",
@@ -82,7 +82,7 @@ def test_batch_job_pipeline(request: pytest.FixtureRequest) -> None:
     assert activity.name == "Set CommonEnvironmentSettings"
     assert activity.type_properties["variableName"] == "CommonEnvironmentSettings"
 
-    common_environment_settings = activity.type_properties["value"].get_json_value()
+    common_environment_settings = activity.type_properties["value"].value
     assert len(common_environment_settings) == 8
     assert common_environment_settings[0]["name"] == "WORKLOAD_APP_PACKAGE"
     assert common_environment_settings[0]["value"] == "test-application"

examples/fabric/batch_job/test_fabric_batchjob_unit.py

@@ -156,7 +156,7 @@ def test_set_common_environment_settings(test_framework: TestFramework, pipeline
     activity.evaluate(state)
 
     # Assert
-    env_settings = activity.type_properties["value"].get_json_value()
+    env_settings = activity.type_properties["value"].value
     assert env_settings[0]["name"] == "WORKLOAD_APP_PACKAGE"
     assert env_settings[0]["value"] == "workload"
     assert env_settings[1]["name"] == "WORKLOAD_APP_PACKAGE_VERSION"
@@ -254,10 +254,10 @@ def test_start_job_pipeline(test_framework: TestFramework, pipeline: Pipeline) -
                 "WorkloadUserAssignedIdentityClientId",
                 "test-application-identity-client-id",
             ),
-            RunParameter[str](
+            RunParameter(
                 RunParameterType.Pipeline,
                 "JobAdditionalEnvironmentSettings",
-                '[{"name": "STORAGE_ACCOUNT_NAME", "value": "teststorage"}]',
+                [{"name": "STORAGE_ACCOUNT_NAME", "value": "teststorage"}],
             ),
         ],
         variables=[
@@ -275,7 +275,7 @@ def test_start_job_pipeline(test_framework: TestFramework, pipeline: Pipeline) -
                 default_value="$AZ_BATCH_APP_PACKAGE_workload_0_13_2/workload.tar.gz",
             ),
             PipelineRunVariable(
-                name="CommonEnvironmentSettings", default_value='[{"name": "COMMON_ENV_SETTING", "value": "dummy"}]'
+                name="CommonEnvironmentSettings", default_value=[{"name": "COMMON_ENV_SETTING", "value": "dummy"}]
             ),
         ],
     )

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "data-factory-testing-framework"
-version = "0.0.0"
+version = "0.0.1"
 description = "A unit test framework that allows you to write unit and functional tests for Data Factory pipelines against the git integrated json resource files."
 authors = ["Arjen Kroezen <arjenkroezen@microsoft.com>", "Yennifer Santos <ysantos@microsoft.com>", "Jaya Kumar <kumarjaya@microsoft.com>", "Leonard Herold <lherold@microsoft.com>"]
 readme = "README.md"

src/data_factory_testing_framework/exceptions/dataset_parameter_not_found_error.py

@@ -0,0 +1,4 @@
+class ParameterNotFoundError(Exception):
+    def __init__(self, pipeline_type: str, name: str) -> None:
+        """Error raised when a parameter is not found."""
+        super().__init__(f"Parameter: '{name}' of type '{pipeline_type}' not found")