Merge pull request #200 from cloudblue/LITE-26686-support-transformations

Lite 26686 support transformations

Author: ffaraone

connect/cli/plugins/project/extension/helpers.py

@@ -92,6 +92,8 @@ def bootstrap_extension_project(  # noqa: CCR001
     if answers['use_github_actions'] == 'n':
         exclude.extend(['.github', '.github/**/*'])
 
+    application_types = answers.get('application_types', [])
+
     if answers.get('webapp_supports_ui') != 'y':
         exclude.extend([
             os.path.join('${package_name}', 'static', '.gitkeep'),
@@ -106,9 +108,25 @@ def bootstrap_extension_project(  # noqa: CCR001
             'jest.config.js.j2',
         ])
 
-    application_types = answers.get('application_types', [])
+    elif 'tfnapp' not in application_types:
+        exclude.extend([
+            'ui/pages/transformations',
+            'ui/pages/transformations/*',
+            'ui/src/pages/transformations',
+            'ui/src/pages/transformations/*',
+            'ui/styles/manual.css.j2',
+        ])
+    else:
+        exclude.extend([
+            'ui/pages/index.html.j2',
+            'ui/pages/settings.html.j2',
+            'ui/src/pages/index.js.j2',
+            'ui/src/pages/settings.js.j2',
+            'ui/tests/pages.spec.js.j2',
+            'ui/tests/utils.spec.js.j2',
+        ])
 
-    for app_type in ['anvil', 'events', 'webapp']:
+    for app_type in ['anvil', 'events', 'webapp', 'tfnapp']:
         if app_type not in application_types:
             exclude.append(
                 os.path.join(

connect/cli/plugins/project/extension/templates/bootstrap/${project_slug}/${package_name}/tfnapp.py.j2

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) {% now 'utc', '%Y' %}, {{ author }}
+# All rights reserved.
+#
+from connect.eaas.core.decorators import manual_transformation, transformation{% if include_variables_example == 'y' -%}, variables{% endif %}
+from connect.eaas.core.extension import TransformationsApplicationBase
+
+
+{% if include_variables_example == 'y' -%}
+@variables([{
+    'name': 'VAR_NAME_1',
+    'initial_value': 'VAR_VALUE_1',
+    'secure': False,
+}])
+{% endif -%}
+class {{ project_slug|replace("_", " ")|title|replace(" ", "") }}TransformationsApplication(TransformationsApplicationBase):
+    @transformation(
+        name='Manual transformation',
+        description=(
+            'This transformation function allows to describe a manual '
+            'procedure to be done.'
+        ),
+        edit_dialog_ui='/static/transformations/manual.html',
+    )
+    @manual_transformation()
+    {% if use_asyncio == 'y' %}async {% endif %}def manual_transformation(self, row: dict):
+        pass
+
+    @transformation(
+        name='Copy Column(s)',
+        description=(
+            'This transformation function allows copy values from Input to Output columns, '
+            'which can be used in case of change column name in the output data or '
+            'create a copy of values in table.'
+        ),
+        edit_dialog_ui='/static/transformations/copy.html',
+    )
+    def copy_columns(self, row: dict):
+        tfn_settings = (
+            self.transformation_request['transformation']['settings']
+        )
+        result = {}
+
+        for setting in tfn_settings:
+            result[setting['to']] = row[setting['from']]
+
+        return result
+

connect/cli/plugins/project/extension/templates/bootstrap/${project_slug}/${package_name}/webapp.py.j2

@@ -3,11 +3,19 @@
 # Copyright (c) {% now 'utc', '%Y' %}, {{ author }}
 # All rights reserved.
 #
+{%- if extension_type == 'transformations' %}
+from connect.eaas.core.decorators import (
+    router,
+    web_app,
+)
+from connect.eaas.core.extension import WebApplicationBase
+from fastapi.responses import JSONResponse
+{%- else %}
 from typing import List
 
 from connect.client import {% if use_asyncio == 'y' %}Async{% endif %}ConnectClient{% if extension_type == 'multiaccount' %}, R{% endif %}
 from connect.eaas.core.decorators import (
-    {%- if webapp_supports_ui == 'y' %}
+    {%- if webapp_supports_ui == 'y'%}
     account_settings_page,
     module_pages,
     {%- endif %}
@@ -35,6 +43,7 @@ from connect.eaas.core.inject.synchronous import get_extension_client
 from fastapi import Depends
 
 from {{ package_name }}.schemas import Marketplace{% if extension_type == 'multiaccount' %}, Settings{% endif %}
+{%- endif %}
 
 
 {% if include_variables_example == 'y' -%}
@@ -52,13 +61,13 @@ from {{ package_name }}.schemas import Marketplace{% if extension_type == 'multi
 ])
 {% endif -%}
 @web_app(router)
-{% if webapp_supports_ui == 'y' -%}
+{% if webapp_supports_ui == 'y' and extension_type != 'transformations' -%}
 @account_settings_page('Chart settings', '/static/settings.html')
 @module_pages('Chart', '/static/index.html')
 {% endif -%}
 class {{ project_slug|replace("_", " ")|title|replace(" ", "") }}WebApplication(WebApplicationBase):
 
-    @router.get(
+    {% if extension_type != 'transformations' -%}@router.get(
         '/marketplaces',
         summary='List all available marketplaces',
         response_model=List[Marketplace],
@@ -77,7 +86,7 @@ class {{ project_slug|replace("_", " ")|title|replace(" ", "") }}WebApplication(
     {% if extension_type == 'multiaccount' -%}
     @router.get(
         '/settings',
-        summary='Retrive charts settings',
+        summary='Retrieve charts settings',
         response_model=Settings,
     )
     {% if use_asyncio == 'y' %}async {% endif %}def retrieve_settings(
@@ -132,4 +141,68 @@ class {{ project_slug|replace("_", " ")|title|replace(" ", "") }}WebApplication(
                 ],
             },
         }
-    {% endif %}
+    {%- endif %}
+
+    {%- else %}def validate_copy_columns(self, data):
+        if (
+            'settings' not in data
+            or not isinstance(data['settings'], list)
+            or 'columns' not in data
+            or 'input' not in data['columns']
+        ):
+            return JSONResponse(status_code=400, content={'error': 'Invalid input data'})
+
+        settings = data['settings']
+        input_columns = data['columns']['input']
+        available_input_columns = [c['name'] for c in input_columns]
+        unique_names = [c['name'] for c in input_columns]
+        overview = []
+
+        for s in settings:
+            if 'from' not in s or 'to' not in s:
+                return JSONResponse(
+                    status_code=400,
+                    content={'error': 'Invalid settings format'},
+                )
+            if s['from'] not in available_input_columns:
+                return JSONResponse(
+                    status_code=400,
+                    content={'error': f'The input column {s["from"]} does not exists'},
+                )
+            if s['to'] in unique_names:
+                return JSONResponse(
+                    status_code=400,
+                    content={
+                        'error': f'Invalid column name {s["to"]}. The to field should be unique',
+                    },
+                )
+            unique_names.append(s['to'])
+            overview.append(f'{s["from"]}  -->  {s["to"]}')
+
+        overview = ''.join([row + '\n' for row in overview])
+
+        return {
+            'overview': overview,
+        }
+
+    @router.post(
+        '/validate/{transformation_function}',
+        summary='Validate settings',
+    )
+    def validate_tfn_settings(
+            self,
+            transformation_function: str,
+            data: dict,
+    ):
+        try:
+            method = getattr(self, f'validate_{transformation_function}')
+            return method(data)
+        except AttributeError:
+            return JSONResponse(
+                status_code=400,
+                content={
+                    'error': f'The validation method {transformation_function} does not exist',
+                },
+            )
+
+{%- endif %}

connect/cli/plugins/project/extension/templates/bootstrap/${project_slug}/pyproject.toml.j2

@@ -19,10 +19,13 @@ readme = "./README.md"
 {%- if 'anvil' in application_types %}
 "anvilapp" = "{{ package_name }}.anvil:{{ project_slug|replace("_", " ")|title|replace(" ", "") }}AnvilApplication"
 {%- endif %}
+{%- if 'tfnapp' in application_types %}
+"tfnapp" = "{{ package_name }}.tfnapp:{{ project_slug|replace("_", " ")|title|replace(" ", "") }}TransformationsApplication"
+{%- endif %}
 
 [tool.poetry.dependencies]
 python = ">=3.8,<4"
-connect-eaas-core = ">=26.13,<27"
+connect-eaas-core = ">=27.11,<28"
 
 [tool.poetry.dev-dependencies]
 pytest = ">=6.1.2,<8"

connect/cli/plugins/project/extension/templates/bootstrap/${project_slug}/tests/test_tfnapp.py.j2

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2022, Globex Corporation
+# All rights reserved.
+#
+from {{ package_name }}.tfnapp import {{ project_slug|replace("_", " ")|title|replace(" ", "") }}TransformationsApplication
+
+
+def test_copy_columns(connect_client, logger, mocker):
+    app = {{ project_slug|replace("_", " ")|title|replace(" ", "") }}TransformationsApplication(
+        connect_client,
+        logger,
+        mocker.MagicMock(),
+        installation_client=connect_client,
+        installation={'id': 'EIN-0000'},
+        context=mocker.MagicMock(),
+        transformation_request={
+            'transformation': {
+                'settings': [
+                    {'from': 'ColumnA', 'to': 'NewColC'},
+                    {'from': 'ColumnB', 'to': 'NewColD'},
+                ],
+            },
+        },
+    )
+
+    assert app.copy_columns(
+        {
+            'ColumnA': 'ContentColumnA',
+            'ColumnB': 'ContentColumnB',
+        },
+    ) == {
+        'NewColC': 'ContentColumnA',
+        'NewColD': 'ContentColumnB',
+    }
+

connect/cli/plugins/project/extension/templates/bootstrap/${project_slug}/tests/test_webapp.py.j2

@@ -3,6 +3,10 @@
 # Copyright (c) {% now 'utc', '%Y' %}, {{ author }}
 # All rights reserved.
 #
+{% if extension_type == 'transformations' -%}
+import pytest
+
+{% endif -%}
 {% if extension_type == 'multiaccount' -%}
 from connect.client import R
 
@@ -11,6 +15,114 @@ from {{ package_name }}.schemas import Marketplace, Settings
 from {{ package_name }}.webapp import {{ project_slug|replace("_", " ")|title|replace(" ", "") }}WebApplication
 
 
+{% if extension_type == 'transformations' -%}
+def test_validate_copy_columns(test_client_factory):
+    data = {
+        'settings': [
+            {
+                'from': 'columnInput1',
+                'to': 'newColumn1',
+            },
+            {
+                'from': 'columnInput2',
+                'to': 'newColumn2',
+            },
+        ],
+        'columns': {
+            'input': [
+                {'name': 'columnInput1'},
+                {'name': 'columnInput2'},
+            ],
+            'output': [],
+        },
+    }
+
+    client = test_client_factory({{ project_slug|replace("_", " ")|title|replace(" ", "") }}WebApplication)
+
+    response = client.post('/api/validate/copy_columns', json=data)
+    assert response.status_code == 200
+
+    data = response.json()
+    assert data == {
+        'overview': 'columnInput1  -->  newColumn1\ncolumnInput2  -->  newColumn2\n',
+    }
+
+
+@pytest.mark.parametrize(
+    'data',
+    (
+        {},
+        {'settings': {}},
+        {'settings': []},
+        {'settings': [], 'columns': {}},
+    ),
+)
+def test_validate_copy_columns_missing_settings_or_invalid(test_client_factory, data):
+
+    client = test_client_factory({{ project_slug|replace("_", " ")|title|replace(" ", "") }}WebApplication)
+
+    response = client.post('/api/validate/copy_columns', json=data)
+    assert response.status_code == 400
+    assert response.json() == {'error': 'Invalid input data'}
+
+
+def test_validate_copy_columns_invalid_settings(test_client_factory):
+    data = {'settings': [{'x': 'y'}], 'columns': {'input': []}}
+
+    client = test_client_factory({{ project_slug|replace("_", " ")|title|replace(" ", "") }}WebApplication)
+
+    response = client.post('/api/validate/copy_columns', json=data)
+    assert response.status_code == 400
+    assert response.json() == {'error': 'Invalid settings format'}
+
+
+def test_validate_copy_columns_invalid_from(test_client_factory):
+    data = {'settings': [{'from': 'Hola', 'to': 'Hola2'}], 'columns': {'input': [{'name': 'Gola'}]}}
+
+    client = test_client_factory({{ project_slug|replace("_", " ")|title|replace(" ", "") }}WebApplication)
+
+    response = client.post('/api/validate/copy_columns', json=data)
+    assert response.status_code == 400
+    assert response.json() == {'error': 'The input column Hola does not exists'}
+
+
+@pytest.mark.parametrize(
+    'data',
+    (
+        {
+            'settings': [
+                {'from': 'A', 'to': 'C'},
+                {'from': 'B', 'to': 'C'},
+            ],
+            'columns': {
+                'input': [
+                    {'name': 'A'},
+                    {'name': 'B'},
+                ],
+            },
+        },
+        {
+            'settings': [
+                {'from': 'A', 'to': 'C'},
+            ],
+            'columns': {
+                'input': [
+                    {'name': 'A'},
+                    {'name': 'C'},
+                ],
+            },
+        },
+    ),
+)
+def test_validate_copy_columns_not_unique_name(test_client_factory, data):
+    client = test_client_factory({{ project_slug|replace("_", " ")|title|replace(" ", "") }}WebApplication)
+
+    response = client.post('/api/validate/copy_columns', json=data)
+    assert response.status_code == 400
+    assert response.json() == {
+        'error': 'Invalid column name C. The to field should be unique',
+    }
+{% else -%}
 def test_list_marketplaces(test_client_factory, {% if use_asyncio == 'y' %}async_{% endif %}client_mocker_factory):
     marketplaces = [
         {
@@ -180,3 +292,4 @@ def test_generate_chart_data(test_client_factory, {% if use_asyncio == 'y' %}asy
         },
     }
 {% endif %}
+{% endif -%}