Complete transform helper documentation

src/engine/test/helper_tests/engine_helper_test/src/documentation_generator/__main__.py

@@ -8,6 +8,7 @@
 from .exporter import IExporter
 from .types import *
 import tempfile
+import subprocess
 
 
 def parse_arguments() -> argparse.Namespace:
@@ -60,10 +61,20 @@ def main():
         sys.exit("the output directory must be a temporary one")
     exporter = ExporterFactory.get_exporter(exporter_type)
     if args.input_file_path:
+        command = f'engine-helper-test-validator --input_file_path {args.input_file_path}'
+        try:
+            subprocess.run(command, check=True, shell=True, stdout=subprocess.PIPE)
+        except subprocess.CalledProcessError as e:
+            sys.exit(e.stderr)
         generate_documentation(parser, exporter, Path(args.input_file_path), output_dir)
     elif args.folder_path:
         for file in Path(args.folder_path).iterdir():
             if file.is_file() and (file.suffix in ['.yml', '.yaml']):
+                command = f'engine-helper-test-validator --folder_path {args.folder_path}'
+                try:
+                    subprocess.run(command, check=True, shell=True, stdout=subprocess.PIPE)
+                except subprocess.CalledProcessError as e:
+                    sys.exit(e.stderr)
                 generate_documentation(parser, exporter, file, output_dir)
     else:
         sys.exit("It is necessary to indicate a file or directory that contains a configuration yaml")

src/engine/test/helper_tests/engine_helper_test/src/documentation_generator/mark_down_generator.py

@@ -47,7 +47,9 @@ def create_table(self, arguments: dict, headers: list):
                 row.append(', '.join(info.restrictions["allowed"]) if isinstance(
                     info.restrictions["allowed"], list) else str(info.restrictions["allowed"]))
             else:
-                if info.arg_type == "object":
+                if isinstance(info.arg_type, list):
+                    row.append("Any object")
+                elif info.arg_type == "object":
                     row.append("Any object")
                 elif info.generate == "integer":
                     row.append("Integers between `-2^63` and `2^63-1`")
@@ -91,8 +93,6 @@ def create_output_table(self, output, headers: list):
             row.append(output.type_)
             if output.type_ == "object":
                 row.append("Any object")
-            elif output.type_ == "array":
-                row.append("Any array")
             elif output.subset == "integer":
                 row.append("Integers between `-2^63` and `2^63-1`")
             elif output.subset == "string":
@@ -103,6 +103,8 @@ def create_output_table(self, output, headers: list):
                 row.append("Any boolean")
             elif output.subset == "hexadecimal":
                 row.append("Any hexadecimal")
+            elif output.subset == "all":
+                row.append("[number, string, boolean, object, array]")
         rows.append(row)
         data_rows = ['| ' + ' | '.join(row) + ' |' for row in rows]
 

src/engine/test/helper_tests/engine_helper_test/src/test_cases_generator/runtime_cases.py

@@ -662,6 +662,7 @@ def generate_unit_test(self):
 
         template = Template(self.parser)
         arguments_list = []
+        target_field_value = None
 
         for test in self.parser.get_tests():
             if "arguments" in test:
@@ -684,7 +685,6 @@ def generate_unit_test(self):
                     combined = list(itertools.zip_longest(arguments_list, case, fillvalue=None))
                     all_arguments = []
                     input = {}
-                    target_field_value = None
                     for (id, value), source in combined:
                         argument = Argument(value)
                         argument.configure_only_value(source)
@@ -711,6 +711,7 @@ def generate_unit_test(self):
 
         arguments_list = []
         for test in self.parser.get_tests():
+            target_field = test.get("target_field", None)
             if "arguments" in test:
                 arguments_list = list(test["arguments"].items())
             if any(isinstance(item[1], dict) for item in arguments_list):
@@ -727,11 +728,12 @@ def generate_unit_test(self):
                             all_arguments.append(f"$eventJson.{val['name']}")
                         else:
                             all_arguments.append(val)
-                    else:
-                        if isinstance(data, list):
-                            target_field_value = list(data)
+
+                    if target_field != None:
+                        if isinstance(target_field, list):
+                            target_field_value = list(target_field)
                         else:
-                            target_field_value = data
+                            target_field_value = target_field
 
                 if len(all_arguments) == 0:
                     break

src/engine/test/helper_tests/helpers_description/transformation/array_append.yml

@@ -1,14 +1,27 @@
 # Name of the helper function
 name: array_append
 
+metadata:
+  description: |
+    Adds the values or references of any Json type to the target array field.
+    This helper function is normally used in the map stage.
+    If the target field exists and is not an array it will be overridden, if it does not exist it will be created.
+    Currently this operations will fails as soon as one reference parameter is not found.
+    As it is an array, only elements of the same type can be added.
+    The type of the first element contained in the array will always be taken or, failing that, the type of the first
+    element to be added.
+
+  keywords:
+    - to define
+
 helper_type: transformation
 
 # Indicates whether the helper function supports a variable number of arguments
 is_variadic: true
 
 # Arguments expected by the helper function
 arguments:
-  1:
+  any_object:
     type:
       - number
       - string
@@ -23,24 +36,24 @@ skipped:
 
 target_field:
   type: array
-  generate: string
+  generate: all
 
 test:
   - arguments:
-      1: 5
-      target_field: [1,2,3,4]
+      any_object: 5
+    target_field: [1,2,3,4]
     should_pass: true
     expected: [1,2,3,4,5]
     description: The type of the value matches the type of the elements of the array
   - arguments:
-      1: hello
-      target_field: [1,2,3,4]
+      any_object: hello
+    target_field: [1,2,3,4]
     should_pass: false
     expected: [1,2,3,4]
     description: The type of the value does not match the type of the array elements
   - arguments:
-      1: [1,2]
-      target_field: [1,2,3,4]
+      any_object: [1,2]
+    target_field: [1,2,3,4]
     should_pass: false
     expected: [1,2,3,4]
     description: The type of the value does not match the type of the array elements
@@ -52,8 +65,8 @@ test:
   #   expected: [1,2,3,4,2.5]
   #   description: The type of the value does match the type of the array elements
   - arguments:
-      1: 2
-      target_field: [1.2,2.3,3.5,4.9]
+      any_object: 2
+    target_field: [1.2,2.3,3.5,4.9]
     should_pass: true
     expected: [1.2,2.3,3.5,4.9,2]
     description: The type of the value does match the type of the array elements

src/engine/test/helper_tests/helpers_description/transformation/array_append_any.yml

@@ -1,14 +1,27 @@
 # Name of the helper function
 name: array_append_any
 
+metadata:
+  description: |
+    Adds the values or references of any Json type to the target array field.
+    This helper function is normally used in the map stage.
+    If the target field exists and is not an array it will be overridden, if it does not exist it will be created.
+    This operation will not fail if a reference parameter is not found, any parameters before or after
+    it will be added to the target field.
+    As it is an array, only elements of the same type can be added. The type of the first element contained in the array
+    will always be taken or, failing that, the type of the first element to be added.
+
+  keywords:
+    - to define
+
 helper_type: transformation
 
 # Indicates whether the helper function supports a variable number of arguments
 is_variadic: true
 
 # Arguments expected by the helper function
 arguments:
-  1:
+  any_object:
     type:
       - number
       - string
@@ -23,35 +36,35 @@ skipped:
 
 target_field:
   type: array
-  generate: string
+  generate: all
 
 test:
   - arguments:
-      1: 5
-      2: 6
-      target_field: [1,2,3,4]
+      any_object: 5
+      any_object_2: 6
+    target_field: [1,2,3,4]
     should_pass: true
     expected: [1,2,3,4,5,6]
     description: The type of the value matches the type of the elements of the array
   - arguments:
-      1:
+      any_object:
         source: reference
         value: null
-      2:
+      any_object_2:
         source: reference
         value: 6
-      target_field: [1,2,3,4]
+    target_field: [1,2,3,4]
     should_pass: true
     expected: [1,2,3,4,6]
     description: The type of the value matches the type of the elements of the array
   - arguments:
-      1:
+      any_object:
         source: value
         value: 5
-      2:
+      any_object_2:
         source: reference
         value: null
-      target_field: [1,2,3,4]
+    target_field: [1,2,3,4]
     should_pass: true
     expected: [1,2,3,4,5]
     description: The type of the value matches the type of the elements of the array

src/engine/test/helper_tests/helpers_description/transformation/array_append_unique.yml

@@ -1,14 +1,25 @@
 # Name of the helper function
 name: array_append_unique
 
+metadata:
+  description: |
+    Appends the string values or any Json type by reference to the target array field.
+    This helper function is typically used in the map stage.
+    If the target field exists and is not an array it will be overridden, if it does not exist it will be created.
+    Currently this operations will fails as soon as one reference parameter is not found or the element trying to be
+    added already exists in the array.
+
+  keywords:
+    - to define
+
 helper_type: transformation
 
 # Indicates whether the helper function supports a variable number of arguments
 is_variadic: true
 
 # Arguments expected by the helper function
 arguments:
-  1:
+  any_object:
     type:
       - number
       - string
@@ -23,24 +34,24 @@ skipped:
 
 target_field:
   type: array
-  generate: string
+  generate: all
 
 test:
   - arguments:
-      1: hello
-      target_field: [hola, hello, salut, ciao]
+      any_object: hello
+    target_field: [hola, hello, salut, ciao]
     should_pass: false
     expected: [hola, hello, salut, ciao]
     description: Failure array append unique
   - arguments:
-      1: 5
-      target_field: [1,2,3,4,5]
+      any_object: 5
+    target_field: [1,2,3,4,5]
     should_pass: false
     expected: [1,2,3,4,5]
     description: Failure array append unique
   - arguments:
-      1: 5
-      target_field: [1,2,3,4]
+      any_object: 5
+    target_field: [1,2,3,4]
     should_pass: true
     expected: [1,2,3,4,5]
     description: Success array append unique

src/engine/test/helper_tests/helpers_description/transformation/array_append_unique_any.yml

@@ -1,14 +1,25 @@
 # Name of the helper function
 name: array_append_unique_any
 
+metadata:
+  description: |
+    Appends the string values or any Json type by reference to the target array field.
+    This helper function is typically used in the map stage.
+    If the target field exists and is not an array it will be overridden, if it does not exist it will be created.
+    This operation will not fail if a reference parameter is not found, any parameters before or after
+    it will be added to the target field.
+
+  keywords:
+    - to define
+
 helper_type: transformation
 
 # Indicates whether the helper function supports a variable number of arguments
 is_variadic: true
 
 # Arguments expected by the helper function
 arguments:
-  1:
+  any_object:
     type:
       - number
       - string
@@ -23,36 +34,36 @@ skipped:
 
 target_field:
   type: array
-  generate: string
+  generate: all
 
 test:
   - arguments:
-      1: 5
-      2: 6
-      target_field: [1,2,3,4,5]
+      any_object: 5
+      any_object_2: 6
+    target_field: [1,2,3,4,5]
     should_pass: true
     expected: [1,2,3,4,5,6]
     description: The type of the value matches the type of the elements of the array
   # TODO: Review this behavior. The helper currently fails if the target_field is the same after applying the operation
   - arguments:
-      1:
+      any_object:
         source: reference
         value: null
-      2:
+      any_object_2:
         source: reference
         value: 6
-      target_field: [1,2,3,4,6]
+    target_field: [1,2,3,4,6]
     should_pass: false
     expected: [1,2,3,4,6]
     description: The type of the value matches the type of the elements of the array
   - arguments:
-      1:
+      any_object:
         source: value
         value: 5
-      2:
+      any_object_2:
         source: reference
         value: null
-      target_field: [1,2,3,4]
+    target_field: [1,2,3,4]
     should_pass: true
     expected: [1,2,3,4,5]
     description: The type of the value matches the type of the elements of the array