Feature-1184- delete edges using blob

[Mr-SabyasachiBose]

Description

additional feature added based on question #1184, now delete_edges() has params to delete a large number of edges from a hierarchy using flat file (.blb) based unbound TI.

delete_edges() can now be called with additional params use_blob and remove_blob, both of type bool. Using this params would write the edges in "parent","child" format in a flat file (.blb). v12 check added to always append .blb at the end. An unbound TI process would be created to read this file, and use HierarchyElementComponentDelete() to delete the edges.

Please note, additional imports are added at the beginning of ElementService.py apart from the specific code. The process would need data admin and ops admin to execute.

Code

    @require_version("11.4")
    def delete_edges(self, dimension_name: str, hierarchy_name: str, edges: Iterable[Tuple[str, str]] = None,
                     use_ti: bool = False, use_blob: bool = False, remove_blob: bool = True, **kwargs):
        if use_ti:
            return self.delete_edges_use_ti(dimension_name, hierarchy_name, edges, **kwargs)
        
        if use_blob:
            return self.delete_edges_use_blob(dimension_name, hierarchy_name, edges, remove_blob, **kwargs)

        h_service = self._get_hierarchy_service()
        h = h_service.get(dimension_name, hierarchy_name, **kwargs)
        for edge in edges:
            h.remove_edge(parent=edge[0], component=edge[1])
        h_service.update(h, **kwargs)

...

    @require_data_admin
    @require_ops_admin
    def delete_edges_use_blob(self, dimension_name: str, hierarchy_name: str, edges: List[str] = None, remove_blob: bool = True, **kwargs):
        """
        Remove edges in TM1 via an unbound TI process having an uploaded CSV as data source
        :param dimension_name as str: dimension name
        :param hierarchy_name as str: hierarchy name
        :param edges as list: 
        :remove_blob as bool: remove the parent child file after use, default True
        :param kwargs:
        :return: Success: bool, Messages: list, ChangeSet: None
        """
        if not edges:
            return
            
        process_service = ProcessService(self._rest)
        file_service = FileService(self._rest)

        unique_name = self.suggest_unique_object_name()

        # Transform cells to format that's consumable for TI
        csv_content = StringIO()
        csv_writer = csv.writer(
            csv_content,
            delimiter=",",
            quoting=csv.QUOTE_ALL)
        csv_writer.writerows(
            list(edge)
            for edge 
            in edges)

        file_name = f'{unique_name}.csv'
        file_service.create(
            file_name=file_name,
            file_content=csv_content.getvalue().encode('utf-8'),
            **kwargs)

        try:
            # Create and execute unbound TI process to delete edges using blob file
            process = self._build_unwind_hierarchy_edges_from_blob_process(
                dimension_name=dimension_name,
                hierarchy_name=hierarchy_name,
                process_name=unique_name,
                blob_filename=file_name)

            success, status, log_file = process_service.execute_process_with_return(process=process, **kwargs)
            if not success:
                if status in ['HasMinorErrors']:
                    raise TM1pyWritePartialFailureException([status], [log_file], 1)
                else:
                    raise TM1pyWriteFailureException([status], [log_file])

        finally:
            if remove_blob:
                file_service.delete(file_name=file_name)

    def _build_unwind_hierarchy_edges_from_blob_process(self, dimension_name: str, hierarchy_name: str, process_name: str, blob_filename: str) -> Process:

        # v11 automatically adds blb file extensions to documents created via the contents api
        if not verify_version(required_version="12", version=self.version):
            blob_filename += ".blb"
        hierarchyupdate_process = Process(
            name=process_name,
            datasource_type='ASCII',
            datasource_ascii_header_records=0,
            datasource_data_source_name_for_server=f"{blob_filename}",
            datasource_data_source_name_for_client=f"{blob_filename}",
            datasource_ascii_delimiter_char=',',
            datasource_ascii_decimal_separator='.',
            datasource_ascii_thousand_separator='',
            datasource_ascii_quote_character='"')

        # Define encoding in Prolog section
        hierarchyupdate_process.prolog_procedure = f"""
        SetInputCharacterSet('TM1CS_UTF8');
         """
        parent_variable="vParent"
        child_variable="vChild"
        hierarchyupdate_process.add_variable(name=parent_variable, variable_type='String')
        hierarchyupdate_process.add_variable(name=child_variable, variable_type='String')

        # Write the statement for delete component in hierarchy
        hierarch_check = f"If( HierarchyExists('{dimension_name}', '{hierarchy_name}') = 1);"
        delete_component = f"\rHierarchyElementComponentDelete('{dimension_name}', '{hierarchy_name}', {parent_variable}, {child_variable});"
        close_if = '\rEndIf;\r'
 
        # Define Metadata section
        metadata_statement = hierarch_check + delete_component + close_if
        hierarchyupdate_process.metadata_procedure = metadata_statement
        return hierarchyupdate_process

    def get_elements(self, dimension_name: str, hierarchy_name: str, **kwargs) -> List[Element]:
        url = format_url(
            "/Dimensions('{}')/Hierarchies('{}')/Elements?select=Name,Type",
            dimension_name,
            hierarchy_name)
        response = self._rest.GET(url, **kwargs)
        return [Element.from_dict(element) for element in response.json()["value"]]

[MariusWirtz]

Nice work! Thank you @Mr-SabyasachiBose

@Cubewise-JoeCHK does this solve #1184 ?

[Cubewise-JoeCHK]

@Mr-SabyasachiBose

Thank YOU!!!!!! It helps me a lot!!!!
I will do in-depth testing about it because the data is in chinese simplified (utf-8 works fine for chinese symbol, just for make sure). will get you back soon!!

@MariusWirtz

If the solution works, I will implement it for the next release on the project. THANK YOU!!!!!

[Mr-SabyasachiBose]

@Cubewise-JoeCHK

You are most welcome! One quick note, please use the code directly from code page (and not the code above) as I have removed the hierarchy validation for each edge.

@MariusWirtz if everything looks Ok, please could you pull the changes to master repository?