feature(serializer): achieve JSON serializer (#283)

.github/workflows/builder_releaser.yml

@@ -35,6 +35,7 @@ jobs:
     runs-on: ubuntu-22.04
 
     env:
+      export_output_folder: tests/ci/ubuntu
       GDAL_VERSION: 3.8.*
 
     steps:
@@ -96,17 +97,37 @@ jobs:
       - name: Install tests fixtures
         run: python -m pip install -U gisdata -t ./tests/fixtures
 
-      - name: Minimal integration test
-        run: |
-          ./dist/DicoGIS-cli.bin --help
-          ./dist/DicoGIS-cli.bin --version
-          ./dist/DicoGIS-cli.bin inventory --input-folder ./tests/ --language EN
+      - name: Get QGIS training data
+        uses: actions/checkout@v4
+        with:
+          repository: qgis/QGIS-Training-Data
+          path: tests/fixtures/qgisdata
+
+      - name: Minimal integration test - help
+        run: ./dist/DicoGIS-cli.bin --help
+
+      - name: Minimal integration test - version
+        run: ./dist/DicoGIS-cli.bin --version
+
+      - name: Minimal integration test - export xlsx
+        run: ./dist/DicoGIS-cli.bin inventory --input-folder ./tests/ --language EN --output-path "${{ env.export_output_folder }}"/ubuntu/dicogis_ubuntu_build_tests.xlsx --verbose
+
+      - name: Minimal integration test - export xlsx
+        run: ./dist/DicoGIS-cli.bin inventory --input-folder ./tests/ --language EN --output-format json --output-path "${{ env.export_output_folder }}" --verbose
+
+      - name: Save generated files as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ubuntu_dicogis_integration_tests
+          path: ${{ env.export_output_folder }}/*
+          if-no-files-found: error
 
   build-windows:
     name: "🏠 Windows"
     runs-on: windows-latest
 
     env:
+      export_output_folder: tests/ci/windows
       GDAL_VERSION: 3.8.4
 
     steps:
@@ -158,11 +179,25 @@ jobs:
       - name: Install tests fixtures
         run: python -m pip install -U gisdata -t ./tests/fixtures
 
+      - name: Get QGIS training data
+        uses: actions/checkout@v4
+        with:
+          repository: qgis/QGIS-Training-Data
+          path: tests/fixtures/qgisdata
+
       - name: Minimal integration test
         run: |
           ./dist/DicoGIS-cli.exe --help
           ./dist/DicoGIS-cli.exe --version
-          ./dist/DicoGIS-cli.exe inventory --input-folder ./tests/  --language EN
+          ./dist/DicoGIS-cli.exe inventory --input-folder ./tests/ --language EN --output-path "${{ env.export_output_folder }}/dicogis_windows_build_tests.xlsx" --verbose
+          ./dist/DicoGIS-cli.exe inventory --input-folder ./tests/ --language EN --output-format json --output-path "${{ env.export_output_folder }}" --verbose
+
+      - name: Save generated files as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: windows_dicogis_integration_tests
+          path: ${{ env.export_output_folder }}/*
+          if-no-files-found: error
 
   release:
     name: "🚀 GitHub Release"

.github/workflows/tester_ubuntu.yml

@@ -27,6 +27,8 @@ env:
 jobs:
   test-ubuntu:
     runs-on: ubuntu-22.04
+    env:
+      export_output_folder: tests/ci/ubuntu
 
     permissions:
       packages: read
@@ -97,3 +99,22 @@ jobs:
           name: Code Coverage for unittests on Ubuntu (Python-${{ env.PYTHON_VERSION }}]
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Minimal integration test - help
+        run: dicogis-cli --help
+
+      - name: Minimal integration test - version
+        run: dicogis-cli --version
+
+      - name: Minimal integration test - export xlsx
+        run: dicogis-cli inventory --input-folder ./tests/ --language EN --output-path "${{ env.export_output_folder }}/dicogis_ubuntu_tests.xlsx" --verbose
+
+      - name: Minimal integration test - export JSON
+        run: dicogis-cli inventory --input-folder ./tests/ --language EN --output-format json --output-path "${{ env.export_output_folder }}" --verbose
+
+      - name: Save generated files as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dicogis_results_ubuntu
+          path: ${{ env.export_output_folder }}/*
+          if-no-files-found: error

dicogis/cli/cmd_inventory.py

@@ -18,6 +18,7 @@
 # project
 from dicogis.__about__ import __package_name__, __title__, __version__
 from dicogis.constants import SUPPORTED_FORMATS, AvailableLocales, OutputFormats
+from dicogis.export.to_json import MetadatasetSerializerJson
 from dicogis.export.to_xlsx import MetadatasetSerializerXlsx
 from dicogis.georeaders.process_files import ProcessingFiles
 from dicogis.georeaders.read_postgis import ReadPostGIS
@@ -75,7 +76,7 @@ def inventory(
     output_path: Annotated[
         Optional[Path],
         typer.Option(
-            dir_okay=False,
+            dir_okay=True,
             envvar="DICOGIS_OUTPUT_FILEPATH",
             file_okay=True,
             help="If not set, the file is created in the current working directory "
@@ -201,13 +202,22 @@ def inventory(
         language = getlocale()
     localized_strings = TextsManager().load_texts(language_code=language)
 
+    # output file path
+    if output_path is None:
+        output_path = Path(f"DicoGIS_{input_folder.name}_{date.today()}.xlsx")
+
     # output format
     if output_format == "excel":
         # creating the Excel workbook
-        xl_workbook = MetadatasetSerializerXlsx(
+        output_serializer = MetadatasetSerializerXlsx(
             translated_texts=localized_strings,
             opt_raw_path=opt_raw_path,
             opt_size_prettify=opt_prettify_size,
+            output_path=output_path,
+        )
+    elif output_format == "json":
+        output_serializer = MetadatasetSerializerJson(
+            translated_texts=localized_strings, output_path=output_path
         )
     else:
         logger.error(
@@ -272,7 +282,7 @@ def inventory(
 
         # instanciate geofiles processor
         geofiles_processor = ProcessingFiles(
-            format_or_serializer=xl_workbook,
+            format_or_serializer=output_serializer,
             localized_strings=localized_strings,
             # list by tabs
             li_vectors=li_vectors,
@@ -314,19 +324,6 @@ def inventory(
 
         geofiles_processor.process_datasets_in_queue()
 
-        # output file path
-        if output_path is None:
-            output_path = Path(f"DicoGIS_{input_folder.name}_{date.today()}.xlsx")
-
-        xl_workbook.post_serializing()
-        saved = Utilities.safe_save(
-            output_object=xl_workbook,
-            dest_dir=f"{output_path.parent.resolve()}",
-            dest_filename=f"{output_path.resolve()}",
-            ftype="Excel Workbook",
-            gui=False,
-        )
-        logger.info(f"Workbook saved: {saved[1]}")
         send_system_notify(
             notification_title="DicoGIS analysis ended",
             notification_message=f"DicoGIS successfully processed {total_files} files.",
@@ -344,7 +341,7 @@ def inventory(
             raise typer.Exit(1)
 
         # configure output workbook
-        xl_workbook.pre_serializing(has_sgbd=True)
+        output_serializer.pre_serializing(has_sgbd=True)
 
         for pg_service in pg_services:
 
@@ -370,16 +367,16 @@ def inventory(
                 layer = sgbd_reader.conn.GetLayerByIndex(idx_layer)
                 metadataset = sgbd_reader.infos_dataset(layer=layer)
                 logger.info(f"Table examined: {metadataset.name}")
-                xl_workbook.serialize_metadaset(metadataset=metadataset)
+                output_serializer.serialize_metadaset(metadataset=metadataset)
                 logger.debug("Layer metadata stored into workbook.")
 
         # output file path
         if output_path is None:
             output_path = Path(f"DicoGIS_PostGIS_{date.today()}.xlsx")
 
-        xl_workbook.post_serializing()
+        output_serializer.post_serializing()
         saved = Utilities.safe_save(
-            output_object=xl_workbook,
+            output_object=output_serializer,
             dest_dir=f"{output_path.parent.resolve()}",
             dest_filename=f"{output_path.resolve()}",
             ftype="Excel Workbook",

dicogis/export/base_serializer.py

@@ -7,6 +7,7 @@
 
 # Standard library
 import logging
+from pathlib import Path
 
 # ############################################################################
 # ######### Globals ############
@@ -25,33 +26,24 @@ class MetadatasetSerializerBase:
     def __init__(
         self,
         translated_texts: dict,
+        output_path: Path | None = None,
         opt_raw_path: bool = False,
         opt_size_prettify: bool = True,
     ) -> None:
-        """Store metadata into JSON files."""
+        """Initialize object."""
         self.translated_texts = translated_texts
 
+        # output path
+        self.output_path = output_path
+
         # options
         self.opt_raw_path = opt_raw_path
         self.opt_size_prettify = opt_size_prettify
 
-    def pre_serializing(
-        self,
-        has_vector: bool = False,
-        has_raster: bool = False,
-        has_filedb: bool = False,
-        has_mapdocs: bool = False,
-        has_cad: bool = False,
-        has_sgbd: bool = False,
-    ):
-        """Set workbook's sheets accordingly to metadata types.
-
-        Args:
-            has_vector (bool, optional): _description_. Defaults to False.
-            has_raster (bool, optional): _description_. Defaults to False.
-            has_filedb (bool, optional): _description_. Defaults to False.
-            has_mapdocs (bool, optional): _description_. Defaults to False.
-            has_cad (bool, optional): _description_. Defaults to False.
-            has_sgbd (bool, optional): _description_. Defaults to False.
-        """
+    def pre_serializing(self, **kwargs):
+        """Operations to run before serialization."""
+        pass
+
+    def post_serializing(self, **kwargs):
+        """Operations to run after serialization."""
         pass

dicogis/export/to_json.py

@@ -8,15 +8,15 @@
 
 
 # Standard library
+import json
 import logging
 from dataclasses import asdict
+from pathlib import Path
 
 # project
 from dicogis.export.base_serializer import MetadatasetSerializerBase
 from dicogis.models.metadataset import MetaDataset
-
-# 3rd party library
-
+from dicogis.utils.slugger import sluggy
 
 # ##############################################################################
 # ############ Globals ############
@@ -34,12 +34,57 @@
 class MetadatasetSerializerJson(MetadatasetSerializerBase):
     """Export to JSON."""
 
-    def __init__(self, translated_texts: dict, opt_size_prettify: bool = True) -> None:
+    def __init__(
+        self,
+        translated_texts: dict,
+        output_path: Path | None = None,
+        opt_size_prettify: bool = True,
+    ) -> None:
         """Store metadata into JSON files."""
 
+        output_path.mkdir(parents=True, exist_ok=True)
+
         super().__init__(
-            translated_texts=translated_texts, opt_size_prettify=opt_size_prettify
+            translated_texts=translated_texts,
+            opt_size_prettify=opt_size_prettify,
+            output_path=output_path,
         )
 
-    def serialize_metadaset(self, metadataset: MetaDataset) -> dict:
-        return asdict(metadataset)
+    def json_encoder_for_unsupported_types(self, obj_to_encode: object) -> str:
+        """Encode objects which are not serializable as they are by json.dump.
+        Typically: pathlib.Path.
+
+        Args:
+            obj_to_encode: object to encode in JSON
+
+        Returns:
+            encoded object as string
+        """
+        if isinstance(obj_to_encode, Path):
+            return str(obj_to_encode)
+
+    def serialize_metadaset(self, metadataset: MetaDataset) -> Path:
+        """Serialize input metadataset as JSON file stored in output_path.
+
+        Args:
+            metadataset: metadataset to serialize
+
+        Returns:
+            path to the generated JSON file
+        """
+        output_json_filepath = self.output_path.joinpath(
+            f"{sluggy(metadataset.name)}.json"
+        )
+        with output_json_filepath.open(mode="w", encoding="UTF-8") as out_json:
+            json.dump(
+                asdict(metadataset),
+                out_json,
+                default=self.json_encoder_for_unsupported_types,
+                sort_keys=True,
+            )
+
+        logger.debug(
+            f"Metadataset {metadataset.path} serialized into JSON file "
+            f"{output_json_filepath}"
+        )
+        return output_json_filepath

dicogis/export/to_xlsx.py

@@ -149,6 +149,7 @@ class MetadatasetSerializerXlsx(MetadatasetSerializerBase):
     def __init__(
         self,
         translated_texts: dict,
+        output_path: Path | None = None,
         opt_raw_path: bool = False,
         opt_size_prettify: bool = True,
     ) -> None:
@@ -174,6 +175,7 @@ def __init__(
         # initiate with parent
         super().__init__(
             translated_texts=translated_texts,
+            output_path=output_path,
             opt_raw_path=opt_raw_path,
             opt_size_prettify=opt_size_prettify,
         )
@@ -290,6 +292,13 @@ def pre_serializing(
             self.row_index_server_geodatabases = 1
 
     def post_serializing(self):
+        """Run post serialization steps."""
+        self.tunning_workbook()
+        self.output_path.parent.mkdir(parents=True, exist_ok=True)
+        self.workbook.save(filename=self.output_path)
+        logger.info(f"Workbook saved under {self.output_path}")
+
+    def tunning_workbook(self):
         """Clean up and tunning worksheet."""
         for sheet in self.workbook.worksheets:
             # Freezing panes
@@ -329,6 +338,18 @@ def post_serializing(self):
             # for col, value in dims.items():
             #     sheet.column_dimensions[col].width = value
 
+    @lru_cache
+    def is_style_registered(self, style_name: str) -> bool:
+        """Function to check if a named style already exists/
+
+        Args:
+            style_name: name of style to check
+
+        Returns:
+            True if a style with the same name already exist in the worksheet
+        """
+        return any(style.name == style_name for style in self.workbook.named_styles)
+
     @lru_cache(maxsize=128, typed=True)
     def format_as_hyperlink(self, target: Union[str, Path], label: str) -> str:
         """Format a cell hyperlink with a target and a label.