#258 Move file paths cache management to SettingsManager

isogeo.py

@@ -329,8 +329,8 @@ def initGui(self):
     # -------------------------------------------------------------------------
     def onClosePlugin(self):
         """Cleanup necessary items here when plugin dockwidget is closed."""
-        # save cache
-        self.form_mng.results_mng.cache_mng.dumper()
+        # delete geoservices cache
+        self.form_mng.results_mng.cache_mng.clean_geoservice_cache()
         # disconnects
         self.form_mng.closingPlugin.disconnect(self.onClosePlugin)
 

modules/results/cache.py

@@ -9,10 +9,14 @@
 # PyQGIS
 from qgis.utils import iface
 
+# Plugin modules
+from ..settings_manager import SettingsManager
+
 # ############################################################################
 # ########## Globals ###############
 # ##################################
 
+settings_mng = SettingsManager()
 logger = logging.getLogger("IsogeoQgisPlugin")
 
 msgBar = iface.messageBar()
@@ -26,109 +30,46 @@ class CacheManager:
     """Basic class to manage the cache system of the layer addition."""
 
     def __init__(self, geo_service_manager: object):
-        # Path to JSON cache file
-        self.cache_file = Path(__file__).parents[2] / "_user" / "cache.json"
-        # Objects for storing inaccessible elements
-        self.cached_dict = dict()
-        self.cached_unreach_paths = list()
-        self.cached_unreach_postgis = list()
-        self.cached_unreach_service = {
-            "WFS": [],
-            "WMS": [],
-            "WMTS": [],
-            "EFS": [],
-            "EMS": [],
-        }
+
+        self.cached_unreached_paths = list()
         # Translator
         self.tr = object
-        # GeoServiceManagerModule used by LayerAdder wich temporary cache has to be cleaned
+        # GeoServiceManagerModule used by LayerAdder which temporary cache has to be cleaned
         self.geo_srv_mng = geo_service_manager
 
-    def dumper(self):
-        """Builds a dict from the stored inaccessible elements
-        and dumps it into the JSON cache file.
-
-        :returns: the dict dumped in the JSON file
-
-        :rtype: dict
-        """
-        # prepare JSON file content
-        self.cached_dict = {
-            "files": list(set(self.cached_unreach_paths)),
-            "PostGIS": list(set(self.cached_unreach_postgis)),
-            "services": {
-                "WFS": list(set(self.cached_unreach_service.get("WFS"))),
-                "WMS": list(set(self.cached_unreach_service.get("WMS"))),
-                "WMTS": list(set(self.cached_unreach_service.get("WMTS"))),
-                "EFS": list(set(self.cached_unreach_service.get("EFS"))),
-                "EMS": list(set(self.cached_unreach_service.get("EMS"))),
-            },
-        }
-        with open(self.cache_file, "w") as cache:
-            json.dump([self.cached_dict], cache, indent=4)
-        logger.debug("Cache has been dumped")
+    def clean_geoservice_cache(self):
         self.geo_srv_mng.service_cached_dict = {
-            "WFS": dict(),
-            "WMS": dict(),
-            "WMTS": dict(),
-            "EFS": dict(),
-            "EMS": dict(),
+            "WFS": {},
+            "WMS": {},
+            "WMTS": {},
+            "EFS": {},
+            "EMS": {},
         }
 
+    def save_cache_to_qsettings(self):
+        settings_mng.set_value(settings_mng.cache_qsetting_key, self.cached_unreached_paths)
+        return
+
+    def add_file_path(self, file_path: Path):
+        self.cached_unreached_paths.append(file_path)
+        self.save_cache_to_qsettings()
+        return
+
     def loader(self):
-        """Load and store ignored elements from the JSON cache file."""
-        try:
-            with open(self.cache_file, "r") as cache:
-                cache_loaded = json.load(cache)
-            if len(cache_loaded) == 0:
-                logger.debug("Empty cache file.")
-            elif isinstance(cache_loaded[0], dict):
-                self.cached_unreach_paths = cache_loaded[0].get("files")
-                logger.debug("Cached unreachable file path has been successfuly loaded.")
-                self.cached_unreach_postgis = cache_loaded[0].get("PostGIS")
-                self.cached_unreach_service = cache_loaded[0].get("services")
-                for srv_type, li_unreachable_srv in cache_loaded[0].get("services").items():
-                    if len(li_unreachable_srv):
-                        cached_srv_content = [
-                            tuple(unreachable_srv) for unreachable_srv in li_unreachable_srv
-                        ]
-                        self.cached_unreach_service[srv_type] = cached_srv_content
-                    else:
-                        self.cached_unreach_service[srv_type] = []
-            else:
-                logger.debug("Old cache file format detected.")
-                self.cached_unreach_paths = cache_loaded
-            return cache_loaded
-
-        except ValueError:
-            logger.error("Path JSON corrupted")
-        except IOError:
-            logger.debug("Cache file not found. Maybe because of first launch.")
-            self.dumper()
+        """Load ignored elements calling SettingsManager."""
+        self.cached_unreached_paths = settings_mng.load_cache()
+        return
 
     def cleaner(self):
         """Removes the stored elements and empties the JSON cache file."""
-        self.cached_unreach_paths = []
-        self.cached_unreach_postgis = []
-        self.cached_unreach_service = {
-            "WFS": [],
-            "WMS": [],
-            "WMTS": [],
-            "EFS": [],
-            "EMS": [],
-        }
-        self.dumper()
+        self.cached_unreached_paths = []
+        self.save_cache_to_qsettings()
+        self.clean_geoservice_cache()
         logger.debug("Cache has been cleaned")
-        self.geo_srv_mng.service_cached_dict = {
-            "WFS": dict(),
-            "WMS": dict(),
-            "WMTS": dict(),
-            "EFS": dict(),
-            "EMS": dict(),
-        }
         msgBar.pushMessage(
             self.tr("Cache has been cleaned.", context=__class__.__name__), duration=3
         )
+        return
 
 
 # #############################################################################

modules/results/display.py

@@ -7,7 +7,6 @@
 from pathlib import Path
 
 # PyQT
-# from QByteArray
 from qgis.PyQt.QtCore import QSettings, QObject, pyqtSignal, Qt
 from qgis.PyQt.QtGui import QIcon, QPixmap
 from qgis.PyQt.QtWidgets import QTableWidgetItem, QComboBox, QPushButton, QLabel
@@ -19,10 +18,10 @@
 from ..layer.limitations_checker import LimitationsChecker
 from ..layer.geo_service import GeoServiceManager
 from ..layer.database import DataBaseManager
+from ..settings_manager import SettingsManager
 
 # isogeo-pysdk
 from ..isogeo_pysdk import Metadata
-from ..settings_manager import SettingsManager
 
 # ############################################################################
 # ########## Globals ###############
@@ -117,7 +116,7 @@ def __init__(self, search_form_manager: object):
             line_list: {"tooltip": "Line", "pix": pix_line},
             multi_list: {"tooltip": "MultiPolygon", "pix": pix_multi},
         }
-        # set instanciate and load JSON file cache content
+        # set instantiate and load JSON file cache content
         self.cache_mng = CacheManager(self.layer_adder.geo_srv_mng)
         self.cache_mng.loader()
         self.cache_mng.tr = self.tr
@@ -668,27 +667,28 @@ def section_resized_slot(self, *args):
     def _filepath_builder(self, metadata_path: str):
         """Build filepath from metadata path handling various cases. See: #129.
 
-        :param str metadata_path: path found in metadata
+        :param str metadata_path: path to the dataset found in metadata
         """
         # building
         filepath = Path(metadata_path)
         try:
             dir_file = str(filepath.parent.resolve())
         except OSError as e:
-            logger.debug("'{}' is not a reguler path : {}".format(metadata_path, e))
+            logger.debug("'{}' is not a regular path : {}".format(metadata_path, e))
             return False
-        if dir_file not in self.cache_mng.cached_unreach_paths:
+        if dir_file not in self.cache_mng.cached_unreached_paths:
             try:
                 with open(filepath):
                     pass
             except Exception as e:
-                self.cache_mng.cached_unreach_paths.append(dir_file)
-                logger.info("Path is not reachable and has been cached:{} / {}".format(dir_file, e))
+                self.cache_mng.add_file_path(dir_file)
+                logger.info("{} path is not reachable and has been cached:".format(dir_file))
+                logger.info("{}".format(e))
                 return False
+            return str(filepath)
         else:
             logger.debug("Path has been ignored because it's cached.")
             return False
-        return str(filepath)
 
     def build_md_portal_url(self, metadata_id: str):
         """Build the URL of the metadata into Isogeo Portal (see https://github.com/isogeo/isogeo-plugin-qgis/issues/312)

modules/settings_manager.py

@@ -2,6 +2,7 @@
 #! python3  # noqa: E265
 
 # Standard library
+import os
 import logging
 import json
 from pathlib import Path
@@ -37,6 +38,9 @@ def __init__(self):
         self.api_base_url = str
         self.tr = object
 
+        self.cache_json_path = Path(__file__).parents[1] / "_user" / "cache.json"
+        self.cache_qsetting_key = "isogeo/user/unreachable_filepath"
+
         self.db_connections_json_path = Path(__file__).parents[1] / "_user" / "db_connections.json"
         self.db_connections = {}
 
@@ -137,8 +141,51 @@ def dump_json_file(self, file_path: Path, content: dict):
             json.dump(content, outfile, sort_keys=True, indent=4)
         return
 
+    def check_cache_json_content(self, json_content):
+        if not isinstance(json_content, list):
+            return 0
+        elif not len(json_content) == 1:
+            return 0
+        elif not isinstance(json_content[0], dict):
+            return 0
+        elif "files" not in json_content[0]:
+            return 0
+        elif not isinstance(json_content[0].get("file"), list):
+            return 0
+        else:
+            return 1
+
     def load_cache(self):
-        return
+
+        cached_unreached_paths = []
+        if self.cache_qsetting_key in self.allKeys():
+            cached_unreached_paths = self.get_value(self.cache_qsetting_key, [], list)
+        else:
+            json_content = self.load_json_file(self.cache_json_path)
+            if not json_content:
+                self.set_value(self.cache_qsetting_key, cached_unreached_paths)
+            elif json_content == -1:
+                self.set_value(self.cache_qsetting_key, cached_unreached_paths)
+                try:
+                    os.remove(self.cache_json_path)
+                    logger.info("{} old cache file has been deleted successfully".format(self.cache_json_path))
+                except Exception as e:
+                    logger.warning("{} file deletion failed:")
+                    logger.warning(str(e))
+            else:
+                if self.check_cache_json_content(json_content):
+                    cached_unreached_paths = json_content[0].get("files")
+                else:
+                    pass
+                self.set_value(self.cache_qsetting_key, cached_unreached_paths)
+                try:
+                    os.remove(self.cache_json_path)
+                    logger.info("{} old cache file has been deleted successfully".format(self.cache_json_path))
+                except Exception as e:
+                    logger.warning("{} file deletion failed:")
+                    logger.warning(str(e))
+
+        return cached_unreached_paths
 
     def check_db_connections_json_content(self, json_content):
         li_expected_keys = ["Oracle", "PostgreSQL"]
@@ -150,7 +197,8 @@ def check_db_connections_json_content(self, json_content):
             return 0
         elif not all(isinstance(json_content.get(key), list) for key in li_expected_keys):
             return 0
-        return 1
+        else:
+            return 1
 
     def get_default_db_connections_content(self):
 
@@ -385,7 +433,8 @@ def check_quicksearch_json_content(self, json_content):
             return 0
         elif not all(all(isinstance(sub_key, str) for sub_key in json_content[key]) for key in json_content):
             return 0
-        return 1
+        else:
+            return 1
 
     def load_quicksearches_from_json(self):