Modularization 2 - manage layer adding cache

isogeo.py

@@ -171,6 +171,7 @@ def __init__(self, iface):
 
         # initialize plugin directory
         self.plugin_dir = os.path.dirname(__file__)
+        self.plg_basepath = os.path.dirname(os.path.realpath(__file__))
 
         # initialize locale
         locale = qsettings.value('locale/userLocale')[0:2]
@@ -216,13 +217,10 @@ def __init__(self, iface):
         # submodules
         self.md_display = MetadataDisplayer(IsogeoMdDetails())
         self.results_mng = ResultsManager(self)
+        self.results_mng.cache_mng.loader()
 
         # link UI and submodules
         plg_api_mngr.ui_auth_form = self.auth_prompt_form
-        self.results_mng.paths_cache = os.path.realpath(os.path.join(plg_basepath,
-                                                                     "_user",
-                                                                     "paths_cache.json"))
-        self.results_mng._cache_loader()
 
         # start variables
         self.savedSearch = "first"
@@ -232,7 +230,7 @@ def __init__(self, iface):
         self.showDetails = False
         self.store = False
         self.settingsRequest = False
-        self.PostGISdict = self.results_mng.layer_adder.build_postgis_dict(qsettings)
+        self.PostGISdict = self.results_mng.build_postgis_dict(qsettings)
 
         self.currentUrl = ""
         # self.currentUrl = "https://v1.api.isogeo.com/resources/search?
@@ -318,7 +316,7 @@ def initGui(self):
     def onClosePlugin(self):
         """Cleanup necessary items here when plugin dockwidget is closed."""
         # save cache
-        self.results_mng._cache_dumper()
+        self.results_mng.cache_mng.dumper()
         # disconnects
         self.dockwidget.closingPlugin.disconnect(self.onClosePlugin)
 
@@ -1457,7 +1455,7 @@ def run(self):
 
         # # -- Settings tab - Search --------------------------------------------
         # button to empty the cache of filepaths #135
-        self.dockwidget.btn_cache_trash.pressed.connect(self.results_mng._cache_cleaner)
+        self.dockwidget.btn_cache_trash.pressed.connect(self.results_mng.cache_mng.cleaner)
 
         # -- Settings tab - Application authentication ------------------------
         # Change user -> see below for authentication form

modules/__init__.py

@@ -1,6 +1,6 @@
 from .api import IsogeoPlgApiMngr
 from .metadata_display import MetadataDisplayer
-from .results import ResultsManager
+from .results import ResultsManager, CacheManager 
 from .tools import IsogeoPlgTools
 from .quick_search import QuickSearchManager
-from .layer.add_layer import LayerAdder
+from .layer import LayerAdder

modules/layer/add_layer.py

@@ -88,62 +88,6 @@ def __init__(self):
         self.tr = None
         self.md_sync = MetadataSynchronizer()
 
-    def build_postgis_dict(self, input_dict):
-        """Build the dict that stores informations about PostGIS connexions."""
-        # input_dict.beginGroup("PostgreSQL/connections")
-        final_dict = {}
-        for k in sorted(input_dict.allKeys()):
-            if k.startswith("PostgreSQL/connections/")\
-                    and k.endswith("/database"):
-                if len(k.split("/")) == 4:
-                    connection_name = k.split("/")[2]
-                    password_saved = input_dict.value(
-                        'PostgreSQL/connections/' +
-                        connection_name +
-                        '/savePassword')
-                    user_saved = input_dict.value(
-                        'PostgreSQL/connections/' +
-                        connection_name +
-                        '/saveUsername')
-                    if password_saved == 'true' and user_saved == 'true':
-                        dictionary = {'name':
-                                      input_dict.value(
-                                          'PostgreSQL/connections/' +
-                                          connection_name +
-                                          '/database'),
-                                      'host':
-                                      input_dict.value(
-                                          'PostgreSQL/connections/' +
-                                          connection_name +
-                                          '/host'),
-                                      'port':
-                                      input_dict.value(
-                                          'PostgreSQL/connections/' +
-                                          connection_name +
-                                          '/port'),
-                                      'username':
-                                      input_dict.value(
-                                          'PostgreSQL/connections/' +
-                                          connection_name +
-                                          '/username'),
-                                      'password':
-                                      input_dict.value(
-                                          'PostgreSQL/connections/' +
-                                          connection_name +
-                                          '/password')}
-                        final_dict[
-                            input_dict.value('PostgreSQL/connections/' +
-                                             connection_name +
-                                             '/database')
-                        ] = dictionary
-                    else:
-                        continue
-                else:
-                    pass
-            else:
-                pass
-        return final_dict
-
     def build_efs_url(self, api_layer, srv_details, rsc_type="ds_dyn_lyr_srv", mode="complete"):
         """Reformat the input Esri Feature Service url so it fits QGIS criterias.
 

modules/results/__init__.py

@@ -0,0 +1,2 @@
+from .display import ResultsManager
+from .cache import CacheManager

modules/results/cache.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
+
+# Standard library
+import logging
+import json
+
+# ############################################################################
+# ########## Globals ###############
+# ##################################
+
+logger = logging.getLogger("IsogeoQgisPlugin")
+
+# ############################################################################
+# ########## Classes ###############
+# ##################################
+
+
+class CacheManager():
+    """Basic class to manage the cache system of the layer addition. 
+    """
+
+    def __init__(self, path_cache_file : str = None):
+        # Path to JSON cache file
+        if isinstance(path_cache_file, str):
+            self.cache_file = path_cache_file
+        elif path_cache_file != None:
+            raise TypeError("str expected")
+        else:
+            raise ValueError("path to JSON cache file must be given to instantiate CacheManager")
+
+        # Objects for storing inaccessible elements 
+        self.cached_dict = {}
+        self.cached_unreach_paths = []
+        self.cached_unreach_postgis = []
+        self.cached_unreach_srv = []
+
+    def dumper(self):
+        """Builds a dict from the stored inaccessible elements 
+        and dumps it into the JSON cache file.
+
+        :returns: the dict dumped in the JSON file
+
+        :rtype: dict
+        """
+        self.cached_dict = {"files" : list(set(self.cached_unreach_paths)),
+                            "PostGIS" : list(set(self.cached_unreach_postgis)),
+                            "services" : list(set(self.cached_unreach_srv))}
+        logger.debug("cached_dict : {}".format(self.cached_dict))
+
+        with open(self.cache_file, 'w') as cache:
+            json.dump([self.cached_dict], cache, indent=4)
+        logger.debug("Paths cache has been dumped")
+
+        return self.cached_dict
+
+    def loader(self):
+        """Load and store ignored elements from the JSON cache file.
+
+        :returns: the content of the JSON cache file
+
+        :rtype: dict
+        """
+        try:
+            with open(self.cache_file, 'r') as cache:
+                cache_loaded = json.load(cache)
+            logger.debug("cache_loaded : {}".format(cache_loaded))
+            if isinstance(cache_loaded[0], dict) :
+                self.cached_unreach_paths = cache_loaded[0].get("files")
+                self.cached_unreach_postgis = cache_loaded[0].get("PostGIS")
+                self.cached_unreach_srv = cache_loaded[0].get("services")
+                logger.debug("Paths cache has been loaded")
+            else:
+                logger.debug("Old cache file format detected")
+                self.cached_unreach_paths = cache_loaded
+            return cache_loaded
+
+        except ValueError as e:
+            logger.error("Path JSON corrupted")
+        except IOError:
+        	logger.debug("Paths cache file not found. Maybe because of first launch.")
+        	self.dumper()
+
+    def cleaner(self):
+        """Removes the stored elements and empties the JSON cache file."""
+        self.cached_unreach_paths = []
+        self.cached_unreach_postgis = []
+        self.cached_unreach_srv = []
+        self.dumper()
+        logger.debug("Cache has been cleaned")
+
+# #############################################################################
+# ##### Stand alone program ########
+# ##################################
+if __name__ == '__main__':
+    """Standalone execution."""

modules/results.py → modules/results/display.py

@@ -18,8 +18,9 @@
 from qgis.utils import iface
 
 # Plugin modules
-from .tools import IsogeoPlgTools
-from .layer.add_layer import LayerAdder
+from .cache import CacheManager
+from ..tools import IsogeoPlgTools
+from ..layer.add_layer import LayerAdder
 
 # ############################################################################
 # ########## Globals ###############
@@ -67,19 +68,20 @@
 
 class ResultsManager(object):
     """Basic class that holds utilitary methods for the plugin."""
-    paths_cache = ""
 
     def __init__(self, isogeo_plugin):
         """Class constructor."""
         self.isogeo_widget = isogeo_plugin.dockwidget
         self.send_details_request = isogeo_plugin.send_details_request
         self.tr = isogeo_plugin.tr
-        self.cached_unreach_paths = []
 
         self.layer_adder = LayerAdder()
         self.layer_adder.tr = self.tr
         self.add_layer = self.layer_adder.adding
-        self.pg_connections = self.layer_adder.build_postgis_dict(qsettings)
+        self.pg_connections = self.build_postgis_dict(qsettings)
+
+        self.path_cache_file = os.path.realpath(os.path.join(isogeo_plugin.plg_basepath, "_user", "paths_cache.json"))
+        self.cache_mng = CacheManager(self.path_cache_file)
 
     def show_results(self, api_results, tbl_result=None, pg_connections=dict(), progress_bar=QProgressBar):
         """Display the results in a table ."""
@@ -397,7 +399,7 @@ def show_results(self, api_results, tbl_result=None, pg_connections=dict(), prog
     def _filepath_builder(self, metadata_path, mode=1):
         """Build filepath from metadata path handling various cases. See: #129.
 
-        :param dict metadata: path found in metadata
+        :param dict metadata_path: path found in metadata
         :param int mode: mode to apply. Options:
           1 = only with standard path.normpath
           2 = using raw string (useful for Windows systems)
@@ -409,9 +411,7 @@ def _filepath_builder(self, metadata_path, mode=1):
         if mode == 1:
             filepath = os.path.normpath(metadata_path)
             dir_file = os.path.dirname(filepath)
-            # logger.debug("*======* METADATA_PATH : {}".format(metadata_path))
-            # logger.debug("*======* FILEPATH : {}".format(filepath))
-            if dir_file not in self.cached_unreach_paths:
+            if dir_file not in self.cache_mng.cached_unreach_paths:
                 try:
                     with open(filepath) as f:
                         return filepath
@@ -423,45 +423,74 @@ def _filepath_builder(self, metadata_path, mode=1):
                 return False
         elif mode == 2:
             logger.debug("Using forced raw string")
-            dir_file = os.path.dirname(metadata_path)
             filepath = plg_tools._to_raw_string(metadata_path)
+            dir_file = os.path.dirname(metadata_path)
             try:
                 with open(filepath) as f:
                     return filepath
             except IOError:
-                self.cached_unreach_paths.append(dir_file)
+                self.cache_mng.cached_unreach_paths.append(dir_file)
                 logger.debug("Path is not reachable and has been cached:{}".format(dir_file))
                 return False
         else:
             logger.debug("Incorrect mode: {}".format(mode))
             raise ValueError
 
-    def _cache_dumper(self):
-        """Dumps paths ignored into a JSON file. See: #135"""
-        cached_filepaths_unique = set(self.cached_unreach_paths)
-        with open(self.paths_cache, 'w') as cached_path_file:
-            json.dump(list(cached_filepaths_unique), cached_path_file,
-                      sort_keys=True, indent=4)
-        logger.debug("Paths cache has been dumped")
-
-    def _cache_loader(self):
-        """Loads paths ignored into a JSON file."""
-        try:
-            with open(self.paths_cache, 'r') as cached_path_file:
-                self.cached_unreach_paths = json.load(cached_path_file)
-            logger.debug("Paths cache has been loaded")
-        except ValueError as e:
-            logger.error("Path JSON corrupted")
-            self.cached_unreach_paths = []
-        except IOError:
-        	logger.debug("Paths cache file not found. Maybe because of first launch.")
-        	self._cache_dumper()
-
-    def _cache_cleaner(self):
-        """Clean cached paths."""
-        self.cached_unreach_paths = []
-        self._cache_dumper()
-        logger.debug("Cache has been cleaned")
+    def build_postgis_dict(self, input_dict):
+        """Build the dict that stores informations about PostGIS connexions."""
+        # input_dict.beginGroup("PostgreSQL/connections")
+        final_dict = {}
+        for k in sorted(input_dict.allKeys()):
+            if k.startswith("PostgreSQL/connections/")\
+                    and k.endswith("/database"):
+                if len(k.split("/")) == 4:
+                    connection_name = k.split("/")[2]
+                    password_saved = input_dict.value(
+                        'PostgreSQL/connections/' +
+                        connection_name +
+                        '/savePassword')
+                    user_saved = input_dict.value(
+                        'PostgreSQL/connections/' +
+                        connection_name +
+                        '/saveUsername')
+                    if password_saved == 'true' and user_saved == 'true':
+                        dictionary = {'name':
+                                      input_dict.value(
+                                          'PostgreSQL/connections/' +
+                                          connection_name +
+                                          '/database'),
+                                      'host':
+                                      input_dict.value(
+                                          'PostgreSQL/connections/' +
+                                          connection_name +
+                                          '/host'),
+                                      'port':
+                                      input_dict.value(
+                                          'PostgreSQL/connections/' +
+                                          connection_name +
+                                          '/port'),
+                                      'username':
+                                      input_dict.value(
+                                          'PostgreSQL/connections/' +
+                                          connection_name +
+                                          '/username'),
+                                      'password':
+                                      input_dict.value(
+                                          'PostgreSQL/connections/' +
+                                          connection_name +
+                                          '/password')}
+                        final_dict[
+                            input_dict.value('PostgreSQL/connections/' +
+                                             connection_name +
+                                             '/database')
+                        ] = dictionary
+                    else:
+                        continue
+                else:
+                    pass
+            else:
+                pass
+        return final_dict
 
 
 # #############################################################################