Skip to content

Commit

Permalink
Merge pull request #212 from isogeo/dev_modularization
Browse files Browse the repository at this point in the history
Modularization 2 - manage layer adding cache
  • Loading branch information
SimonSAMPERE authored Jul 31, 2019
2 parents 492bb93 + 2ce825e commit 3abd19d
Show file tree
Hide file tree
Showing 7 changed files with 240 additions and 102 deletions.
12 changes: 5 additions & 7 deletions isogeo.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ def __init__(self, iface):

# initialize plugin directory
self.plugin_dir = os.path.dirname(__file__)
self.plg_basepath = os.path.dirname(os.path.realpath(__file__))

# initialize locale
locale = qsettings.value('locale/userLocale')[0:2]
Expand Down Expand Up @@ -216,13 +217,10 @@ def __init__(self, iface):
# submodules
self.md_display = MetadataDisplayer(IsogeoMdDetails())
self.results_mng = ResultsManager(self)
self.results_mng.cache_mng.loader()

# link UI and submodules
plg_api_mngr.ui_auth_form = self.auth_prompt_form
self.results_mng.paths_cache = os.path.realpath(os.path.join(plg_basepath,
"_user",
"paths_cache.json"))
self.results_mng._cache_loader()

# start variables
self.savedSearch = "first"
Expand All @@ -232,7 +230,7 @@ def __init__(self, iface):
self.showDetails = False
self.store = False
self.settingsRequest = False
self.PostGISdict = self.results_mng.layer_adder.build_postgis_dict(qsettings)
self.PostGISdict = self.results_mng.build_postgis_dict(qsettings)

self.currentUrl = ""
# self.currentUrl = "https://v1.api.isogeo.com/resources/search?
Expand Down Expand Up @@ -318,7 +316,7 @@ def initGui(self):
def onClosePlugin(self):
"""Cleanup necessary items here when plugin dockwidget is closed."""
# save cache
self.results_mng._cache_dumper()
self.results_mng.cache_mng.dumper()
# disconnects
self.dockwidget.closingPlugin.disconnect(self.onClosePlugin)

Expand Down Expand Up @@ -1457,7 +1455,7 @@ def run(self):

# # -- Settings tab - Search --------------------------------------------
# button to empty the cache of filepaths #135
self.dockwidget.btn_cache_trash.pressed.connect(self.results_mng._cache_cleaner)
self.dockwidget.btn_cache_trash.pressed.connect(self.results_mng.cache_mng.cleaner)

# -- Settings tab - Application authentication ------------------------
# Change user -> see below for authentication form
Expand Down
4 changes: 2 additions & 2 deletions modules/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from .api import IsogeoPlgApiMngr
from .metadata_display import MetadataDisplayer
from .results import ResultsManager
from .results import ResultsManager, CacheManager
from .tools import IsogeoPlgTools
from .quick_search import QuickSearchManager
from .layer.add_layer import LayerAdder
from .layer import LayerAdder
56 changes: 0 additions & 56 deletions modules/layer/add_layer.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,62 +88,6 @@ def __init__(self):
self.tr = None
self.md_sync = MetadataSynchronizer()

def build_postgis_dict(self, input_dict):
"""Build the dict that stores informations about PostGIS connexions."""
# input_dict.beginGroup("PostgreSQL/connections")
final_dict = {}
for k in sorted(input_dict.allKeys()):
if k.startswith("PostgreSQL/connections/")\
and k.endswith("/database"):
if len(k.split("/")) == 4:
connection_name = k.split("/")[2]
password_saved = input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/savePassword')
user_saved = input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/saveUsername')
if password_saved == 'true' and user_saved == 'true':
dictionary = {'name':
input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/database'),
'host':
input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/host'),
'port':
input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/port'),
'username':
input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/username'),
'password':
input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/password')}
final_dict[
input_dict.value('PostgreSQL/connections/' +
connection_name +
'/database')
] = dictionary
else:
continue
else:
pass
else:
pass
return final_dict

def build_efs_url(self, api_layer, srv_details, rsc_type="ds_dyn_lyr_srv", mode="complete"):
"""Reformat the input Esri Feature Service url so it fits QGIS criterias.
Expand Down
2 changes: 2 additions & 0 deletions modules/results/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
from .display import ResultsManager
from .cache import CacheManager
97 changes: 97 additions & 0 deletions modules/results/cache.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
# -*- coding: utf-8 -*-
from __future__ import (absolute_import, division,
print_function, unicode_literals)

# Standard library
import logging
import json

# ############################################################################
# ########## Globals ###############
# ##################################

logger = logging.getLogger("IsogeoQgisPlugin")

# ############################################################################
# ########## Classes ###############
# ##################################


class CacheManager():
"""Basic class to manage the cache system of the layer addition.
"""

def __init__(self, path_cache_file : str = None):
# Path to JSON cache file
if isinstance(path_cache_file, str):
self.cache_file = path_cache_file
elif path_cache_file != None:
raise TypeError("str expected")
else:
raise ValueError("path to JSON cache file must be given to instantiate CacheManager")

# Objects for storing inaccessible elements
self.cached_dict = {}
self.cached_unreach_paths = []
self.cached_unreach_postgis = []
self.cached_unreach_srv = []

def dumper(self):
"""Builds a dict from the stored inaccessible elements
and dumps it into the JSON cache file.
:returns: the dict dumped in the JSON file
:rtype: dict
"""
self.cached_dict = {"files" : list(set(self.cached_unreach_paths)),
"PostGIS" : list(set(self.cached_unreach_postgis)),
"services" : list(set(self.cached_unreach_srv))}
logger.debug("cached_dict : {}".format(self.cached_dict))

with open(self.cache_file, 'w') as cache:
json.dump([self.cached_dict], cache, indent=4)
logger.debug("Paths cache has been dumped")

return self.cached_dict

def loader(self):
"""Load and store ignored elements from the JSON cache file.
:returns: the content of the JSON cache file
:rtype: dict
"""
try:
with open(self.cache_file, 'r') as cache:
cache_loaded = json.load(cache)
logger.debug("cache_loaded : {}".format(cache_loaded))
if isinstance(cache_loaded[0], dict) :
self.cached_unreach_paths = cache_loaded[0].get("files")
self.cached_unreach_postgis = cache_loaded[0].get("PostGIS")
self.cached_unreach_srv = cache_loaded[0].get("services")
logger.debug("Paths cache has been loaded")
else:
logger.debug("Old cache file format detected")
self.cached_unreach_paths = cache_loaded
return cache_loaded

except ValueError as e:
logger.error("Path JSON corrupted")
except IOError:
logger.debug("Paths cache file not found. Maybe because of first launch.")
self.dumper()

def cleaner(self):
"""Removes the stored elements and empties the JSON cache file."""
self.cached_unreach_paths = []
self.cached_unreach_postgis = []
self.cached_unreach_srv = []
self.dumper()
logger.debug("Cache has been cleaned")

# #############################################################################
# ##### Stand alone program ########
# ##################################
if __name__ == '__main__':
"""Standalone execution."""
103 changes: 66 additions & 37 deletions modules/results.py → modules/results/display.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,9 @@
from qgis.utils import iface

# Plugin modules
from .tools import IsogeoPlgTools
from .layer.add_layer import LayerAdder
from .cache import CacheManager
from ..tools import IsogeoPlgTools
from ..layer.add_layer import LayerAdder

# ############################################################################
# ########## Globals ###############
Expand Down Expand Up @@ -67,19 +68,20 @@

class ResultsManager(object):
"""Basic class that holds utilitary methods for the plugin."""
paths_cache = ""

def __init__(self, isogeo_plugin):
"""Class constructor."""
self.isogeo_widget = isogeo_plugin.dockwidget
self.send_details_request = isogeo_plugin.send_details_request
self.tr = isogeo_plugin.tr
self.cached_unreach_paths = []

self.layer_adder = LayerAdder()
self.layer_adder.tr = self.tr
self.add_layer = self.layer_adder.adding
self.pg_connections = self.layer_adder.build_postgis_dict(qsettings)
self.pg_connections = self.build_postgis_dict(qsettings)

self.path_cache_file = os.path.realpath(os.path.join(isogeo_plugin.plg_basepath, "_user", "paths_cache.json"))
self.cache_mng = CacheManager(self.path_cache_file)

def show_results(self, api_results, tbl_result=None, pg_connections=dict(), progress_bar=QProgressBar):
"""Display the results in a table ."""
Expand Down Expand Up @@ -397,7 +399,7 @@ def show_results(self, api_results, tbl_result=None, pg_connections=dict(), prog
def _filepath_builder(self, metadata_path, mode=1):
"""Build filepath from metadata path handling various cases. See: #129.
:param dict metadata: path found in metadata
:param dict metadata_path: path found in metadata
:param int mode: mode to apply. Options:
1 = only with standard path.normpath
2 = using raw string (useful for Windows systems)
Expand All @@ -409,9 +411,7 @@ def _filepath_builder(self, metadata_path, mode=1):
if mode == 1:
filepath = os.path.normpath(metadata_path)
dir_file = os.path.dirname(filepath)
# logger.debug("*======* METADATA_PATH : {}".format(metadata_path))
# logger.debug("*======* FILEPATH : {}".format(filepath))
if dir_file not in self.cached_unreach_paths:
if dir_file not in self.cache_mng.cached_unreach_paths:
try:
with open(filepath) as f:
return filepath
Expand All @@ -423,45 +423,74 @@ def _filepath_builder(self, metadata_path, mode=1):
return False
elif mode == 2:
logger.debug("Using forced raw string")
dir_file = os.path.dirname(metadata_path)
filepath = plg_tools._to_raw_string(metadata_path)
dir_file = os.path.dirname(metadata_path)
try:
with open(filepath) as f:
return filepath
except IOError:
self.cached_unreach_paths.append(dir_file)
self.cache_mng.cached_unreach_paths.append(dir_file)
logger.debug("Path is not reachable and has been cached:{}".format(dir_file))
return False
else:
logger.debug("Incorrect mode: {}".format(mode))
raise ValueError

def _cache_dumper(self):
"""Dumps paths ignored into a JSON file. See: #135"""
cached_filepaths_unique = set(self.cached_unreach_paths)
with open(self.paths_cache, 'w') as cached_path_file:
json.dump(list(cached_filepaths_unique), cached_path_file,
sort_keys=True, indent=4)
logger.debug("Paths cache has been dumped")

def _cache_loader(self):
"""Loads paths ignored into a JSON file."""
try:
with open(self.paths_cache, 'r') as cached_path_file:
self.cached_unreach_paths = json.load(cached_path_file)
logger.debug("Paths cache has been loaded")
except ValueError as e:
logger.error("Path JSON corrupted")
self.cached_unreach_paths = []
except IOError:
logger.debug("Paths cache file not found. Maybe because of first launch.")
self._cache_dumper()

def _cache_cleaner(self):
"""Clean cached paths."""
self.cached_unreach_paths = []
self._cache_dumper()
logger.debug("Cache has been cleaned")
def build_postgis_dict(self, input_dict):
"""Build the dict that stores informations about PostGIS connexions."""
# input_dict.beginGroup("PostgreSQL/connections")
final_dict = {}
for k in sorted(input_dict.allKeys()):
if k.startswith("PostgreSQL/connections/")\
and k.endswith("/database"):
if len(k.split("/")) == 4:
connection_name = k.split("/")[2]
password_saved = input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/savePassword')
user_saved = input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/saveUsername')
if password_saved == 'true' and user_saved == 'true':
dictionary = {'name':
input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/database'),
'host':
input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/host'),
'port':
input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/port'),
'username':
input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/username'),
'password':
input_dict.value(
'PostgreSQL/connections/' +
connection_name +
'/password')}
final_dict[
input_dict.value('PostgreSQL/connections/' +
connection_name +
'/database')
] = dictionary
else:
continue
else:
pass
else:
pass
return final_dict


# #############################################################################
Expand Down
Loading

0 comments on commit 3abd19d

Please sign in to comment.