From 8408d61823c7e6975031a6245e74b1cd78bde65f Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 7 Sep 2018 10:14:22 -0400
Subject: [PATCH 001/124] add remote resources from the groups view

---
 magpie/ui/management/sync_resources.py        | 60 +++++++++++++++++++
 .../ui/management/templates/tree_scripts.mako |  1 +
 magpie/ui/management/views.py                 | 39 +++++++++++-
 requirements.txt                              |  1 +
 4 files changed, 100 insertions(+), 1 deletion(-)
 create mode 100644 magpie/ui/management/sync_resources.py

diff --git a/magpie/ui/management/sync_resources.py b/magpie/ui/management/sync_resources.py
new file mode 100644
index 000000000..7ecbc42f7
--- /dev/null
+++ b/magpie/ui/management/sync_resources.py
@@ -0,0 +1,60 @@
+import threddsclient
+
+
+def merge_db_and_remote_resources(current_resources, service_url, service_name):
+    depth = 2  # replace with global parameter?
+
+    external_resources = {}
+
+    if service_name == "thredds":
+        external_resources = thredds_get_references(service_url, depth)
+
+    if external_resources:
+        # don't compare the name of the service, only the resources
+        service_name = list(current_resources)[0]
+        external_service_name = list(external_resources)[0]
+        external_resources[service_name] = external_resources[external_service_name]
+        del external_resources[external_service_name]
+
+        merged_resources = traverse_and_merge_ressources(current_resources, external_resources)
+
+        return merged_resources
+    else:
+        return current_resources
+
+
+def thredds_get_references(url, depth=0, **kwargs):
+    cat = threddsclient.read_url(url, **kwargs)
+    name = cat.name
+
+    tree_item = {name: {'children': {}}}
+
+    if depth > 0:
+        for reference in cat.flat_references():
+            tree_item[name]['children'].update(thredds_get_references(reference.url, depth - 1))
+
+    return tree_item
+
+
+def traverse_and_merge_ressources(current_resources, external_resources, remote_path=""):
+    for resource_name, values in current_resources.items():
+        matches_remote = resource_name in external_resources
+        values["matches_remote"] = matches_remote
+
+        external_resource_children = external_resources.get(resource_name, {}).get('children', {})
+        new_path = "/".join([remote_path, resource_name])
+        traverse_and_merge_ressources(values['children'], external_resource_children, new_path)
+
+    for resource_name, values in external_resources.items():
+        if not resource_name in current_resources:
+            new_path = "/".join([remote_path, resource_name])
+            new_resource = {'permission_names': [],
+                            'children': {},
+                            'id': new_path,
+                            'remote_path': new_path,
+                            'matches_remote': True}
+
+            current_resources[resource_name] = new_resource
+            traverse_and_merge_ressources(new_resource['children'], values['children'], new_path)
+
+    return current_resources
diff --git a/magpie/ui/management/templates/tree_scripts.mako b/magpie/ui/management/templates/tree_scripts.mako
index bca77e8f7..216b1eb93 100644
--- a/magpie/ui/management/templates/tree_scripts.mako
+++ b/magpie/ui/management/templates/tree_scripts.mako
@@ -50,6 +50,7 @@ li.Collapsed {
             % endif
                 <div class="tree_item">${key}</div>
                 <input type="hidden" value=${tree[key]['id']} name="resource_id">
+                <input type="hidden" value=${tree[key].get('remote_path', '')} name="remote_path">
                 ${item_renderer(key, tree[key], level)}
             </li>
         </form>
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index c683252bb..3d3fed5c1 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -4,6 +4,7 @@
 from magpie.services import service_type_dict
 from magpie.models import resource_type_dict
 from magpie.ui.management import check_response
+from magpie.ui.management.sync_resources import merge_db_and_remote_resources
 from magpie.ui.home import add_template_data
 from magpie import register, __meta__
 from distutils.version import LooseVersion
@@ -474,6 +475,9 @@ def edit_group(self):
             elif u'goto_service' in self.request.POST:
                 return self.goto_service(res_id)
             elif u'resource_id' in self.request.POST:
+                remote_path = self.request.POST.get('remote_path')
+                if remote_path:
+                    res_id = self.add_external_resource(cur_svc_type, group_name, remote_path)
                 self.edit_user_or_group_resource_permissions(group_name, res_id, is_user=False)
             else:
                 self.edit_group_users(group_name)
@@ -486,16 +490,49 @@ def edit_group(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
+        service_url = self.get_service_data(cur_svc_type)['service_url']
+        resources = merge_db_and_remote_resources(res_perms, service_url, cur_svc_type)
+
         group_info[u'group_name'] = group_name
         group_info[u'cur_svc_type'] = cur_svc_type
         group_info[u'users'] = self.get_user_names()
         group_info[u'members'] = self.get_group_users(group_name)
         group_info[u'svc_types'] = svc_types
         group_info[u'cur_svc_type'] = cur_svc_type
-        group_info[u'resources'] = res_perms
+        group_info[u'resources'] = resources
         group_info[u'permissions'] = res_perm_names
         return add_template_data(self.request, data=group_info)
 
+    def add_external_resource(self, service, group_name, resource_path):
+        try:
+            res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(group_name,
+                                                                                          services=[service],
+                                                                                          service_type=service,
+                                                                                          is_user=False)
+        except Exception as e:
+            raise HTTPBadRequest(detail=repr(e))
+
+        def parse_resources_and_put(resources, path, parent_id=None):
+            current_name = path.pop(0)
+            if current_name in resources:
+                res_id = parse_resources_and_put(resources[current_name]['children'],
+                                                 path,
+                                                 parent_id=resources[current_name]['id'])
+            else:
+                resources_url = '{url}/resources'.format(url=self.magpie_url)
+                data = {
+                    'resource_name': current_name,
+                    'resource_type': "directory",
+                    'parent_id': parent_id,
+                }
+                response = check_response(requests.post(resources_url, data=data, cookies=self.request.cookies))
+                res_id = response.json()['resource']['resource_id']
+                if path:
+                    res_id = parse_resources_and_put({}, path, parent_id=res_id)
+            return res_id
+
+        return parse_resources_and_put(res_perms, resource_path.split("/")[1:])
+
     @view_config(route_name='view_services', renderer='templates/view_services.mako')
     def view_services(self):
         if 'delete' in self.request.POST:
diff --git a/requirements.txt b/requirements.txt
index 2a8bdbfb0..dea01f4fa 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -29,3 +29,4 @@ python-dotenv
 cornice_swagger>=0.7.0
 cornice
 colander
+threddsclient

From eb889b5f402a8cb3a5141bef4a114e31842e5807 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 7 Sep 2018 14:24:47 -0400
Subject: [PATCH 002/124] default service_url bug

---
 magpie/ui/management/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 3d3fed5c1..ef7d363df 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -490,7 +490,7 @@ def edit_group(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        service_url = self.get_service_data(cur_svc_type)['service_url']
+        service_url = services[list(services)[0]].get('service_url', '')
         resources = merge_db_and_remote_resources(res_perms, service_url, cur_svc_type)
 
         group_info[u'group_name'] = group_name

From 446ca7eb4687b0717f69c3c49eb93a68642a6139 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 7 Sep 2018 14:30:13 -0400
Subject: [PATCH 003/124] groups view: add flag + button to remove a resource
 not on remote server

---
 magpie/ui/home/static/style.css                  | 14 ++++++++++++++
 magpie/ui/management/sync_resources.py           |  4 ++--
 magpie/ui/management/templates/edit_group.mako   | 11 +++++++++++
 magpie/ui/management/templates/tree_scripts.mako |  1 +
 magpie/ui/management/views.py                    |  5 ++++-
 5 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/magpie/ui/home/static/style.css b/magpie/ui/home/static/style.css
index d0f2b8868..6aab54537 100644
--- a/magpie/ui/home/static/style.css
+++ b/magpie/ui/home/static/style.css
@@ -403,6 +403,20 @@ div.perm_title {
     margin: 0.1em 0 0 -4em;
 }
 
+.tree_item_message {
+    text-align: left;
+    color: gray;
+    display: inline-flex;
+    float: right;
+    margin: 0.2em 0 0 0;
+}
+
+.tree_item_message>img {
+    width: 1.5em;
+    height: 1.5em;
+    margin: -0.15em 0 0 0;
+}
+
 div.perm_checkbox {
     width:3em;
     float:right;
diff --git a/magpie/ui/management/sync_resources.py b/magpie/ui/management/sync_resources.py
index 7ecbc42f7..495dc684a 100644
--- a/magpie/ui/management/sync_resources.py
+++ b/magpie/ui/management/sync_resources.py
@@ -19,8 +19,8 @@ def merge_db_and_remote_resources(current_resources, service_url, service_name):
         merged_resources = traverse_and_merge_ressources(current_resources, external_resources)
 
         return merged_resources
-    else:
-        return current_resources
+
+    return current_resources
 
 
 def thredds_get_references(url, depth=0, **kwargs):
diff --git a/magpie/ui/management/templates/edit_group.mako b/magpie/ui/management/templates/edit_group.mako
index d8b8e7169..4da40f1c8 100644
--- a/magpie/ui/management/templates/edit_group.mako
+++ b/magpie/ui/management/templates/edit_group.mako
@@ -15,6 +15,17 @@
             </div>
         % endif
     %endfor
+    % if not value.get('matches_remote', True):
+        <form id="delete_resource_${value['id']}" action="${request.path}" method="post">
+            <div class="tree_button">
+                <input type="submit" class="button delete" value="Clean" name="clean_resource">
+            </div>
+            <p class="tree_item_message">
+                <img title="This resource is absent from the remote server."
+                     src="${request.static_url('magpie.ui.home:static/warning_exclamation.png')}" />
+            </p>
+        </form>
+    % endif
     % if level == 0:
         <div class="tree_button">
             <input type="submit" class="tree_button goto_service" value="Edit Service" name="goto_service">
diff --git a/magpie/ui/management/templates/tree_scripts.mako b/magpie/ui/management/templates/tree_scripts.mako
index 216b1eb93..fd47a526b 100644
--- a/magpie/ui/management/templates/tree_scripts.mako
+++ b/magpie/ui/management/templates/tree_scripts.mako
@@ -51,6 +51,7 @@ li.Collapsed {
                 <div class="tree_item">${key}</div>
                 <input type="hidden" value=${tree[key]['id']} name="resource_id">
                 <input type="hidden" value=${tree[key].get('remote_path', '')} name="remote_path">
+                <input type="hidden" value=${tree[key].get('matches_remote', '')} name="matches_remote">
                 ${item_renderer(key, tree[key], level)}
             </li>
         </form>
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index ef7d363df..e7721dca6 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -474,11 +474,14 @@ def edit_group(self):
                 return HTTPFound(self.request.route_url('edit_group', **group_info))
             elif u'goto_service' in self.request.POST:
                 return self.goto_service(res_id)
-            elif u'resource_id' in self.request.POST:
+            elif u'permission' in self.request.POST:
                 remote_path = self.request.POST.get('remote_path')
                 if remote_path:
                     res_id = self.add_external_resource(cur_svc_type, group_name, remote_path)
                 self.edit_user_or_group_resource_permissions(group_name, res_id, is_user=False)
+            elif u'clean_resource' in self.request.POST:
+                url = '{url}/resources/{resource_id}'.format(url=self.magpie_url, resource_id=res_id)
+                check_response(requests.delete(url, cookies=self.request.cookies))
             else:
                 self.edit_group_users(group_name)
 

From 6106f24c28aeab5ceea7ddb1276508f9cf837b3e Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 7 Sep 2018 14:56:15 -0400
Subject: [PATCH 004/124] add warning when there was an error fetching the data
 from remote server

---
 magpie/ui/management/templates/edit_group.mako | 11 +++++++----
 magpie/ui/management/views.py                  | 12 ++++++++++--
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/magpie/ui/management/templates/edit_group.mako b/magpie/ui/management/templates/edit_group.mako
index 4da40f1c8..60addc6fa 100644
--- a/magpie/ui/management/templates/edit_group.mako
+++ b/magpie/ui/management/templates/edit_group.mako
@@ -110,11 +110,14 @@
 
     <div class="current_tab_panel">
         <div class="clear"/>
+        %if error_message:
+            <div class="alert danger visible">${error_message}</div>
+        %endif
         <div class="tree_header">
-        <div class="tree_item">Resources</div>
-        %for perm in permissions:
-            <div class="perm_title">${perm}</div>
-        %endfor
+            <div class="tree_item">Resources</div>
+            %for perm in permissions:
+                <div class="perm_title">${perm}</div>
+            %endfor
         </div>
         <div class="tree">
             ${tree.render_tree(render_item, resources)}
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index e7721dca6..9befc8a0f 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -494,15 +494,23 @@ def edit_group(self):
             raise HTTPBadRequest(detail=repr(e))
 
         service_url = services[list(services)[0]].get('service_url', '')
-        resources = merge_db_and_remote_resources(res_perms, service_url, cur_svc_type)
 
+        error_message = None
+
+        try:
+            res_perms = merge_db_and_remote_resources(res_perms, service_url, cur_svc_type)
+        except Exception:
+            error_message = ("Couldn't get resources from the remote service ({}) "
+                             "Only local resources are displayed. ".format(service_url))
+
+        group_info[u'error_message'] = error_message
         group_info[u'group_name'] = group_name
         group_info[u'cur_svc_type'] = cur_svc_type
         group_info[u'users'] = self.get_user_names()
         group_info[u'members'] = self.get_group_users(group_name)
         group_info[u'svc_types'] = svc_types
         group_info[u'cur_svc_type'] = cur_svc_type
-        group_info[u'resources'] = resources
+        group_info[u'resources'] = res_perms
         group_info[u'permissions'] = res_perm_names
         return add_template_data(self.request, data=group_info)
 

From 487de88a7137016e87a543f80710560b9d6c5399 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 10 Sep 2018 14:37:03 -0400
Subject: [PATCH 005/124] Clarify how to implement a new service by using an
 abstract class

---
 magpie/ui/management/sync_resources.py | 172 +++++++++++++++++++------
 magpie/ui/management/views.py          |   4 +-
 2 files changed, 134 insertions(+), 42 deletions(-)

diff --git a/magpie/ui/management/sync_resources.py b/magpie/ui/management/sync_resources.py
index 495dc684a..e4692edaf 100644
--- a/magpie/ui/management/sync_resources.py
+++ b/magpie/ui/management/sync_resources.py
@@ -1,60 +1,152 @@
+"""
+Sychronize local and remote resources.
+
+To implement a new service, see the _SyncServiceInterface class.
+"""
+
+import abc
+import copy
+
 import threddsclient
 
 
-def merge_db_and_remote_resources(current_resources, service_url, service_name):
-    depth = 2  # replace with global parameter?
+def merge_local_and_remote_resources(resources_local, service_name, service_url):
+    """Main function to sync resources with remote server"""
+    if service_url.endswith("/"):  # remove trailing slash
+        service_url = service_url[:-1]
+
+    synchronizers = {
+        "thredds": _SyncServiceThreads(service_url),
+    }
+
+    sync_service = synchronizers.get(service_name.lower(), _SyncServiceDefault())
+
+    remote_resources = sync_service.get_resources()
+    merged_resources = _merge_resources(resources_local, remote_resources)
+
+    return merged_resources
+
+
+def _merge_resources(resources_local, resources_remote):
+    """
+    Merge resources_local and resources_remote, adding the following keys to the output:
+
+        - remote_path: '/' separated string representing the remote path of the resource
+        - matches_remote: True or False depending if the resource is present on the remote server
+        - id: set to the value of 'remote_path' if the resource if remote only
+
+    returns a dictionary of the form validated by 'is_valid_resource_schema'
+
+    """
+    if not resources_remote:
+        return resources_local
+
+    assert _is_valid_resource_schema(resources_local)
+    assert _is_valid_resource_schema(resources_remote)
+
+    if not resources_local:
+        raise ValueError("The resources must contain at least the service name.")
+
+    # The first item is the service name. It is skipped so that only the resources are compared.
+    service_name = resources_local.keys()[0]
+    _, remote_values = resources_remote.popitem()
+    resources_remote = {service_name: remote_values}
+
+    # don't overwrite the input arguments
+    merged_resources = copy.deepcopy(resources_local)
+
+    def recurse(_resources_local, _resources_remote, remote_path=""):
+        for resource_name_local, values in _resources_local.items():
+            current_path = "/".join([remote_path, resource_name_local])
+            matches_remote = resource_name_local in _resources_remote
+
+            values["remote_path"] = current_path
+            values["matches_remote"] = matches_remote
+
+            resource_remote_children = _resources_remote[resource_name_local]['children'] if matches_remote else {}
+
+            recurse(values['children'], resource_remote_children, current_path)
+
+        for resource_name_remote, values in _resources_remote.items():
+            if resource_name_remote not in _resources_local:
+                current_path = "/".join([remote_path, resource_name_remote])
+                new_resource = {'permission_names': [],
+                                'children': {},
+                                'id': current_path,
+                                'remote_path': current_path,
+                                'matches_remote': True}
+
+                _resources_local[resource_name_remote] = new_resource
+                recurse(new_resource['children'], values['children'], current_path)
+
+    recurse(merged_resources, resources_remote)
+
+    return merged_resources
 
-    external_resources = {}
 
-    if service_name == "thredds":
-        external_resources = thredds_get_references(service_url, depth)
+def _is_valid_resource_schema(resources):
+    """
+    Returns True if the structure of the input dictionary is a tree of the form:
 
-    if external_resources:
-        # don't compare the name of the service, only the resources
-        service_name = list(current_resources)[0]
-        external_service_name = list(external_resources)[0]
-        external_resources[service_name] = external_resources[external_service_name]
-        del external_resources[external_service_name]
+    {'resource_name_1': {'children': {'resource_name_3': {'children': {}},
+                                      'resource_name_4': {'children': {}}
+                                      }
+                         },
+     'resource_name_2': {'children': {}}
+     }
+    :return: bool
+    """
+    for resource_name, values in resources.items():
+        if not isinstance(resource_name, basestring):
+            return False
+        if 'children' not in values:
+            return False
+        if not isinstance(values['children'], dict):
+            return False
+        return _is_valid_resource_schema(values['children'])
+    return True
 
-        merged_resources = traverse_and_merge_ressources(current_resources, external_resources)
 
-        return merged_resources
+class _SyncServiceInterface:
+    __metaclass__ = abc.ABCMeta
 
-    return current_resources
+    @abc.abstractmethod
+    def get_resources(self):
+        """
+        This is the function actually fetching the data from the remote service.
+        Implement this for every specific service.
 
+        :return: The returned dictionary must be validated by 'is_valid_resource_schema'
+        """
+        pass
 
-def thredds_get_references(url, depth=0, **kwargs):
-    cat = threddsclient.read_url(url, **kwargs)
-    name = cat.name
 
-    tree_item = {name: {'children': {}}}
+class _SyncServiceThreads(_SyncServiceInterface):
+    DEPTH_DEFAULT = 2
 
-    if depth > 0:
-        for reference in cat.flat_references():
-            tree_item[name]['children'].update(thredds_get_references(reference.url, depth - 1))
+    def __init__(self, thredds_url, depth=DEPTH_DEFAULT, **kwargs):
+        self.thredds_url = thredds_url
+        self.depth = depth
+        self.kwargs = kwargs  # kwargs is passed to the requests.get method.
 
-    return tree_item
+    def get_resources(self):
+        def thredds_get_resources(url, depth, **kwargs):
+            cat = threddsclient.read_url(url, **kwargs)
+            name = cat.name
 
+            tree_item = {name: {'children': {}}}
 
-def traverse_and_merge_ressources(current_resources, external_resources, remote_path=""):
-    for resource_name, values in current_resources.items():
-        matches_remote = resource_name in external_resources
-        values["matches_remote"] = matches_remote
+            if depth > 0:
+                for reference in cat.flat_references():
+                    tree_item[name]['children'].update(thredds_get_resources(reference.url, depth - 1, **kwargs))
 
-        external_resource_children = external_resources.get(resource_name, {}).get('children', {})
-        new_path = "/".join([remote_path, resource_name])
-        traverse_and_merge_ressources(values['children'], external_resource_children, new_path)
+            return tree_item
 
-    for resource_name, values in external_resources.items():
-        if not resource_name in current_resources:
-            new_path = "/".join([remote_path, resource_name])
-            new_resource = {'permission_names': [],
-                            'children': {},
-                            'id': new_path,
-                            'remote_path': new_path,
-                            'matches_remote': True}
+        resources = thredds_get_resources(self.thredds_url, self.depth, **self.kwargs)
+        assert _is_valid_resource_schema(resources), "Error in Interface implementation"
+        return resources
 
-            current_resources[resource_name] = new_resource
-            traverse_and_merge_ressources(new_resource['children'], values['children'], new_path)
 
-    return current_resources
+class _SyncServiceDefault(_SyncServiceInterface):
+    def get_resources(self):
+        return {}
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 9befc8a0f..cdd3f3073 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -4,7 +4,7 @@
 from magpie.services import service_type_dict
 from magpie.models import resource_type_dict
 from magpie.ui.management import check_response
-from magpie.ui.management.sync_resources import merge_db_and_remote_resources
+from magpie.ui.management.sync_resources import merge_local_and_remote_resources
 from magpie.ui.home import add_template_data
 from magpie import register, __meta__
 from distutils.version import LooseVersion
@@ -498,7 +498,7 @@ def edit_group(self):
         error_message = None
 
         try:
-            res_perms = merge_db_and_remote_resources(res_perms, service_url, cur_svc_type)
+            res_perms = merge_local_and_remote_resources(res_perms, cur_svc_type, service_url)
         except Exception:
             error_message = ("Couldn't get resources from the remote service ({}) "
                              "Only local resources are displayed. ".format(service_url))

From 47149b0c2412c37c32232a0148e2b068c4e97995 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 10 Sep 2018 14:37:44 -0400
Subject: [PATCH 006/124] Implement Geoserver

---
 magpie/ui/management/sync_resources.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/magpie/ui/management/sync_resources.py b/magpie/ui/management/sync_resources.py
index e4692edaf..e6d31b3fa 100644
--- a/magpie/ui/management/sync_resources.py
+++ b/magpie/ui/management/sync_resources.py
@@ -7,6 +7,7 @@
 import abc
 import copy
 
+import requests
 import threddsclient
 
 
@@ -17,6 +18,7 @@ def merge_local_and_remote_resources(resources_local, service_name, service_url)
 
     synchronizers = {
         "thredds": _SyncServiceThreads(service_url),
+        "geoserver-api": _SyncServiceGeoserver(service_url),
     }
 
     sync_service = synchronizers.get(service_name.lower(), _SyncServiceDefault())
@@ -121,6 +123,24 @@ def get_resources(self):
         pass
 
 
+class _SyncServiceGeoserver:
+    def __init__(self, geoserver_url):
+        self.geoserver_url = geoserver_url
+
+    def get_resources(self):
+        # Only workspaces are fetched for now
+        workspaces_url = "{}/{}".format(self.geoserver_url, "workspaces")
+        resp = requests.get(workspaces_url, headers={"Accept": "application/json"})
+        resp.raise_for_status()
+        workspaces_list = resp.json().get("workspaces", {}).get("workspace", {})
+
+        workspaces = {w["name"]: {"children": {}} for w in workspaces_list}
+
+        resources = {"geoserver-api": {"children": workspaces}}
+        assert _is_valid_resource_schema(resources), "Error in Interface implementation"
+        return resources
+
+
 class _SyncServiceThreads(_SyncServiceInterface):
     DEPTH_DEFAULT = 2
 

From 890f521ad4639b2467daee322bb2d7a7169b1ca7 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 10 Sep 2018 16:40:02 -0400
Subject: [PATCH 007/124] allow integers for resource_names

---
 magpie/{ui/management => helpers}/sync_resources.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)
 rename magpie/{ui/management => helpers}/sync_resources.py (96%)

diff --git a/magpie/ui/management/sync_resources.py b/magpie/helpers/sync_resources.py
similarity index 96%
rename from magpie/ui/management/sync_resources.py
rename to magpie/helpers/sync_resources.py
index e6d31b3fa..64b6db7d1 100644
--- a/magpie/ui/management/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -59,7 +59,7 @@ def _merge_resources(resources_local, resources_remote):
 
     def recurse(_resources_local, _resources_remote, remote_path=""):
         for resource_name_local, values in _resources_local.items():
-            current_path = "/".join([remote_path, resource_name_local])
+            current_path = "/".join([remote_path, str(resource_name_local)])
             matches_remote = resource_name_local in _resources_remote
 
             values["remote_path"] = current_path
@@ -71,7 +71,7 @@ def recurse(_resources_local, _resources_remote, remote_path=""):
 
         for resource_name_remote, values in _resources_remote.items():
             if resource_name_remote not in _resources_local:
-                current_path = "/".join([remote_path, resource_name_remote])
+                current_path = "/".join([remote_path, str(resource_name_remote)])
                 new_resource = {'permission_names': [],
                                 'children': {},
                                 'id': current_path,
@@ -99,8 +99,6 @@ def _is_valid_resource_schema(resources):
     :return: bool
     """
     for resource_name, values in resources.items():
-        if not isinstance(resource_name, basestring):
-            return False
         if 'children' not in values:
             return False
         if not isinstance(values['children'], dict):

From 71c464966e16a99e4463e1bfbd9f911219904491 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Wed, 12 Sep 2018 14:59:58 -0400
Subject: [PATCH 008/124] Save remote resources in a separate table.

This is the base for a cron service to be ran periodically.
The local and remote resource tables are compared when the
view is requested.
---
 .../d01af1f2e445_create_remote_sync_tables.py |  64 +++++++
 magpie/definitions/ziggurat_definitions.py    |   2 +-
 magpie/helpers/sync_resources.py              | 173 ++++++++++++++++--
 magpie/models.py                              |  76 +++++++-
 magpie/ui/management/views.py                 |   4 +-
 5 files changed, 299 insertions(+), 20 deletions(-)
 create mode 100644 magpie/alembic/versions/d01af1f2e445_create_remote_sync_tables.py

diff --git a/magpie/alembic/versions/d01af1f2e445_create_remote_sync_tables.py b/magpie/alembic/versions/d01af1f2e445_create_remote_sync_tables.py
new file mode 100644
index 000000000..bd03a779b
--- /dev/null
+++ b/magpie/alembic/versions/d01af1f2e445_create_remote_sync_tables.py
@@ -0,0 +1,64 @@
+"""create remote sync tables
+
+Revision ID: d01af1f2e445
+Revises: c352a98d570e
+Create Date: 2018-09-11 10:56:23.779143
+
+"""
+import os, sys
+
+cur_file = os.path.abspath(__file__)
+root_dir = os.path.dirname(cur_file)  # version
+root_dir = os.path.dirname(root_dir)  # alembic
+root_dir = os.path.dirname(root_dir)  # magpie
+root_dir = os.path.dirname(root_dir)  # root
+sys.path.insert(0, root_dir)
+
+from alembic import op
+from magpie.definitions.sqlalchemy_definitions import *
+
+Session = sessionmaker()
+
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'd01af1f2e445'
+down_revision = 'c352a98d570e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('remote_resources',
+                    sa.Column('resource_id', sa.Integer(), primary_key=True,
+                              nullable=False, autoincrement=True),
+                    sa.Column('service_id',
+                              sa.Integer(),
+                              sa.ForeignKey('services.resource_id', onupdate='CASCADE', ondelete='CASCADE'),
+                              index=True,
+                              nullable=False),
+                    sa.Column('parent_id',
+                              sa.Integer(),
+                              sa.ForeignKey('remote_resources.resource_id', onupdate='CASCADE', ondelete='SET NULL'),
+                              nullable=True),
+                    sa.Column('ordering', sa.Integer(), default=0, nullable=False),
+                    sa.Column('resource_name', sa.Unicode(100), nullable=False),
+                    sa.Column('resource_type', sa.Unicode(30), nullable=False),
+                    )
+    op.create_table('remote_resources_sync_info',
+                    sa.Column('id', sa.Integer(), primary_key=True, nullable=False, autoincrement=True),
+                    sa.Column('service_id',
+                              sa.Integer(),
+                              sa.ForeignKey('services.resource_id', onupdate='CASCADE', ondelete='CASCADE'),
+                              index=True,
+                              nullable=False),
+                    sa.Column('remote_resource_id',
+                              sa.Integer(),
+                              sa.ForeignKey('remote_resources.resource_id', onupdate='CASCADE', ondelete='CASCADE')),
+                    sa.Column('last_sync', sa.DateTime(), nullable=True),
+                    )
+
+
+def downgrade():
+    op.drop_table('remote_resources_sync_info')
+    op.drop_table('remote_resources')
diff --git a/magpie/definitions/ziggurat_definitions.py b/magpie/definitions/ziggurat_definitions.py
index a6e6b0dd4..5936178b4 100644
--- a/magpie/definitions/ziggurat_definitions.py
+++ b/magpie/definitions/ziggurat_definitions.py
@@ -1,7 +1,7 @@
 # import required definitions
 from ziggurat_foundations import ziggurat_model_init
 from ziggurat_foundations.models import groupfinder
-from ziggurat_foundations.models.base import get_db_session
+from ziggurat_foundations.models.base import get_db_session, BaseModel
 from ziggurat_foundations.models.external_identity import ExternalIdentityMixin
 from ziggurat_foundations.models.group import GroupMixin
 from ziggurat_foundations.models.group_permission import GroupPermissionMixin
diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 64b6db7d1..99da0a2e5 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -6,26 +6,20 @@
 
 import abc
 import copy
+from collections import OrderedDict
 
 import requests
 import threddsclient
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
 
+from magpie import db, models
 
-def merge_local_and_remote_resources(resources_local, service_name, service_url):
-    """Main function to sync resources with remote server"""
-    if service_url.endswith("/"):  # remove trailing slash
-        service_url = service_url[:-1]
 
-    synchronizers = {
-        "thredds": _SyncServiceThreads(service_url),
-        "geoserver-api": _SyncServiceGeoserver(service_url),
-    }
-
-    sync_service = synchronizers.get(service_name.lower(), _SyncServiceDefault())
-
-    remote_resources = sync_service.get_resources()
+def merge_local_and_remote_resources(resources_local, service_name, session):
+    """Main function to sync resources with remote server"""
+    remote_resources = query_resources(service_name, session=session)
     merged_resources = _merge_resources(resources_local, remote_resources)
-
     return merged_resources
 
 
@@ -58,17 +52,19 @@ def _merge_resources(resources_local, resources_remote):
     merged_resources = copy.deepcopy(resources_local)
 
     def recurse(_resources_local, _resources_remote, remote_path=""):
+        # loop local resources, looking for matches in remote resources
         for resource_name_local, values in _resources_local.items():
             current_path = "/".join([remote_path, str(resource_name_local)])
             matches_remote = resource_name_local in _resources_remote
 
-            values["remote_path"] = current_path
+            values["remote_path"] = current_path if matches_remote else ""
             values["matches_remote"] = matches_remote
 
             resource_remote_children = _resources_remote[resource_name_local]['children'] if matches_remote else {}
 
             recurse(values['children'], resource_remote_children, current_path)
 
+        # loop remote resources, looking for matches in local resources
         for resource_name_remote, values in _resources_remote.items():
             if resource_name_remote not in _resources_local:
                 current_path = "/".join([remote_path, str(resource_name_remote)])
@@ -77,7 +73,6 @@ def recurse(_resources_local, _resources_remote, remote_path=""):
                                 'id': current_path,
                                 'remote_path': current_path,
                                 'matches_remote': True}
-
                 _resources_local[resource_name_remote] = new_resource
                 recurse(new_resource['children'], values['children'], current_path)
 
@@ -166,5 +161,153 @@ def thredds_get_resources(url, depth, **kwargs):
 
 
 class _SyncServiceDefault(_SyncServiceInterface):
+    def __init__(self, _):
+        pass
+
     def get_resources(self):
         return {}
+
+
+SYNC_SERVICES = {
+    "thredds": _SyncServiceThreads,
+    "geoserver-api": _SyncServiceGeoserver,
+}
+
+
+def _resource_tree_parser(resources, permissions):
+    resources_tree = {}
+    for r_id, resource in resources.items():
+        children = _resource_tree_parser(resource['children'], permissions)
+        resources_tree[resource['resource_name']] = dict(id=r_id, permission_names=permissions, children=children)
+    return resources_tree
+
+
+def get_resource_children(resource, db_session):
+    query = models.remote_resource_tree_service.from_parent_deeper(resource.resource_id, db_session=db_session)
+
+    def build_subtree_strut(result):
+        """
+        Returns a dictionary in form of
+        {node:Resource, children:{node_id: RemoteResource}}
+        """
+        items = list(result)
+        root_elem = {'node': None, 'children': OrderedDict()}
+        if len(items) == 0:
+            return root_elem
+        for i, node in enumerate(items):
+            new_elem = {'node': node.RemoteResource, 'children': OrderedDict()}
+            path = list(map(int, node.path.split('/')))
+            parent_node = root_elem
+            normalized_path = path[:-1]
+            if normalized_path:
+                for path_part in normalized_path:
+                    parent_node = parent_node['children'][path_part]
+            parent_node['children'][new_elem['node'].resource_id] = new_elem
+        return root_elem
+
+    return build_subtree_strut(query)[u'children']
+
+
+def ensure_sync_info_exists(service_name, session):
+    service = models.Service.by_service_name(service_name, db_session=session)
+    service_sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
+    if not service_sync_info:
+        sync_info = models.RemoteResourcesSyncInfo(service_id=service.resource_id)
+        session.add(sync_info)
+        session.flush()
+        create_main_resource(service.resource_id, session)
+
+
+def get_remote_resources(service, service_name):
+    service_url = service.url
+    if service_url.endswith("/"):  # remove trailing slash
+        service_url = service_url[:-1]
+    sync_service = SYNC_SERVICES.get(service_name.lower(), _SyncServiceDefault)(service_url)
+    return sync_service.get_resources()
+
+
+def delete_records(service_id, session):
+    session.query(models.RemoteResource).filter_by(service_id=service_id).delete()
+    session.flush()
+
+
+def create_main_resource(service_id, session):
+    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session)
+    main_resource = models.RemoteResource(service_id=service_id,
+                                          resource_name=unicode(sync_info.service.resource_name),
+                                          resource_type=u"directory")
+    session.add(main_resource)
+    session.flush()
+    sync_info.remote_resource_id = main_resource.resource_id
+    session.flush()
+
+
+def update_db(remote_resources, service_id, session):
+    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session)
+
+    def add_children(resources, parent_id, position=0):
+        for resource_name, values in resources.items():
+            new_resource = models.RemoteResource(service_id=sync_info.service_id,
+                                                 resource_name=unicode(resource_name),
+                                                 resource_type=u"directory",  # todo: fixme
+                                                 parent_id=parent_id,
+                                                 ordering=position)
+            session.add(new_resource)
+            session.flush()
+            position += 1
+            add_children(values['children'], new_resource.resource_id)
+
+    first_item = list(remote_resources)[0]
+    add_children(remote_resources[first_item]['children'], sync_info.remote_resource_id)
+
+    session.flush()
+
+
+def fetch():
+    url = db.get_db_url()
+    engine = create_engine(url)
+
+    session = Session(bind=engine)
+
+    for service_name in SYNC_SERVICES:
+        service = models.Service.by_service_name(service_name, db_session=session)
+
+        remote_resources = get_remote_resources(service, service_name)
+
+        service_id = service.resource_id
+
+        delete_records(service_id, session)
+
+        ensure_sync_info_exists(service_name, session)
+
+        update_db(remote_resources, service_id, session)
+
+    session.commit()
+    session.close()
+
+
+def format_resource_tree(children):
+    fmt_res_tree = {}
+    for child_id, child_dict in children.items():
+        resource = child_dict[u'node']
+        new_children = child_dict[u'children']
+        fmt_res_tree[resource.resource_name] = {}
+        fmt_res_tree[resource.resource_name][u'children'] = format_resource_tree(new_children)
+
+    return fmt_res_tree
+
+
+def query_resources(service_name, session):
+    service = models.Service.by_service_name(service_name, db_session=session)
+    ensure_sync_info_exists(service_name, session)
+
+    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
+    main_resource = session.query(models.RemoteResource).filter_by(
+        resource_id=sync_info.remote_resource_id).first()
+    tree = get_resource_children(main_resource, session)
+    remote_resources = format_resource_tree(tree)
+    return {service_name: {'children': remote_resources}}
+
+
+if __name__ == '__main__':
+    fetch()
diff --git a/magpie/models.py b/magpie/models.py
index 822858056..2885b79e5 100644
--- a/magpie/models.py
+++ b/magpie/models.py
@@ -3,7 +3,6 @@
 from magpie.definitions.sqlalchemy_definitions import *
 from magpie.api.api_except import *
 
-
 Base = declarative_base()
 
 
@@ -170,11 +169,83 @@ class Route(Resource):
     resource_type_name = u'route'
 
 
+class RemoteResource(BaseModel, Base):
+    __tablename__ = "remote_resources"
+
+    __possible_permissions__ = ()
+    _ziggurat_services = [ResourceTreeService]
+
+    resource_id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True)
+    service_id = sa.Column(sa.Integer(),
+                           sa.ForeignKey('services.resource_id',
+                                         onupdate='CASCADE',
+                                         ondelete='CASCADE'),
+                           index=True,
+                           nullable=False)
+    parent_id = sa.Column(sa.Integer(),
+                          sa.ForeignKey('remote_resources.resource_id',
+                                        onupdate='CASCADE',
+                                        ondelete='SET NULL'),
+                          nullable=True)
+    ordering = sa.Column(sa.Integer(), default=0, nullable=False)
+    resource_name = sa.Column(sa.Unicode(100), nullable=False)
+    resource_type = sa.Column(sa.Unicode(30), nullable=False)
+
+    def __repr__(self):
+        info = self.resource_type, self.resource_name, self.resource_id, self.ordering, self.parent_id
+        return '<RemoteResource: %s, %s, id: %s position: %s, parent_id: %s>' % info
+
+
+class RemoteResourcesSyncInfo(Base):
+    __tablename__ = "remote_resources_sync_info"
+
+    id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True)
+    service_id = sa.Column(sa.Integer(),
+                           sa.ForeignKey('services.resource_id',
+                                         onupdate='CASCADE',
+                                         ondelete='CASCADE'),
+                           index=True,
+                           nullable=False)
+    service = relationship("Service", foreign_keys=[service_id])
+    remote_resource_id = sa.Column(sa.Integer(),
+                                   sa.ForeignKey('remote_resources.resource_id', onupdate='CASCADE',
+                                                 ondelete='CASCADE'))
+    last_sync = sa.Column(sa.DateTime(), nullable=True)
+
+    @staticmethod
+    def by_service_id(service_id, session):
+        condition = RemoteResourcesSyncInfo.service_id == service_id
+        service_info = session.query(RemoteResourcesSyncInfo).filter(condition).first()
+        return service_info
+
+    def __repr__(self):
+        last_modified = self.last_sync.strftime("%Y-%m-%dT%H:%M:%S") if self.last_sync else None
+        info = self.service_id, last_modified, self.id
+        return '<RemoteResourcesSyncInfo service_id: %s, last_sync: %s, id: %s>' % info
+
+
+class RemoteResourceTreeService(ResourceTreeService):
+    def __init__(self, service_cls):
+        self.model = RemoteResource
+        super(RemoteResourceTreeService, self).__init__(service_cls)
+
+
+class RemoteResourceTreeServicePostgreSQL(ResourceTreeServicePostgreSQL):
+    """
+    This is necessary, because ResourceTreeServicePostgreSQL.model is the Resource class.
+    If we want to change it for a RemoteResource, we need this class.
+
+    The ResourceTreeService.__init__ call sets the model.
+    """
+    pass
+
+
 ziggurat_model_init(User, Group, UserGroup, GroupPermission, UserPermission,
                     UserResourcePermission, GroupResourcePermission, Resource,
                     ExternalIdentity, passwordmanager=None)
 
 resource_tree_service = ResourceTreeService(ResourceTreeServicePostgreSQL)
+remote_resource_tree_service = RemoteResourceTreeService(RemoteResourceTreeServicePostgreSQL)
 
 resource_type_dict = {
     Service.resource_type_name:     Service,
@@ -204,5 +275,6 @@ def get_all_resource_permission_names():
 def find_children_by_name(name, parent_id, db_session):
     tree_struct = resource_tree_service.from_parent_deeper(parent_id=parent_id, limit_depth=1, db_session=db_session)
     tree_level_entries = [node for node in tree_struct]
-    tree_level_filtered = [node.Resource for node in tree_level_entries if node.Resource.resource_name.lower() == name.lower()]
+    tree_level_filtered = [node.Resource for node in tree_level_entries if
+                           node.Resource.resource_name.lower() == name.lower()]
     return tree_level_filtered.pop() if len(tree_level_filtered) else None
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index cdd3f3073..8788485a8 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -4,7 +4,7 @@
 from magpie.services import service_type_dict
 from magpie.models import resource_type_dict
 from magpie.ui.management import check_response
-from magpie.ui.management.sync_resources import merge_local_and_remote_resources
+from magpie.helpers.sync_resources import merge_local_and_remote_resources
 from magpie.ui.home import add_template_data
 from magpie import register, __meta__
 from distutils.version import LooseVersion
@@ -498,7 +498,7 @@ def edit_group(self):
         error_message = None
 
         try:
-            res_perms = merge_local_and_remote_resources(res_perms, cur_svc_type, service_url)
+            res_perms = merge_local_and_remote_resources(res_perms, cur_svc_type, self.request.db)
         except Exception:
             error_message = ("Couldn't get resources from the remote service ({}) "
                              "Only local resources are displayed. ".format(service_url))

From 3b8e1b86ad2f52626b1074b4b6a2dd3182e73a9e Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Wed, 12 Sep 2018 17:15:07 -0400
Subject: [PATCH 009/124] Permission checkboxes post a 'permission' value

instead of a 'resource_id'
To distinguish permission requests and 'clean_resource' requests
---
 .../ui/management/templates/edit_group.mako   | 23 ++++++++++---------
 magpie/ui/management/views.py                 |  8 +++++--
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/magpie/ui/management/templates/edit_group.mako b/magpie/ui/management/templates/edit_group.mako
index 60addc6fa..f4ebd9b92 100644
--- a/magpie/ui/management/templates/edit_group.mako
+++ b/magpie/ui/management/templates/edit_group.mako
@@ -3,17 +3,18 @@
 
 <%def name="render_item(key, value, level)">
     %for perm in permissions:
-        % if perm in value['permission_names']:
-            <div class="perm_checkbox">
-                <input type="checkbox" value="${perm}" name="permission"
-                       onchange="document.getElementById('resource_${value['id']}').submit()" checked>
-           </div>
-        % else:
-            <div class="perm_checkbox">
-                <input type="checkbox" value="${perm}" name="permission"
-                       onchange="document.getElementById('resource_${value['id']}').submit()">
-            </div>
-        % endif
+
+        <div class="perm_checkbox">
+            % if perm in value['permission_names']:
+            <input type="hidden" value="${perm}_unchecked" name="permission"/>
+            <input type="checkbox" value="${perm}_checked" name="permission"
+                   onchange="document.getElementById('resource_${value['id']}').submit()" checked>
+            % else:
+            <input type="checkbox" value="${perm}_checked" name="permission"
+                   onchange="document.getElementById('resource_${value['id']}').submit()">
+            % endif
+        </div>
+
     %endfor
     % if not value.get('matches_remote', True):
         <form id="delete_resource_${value['id']}" action="${request.path}" method="post">
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 8788485a8..66566a44c 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -399,7 +399,9 @@ def edit_user_or_group_resource_permissions(self, user_or_group_name, resource_i
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        selected_perms = self.request.POST.getall('permission')
+        permission_selections = self.request.POST.getall('permission')
+        selected_perms = [p.replace("_checked", "") for p in permission_selections if p.endswith("_checked")]
+
         removed_perms = list(set(res_perms) - set(selected_perms))
         new_perms = list(set(selected_perms) - set(res_perms))
 
@@ -482,8 +484,10 @@ def edit_group(self):
             elif u'clean_resource' in self.request.POST:
                 url = '{url}/resources/{resource_id}'.format(url=self.magpie_url, resource_id=res_id)
                 check_response(requests.delete(url, cookies=self.request.cookies))
-            else:
+            elif u'member' in self.request.POST:
                 self.edit_group_users(group_name)
+            else:
+                return HTTPBadRequest(detail="Invalid POST request.")
 
         # display resources permissions per service type tab
         try:

From 4b1df51b214194c1d2d3980938d41d73dbe3d23f Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 10:44:30 -0400
Subject: [PATCH 010/124] Fix post request to edit permissions

---
 magpie/helpers/sync_resources.py               | 2 +-
 magpie/ui/management/templates/edit_group.mako | 8 +++-----
 magpie/ui/management/views.py                  | 7 ++++---
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 99da0a2e5..f1bdab7d0 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -57,7 +57,7 @@ def recurse(_resources_local, _resources_remote, remote_path=""):
             current_path = "/".join([remote_path, str(resource_name_local)])
             matches_remote = resource_name_local in _resources_remote
 
-            values["remote_path"] = current_path if matches_remote else ""
+            values["remote_path"] = "" if matches_remote else current_path
             values["matches_remote"] = matches_remote
 
             resource_remote_children = _resources_remote[resource_name_local]['children'] if matches_remote else {}
diff --git a/magpie/ui/management/templates/edit_group.mako b/magpie/ui/management/templates/edit_group.mako
index f4ebd9b92..ee76eec08 100644
--- a/magpie/ui/management/templates/edit_group.mako
+++ b/magpie/ui/management/templates/edit_group.mako
@@ -2,19 +2,17 @@
 <%namespace name="tree" file="ui.management:templates/tree_scripts.mako"/>
 
 <%def name="render_item(key, value, level)">
+    <input type="hidden" value="" name="edit_permissions">
     %for perm in permissions:
-
         <div class="perm_checkbox">
             % if perm in value['permission_names']:
-            <input type="hidden" value="${perm}_unchecked" name="permission"/>
-            <input type="checkbox" value="${perm}_checked" name="permission"
+            <input type="checkbox" value="${perm}" name="permission"
                    onchange="document.getElementById('resource_${value['id']}').submit()" checked>
             % else:
-            <input type="checkbox" value="${perm}_checked" name="permission"
+            <input type="checkbox" value="${perm}" name="permission"
                    onchange="document.getElementById('resource_${value['id']}').submit()">
             % endif
         </div>
-
     %endfor
     % if not value.get('matches_remote', True):
         <form id="delete_resource_${value['id']}" action="${request.path}" method="post">
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 66566a44c..e03c14a6a 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -399,8 +399,7 @@ def edit_user_or_group_resource_permissions(self, user_or_group_name, resource_i
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        permission_selections = self.request.POST.getall('permission')
-        selected_perms = [p.replace("_checked", "") for p in permission_selections if p.endswith("_checked")]
+        selected_perms = self.request.POST.getall('permission')
 
         removed_perms = list(set(res_perms) - set(selected_perms))
         new_perms = list(set(selected_perms) - set(res_perms))
@@ -476,7 +475,7 @@ def edit_group(self):
                 return HTTPFound(self.request.route_url('edit_group', **group_info))
             elif u'goto_service' in self.request.POST:
                 return self.goto_service(res_id)
-            elif u'permission' in self.request.POST:
+            elif u'edit_permissions' in self.request.POST:
                 remote_path = self.request.POST.get('remote_path')
                 if remote_path:
                     res_id = self.add_external_resource(cur_svc_type, group_name, remote_path)
@@ -528,6 +527,8 @@ def add_external_resource(self, service, group_name, resource_path):
             raise HTTPBadRequest(detail=repr(e))
 
         def parse_resources_and_put(resources, path, parent_id=None):
+            if not path:
+                return parent_id
             current_name = path.pop(0)
             if current_name in resources:
                 res_id = parse_resources_and_put(resources[current_name]['children'],

From 11755f51ba78dc31713d588a449ec8ffc365f65c Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 11:05:18 -0400
Subject: [PATCH 011/124] Display resources sorted alphabetically

---
 magpie/helpers/sync_resources.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index f1bdab7d0..c88a8dc8b 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -20,6 +20,7 @@ def merge_local_and_remote_resources(resources_local, service_name, session):
     """Main function to sync resources with remote server"""
     remote_resources = query_resources(service_name, session=session)
     merged_resources = _merge_resources(resources_local, remote_resources)
+    _sort_resources(merged_resources)
     return merged_resources
 
 
@@ -96,12 +97,22 @@ def _is_valid_resource_schema(resources):
     for resource_name, values in resources.items():
         if 'children' not in values:
             return False
-        if not isinstance(values['children'], dict):
+        if not isinstance(values['children'], (OrderedDict, dict)):
             return False
         return _is_valid_resource_schema(values['children'])
     return True
 
 
+def _sort_resources(resources):
+    """
+    Sorts a resource dictionary by using OrderedDict
+    :return: None
+    """
+    for resource_name, values in resources.items():
+        values['children'] = OrderedDict(sorted(values['children'].iteritems()))
+        return _sort_resources(values['children'])
+
+
 class _SyncServiceInterface:
     __metaclass__ = abc.ABCMeta
 

From e4cdb3e62726d5927b5410622cea0752bd53edaa Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 11:20:17 -0400
Subject: [PATCH 012/124] Cleanup and document

---
 magpie/helpers/sync_resources.py | 189 +++++++++++++++++++------------
 magpie/ui/management/views.py    |   3 +-
 2 files changed, 116 insertions(+), 76 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index c88a8dc8b..3c4c914da 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -18,7 +18,7 @@
 
 def merge_local_and_remote_resources(resources_local, service_name, session):
     """Main function to sync resources with remote server"""
-    remote_resources = query_resources(service_name, session=session)
+    remote_resources = _query_resources(service_name, session=session)
     merged_resources = _merge_resources(resources_local, remote_resources)
     _sort_resources(merged_resources)
     return merged_resources
@@ -105,7 +105,8 @@ def _is_valid_resource_schema(resources):
 
 def _sort_resources(resources):
     """
-    Sorts a resource dictionary by using OrderedDict
+    Sorts a resource dictionary of the type validated by '_is_valid_resource_schema'
+    by using an OrderedDict
     :return: None
     """
     for resource_name, values in resources.items():
@@ -185,51 +186,28 @@ def get_resources(self):
 }
 
 
-def _resource_tree_parser(resources, permissions):
-    resources_tree = {}
-    for r_id, resource in resources.items():
-        children = _resource_tree_parser(resource['children'], permissions)
-        resources_tree[resource['resource_name']] = dict(id=r_id, permission_names=permissions, children=children)
-    return resources_tree
-
-
-def get_resource_children(resource, db_session):
-    query = models.remote_resource_tree_service.from_parent_deeper(resource.resource_id, db_session=db_session)
-
-    def build_subtree_strut(result):
-        """
-        Returns a dictionary in form of
-        {node:Resource, children:{node_id: RemoteResource}}
-        """
-        items = list(result)
-        root_elem = {'node': None, 'children': OrderedDict()}
-        if len(items) == 0:
-            return root_elem
-        for i, node in enumerate(items):
-            new_elem = {'node': node.RemoteResource, 'children': OrderedDict()}
-            path = list(map(int, node.path.split('/')))
-            parent_node = root_elem
-            normalized_path = path[:-1]
-            if normalized_path:
-                for path_part in normalized_path:
-                    parent_node = parent_node['children'][path_part]
-            parent_node['children'][new_elem['node'].resource_id] = new_elem
-        return root_elem
-
-    return build_subtree_strut(query)[u'children']
-
-
-def ensure_sync_info_exists(service_name, session):
+def _ensure_sync_info_exists(service_name, session):
+    """
+    Make sure the RemoteResourcesSyncInfo entry exists in the database.
+    :param service_name:
+    :param session:
+    """
     service = models.Service.by_service_name(service_name, db_session=session)
     service_sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
     if not service_sync_info:
         sync_info = models.RemoteResourcesSyncInfo(service_id=service.resource_id)
         session.add(sync_info)
         session.flush()
-        create_main_resource(service.resource_id, session)
+        _create_main_resource(service.resource_id, session)
 
 
-def get_remote_resources(service, service_name):
+def _get_remote_resources(service, service_name):
+    """
+    Request rmeote resources, depending on service type.
+    :param service:
+    :param service_name:
+    :return:
+    """
     service_url = service.url
     if service_url.endswith("/"):  # remove trailing slash
         service_url = service_url[:-1]
@@ -237,12 +215,25 @@ def get_remote_resources(service, service_name):
     return sync_service.get_resources()
 
 
-def delete_records(service_id, session):
+def _delete_records(service_id, session):
+    """
+    Delete all RemoteResource based on a Service.resource_id
+    :param service_id:
+    :param session:
+    """
     session.query(models.RemoteResource).filter_by(service_id=service_id).delete()
     session.flush()
 
 
-def create_main_resource(service_id, session):
+def _create_main_resource(service_id, session):
+    """
+    Creates a main resource for a service, whether one currently exists or not.
+
+    Each RemoteResourcesSyncInfo has a main RemoteResource of the same name as the service.
+    This is similar to the Service and Resource relationship.
+    :param service_id:
+    :param session:
+    """
     sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session)
     main_resource = models.RemoteResource(service_id=service_id,
                                           resource_name=unicode(sync_info.service.resource_name),
@@ -253,7 +244,13 @@ def create_main_resource(service_id, session):
     session.flush()
 
 
-def update_db(remote_resources, service_id, session):
+def _update_db(remote_resources, service_id, session):
+    """
+    Writes remote resources to database.
+    :param remote_resources:
+    :param service_id:
+    :param session:
+    """
     sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session)
 
     def add_children(resources, parent_id, position=0):
@@ -274,50 +271,94 @@ def add_children(resources, parent_id, position=0):
     session.flush()
 
 
-def fetch():
-    url = db.get_db_url()
-    engine = create_engine(url)
-
-    session = Session(bind=engine)
-
-    for service_name in SYNC_SERVICES:
-        service = models.Service.by_service_name(service_name, db_session=session)
+def _get_resource_children(resource, db_session):
+    """
+    Mostly copied from ziggurat_foundations to use RemoteResource instead of Resource
+    :param resource:
+    :param db_session:
+    :return:
+    """
+    query = models.remote_resource_tree_service.from_parent_deeper(resource.resource_id, db_session=db_session)
 
-        remote_resources = get_remote_resources(service, service_name)
+    def build_subtree_strut(result):
+        """
+        Returns a dictionary in form of
+        {node:Resource, children:{node_id: RemoteResource}}
+        """
+        items = list(result)
+        root_elem = {'node': None, 'children': OrderedDict()}
+        if len(items) == 0:
+            return root_elem
+        for i, node in enumerate(items):
+            new_elem = {'node': node.RemoteResource, 'children': OrderedDict()}
+            path = list(map(int, node.path.split('/')))
+            parent_node = root_elem
+            normalized_path = path[:-1]
+            if normalized_path:
+                for path_part in normalized_path:
+                    parent_node = parent_node['children'][path_part]
+            parent_node['children'][new_elem['node'].resource_id] = new_elem
+        return root_elem
 
-        service_id = service.resource_id
+    return build_subtree_strut(query)['children']
 
-        delete_records(service_id, session)
 
-        ensure_sync_info_exists(service_name, session)
+def _query_resources(service_name, session):
+    """
+    Reads remote resources from database. No external request is made.
+    :param service_name:
+    :param session:
+    :return: a dictionary of the form defined in '_is_valid_resource_schema'
+    """
+    service = models.Service.by_service_name(service_name, db_session=session)
+    _ensure_sync_info_exists(service_name, session)
 
-        update_db(remote_resources, service_id, session)
+    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
+    main_resource = session.query(models.RemoteResource).filter_by(
+        resource_id=sync_info.remote_resource_id).first()
+    tree = _get_resource_children(main_resource, session)
+
+    def _format_resource_tree(children):
+        fmt_res_tree = {}
+        for child_id, child_dict in children.items():
+            resource = child_dict[u'node']
+            new_children = child_dict[u'children']
+            fmt_res_tree[resource.resource_name] = {}
+            fmt_res_tree[resource.resource_name][u'children'] = _format_resource_tree(new_children)
+        return fmt_res_tree
+
+    remote_resources = _format_resource_tree(tree)
+    return {service_name: {'children': remote_resources}}
 
-    session.commit()
-    session.close()
 
+def fetch_single_service(service_name, session):
+    """
+    Get remote resources for a single service.
+    :param service_name:
+    :param session:
+    """
+    service = models.Service.by_service_name(service_name, db_session=session)
+    remote_resources = _get_remote_resources(service, service_name)
+    service_id = service.resource_id
+    _delete_records(service_id, session)
+    _ensure_sync_info_exists(service_name, session)
+    _update_db(remote_resources, service_id, session)
 
-def format_resource_tree(children):
-    fmt_res_tree = {}
-    for child_id, child_dict in children.items():
-        resource = child_dict[u'node']
-        new_children = child_dict[u'children']
-        fmt_res_tree[resource.resource_name] = {}
-        fmt_res_tree[resource.resource_name][u'children'] = format_resource_tree(new_children)
 
-    return fmt_res_tree
+def fetch():
+    """
+    Main entry point to get all remote resources for each service and write to database.
+    """
+    url = db.get_db_url()
+    engine = create_engine(url)
 
+    session = Session(bind=engine)
 
-def query_resources(service_name, session):
-    service = models.Service.by_service_name(service_name, db_session=session)
-    ensure_sync_info_exists(service_name, session)
+    for service_name in SYNC_SERVICES:
+        fetch_single_service(service_name, session)
 
-    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
-    main_resource = session.query(models.RemoteResource).filter_by(
-        resource_id=sync_info.remote_resource_id).first()
-    tree = get_resource_children(main_resource, session)
-    remote_resources = format_resource_tree(tree)
-    return {service_name: {'children': remote_resources}}
+    session.commit()
+    session.close()
 
 
 if __name__ == '__main__':
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index e03c14a6a..e8dd5a666 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -496,13 +496,12 @@ def edit_group(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        service_url = services[list(services)[0]].get('service_url', '')
-
         error_message = None
 
         try:
             res_perms = merge_local_and_remote_resources(res_perms, cur_svc_type, self.request.db)
         except Exception:
+            service_url = services[list(services)[0]].get('service_url', '')
             error_message = ("Couldn't get resources from the remote service ({}) "
                              "Only local resources are displayed. ".format(service_url))
 

From aeea8aad24dc9d64688193e3f13e5e71ebcbc4cd Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 11:27:23 -0400
Subject: [PATCH 013/124] Display resources sorted in edit service view

---
 magpie/ui/management/views.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index e8dd5a666..8166214e5 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -1,3 +1,5 @@
+from collections import OrderedDict
+
 from magpie.definitions.pyramid_definitions import *
 from magpie.constants import get_constant
 from magpie.common import str2bool
@@ -360,6 +362,7 @@ def resource_tree_parser(self, raw_resources_tree, permission):
         for r_id, resource in raw_resources_tree.items():
             perm_names = self.default_get(permission, r_id, [])
             children = self.resource_tree_parser(resource['children'], permission)
+            children = OrderedDict(sorted(children.items()))
             resources_tree[resource['resource_name']] = dict(id=r_id, permission_names=perm_names, children=children)
         return resources_tree
 

From fb2c6c25887babc06f468415a9f463d05c271051 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 11:57:32 -0400
Subject: [PATCH 014/124] ui color change

---
 magpie/ui/home/static/style.css                  |   2 +-
 .../home/static/warning_exclamation_orange.png   | Bin 0 -> 2612 bytes
 magpie/ui/management/templates/edit_group.mako   |   4 ++--
 3 files changed, 3 insertions(+), 3 deletions(-)
 create mode 100644 magpie/ui/home/static/warning_exclamation_orange.png

diff --git a/magpie/ui/home/static/style.css b/magpie/ui/home/static/style.css
index 6aab54537..1c936469b 100644
--- a/magpie/ui/home/static/style.css
+++ b/magpie/ui/home/static/style.css
@@ -408,7 +408,7 @@ div.perm_title {
     color: gray;
     display: inline-flex;
     float: right;
-    margin: 0.2em 0 0 0;
+    margin: 0.25em 0 0 0;
 }
 
 .tree_item_message>img {
diff --git a/magpie/ui/home/static/warning_exclamation_orange.png b/magpie/ui/home/static/warning_exclamation_orange.png
new file mode 100644
index 0000000000000000000000000000000000000000..4fe4ea20f13dd5bb2df71d856924f5efbb74a71c
GIT binary patch
literal 2612
zcmc&$do<MB9^YePLX1jAG^3|UUN@9uJY$$T;ws@vgC2}m9`*B>VLVcvy;H8o$P7`4
zRK$4ah7dCe9b%&JbMiJAlsVJwtaI<Wf8BN0S?lh#*Z2GRe)sqL-D`dK+Mm4>?QBUh
zQc6+)0GT6)iN^sD6G1Vs9w}0ddAnOgD$MTa2}_ZF;|bc&fKdS2&w(KT>i!8txXBg3
z0F;U%0B{B15&$Cr3;-|)K<tk!56}ZZIRN9}dt?IuB$59t=y3loZvl`2z&la3vmgb4
zRsiS#i1uyw0PXHT@~eWMsP10qxUD@9dLcW+5SKEQB@m*4M2aEOV%8xfMcO8zT;zxq
zLm<|r+bI_B3j;vX;t283iLlJc58ffCnm3A{F2_^LHTviLs40te77SHR$>FU#<02hy
zz0j$Z*>331L<{An2anv|{b$U@i&V*#!;Hxb%MHo*gzwRGDdn)nnMf7CPiqVGb)~k-
zJdzyDU!WuRYni4$uPxc$@OhRl_CV8_r*GH&a+xkRqT$NJ+ewwy-pJk%7<nH|HP3u?
zf?}KCaWs^W1$UbgU%+E{{%798R&p+eHihRNTggscbA|q{3a6RI*mfamW_4ksjH<6G
zIDxZij`HMH+>%G6Eu~MPW+Iu(TqKsPC_#l-Zbt1UhOaI^GhkMl&}Ri}@)A1AWoRw>
zIgRHpWzEV$Y%E84FupkThU<(!RPV>9Q3QLi`xR>j4B(_nc*NEseuHIa$WDUCZrXEu
z?*4gBGUK|eQe2j=7{ObfR-^_$JOn>dWj@OjY%zYPSC`jimF<+C=&u}u^WI1B2Q=1e
zxO#+(v>~C2uenkr2^UWD^?W&ZrE~I~FL#_$FmBg@U>5AGz_NGZE7MZS;13I&WDTn^
zt$5ry147t-{x(3Ms__-q6KFg5@^Xamty_FCM&54fXPhEHYG`dt;hx9VCSa-V<Q?`X
zaz?yp74;EEnWQ+t1?Z3SY7Q5Rem%8_fP_+i>#3YT!r#PtJc?HVp)UH$T{_sJS#(N(
z;L{Q6yPP5c*{%L<(*qT=PRx^oL6a!Xthk2O1oEM0(9keCdq{e(YlVQi?1(B}1MHgU
z5jq`|X-vDUA^0@Jm-(J3z;Fe2y(qhufEYFco!u$DeRu^(?HE=yiq_Hw-paEey*fIf
zd?U&8%v(>RXjyHrPk9a~RY%`^x`Q&xe3@qyJ>Uj<x>0t>AL76$V&a=vMUhGLrivm1
zwEGwaI4&Sg07n@bSgi#*1?X&`<MS5zenC9qmi&S79(k{>ho~px(g{9F!G014AE80B
z>H^~9`&?(Vdou=L77(0|tkY=sMvR=J-o%F$INKZTUWGAR;v=5GnN+m<GmKd;A5jmd
zn*asu{3Zb?ry-!WWHhkMuWmJ@w#siHj5rLww$F3tsFf_5R{n+kxqA(du}ZD(6MkmH
zH>VF&RBEt4h0FMBB~J-lV>1d_x+ZWSbPRfh!d}-(Di&lnaxJ-ARG^>Os)XSj1*cS^
ze#E=sUvI;4H%^`+Z)D7<&QGY_DYC>@j|YVuf*LY#6bupbFQeB2WpsJa*dygG939W8
z%D?(_6Lv<9wv=+rk%h#WPiN`0{tVSy+^K3~u42<akYxv^IiII4z>)E)p{W*2hI*A%
zN(A0bnjpowhdD3vDE_Z$9@ziM!5Nu31Jf*08<Uw|WXQ7GamD(OLA4@#@c*=si!<@K
z7Ys*O{5oXs=^Wpo|6e+;{CjV<v3bnb10$SovNls%@l&B*7eWo7y5HORM|a@?dn@Ne
z*|7CB;jMeEx#9t|OtKt#Ca?Jy@&7b*I<s_MpMe@ln#Ed){(^_Ey7zEpRLUyYyV*mc
z<L@PsA~-{$`=5j+Y$MB)_bVC?7(|jNS1q+e71(DrXcf;m*t6V5#>f)XV#oI%V_L_t
zD28aVj=XU{6~3JYciR&4+leLL2}+e#rpwjb#m!+P`fp9dO`d?-&G_fWSiTZ0ykV~j
zoEcP`i#Oalpet+q{ueka2yU<?p=2yNgH708inL-$IQJM_>Y1Hk#cWQG@lU@z=p#k&
z*+eTefD1$vH=12&!sL@zna?68K6R}fzCXXi>;5=g96vmEeian#zFwn^BQGABiy2xY
z=GPcGrPfaQyiwS|o}3RB9#yGWLe{(0<M0yJ2^oQBixSEu_48J&&0Xf222_AjXvr>n
z%iL=}>6{?;C;Sc_%ukLs;Z{jTV6;W8Qu^mV@|u3Y%M}Q`gmgmH*MB&o-Km2sn8m&*
zDsK)I($7N$3a<<mjFH3L>b}u|%JP@xm^K$B5qZhffe5g~coEx|=<D_({%xYQ%8TUI
zYwEo68|+)X<Q}T<Wm+4qJnbn%Br{md<gBus+mEbqWshz@Rv&!+*rQZ^t@lYH`Qp1m
z@-(Z);R|^(Yf@9%bplO1+M#sLdG2`BPET6T_BNKOTMb8zk@DJg^rnB(Ww(%5JcZyl
zr5f~Wrdz6>_$6IgxZfcRU5TDjSQHAy@2v$cMfLS<)Y=pI)Z}>>N{UgbMN2yXw?5#0
z(lEnoUC*Kr2!T|-*&Qww$=PgU*=%DLptpJ8;5y?jJ?On6-|h~_IhJE@bBJ`!{6=tX
z6z516Czdgx&ZgjKPeqRwk{lu;vUWSZABAZ+x`H<zP=k}W@T{#Fegmm9RGZC{fF-BR
zoBPF#hp{k}kxhvxjN%TJ&j!$1^_hj|ui)kv)-3S-GRCkvw=#qnup$K&*l?fKDq?+H
z+}^g;wnU8|b;;?aD`36C`Stg$dc9+C%bQ_68;)+2HsPp>Feg9qWiOQC3D?xcc5pY=
zETdp~GPlZ<r3+aFL<L2r&nMH?@tb=-ew{Rk3NO|k>wQNui#k|y^-J4A@Zhd&(}cl^
rj*+^cuVjHsNyD2xf5smsH?CHEoBPh;Pk6!i{~<VHX-h0Ncfa%-TH@E!

literal 0
HcmV?d00001

diff --git a/magpie/ui/management/templates/edit_group.mako b/magpie/ui/management/templates/edit_group.mako
index ee76eec08..3a6d11a8a 100644
--- a/magpie/ui/management/templates/edit_group.mako
+++ b/magpie/ui/management/templates/edit_group.mako
@@ -17,11 +17,11 @@
     % if not value.get('matches_remote', True):
         <form id="delete_resource_${value['id']}" action="${request.path}" method="post">
             <div class="tree_button">
-                <input type="submit" class="button delete" value="Clean" name="clean_resource">
+                <input type="submit" class="button warning" value="Clean" name="clean_resource">
             </div>
             <p class="tree_item_message">
                 <img title="This resource is absent from the remote server."
-                     src="${request.static_url('magpie.ui.home:static/warning_exclamation.png')}" />
+                     src="${request.static_url('magpie.ui.home:static/warning_exclamation_orange.png')}" />
             </p>
         </form>
     % endif

From d9b58c7a4d2cf86ece6586d9df30191f6338b433 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 14:10:44 -0400
Subject: [PATCH 015/124] last sync info and clean all button

---
 magpie/helpers/sync_resources.py              | 13 +++++
 .../ui/management/templates/edit_group.mako   | 16 ++++++
 magpie/ui/management/views.py                 | 51 +++++++++++++++----
 requirements.txt                              |  1 +
 4 files changed, 70 insertions(+), 11 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 3c4c914da..632af3f69 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -6,6 +6,7 @@
 
 import abc
 import copy
+import datetime
 from collections import OrderedDict
 
 import requests
@@ -268,6 +269,8 @@ def add_children(resources, parent_id, position=0):
     first_item = list(remote_resources)[0]
     add_children(remote_resources[first_item]['children'], sync_info.remote_resource_id)
 
+    sync_info.last_sync = datetime.datetime.now()
+
     session.flush()
 
 
@@ -331,6 +334,16 @@ def _format_resource_tree(children):
     return {service_name: {'children': remote_resources}}
 
 
+def get_last_sync(service_name, session):
+    last_sync = None
+    service = models.Service.by_service_name(service_name, db_session=session)
+    _ensure_sync_info_exists(service_name, session)
+    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
+    if sync_info:
+        last_sync = sync_info.last_sync
+    return last_sync
+
+
 def fetch_single_service(service_name, session):
     """
     Get remote resources for a single service.
diff --git a/magpie/ui/management/templates/edit_group.mako b/magpie/ui/management/templates/edit_group.mako
index 3a6d11a8a..8945effd1 100644
--- a/magpie/ui/management/templates/edit_group.mako
+++ b/magpie/ui/management/templates/edit_group.mako
@@ -112,6 +112,22 @@
         %if error_message:
             <div class="alert danger visible">${error_message}</div>
         %endif
+        <form id="sync_info" action="${request.path}" method="post">
+            <p class="panel_line">
+                <span class="panel_entry">Last synchronization with remote service: </span>
+                <span class="panel_value">${last_sync} </span>
+                <input type="submit" value="Sync now" name="force_sync">
+            </p>
+            %if ids_to_clean:
+                <p class="panel_line">
+                    <span class="panel_entry">Note: </span>
+                    <span class="panel_value">Some resources are absent from the remote server </span>
+                    <input type="hidden" value="${ids_to_clean}" name="ids_to_clean">
+                    <input type="submit" class="button warning" value="Clean all" name="clean_all">
+                </p>
+            %endif
+        </form>
+
         <div class="tree_header">
             <div class="tree_item">Resources</div>
             %for perm in permissions:
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 8166214e5..d74501891 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -1,12 +1,15 @@
+import datetime
 from collections import OrderedDict
 
+import humanize
+
 from magpie.definitions.pyramid_definitions import *
 from magpie.constants import get_constant
 from magpie.common import str2bool
 from magpie.services import service_type_dict
 from magpie.models import resource_type_dict
 from magpie.ui.management import check_response
-from magpie.helpers.sync_resources import merge_local_and_remote_resources
+from magpie.helpers import sync_resources
 from magpie.ui.home import add_template_data
 from magpie import register, __meta__
 from distutils.version import LooseVersion
@@ -484,10 +487,15 @@ def edit_group(self):
                     res_id = self.add_external_resource(cur_svc_type, group_name, remote_path)
                 self.edit_user_or_group_resource_permissions(group_name, res_id, is_user=False)
             elif u'clean_resource' in self.request.POST:
-                url = '{url}/resources/{resource_id}'.format(url=self.magpie_url, resource_id=res_id)
-                check_response(requests.delete(url, cookies=self.request.cookies))
+                self.delete_resource(res_id)
             elif u'member' in self.request.POST:
                 self.edit_group_users(group_name)
+            elif u'force_sync' in self.request.POST:
+                sync_resources.fetch_single_service(cur_svc_type, session=self.request.db)
+            elif u'clean_all' in self.request.POST:
+                ids_to_clean = self.request.POST.get('ids_to_clean').split(";")
+                for id_ in ids_to_clean:
+                    self.delete_resource(id_)
             else:
                 return HTTPBadRequest(detail="Invalid POST request.")
 
@@ -499,16 +507,19 @@ def edit_group(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        error_message = None
+        res_perms = sync_resources.merge_local_and_remote_resources(res_perms,
+                                                                    cur_svc_type,
+                                                                    self.request.db)
 
-        try:
-            res_perms = merge_local_and_remote_resources(res_perms, cur_svc_type, self.request.db)
-        except Exception:
-            service_url = services[list(services)[0]].get('service_url', '')
-            error_message = ("Couldn't get resources from the remote service ({}) "
-                             "Only local resources are displayed. ".format(service_url))
+        last_sync_datetime = sync_resources.get_last_sync(cur_svc_type, self.request.db)
+        now = datetime.datetime.now()
+        last_sync = humanize.naturaltime(now - last_sync_datetime) if last_sync_datetime else "Never"
 
-        group_info[u'error_message'] = error_message
+        ids = self.get_ids_to_clean(res_perms)
+
+        group_info[u'error_message'] = None
+        group_info[u'ids_to_clean'] = ";".join(ids)
+        group_info[u'last_sync'] = last_sync
         group_info[u'group_name'] = group_name
         group_info[u'cur_svc_type'] = cur_svc_type
         group_info[u'users'] = self.get_user_names()
@@ -519,6 +530,24 @@ def edit_group(self):
         group_info[u'permissions'] = res_perm_names
         return add_template_data(self.request, data=group_info)
 
+    def delete_resource(self, res_id):
+        url = '{url}/resources/{resource_id}'.format(url=self.magpie_url, resource_id=res_id)
+        try:
+            check_response(requests.delete(url, cookies=self.request.cookies))
+        except HTTPNotFound:
+            # Some resource ids are already deleted because they were a child
+            # of another just deleted parent resource.
+            # We just skip them.
+            pass
+
+    def get_ids_to_clean(self, resources):
+        ids = []
+        for resource_name, values in resources.iteritems():
+            if "matches_remote" in values and not values["matches_remote"]:
+                ids.append(values['id'])
+            ids += self.get_ids_to_clean(values['children'])
+        return ids
+
     def add_external_resource(self, service, group_name, resource_path):
         try:
             res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(group_name,
diff --git a/requirements.txt b/requirements.txt
index dea01f4fa..e6cea82a1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -30,3 +30,4 @@ cornice_swagger>=0.7.0
 cornice
 colander
 threddsclient
+humanize

From 49ddb13bf97c310fd452be279888c2f867d6f80c Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 14:39:21 -0400
Subject: [PATCH 016/124] also write resource type to database

---
 magpie/helpers/sync_resources.py | 44 ++++++++++++++++++++------------
 1 file changed, 28 insertions(+), 16 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 632af3f69..8b0477ae6 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -39,7 +39,7 @@ def _merge_resources(resources_local, resources_remote):
     if not resources_remote:
         return resources_local
 
-    assert _is_valid_resource_schema(resources_local)
+    assert _is_valid_resource_schema(resources_local, ignore_resource_type=True)
     assert _is_valid_resource_schema(resources_remote)
 
     if not resources_local:
@@ -61,6 +61,7 @@ def recurse(_resources_local, _resources_remote, remote_path=""):
 
             values["remote_path"] = "" if matches_remote else current_path
             values["matches_remote"] = matches_remote
+            values["resource_type"] = _resources_remote[resource_name_local]['children'] if matches_remote else ""
 
             resource_remote_children = _resources_remote[resource_name_local]['children'] if matches_remote else {}
 
@@ -72,6 +73,7 @@ def recurse(_resources_local, _resources_remote, remote_path=""):
                 current_path = "/".join([remote_path, str(resource_name_remote)])
                 new_resource = {'permission_names': [],
                                 'children': {},
+                                'resource_type': values['resource_type'],
                                 'id': current_path,
                                 'remote_path': current_path,
                                 'matches_remote': True}
@@ -80,27 +82,33 @@ def recurse(_resources_local, _resources_remote, remote_path=""):
 
     recurse(merged_resources, resources_remote)
 
+    assert _is_valid_resource_schema(merged_resources)
+
     return merged_resources
 
 
-def _is_valid_resource_schema(resources):
+def _is_valid_resource_schema(resources, ignore_resource_type=False):
     """
     Returns True if the structure of the input dictionary is a tree of the form:
 
-    {'resource_name_1': {'children': {'resource_name_3': {'children': {}},
-                                      'resource_name_4': {'children': {}}
-                                      }
+    {'resource_name_1': {'children': {'resource_name_3': {'children': {}, 'resource_type': ...},
+                                      'resource_name_4': {'children': {}, 'resource_type': ...}
+                                      },
+                         'resource_type': ...
                          },
-     'resource_name_2': {'children': {}}
+     'resource_name_2': {'children': {}, resource_type': ...}
      }
     :return: bool
     """
     for resource_name, values in resources.items():
         if 'children' not in values:
             return False
+        if not ignore_resource_type and 'resource_type' not in values:
+            return False
         if not isinstance(values['children'], (OrderedDict, dict)):
             return False
-        return _is_valid_resource_schema(values['children'])
+        return _is_valid_resource_schema(values['children'],
+                                         ignore_resource_type=ignore_resource_type)
     return True
 
 
@@ -140,9 +148,10 @@ def get_resources(self):
         resp.raise_for_status()
         workspaces_list = resp.json().get("workspaces", {}).get("workspace", {})
 
-        workspaces = {w["name"]: {"children": {}} for w in workspaces_list}
+        workspaces = {w["name"]: {"children": {}, "resource_type": "directory"} for w in workspaces_list}
 
-        resources = {"geoserver-api": {"children": workspaces}}
+        resources = {"geoserver-api": {"children": workspaces,
+                                       "resource_type": "directory"}}
         assert _is_valid_resource_schema(resources), "Error in Interface implementation"
         return resources
 
@@ -159,8 +168,10 @@ def get_resources(self):
         def thredds_get_resources(url, depth, **kwargs):
             cat = threddsclient.read_url(url, **kwargs)
             name = cat.name
-
-            tree_item = {name: {'children': {}}}
+            resource_type = 'directory'
+            if cat.datasets:
+                resource_type = cat.datasets[0].content_type.replace('application/', '')
+            tree_item = {name: {'children': {}, 'resource_type': resource_type}}
 
             if depth > 0:
                 for reference in cat.flat_references():
@@ -169,7 +180,7 @@ def thredds_get_resources(url, depth, **kwargs):
             return tree_item
 
         resources = thredds_get_resources(self.thredds_url, self.depth, **self.kwargs)
-        assert _is_valid_resource_schema(resources), "Error in Interface implementation"
+        assert _is_valid_resource_schema(resources), 'Error in Interface implementation'
         return resources
 
 
@@ -258,7 +269,7 @@ def add_children(resources, parent_id, position=0):
         for resource_name, values in resources.items():
             new_resource = models.RemoteResource(service_id=sync_info.service_id,
                                                  resource_name=unicode(resource_name),
-                                                 resource_type=u"directory",  # todo: fixme
+                                                 resource_type=values['resource_type'],
                                                  parent_id=parent_id,
                                                  ordering=position)
             session.add(new_resource)
@@ -326,12 +337,13 @@ def _format_resource_tree(children):
         for child_id, child_dict in children.items():
             resource = child_dict[u'node']
             new_children = child_dict[u'children']
-            fmt_res_tree[resource.resource_name] = {}
-            fmt_res_tree[resource.resource_name][u'children'] = _format_resource_tree(new_children)
+            resource_dict = {'children': _format_resource_tree(new_children),
+                             'resource_type': resource.resource_type}
+            fmt_res_tree[resource.resource_name] = resource_dict
         return fmt_res_tree
 
     remote_resources = _format_resource_tree(tree)
-    return {service_name: {'children': remote_resources}}
+    return {service_name: {'children': remote_resources, 'resource_type': 'directory'}}
 
 
 def get_last_sync(service_name, session):

From dad9f72f10cd112127f23173391dbbd50043e891 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 15:36:50 -0400
Subject: [PATCH 017/124] rename

---
 magpie/ui/management/views.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index d74501891..e1f730c76 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -484,7 +484,7 @@ def edit_group(self):
             elif u'edit_permissions' in self.request.POST:
                 remote_path = self.request.POST.get('remote_path')
                 if remote_path:
-                    res_id = self.add_external_resource(cur_svc_type, group_name, remote_path)
+                    res_id = self.add_remote_resource(cur_svc_type, group_name, remote_path)
                 self.edit_user_or_group_resource_permissions(group_name, res_id, is_user=False)
             elif u'clean_resource' in self.request.POST:
                 self.delete_resource(res_id)
@@ -548,23 +548,23 @@ def get_ids_to_clean(self, resources):
             ids += self.get_ids_to_clean(values['children'])
         return ids
 
-    def add_external_resource(self, service, group_name, resource_path):
+    def add_remote_resource(self, service_type, group_name, resource_path):
         try:
             res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(group_name,
-                                                                                          services=[service],
-                                                                                          service_type=service,
+                                                                                          services=[service_type],
+                                                                                          service_type=service_type,
                                                                                           is_user=False)
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        def parse_resources_and_put(resources, path, parent_id=None):
+        def traverse_path_and_post(resources, path, parent_id=None):
             if not path:
                 return parent_id
             current_name = path.pop(0)
             if current_name in resources:
-                res_id = parse_resources_and_put(resources[current_name]['children'],
-                                                 path,
-                                                 parent_id=resources[current_name]['id'])
+                res_id = traverse_path_and_post(resources[current_name]['children'],
+                                                path,
+                                                parent_id=resources[current_name]['id'])
             else:
                 resources_url = '{url}/resources'.format(url=self.magpie_url)
                 data = {
@@ -575,10 +575,10 @@ def parse_resources_and_put(resources, path, parent_id=None):
                 response = check_response(requests.post(resources_url, data=data, cookies=self.request.cookies))
                 res_id = response.json()['resource']['resource_id']
                 if path:
-                    res_id = parse_resources_and_put({}, path, parent_id=res_id)
+                    res_id = traverse_path_and_post({}, path, parent_id=res_id)
             return res_id
 
-        return parse_resources_and_put(res_perms, resource_path.split("/")[1:])
+        return traverse_path_and_post(res_perms, resource_path.split("/")[1:])
 
     @view_config(route_name='view_services', renderer='templates/view_services.mako')
     def view_services(self):

From bd08cae2f53a4f39a8bfe604d2481ec7eb117c4f Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 15:55:51 -0400
Subject: [PATCH 018/124] add resource_type attribute to resources

---
 magpie/helpers/sync_resources.py              | 24 ++++++++++++-------
 .../ui/management/templates/tree_scripts.mako |  1 +
 magpie/ui/management/views.py                 | 15 ++++++++----
 3 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 8b0477ae6..dd00cf203 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -53,32 +53,38 @@ def _merge_resources(resources_local, resources_remote):
     # don't overwrite the input arguments
     merged_resources = copy.deepcopy(resources_local)
 
-    def recurse(_resources_local, _resources_remote, remote_path=""):
+    def recurse(_resources_local, _resources_remote, remote_path="", remote_type_path=""):
         # loop local resources, looking for matches in remote resources
         for resource_name_local, values in _resources_local.items():
             current_path = "/".join([remote_path, str(resource_name_local)])
+
             matches_remote = resource_name_local in _resources_remote
+            resource_type = _resources_remote[resource_name_local]['resource_type'] if matches_remote else ""
+            current_type_path = "/".join([remote_type_path, resource_type])
 
             values["remote_path"] = "" if matches_remote else current_path
+            values["remote_type_path"] = current_type_path
             values["matches_remote"] = matches_remote
-            values["resource_type"] = _resources_remote[resource_name_local]['children'] if matches_remote else ""
+            values["resource_type"] = resource_type
 
             resource_remote_children = _resources_remote[resource_name_local]['children'] if matches_remote else {}
 
-            recurse(values['children'], resource_remote_children, current_path)
+            recurse(values['children'], resource_remote_children, current_path, current_type_path)
 
         # loop remote resources, looking for matches in local resources
         for resource_name_remote, values in _resources_remote.items():
             if resource_name_remote not in _resources_local:
                 current_path = "/".join([remote_path, str(resource_name_remote)])
+                current_type_path = "/".join([remote_type_path, values['resource_type']])
                 new_resource = {'permission_names': [],
                                 'children': {},
                                 'resource_type': values['resource_type'],
                                 'id': current_path,
                                 'remote_path': current_path,
+                                'remote_type_path': current_type_path,
                                 'matches_remote': True}
                 _resources_local[resource_name_remote] = new_resource
-                recurse(new_resource['children'], values['children'], current_path)
+                recurse(new_resource['children'], values['children'], current_path, current_type_path)
 
     recurse(merged_resources, resources_remote)
 
@@ -143,15 +149,16 @@ def __init__(self, geoserver_url):
 
     def get_resources(self):
         # Only workspaces are fetched for now
+        resource_type = "route"
         workspaces_url = "{}/{}".format(self.geoserver_url, "workspaces")
         resp = requests.get(workspaces_url, headers={"Accept": "application/json"})
         resp.raise_for_status()
         workspaces_list = resp.json().get("workspaces", {}).get("workspace", {})
 
-        workspaces = {w["name"]: {"children": {}, "resource_type": "directory"} for w in workspaces_list}
+        workspaces = {w["name"]: {"children": {}, "resource_type": resource_type} for w in workspaces_list}
 
         resources = {"geoserver-api": {"children": workspaces,
-                                       "resource_type": "directory"}}
+                                       "resource_type": resource_type}}
         assert _is_valid_resource_schema(resources), "Error in Interface implementation"
         return resources
 
@@ -169,8 +176,9 @@ def thredds_get_resources(url, depth, **kwargs):
             cat = threddsclient.read_url(url, **kwargs)
             name = cat.name
             resource_type = 'directory'
-            if cat.datasets:
-                resource_type = cat.datasets[0].content_type.replace('application/', '')
+            if cat.datasets and cat.datasets[0].content_type != "application/directory":
+                resource_type = 'file'
+
             tree_item = {name: {'children': {}, 'resource_type': resource_type}}
 
             if depth > 0:
diff --git a/magpie/ui/management/templates/tree_scripts.mako b/magpie/ui/management/templates/tree_scripts.mako
index fd47a526b..6889da0c9 100644
--- a/magpie/ui/management/templates/tree_scripts.mako
+++ b/magpie/ui/management/templates/tree_scripts.mako
@@ -51,6 +51,7 @@ li.Collapsed {
                 <div class="tree_item">${key}</div>
                 <input type="hidden" value=${tree[key]['id']} name="resource_id">
                 <input type="hidden" value=${tree[key].get('remote_path', '')} name="remote_path">
+                <input type="hidden" value=${tree[key].get('remote_type_path', '')} name="remote_type_path">
                 <input type="hidden" value=${tree[key].get('matches_remote', '')} name="matches_remote">
                 ${item_renderer(key, tree[key], level)}
             </li>
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index e1f730c76..6c8c9a5f0 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -484,7 +484,8 @@ def edit_group(self):
             elif u'edit_permissions' in self.request.POST:
                 remote_path = self.request.POST.get('remote_path')
                 if remote_path:
-                    res_id = self.add_remote_resource(cur_svc_type, group_name, remote_path)
+                    remote_type_path = self.request.POST.get('remote_type_path')
+                    res_id = self.add_remote_resource(cur_svc_type, group_name, remote_path, remote_type_path)
                 self.edit_user_or_group_resource_permissions(group_name, res_id, is_user=False)
             elif u'clean_resource' in self.request.POST:
                 self.delete_resource(res_id)
@@ -548,7 +549,7 @@ def get_ids_to_clean(self, resources):
             ids += self.get_ids_to_clean(values['children'])
         return ids
 
-    def add_remote_resource(self, service_type, group_name, resource_path):
+    def add_remote_resource(self, service_type, group_name, resource_path, remote_type_path):
         try:
             res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(group_name,
                                                                                           services=[service_type],
@@ -557,19 +558,21 @@ def add_remote_resource(self, service_type, group_name, resource_path):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        def traverse_path_and_post(resources, path, parent_id=None):
+        def traverse_path_and_post(resources, path, type_path, parent_id=None):
             if not path:
                 return parent_id
             current_name = path.pop(0)
+            current_type = type_path.pop(0)
             if current_name in resources:
                 res_id = traverse_path_and_post(resources[current_name]['children'],
                                                 path,
+                                                type_path,
                                                 parent_id=resources[current_name]['id'])
             else:
                 resources_url = '{url}/resources'.format(url=self.magpie_url)
                 data = {
                     'resource_name': current_name,
-                    'resource_type': "directory",
+                    'resource_type': current_type,
                     'parent_id': parent_id,
                 }
                 response = check_response(requests.post(resources_url, data=data, cookies=self.request.cookies))
@@ -578,7 +581,9 @@ def traverse_path_and_post(resources, path, parent_id=None):
                     res_id = traverse_path_and_post({}, path, parent_id=res_id)
             return res_id
 
-        return traverse_path_and_post(res_perms, resource_path.split("/")[1:])
+        path_list = resource_path.split("/")[1:]
+        remote_type_path_list = remote_type_path.split("/")[1:]
+        return traverse_path_and_post(res_perms, path_list, remote_type_path_list)
 
     @view_config(route_name='view_services', renderer='templates/view_services.mako')
     def view_services(self):

From 0b076b1d0e78cc1f41ca42c0b08c9532ed103f0b Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 15:58:23 -0400
Subject: [PATCH 019/124] reformat

---
 magpie/helpers/sync_resources.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index dd00cf203..e4501f794 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -19,7 +19,7 @@
 
 def merge_local_and_remote_resources(resources_local, service_name, session):
     """Main function to sync resources with remote server"""
-    remote_resources = _query_resources(service_name, session=session)
+    remote_resources = _query_remote_resources_in_database(service_name, session=session)
     merged_resources = _merge_resources(resources_local, remote_resources)
     _sort_resources(merged_resources)
     return merged_resources
@@ -325,9 +325,20 @@ def build_subtree_strut(result):
     return build_subtree_strut(query)['children']
 
 
-def _query_resources(service_name, session):
+def _format_resource_tree(children):
+    fmt_res_tree = {}
+    for child_id, child_dict in children.items():
+        resource = child_dict[u'node']
+        new_children = child_dict[u'children']
+        resource_dict = {'children': _format_resource_tree(new_children),
+                         'resource_type': resource.resource_type}
+        fmt_res_tree[resource.resource_name] = resource_dict
+    return fmt_res_tree
+
+
+def _query_remote_resources_in_database(service_name, session):
     """
-    Reads remote resources from database. No external request is made.
+    Reads remote resources from the RemoteResources table. No external request is made.
     :param service_name:
     :param session:
     :return: a dictionary of the form defined in '_is_valid_resource_schema'
@@ -340,16 +351,6 @@ def _query_resources(service_name, session):
         resource_id=sync_info.remote_resource_id).first()
     tree = _get_resource_children(main_resource, session)
 
-    def _format_resource_tree(children):
-        fmt_res_tree = {}
-        for child_id, child_dict in children.items():
-            resource = child_dict[u'node']
-            new_children = child_dict[u'children']
-            resource_dict = {'children': _format_resource_tree(new_children),
-                             'resource_type': resource.resource_type}
-            fmt_res_tree[resource.resource_name] = resource_dict
-        return fmt_res_tree
-
     remote_resources = _format_resource_tree(tree)
     return {service_name: {'children': remote_resources, 'resource_type': 'directory'}}
 

From 23f03d196f8e13599b033bcd2c05ee78a48e7d16 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 15:58:35 -0400
Subject: [PATCH 020/124] basic housekeeping function

---
 magpie/helpers/sync_resources.py | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index e4501f794..3252a2174 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -381,7 +381,7 @@ def fetch_single_service(service_name, session):
 
 def fetch():
     """
-    Main entry point to get all remote resources for each service and write to database.
+    Main function to get all remote resources for each service and write to database.
     """
     url = db.get_db_url()
     engine = create_engine(url)
@@ -395,5 +395,33 @@ def fetch():
     session.close()
 
 
-if __name__ == '__main__':
+def housekeeping():
+    """
+    Clean resources that are in the Resource table but have no
+    group or user permissions associated to them.
+    """
+    url = db.get_db_url()
+    engine = create_engine(url)
+
+    session = Session(bind=engine)
+
+    for resource in session.query(models.Resource):
+        if resource.resource_type_name == 'service':
+            continue
+        if not resource.group_permissions and not resource.user_permissions:
+            session.delete(resource)
+
+    session.commit()
+    session.close()
+
+
+def main():
+    """
+    Main entry point for cron service.
+    """
     fetch()
+    housekeeping()
+
+
+if __name__ == '__main__':
+    main()

From a7de1dadd11fd85b6707c03feef93ca54d5d22ae Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 16:37:21 -0400
Subject: [PATCH 021/124] use service type instead of service name

---
 magpie/helpers/sync_resources.py | 41 ++++++++++++++++----------------
 1 file changed, 20 insertions(+), 21 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 3252a2174..c7714c289 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -17,9 +17,9 @@
 from magpie import db, models
 
 
-def merge_local_and_remote_resources(resources_local, service_name, session):
+def merge_local_and_remote_resources(resources_local, service_type, session):
     """Main function to sync resources with remote server"""
-    remote_resources = _query_remote_resources_in_database(service_name, session=session)
+    remote_resources = _query_remote_resources_in_database(service_type, session=session)
     merged_resources = _merge_resources(resources_local, remote_resources)
     _sort_resources(merged_resources)
     return merged_resources
@@ -206,19 +206,18 @@ def get_resources(self):
 }
 
 
-def _ensure_sync_info_exists(service_name, session):
+def _ensure_sync_info_exists(service_resource_id, session):
     """
     Make sure the RemoteResourcesSyncInfo entry exists in the database.
-    :param service_name:
+    :param service_resource_id:
     :param session:
     """
-    service = models.Service.by_service_name(service_name, db_session=session)
-    service_sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
+    service_sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_resource_id, session)
     if not service_sync_info:
-        sync_info = models.RemoteResourcesSyncInfo(service_id=service.resource_id)
+        sync_info = models.RemoteResourcesSyncInfo(service_id=service_resource_id)
         session.add(sync_info)
         session.flush()
-        _create_main_resource(service.resource_id, session)
+        _create_main_resource(service_resource_id, session)
 
 
 def _get_remote_resources(service, service_name):
@@ -336,15 +335,15 @@ def _format_resource_tree(children):
     return fmt_res_tree
 
 
-def _query_remote_resources_in_database(service_name, session):
+def _query_remote_resources_in_database(service_type, session):
     """
     Reads remote resources from the RemoteResources table. No external request is made.
-    :param service_name:
+    :param service_type:
     :param session:
     :return: a dictionary of the form defined in '_is_valid_resource_schema'
     """
-    service = models.Service.by_service_name(service_name, db_session=session)
-    _ensure_sync_info_exists(service_name, session)
+    service = session.query(models.Service).filter_by(type=service_type).first()
+    _ensure_sync_info_exists(service.resource_id, session)
 
     sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
     main_resource = session.query(models.RemoteResource).filter_by(
@@ -352,30 +351,30 @@ def _query_remote_resources_in_database(service_name, session):
     tree = _get_resource_children(main_resource, session)
 
     remote_resources = _format_resource_tree(tree)
-    return {service_name: {'children': remote_resources, 'resource_type': 'directory'}}
+    return {service_type: {'children': remote_resources, 'resource_type': 'directory'}}
 
 
-def get_last_sync(service_name, session):
+def get_last_sync(service_type, session):
     last_sync = None
-    service = models.Service.by_service_name(service_name, db_session=session)
-    _ensure_sync_info_exists(service_name, session)
+    service = session.query(models.Service).filter_by(type=service_type).first()
+    _ensure_sync_info_exists(service.resource_id, session)
     sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
     if sync_info:
         last_sync = sync_info.last_sync
     return last_sync
 
 
-def fetch_single_service(service_name, session):
+def fetch_single_service(service_type, session):
     """
     Get remote resources for a single service.
-    :param service_name:
+    :param service_type:
     :param session:
     """
-    service = models.Service.by_service_name(service_name, db_session=session)
-    remote_resources = _get_remote_resources(service, service_name)
+    service = session.query(models.Service).filter_by(type=service_type).first()
+    remote_resources = _get_remote_resources(service, service_type)
     service_id = service.resource_id
     _delete_records(service_id, session)
-    _ensure_sync_info_exists(service_name, session)
+    _ensure_sync_info_exists(service.resource_id, session)
     _update_db(remote_resources, service_id, session)
 
 

From 20bda2cd93c7473d868f81ca27713cc7653a9233 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 16:38:03 -0400
Subject: [PATCH 022/124] show message if sync is not implemented

---
 magpie/ui/management/templates/edit_group.mako |  9 +++++++--
 magpie/ui/management/views.py                  | 10 ++++++++--
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/magpie/ui/management/templates/edit_group.mako b/magpie/ui/management/templates/edit_group.mako
index 8945effd1..eb2008014 100644
--- a/magpie/ui/management/templates/edit_group.mako
+++ b/magpie/ui/management/templates/edit_group.mako
@@ -115,8 +115,13 @@
         <form id="sync_info" action="${request.path}" method="post">
             <p class="panel_line">
                 <span class="panel_entry">Last synchronization with remote service: </span>
-                <span class="panel_value">${last_sync} </span>
-                <input type="submit" value="Sync now" name="force_sync">
+                %if sync_implemented:
+                    <span class="panel_value">${last_sync} </span>
+                    <input type="submit" value="Sync now" name="force_sync">
+                %else:
+                    <span class="panel_value">Not implemented for this service type.</span>
+                %endif
+
             </p>
             %if ids_to_clean:
                 <p class="panel_line">
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 6c8c9a5f0..b10e5e094 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -464,6 +464,8 @@ def edit_group(self):
         cur_svc_type = self.request.matchdict['cur_svc_type']
         group_info = {u'edit_mode': u'no_edit', u'group_name': group_name, u'cur_svc_type': cur_svc_type}
 
+        error_message = None
+
         # move to service or edit requested group/permission changes
         if self.request.method == 'POST':
             res_id = self.request.POST.get('resource_id')
@@ -492,7 +494,10 @@ def edit_group(self):
             elif u'member' in self.request.POST:
                 self.edit_group_users(group_name)
             elif u'force_sync' in self.request.POST:
-                sync_resources.fetch_single_service(cur_svc_type, session=self.request.db)
+                try:
+                    sync_resources.fetch_single_service(cur_svc_type, session=self.request.db)
+                except:
+                    error_message = "There was an error when trying to get remote resources."
             elif u'clean_all' in self.request.POST:
                 ids_to_clean = self.request.POST.get('ids_to_clean').split(";")
                 for id_ in ids_to_clean:
@@ -518,9 +523,10 @@ def edit_group(self):
 
         ids = self.get_ids_to_clean(res_perms)
 
-        group_info[u'error_message'] = None
+        group_info[u'error_message'] = error_message
         group_info[u'ids_to_clean'] = ";".join(ids)
         group_info[u'last_sync'] = last_sync
+        group_info[u'sync_implemented'] = cur_svc_type in sync_resources.SYNC_SERVICES
         group_info[u'group_name'] = group_name
         group_info[u'cur_svc_type'] = cur_svc_type
         group_info[u'users'] = self.get_user_names()

From 52a82b75eef0762b4d201c64dcf23e0f6dcca662 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Thu, 13 Sep 2018 16:38:25 -0400
Subject: [PATCH 023/124] implement project-api

---
 magpie/helpers/sync_resources.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index c7714c289..7d1e85d25 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -163,6 +163,24 @@ def get_resources(self):
         return resources
 
 
+class _SyncServiceProjectAPI:
+    def __init__(self, project_api_url):
+        self.project_api_url = project_api_url
+
+    def get_resources(self):
+        # Only workspaces are fetched for now
+        resource_type = "route"
+        projects_url = "/".join([self.project_api_url, "api", "Projects"])
+        resp = requests.get(projects_url)
+        resp.raise_for_status()
+
+        projects = {p["id"]: {"children": {}, "resource_type": resource_type} for p in resp.json()}
+
+        resources = {"project-api": {"children": projects, "resource_type": resource_type}}
+        assert _is_valid_resource_schema(resources), "Error in Interface implementation"
+        return resources
+
+
 class _SyncServiceThreads(_SyncServiceInterface):
     DEPTH_DEFAULT = 2
 
@@ -203,6 +221,7 @@ def get_resources(self):
 SYNC_SERVICES = {
     "thredds": _SyncServiceThreads,
     "geoserver-api": _SyncServiceGeoserver,
+    "project-api": _SyncServiceProjectAPI,
 }
 
 

From 94889cdc33c8b2d90b85b6496d97ae94ee8b334b Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 14 Sep 2018 10:30:15 -0400
Subject: [PATCH 024/124] refactoring

---
 magpie/helpers/sync_resources.py | 154 ++++---------------------------
 magpie/helpers/sync_services.py  | 119 ++++++++++++++++++++++++
 2 files changed, 139 insertions(+), 134 deletions(-)
 create mode 100644 magpie/helpers/sync_services.py

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 7d1e85d25..883c73cde 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -4,17 +4,23 @@
 To implement a new service, see the _SyncServiceInterface class.
 """
 
-import abc
 import copy
 import datetime
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
 
-import requests
-import threddsclient
 from sqlalchemy import create_engine
 from sqlalchemy.orm import Session
 
 from magpie import db, models
+from magpie.helpers import sync_services
+
+SYNC_SERVICES = defaultdict(lambda: sync_services._SyncServiceDefault)
+# noinspection PyTypeChecker
+SYNC_SERVICES.update({
+    "thredds": sync_services._SyncServiceThreads,
+    "geoserver-api": sync_services._SyncServiceGeoserver,
+    "project-api": sync_services._SyncServiceProjectAPI,
+})
 
 
 def merge_local_and_remote_resources(resources_local, service_type, session):
@@ -33,14 +39,14 @@ def _merge_resources(resources_local, resources_remote):
         - matches_remote: True or False depending if the resource is present on the remote server
         - id: set to the value of 'remote_path' if the resource if remote only
 
-    returns a dictionary of the form validated by 'is_valid_resource_schema'
+    returns a dictionary of the form validated by 'sync_services.is_valid_resource_schema'
 
     """
     if not resources_remote:
         return resources_local
 
-    assert _is_valid_resource_schema(resources_local, ignore_resource_type=True)
-    assert _is_valid_resource_schema(resources_remote)
+    assert sync_services.is_valid_resource_schema(resources_local, ignore_resource_type=True)
+    assert sync_services.is_valid_resource_schema(resources_remote)
 
     if not resources_local:
         raise ValueError("The resources must contain at least the service name.")
@@ -88,39 +94,14 @@ def recurse(_resources_local, _resources_remote, remote_path="", remote_type_pat
 
     recurse(merged_resources, resources_remote)
 
-    assert _is_valid_resource_schema(merged_resources)
+    assert sync_services.is_valid_resource_schema(merged_resources)
 
     return merged_resources
 
 
-def _is_valid_resource_schema(resources, ignore_resource_type=False):
-    """
-    Returns True if the structure of the input dictionary is a tree of the form:
-
-    {'resource_name_1': {'children': {'resource_name_3': {'children': {}, 'resource_type': ...},
-                                      'resource_name_4': {'children': {}, 'resource_type': ...}
-                                      },
-                         'resource_type': ...
-                         },
-     'resource_name_2': {'children': {}, resource_type': ...}
-     }
-    :return: bool
-    """
-    for resource_name, values in resources.items():
-        if 'children' not in values:
-            return False
-        if not ignore_resource_type and 'resource_type' not in values:
-            return False
-        if not isinstance(values['children'], (OrderedDict, dict)):
-            return False
-        return _is_valid_resource_schema(values['children'],
-                                         ignore_resource_type=ignore_resource_type)
-    return True
-
-
 def _sort_resources(resources):
     """
-    Sorts a resource dictionary of the type validated by '_is_valid_resource_schema'
+    Sorts a resource dictionary of the type validated by 'sync_services.is_valid_resource_schema'
     by using an OrderedDict
     :return: None
     """
@@ -129,102 +110,6 @@ def _sort_resources(resources):
         return _sort_resources(values['children'])
 
 
-class _SyncServiceInterface:
-    __metaclass__ = abc.ABCMeta
-
-    @abc.abstractmethod
-    def get_resources(self):
-        """
-        This is the function actually fetching the data from the remote service.
-        Implement this for every specific service.
-
-        :return: The returned dictionary must be validated by 'is_valid_resource_schema'
-        """
-        pass
-
-
-class _SyncServiceGeoserver:
-    def __init__(self, geoserver_url):
-        self.geoserver_url = geoserver_url
-
-    def get_resources(self):
-        # Only workspaces are fetched for now
-        resource_type = "route"
-        workspaces_url = "{}/{}".format(self.geoserver_url, "workspaces")
-        resp = requests.get(workspaces_url, headers={"Accept": "application/json"})
-        resp.raise_for_status()
-        workspaces_list = resp.json().get("workspaces", {}).get("workspace", {})
-
-        workspaces = {w["name"]: {"children": {}, "resource_type": resource_type} for w in workspaces_list}
-
-        resources = {"geoserver-api": {"children": workspaces,
-                                       "resource_type": resource_type}}
-        assert _is_valid_resource_schema(resources), "Error in Interface implementation"
-        return resources
-
-
-class _SyncServiceProjectAPI:
-    def __init__(self, project_api_url):
-        self.project_api_url = project_api_url
-
-    def get_resources(self):
-        # Only workspaces are fetched for now
-        resource_type = "route"
-        projects_url = "/".join([self.project_api_url, "api", "Projects"])
-        resp = requests.get(projects_url)
-        resp.raise_for_status()
-
-        projects = {p["id"]: {"children": {}, "resource_type": resource_type} for p in resp.json()}
-
-        resources = {"project-api": {"children": projects, "resource_type": resource_type}}
-        assert _is_valid_resource_schema(resources), "Error in Interface implementation"
-        return resources
-
-
-class _SyncServiceThreads(_SyncServiceInterface):
-    DEPTH_DEFAULT = 2
-
-    def __init__(self, thredds_url, depth=DEPTH_DEFAULT, **kwargs):
-        self.thredds_url = thredds_url
-        self.depth = depth
-        self.kwargs = kwargs  # kwargs is passed to the requests.get method.
-
-    def get_resources(self):
-        def thredds_get_resources(url, depth, **kwargs):
-            cat = threddsclient.read_url(url, **kwargs)
-            name = cat.name
-            resource_type = 'directory'
-            if cat.datasets and cat.datasets[0].content_type != "application/directory":
-                resource_type = 'file'
-
-            tree_item = {name: {'children': {}, 'resource_type': resource_type}}
-
-            if depth > 0:
-                for reference in cat.flat_references():
-                    tree_item[name]['children'].update(thredds_get_resources(reference.url, depth - 1, **kwargs))
-
-            return tree_item
-
-        resources = thredds_get_resources(self.thredds_url, self.depth, **self.kwargs)
-        assert _is_valid_resource_schema(resources), 'Error in Interface implementation'
-        return resources
-
-
-class _SyncServiceDefault(_SyncServiceInterface):
-    def __init__(self, _):
-        pass
-
-    def get_resources(self):
-        return {}
-
-
-SYNC_SERVICES = {
-    "thredds": _SyncServiceThreads,
-    "geoserver-api": _SyncServiceGeoserver,
-    "project-api": _SyncServiceProjectAPI,
-}
-
-
 def _ensure_sync_info_exists(service_resource_id, session):
     """
     Make sure the RemoteResourcesSyncInfo entry exists in the database.
@@ -249,7 +134,7 @@ def _get_remote_resources(service, service_name):
     service_url = service.url
     if service_url.endswith("/"):  # remove trailing slash
         service_url = service_url[:-1]
-    sync_service = SYNC_SERVICES.get(service_name.lower(), _SyncServiceDefault)(service_url)
+    sync_service = SYNC_SERVICES.get(service_name.lower(), sync_services._SyncServiceDefault)(service_url)
     return sync_service.get_resources()
 
 
@@ -359,7 +244,7 @@ def _query_remote_resources_in_database(service_type, session):
     Reads remote resources from the RemoteResources table. No external request is made.
     :param service_type:
     :param session:
-    :return: a dictionary of the form defined in '_is_valid_resource_schema'
+    :return: a dictionary of the form defined in 'sync_services.is_valid_resource_schema'
     """
     service = session.query(models.Service).filter_by(type=service_type).first()
     _ensure_sync_info_exists(service.resource_id, session)
@@ -437,8 +322,9 @@ def main():
     """
     Main entry point for cron service.
     """
-    fetch()
-    housekeeping()
+    if db.is_database_ready():
+        fetch()
+        housekeeping()
 
 
 if __name__ == '__main__':
diff --git a/magpie/helpers/sync_services.py b/magpie/helpers/sync_services.py
new file mode 100644
index 000000000..2a44b5644
--- /dev/null
+++ b/magpie/helpers/sync_services.py
@@ -0,0 +1,119 @@
+import abc
+from collections import OrderedDict
+
+import requests
+import threddsclient
+
+
+def is_valid_resource_schema(resources, ignore_resource_type=False):
+    """
+    Returns True if the structure of the input dictionary is a tree of the form:
+
+    {'resource_name_1': {'children': {'resource_name_3': {'children': {}, 'resource_type': ...},
+                                      'resource_name_4': {'children': {}, 'resource_type': ...}
+                                      },
+                         'resource_type': ...
+                         },
+     'resource_name_2': {'children': {}, resource_type': ...}
+     }
+    :return: bool
+    """
+    for resource_name, values in resources.items():
+        if 'children' not in values:
+            return False
+        if not ignore_resource_type and 'resource_type' not in values:
+            return False
+        if not isinstance(values['children'], (OrderedDict, dict)):
+            return False
+        return is_valid_resource_schema(values['children'],
+                                        ignore_resource_type=ignore_resource_type)
+    return True
+
+
+class _SyncServiceInterface:
+    __metaclass__ = abc.ABCMeta
+
+    @abc.abstractmethod
+    def get_resources(self):
+        """
+        This is the function actually fetching the data from the remote service.
+        Implement this for every specific service.
+
+        :return: The returned dictionary must be validated by 'is_valid_resource_schema'
+        """
+        pass
+
+
+class _SyncServiceGeoserver:
+    def __init__(self, geoserver_url):
+        self.geoserver_url = geoserver_url
+
+    def get_resources(self):
+        # Only workspaces are fetched for now
+        resource_type = "route"
+        workspaces_url = "{}/{}".format(self.geoserver_url, "workspaces")
+        resp = requests.get(workspaces_url, headers={"Accept": "application/json"})
+        resp.raise_for_status()
+        workspaces_list = resp.json().get("workspaces", {}).get("workspace", {})
+
+        workspaces = {w["name"]: {"children": {}, "resource_type": resource_type} for w in workspaces_list}
+
+        resources = {"geoserver-api": {"children": workspaces,
+                                       "resource_type": resource_type}}
+        assert is_valid_resource_schema(resources), "Error in Interface implementation"
+        return resources
+
+
+class _SyncServiceProjectAPI:
+    def __init__(self, project_api_url):
+        self.project_api_url = project_api_url
+
+    def get_resources(self):
+        # Only workspaces are fetched for now
+        resource_type = "route"
+        projects_url = "/".join([self.project_api_url, "api", "Projects"])
+        resp = requests.get(projects_url)
+        resp.raise_for_status()
+
+        projects = {p["id"]: {"children": {}, "resource_type": resource_type} for p in resp.json()}
+
+        resources = {"project-api": {"children": projects, "resource_type": resource_type}}
+        assert is_valid_resource_schema(resources), "Error in Interface implementation"
+        return resources
+
+
+class _SyncServiceThreads(_SyncServiceInterface):
+    DEPTH_DEFAULT = 3
+
+    def __init__(self, thredds_url, depth=DEPTH_DEFAULT, **kwargs):
+        self.thredds_url = thredds_url
+        self.depth = depth
+        self.kwargs = kwargs  # kwargs is passed to the requests.get method.
+
+    def get_resources(self):
+        def thredds_get_resources(url, depth, **kwargs):
+            cat = threddsclient.read_url(url, **kwargs)
+            name = cat.name
+            resource_type = 'directory'
+            if cat.datasets and cat.datasets[0].content_type != "application/directory":
+                resource_type = 'file'
+
+            tree_item = {name: {'children': {}, 'resource_type': resource_type}}
+
+            if depth > 0:
+                for reference in cat.flat_references():
+                    tree_item[name]['children'].update(thredds_get_resources(reference.url, depth - 1, **kwargs))
+
+            return tree_item
+
+        resources = thredds_get_resources(self.thredds_url, self.depth, **self.kwargs)
+        assert is_valid_resource_schema(resources), 'Error in Interface implementation'
+        return resources
+
+
+class _SyncServiceDefault(_SyncServiceInterface):
+    def __init__(self, _):
+        pass
+
+    def get_resources(self):
+        return {}

From 3e49695628ec562fb47ce2297de0f838ca5b5484 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 14 Sep 2018 12:01:42 -0400
Subject: [PATCH 025/124] services are uniquely identified using both their
 type and resource_name

---
 magpie/helpers/sync_resources.py | 51 ++++++++++++++++++--------------
 magpie/helpers/sync_services.py  | 19 +++++++-----
 magpie/ui/management/views.py    | 22 ++++++++------
 3 files changed, 53 insertions(+), 39 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 883c73cde..97c4a150f 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -14,18 +14,18 @@
 from magpie import db, models
 from magpie.helpers import sync_services
 
-SYNC_SERVICES = defaultdict(lambda: sync_services._SyncServiceDefault)
+SYNC_SERVICES_TYPES = defaultdict(lambda: sync_services._SyncServiceDefault)
 # noinspection PyTypeChecker
-SYNC_SERVICES.update({
+SYNC_SERVICES_TYPES.update({
     "thredds": sync_services._SyncServiceThreads,
     "geoserver-api": sync_services._SyncServiceGeoserver,
     "project-api": sync_services._SyncServiceProjectAPI,
 })
 
 
-def merge_local_and_remote_resources(resources_local, service_type, session):
+def merge_local_and_remote_resources(resources_local, service_type, service_name, session):
     """Main function to sync resources with remote server"""
-    remote_resources = _query_remote_resources_in_database(service_type, session=session)
+    remote_resources = _query_remote_resources_in_database(service_type, service_name, session=session)
     merged_resources = _merge_resources(resources_local, remote_resources)
     _sort_resources(merged_resources)
     return merged_resources
@@ -51,11 +51,6 @@ def _merge_resources(resources_local, resources_remote):
     if not resources_local:
         raise ValueError("The resources must contain at least the service name.")
 
-    # The first item is the service name. It is skipped so that only the resources are compared.
-    service_name = resources_local.keys()[0]
-    _, remote_values = resources_remote.popitem()
-    resources_remote = {service_name: remote_values}
-
     # don't overwrite the input arguments
     merged_resources = copy.deepcopy(resources_local)
 
@@ -124,17 +119,18 @@ def _ensure_sync_info_exists(service_resource_id, session):
         _create_main_resource(service_resource_id, session)
 
 
-def _get_remote_resources(service, service_name):
+def _get_remote_resources(service):
     """
-    Request rmeote resources, depending on service type.
-    :param service:
-    :param service_name:
+    Request remote resources, depending on service type.
+    :param service: (models.Service)
     :return:
     """
     service_url = service.url
     if service_url.endswith("/"):  # remove trailing slash
         service_url = service_url[:-1]
-    sync_service = SYNC_SERVICES.get(service_name.lower(), sync_services._SyncServiceDefault)(service_url)
+
+    sync_service_class = SYNC_SERVICES_TYPES.get(service.type.lower(), sync_services._SyncServiceDefault)
+    sync_service = sync_service_class(service.resource_name, service_url)
     return sync_service.get_resources()
 
 
@@ -239,14 +235,14 @@ def _format_resource_tree(children):
     return fmt_res_tree
 
 
-def _query_remote_resources_in_database(service_type, session):
+def _query_remote_resources_in_database(service_type, service_name, session):
     """
     Reads remote resources from the RemoteResources table. No external request is made.
     :param service_type:
     :param session:
     :return: a dictionary of the form defined in 'sync_services.is_valid_resource_schema'
     """
-    service = session.query(models.Service).filter_by(type=service_type).first()
+    service = session.query(models.Service).filter_by(type=service_type, resource_name=service_name).first()
     _ensure_sync_info_exists(service.resource_id, session)
 
     sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
@@ -255,7 +251,7 @@ def _query_remote_resources_in_database(service_type, session):
     tree = _get_resource_children(main_resource, session)
 
     remote_resources = _format_resource_tree(tree)
-    return {service_type: {'children': remote_resources, 'resource_type': 'directory'}}
+    return {service_name: {'children': remote_resources, 'resource_type': 'directory'}}
 
 
 def get_last_sync(service_type, session):
@@ -268,14 +264,23 @@ def get_last_sync(service_type, session):
     return last_sync
 
 
-def fetch_single_service(service_type, session):
+def fetch_all_services_by_type(service_type, session):
     """
-    Get remote resources for a single service.
+    Get remote resources for all services of a certain type.
     :param service_type:
     :param session:
     """
-    service = session.query(models.Service).filter_by(type=service_type).first()
-    remote_resources = _get_remote_resources(service, service_type)
+    for service in session.query(models.Service).filter_by(type=service_type):
+        fetch_single_service(service, session)
+
+
+def fetch_single_service(service, session):
+    """
+    Get remote resources for a single service.
+    :param service: (models.Service)
+    :param session:
+    """
+    remote_resources = _get_remote_resources(service)
     service_id = service.resource_id
     _delete_records(service_id, session)
     _ensure_sync_info_exists(service.resource_id, session)
@@ -291,8 +296,8 @@ def fetch():
 
     session = Session(bind=engine)
 
-    for service_name in SYNC_SERVICES:
-        fetch_single_service(service_name, session)
+    for service_type in SYNC_SERVICES_TYPES:
+        fetch_all_services_by_type(service_type, session)
 
     session.commit()
     session.close()
diff --git a/magpie/helpers/sync_services.py b/magpie/helpers/sync_services.py
index 2a44b5644..3dce6a7f6 100644
--- a/magpie/helpers/sync_services.py
+++ b/magpie/helpers/sync_services.py
@@ -45,7 +45,8 @@ def get_resources(self):
 
 
 class _SyncServiceGeoserver:
-    def __init__(self, geoserver_url):
+    def __init__(self, service_name, geoserver_url):
+        self.service_name = service_name
         self.geoserver_url = geoserver_url
 
     def get_resources(self):
@@ -58,14 +59,15 @@ def get_resources(self):
 
         workspaces = {w["name"]: {"children": {}, "resource_type": resource_type} for w in workspaces_list}
 
-        resources = {"geoserver-api": {"children": workspaces,
-                                       "resource_type": resource_type}}
+        resources = {self.service_name: {"children": workspaces,
+                                         "resource_type": resource_type}}
         assert is_valid_resource_schema(resources), "Error in Interface implementation"
         return resources
 
 
 class _SyncServiceProjectAPI:
-    def __init__(self, project_api_url):
+    def __init__(self, service_name, project_api_url):
+        self.service_name = service_name
         self.project_api_url = project_api_url
 
     def get_resources(self):
@@ -77,7 +79,7 @@ def get_resources(self):
 
         projects = {p["id"]: {"children": {}, "resource_type": resource_type} for p in resp.json()}
 
-        resources = {"project-api": {"children": projects, "resource_type": resource_type}}
+        resources = {self.service_name: {"children": projects, "resource_type": resource_type}}
         assert is_valid_resource_schema(resources), "Error in Interface implementation"
         return resources
 
@@ -85,7 +87,8 @@ def get_resources(self):
 class _SyncServiceThreads(_SyncServiceInterface):
     DEPTH_DEFAULT = 3
 
-    def __init__(self, thredds_url, depth=DEPTH_DEFAULT, **kwargs):
+    def __init__(self, service_name, thredds_url, depth=DEPTH_DEFAULT, **kwargs):
+        self.service_name = service_name
         self.thredds_url = thredds_url
         self.depth = depth
         self.kwargs = kwargs  # kwargs is passed to the requests.get method.
@@ -94,6 +97,8 @@ def get_resources(self):
         def thredds_get_resources(url, depth, **kwargs):
             cat = threddsclient.read_url(url, **kwargs)
             name = cat.name
+            if depth == self.depth:
+                name = self.service_name
             resource_type = 'directory'
             if cat.datasets and cat.datasets[0].content_type != "application/directory":
                 resource_type = 'file'
@@ -112,7 +117,7 @@ def thredds_get_resources(url, depth, **kwargs):
 
 
 class _SyncServiceDefault(_SyncServiceInterface):
-    def __init__(self, _):
+    def __init__(self, *_):
         pass
 
     def get_resources(self):
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index b10e5e094..cda7526e6 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -398,7 +398,7 @@ def edit_group_users(self, group_name):
     def edit_user_or_group_resource_permissions(self, user_or_group_name, resource_id, is_user=False):
         usr_grp_type = 'users' if is_user else 'groups'
         res_perms_url = '{url}/{grp_type}/{grp}/resources/{res_id}/permissions' \
-                        .format(url=self.magpie_url, grp_type=usr_grp_type, grp=user_or_group_name, res_id=resource_id)
+            .format(url=self.magpie_url, grp_type=usr_grp_type, grp=user_or_group_name, res_id=resource_id)
         try:
             res_perms_resp = requests.get(res_perms_url, cookies=self.request.cookies)
             res_perms = res_perms_resp.json()['permission_names']
@@ -495,9 +495,10 @@ def edit_group(self):
                 self.edit_group_users(group_name)
             elif u'force_sync' in self.request.POST:
                 try:
-                    sync_resources.fetch_single_service(cur_svc_type, session=self.request.db)
-                except:
-                    error_message = "There was an error when trying to get remote resources."
+                    sync_resources.fetch_all_services_by_type(cur_svc_type, session=self.request.db)
+                except Exception as e:
+                    error_message = "There was an error when trying to get remote resources. "
+                    error_message += "({})".format(repr(e))
             elif u'clean_all' in self.request.POST:
                 ids_to_clean = self.request.POST.get('ids_to_clean').split(";")
                 for id_ in ids_to_clean:
@@ -513,9 +514,12 @@ def edit_group(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        res_perms = sync_resources.merge_local_and_remote_resources(res_perms,
-                                                                    cur_svc_type,
-                                                                    self.request.db)
+        for service_name in services:
+            resources_for_service = sync_resources.merge_local_and_remote_resources(res_perms,
+                                                                                    cur_svc_type,
+                                                                                    service_name,
+                                                                                    self.request.db)
+            res_perms[service_name] = resources_for_service[service_name]
 
         last_sync_datetime = sync_resources.get_last_sync(cur_svc_type, self.request.db)
         now = datetime.datetime.now()
@@ -526,7 +530,7 @@ def edit_group(self):
         group_info[u'error_message'] = error_message
         group_info[u'ids_to_clean'] = ";".join(ids)
         group_info[u'last_sync'] = last_sync
-        group_info[u'sync_implemented'] = cur_svc_type in sync_resources.SYNC_SERVICES
+        group_info[u'sync_implemented'] = cur_svc_type in sync_resources.SYNC_SERVICES_TYPES
         group_info[u'group_name'] = group_name
         group_info[u'cur_svc_type'] = cur_svc_type
         group_info[u'users'] = self.get_user_names()
@@ -635,7 +639,7 @@ def add_service(self):
                     u'service_url': service_url,
                     u'service_type': service_type,
                     u'service_push': service_push}
-            check_response(requests.post(self.magpie_url+'/services', data=data, cookies=self.request.cookies))
+            check_response(requests.post(self.magpie_url + '/services', data=data, cookies=self.request.cookies))
             return HTTPFound(self.request.route_url('view_services', cur_svc_type=service_type))
 
         services_keys_sorted = sorted(service_type_dict)

From 23c52baa851ccbef2773973f64a3d312fcde95df Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 14 Sep 2018 12:05:18 -0400
Subject: [PATCH 026/124] fix adding a nested resource

---
 magpie/ui/management/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index cda7526e6..1022d33f8 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -588,7 +588,7 @@ def traverse_path_and_post(resources, path, type_path, parent_id=None):
                 response = check_response(requests.post(resources_url, data=data, cookies=self.request.cookies))
                 res_id = response.json()['resource']['resource_id']
                 if path:
-                    res_id = traverse_path_and_post({}, path, parent_id=res_id)
+                    res_id = traverse_path_and_post({}, path, type_path, parent_id=res_id)
             return res_id
 
         path_list = resource_path.split("/")[1:]

From f014586b534cdc7e5377eae854c58c13866e09db Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 14 Sep 2018 12:11:00 -0400
Subject: [PATCH 027/124] don't add remote_path if it doesn't match remote
 resource

---
 magpie/helpers/sync_resources.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 97c4a150f..068c0052e 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -63,8 +63,8 @@ def recurse(_resources_local, _resources_remote, remote_path="", remote_type_pat
             resource_type = _resources_remote[resource_name_local]['resource_type'] if matches_remote else ""
             current_type_path = "/".join([remote_type_path, resource_type])
 
-            values["remote_path"] = "" if matches_remote else current_path
-            values["remote_type_path"] = current_type_path
+            values["remote_path"] = current_path if matches_remote else ""
+            values["remote_type_path"] = current_type_path if matches_remote else ""
             values["matches_remote"] = matches_remote
             values["resource_type"] = resource_type
 

From 11c7c165d59f46323af5344de1ccccba560e438e Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 14 Sep 2018 12:21:45 -0400
Subject: [PATCH 028/124] fix 'Clean' button for a single resource

---
 magpie/ui/management/templates/edit_group.mako | 16 +++++++---------
 magpie/ui/management/views.py                  |  5 +++--
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/magpie/ui/management/templates/edit_group.mako b/magpie/ui/management/templates/edit_group.mako
index eb2008014..efbc58888 100644
--- a/magpie/ui/management/templates/edit_group.mako
+++ b/magpie/ui/management/templates/edit_group.mako
@@ -15,15 +15,13 @@
         </div>
     %endfor
     % if not value.get('matches_remote', True):
-        <form id="delete_resource_${value['id']}" action="${request.path}" method="post">
-            <div class="tree_button">
-                <input type="submit" class="button warning" value="Clean" name="clean_resource">
-            </div>
-            <p class="tree_item_message">
-                <img title="This resource is absent from the remote server."
-                     src="${request.static_url('magpie.ui.home:static/warning_exclamation_orange.png')}" />
-            </p>
-        </form>
+        <div class="tree_button">
+            <input type="submit" class="button warning" value="Clean" name="clean_resource">
+        </div>
+        <p class="tree_item_message">
+            <img title="This resource is absent from the remote server."
+                 src="${request.static_url('magpie.ui.home:static/warning_exclamation_orange.png')}" />
+        </p>
     % endif
     % if level == 0:
         <div class="tree_button">
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 1022d33f8..a9287d3c5 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -483,14 +483,15 @@ def edit_group(self):
                 return HTTPFound(self.request.route_url('edit_group', **group_info))
             elif u'goto_service' in self.request.POST:
                 return self.goto_service(res_id)
+            elif u'clean_resource' in self.request.POST:
+                # 'clean_resource' must be above 'edit_permissions' because they're in the same form.
+                self.delete_resource(res_id)
             elif u'edit_permissions' in self.request.POST:
                 remote_path = self.request.POST.get('remote_path')
                 if remote_path:
                     remote_type_path = self.request.POST.get('remote_type_path')
                     res_id = self.add_remote_resource(cur_svc_type, group_name, remote_path, remote_type_path)
                 self.edit_user_or_group_resource_permissions(group_name, res_id, is_user=False)
-            elif u'clean_resource' in self.request.POST:
-                self.delete_resource(res_id)
             elif u'member' in self.request.POST:
                 self.edit_group_users(group_name)
             elif u'force_sync' in self.request.POST:

From e239217bd5d7d03c3df773b75b089dc087199c6f Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 14 Sep 2018 15:04:22 -0400
Subject: [PATCH 029/124] fix housekeeping algorithm

---
 magpie/helpers/sync_resources.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 068c0052e..e42881326 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -13,6 +13,7 @@
 
 from magpie import db, models
 from magpie.helpers import sync_services
+from magpie.models import resource_tree_service
 
 SYNC_SERVICES_TYPES = defaultdict(lambda: sync_services._SyncServiceDefault)
 # noinspection PyTypeChecker
@@ -313,11 +314,15 @@ def housekeeping():
 
     session = Session(bind=engine)
 
-    for resource in session.query(models.Resource):
+    # loop the resource tree by reversed ordering (starting from the leaves)
+    # if the resource doesn't have any children or permissions, delete it
+    for resource in session.query(models.Resource).order_by(models.Resource.ordering.desc()):
         if resource.resource_type_name == 'service':
             continue
-        if not resource.group_permissions and not resource.user_permissions:
-            session.delete(resource)
+        n_children = resource_tree_service.count_children(resource.resource_id, session)
+        if n_children == 0:
+            if not resource.group_permissions and not resource.user_permissions:
+                session.delete(resource)
 
     session.commit()
     session.close()

From 00eed7288e231e6cfba5d55101b9c866fbdb1225 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 14 Sep 2018 15:15:55 -0400
Subject: [PATCH 030/124] add logging

---
 env/magpie.env.example           |  1 +
 magpie/helpers/sync_resources.py | 42 ++++++++++++++++++++++++++++++--
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/env/magpie.env.example b/env/magpie.env.example
index 08ac5c3bf..ba6fbf117 100644
--- a/env/magpie.env.example
+++ b/env/magpie.env.example
@@ -6,6 +6,7 @@ MAGPIE_ADMIN_USER=admin
 MAGPIE_ADMIN_PASSWORD=qwerty
 MAGPIE_ANONYMOUS_USER=anonymous
 MAGPIE_USERS_GROUP=users
+MAGPIE_CRON_LOG=~/magpie_cron.log
 PHOENIX_USER=phoenix
 PHOENIX_PASSWORD=qwerty
 PHOENIX_PORT=8443
diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index e42881326..1e7e892cb 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -7,14 +7,27 @@
 import copy
 import datetime
 from collections import OrderedDict, defaultdict
+import logging
+import os
 
 from sqlalchemy import create_engine
 from sqlalchemy.orm import Session
 
-from magpie import db, models
+from magpie import db, models, constants
 from magpie.helpers import sync_services
 from magpie.models import resource_tree_service
 
+LOGGER = logging.getLogger(__name__)
+log_path = constants.get_constant("MAGPIE_CRON_LOG")
+log_path = os.path.expandvars(log_path)
+log_path = os.path.expanduser(log_path)
+file_handler = logging.FileHandler(log_path)
+file_handler.setLevel(logging.INFO)
+formatter = logging.Formatter("%(asctime)s %(levelname)8s %(message)s")
+file_handler.setFormatter(formatter)
+LOGGER.addHandler(file_handler)
+LOGGER.setLevel(logging.INFO)
+
 SYNC_SERVICES_TYPES = defaultdict(lambda: sync_services._SyncServiceDefault)
 # noinspection PyTypeChecker
 SYNC_SERVICES_TYPES.update({
@@ -281,10 +294,13 @@ def fetch_single_service(service, session):
     :param service: (models.Service)
     :param session:
     """
+    LOGGER.info("Requesting remote resources")
     remote_resources = _get_remote_resources(service)
     service_id = service.resource_id
+    LOGGER.info("Deleting RemoteResource records for service: %s" % service.resource_name)
     _delete_records(service_id, session)
     _ensure_sync_info_exists(service.resource_id, session)
+    LOGGER.info("Writing RemoteResource records to database")
     _update_db(remote_resources, service_id, session)
 
 
@@ -292,12 +308,16 @@ def fetch():
     """
     Main function to get all remote resources for each service and write to database.
     """
+    LOGGER.info("Getting database url")
     url = db.get_db_url()
+
+    LOGGER.debug("Database url: %s" % url)
     engine = create_engine(url)
 
     session = Session(bind=engine)
 
     for service_type in SYNC_SERVICES_TYPES:
+        LOGGER.info("Fetching data for service type: %s" % service_type)
         fetch_all_services_by_type(service_type, session)
 
     session.commit()
@@ -332,9 +352,27 @@ def main():
     """
     Main entry point for cron service.
     """
-    if db.is_database_ready():
+    LOGGER.info("Magpie cron started.")
+    db_ready = db.is_database_ready()
+    if not db_ready:
+        LOGGER.info("Database isn't ready")
+        return
+
+    try:
+        LOGGER.info("Starting to fetch data for all service types")
         fetch()
+    except Exception:
+        LOGGER.exception("There was an error when fetching the data from the remote services")
+        raise
+
+    try:
+        LOGGER.info("Starting housekeeping script")
         housekeeping()
+    except Exception:
+        LOGGER.exception("There was an error running the housekeeping script")
+        raise
+
+    LOGGER.info("Success, exiting.")
 
 
 if __name__ == '__main__':

From 5f87d28fe42745d933f453437a3763e7acb6711f Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 17 Sep 2018 11:52:33 -0400
Subject: [PATCH 031/124] cron service to fetch remote resources

---
 Dockerfile                       |  9 +++++++-
 magpie-cron                      |  1 +
 magpie/constants.py              |  1 +
 magpie/helpers/sync_resources.py | 35 ++++++++++++++++----------------
 4 files changed, 28 insertions(+), 18 deletions(-)
 create mode 100644 magpie-cron

diff --git a/Dockerfile b/Dockerfile
index ec871796b..34dbef357 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,6 +4,7 @@ MAINTAINER Francis Charette-Migneault
 RUN apt-get update && apt-get install -y \
 	build-essential \
 	supervisor \
+	cron \
 	curl \
 	libssl-dev \
 	libffi-dev \
@@ -23,4 +24,10 @@ COPY ./ $MAGPIE_DIR
 RUN make install -f $MAGPIE_DIR/Makefile
 RUN make docs -f $MAGPIE_DIR/Makefile
 
-CMD ["make", "start"]
+ADD magpie-cron /etc/cron.d/magpie-cron
+RUN chmod 0644 /etc/cron.d/magpie-cron
+RUN touch ~/magpie_cron_status.log
+
+CMD env >> /etc/environment && \
+    cron && \
+    make start
diff --git a/magpie-cron b/magpie-cron
new file mode 100644
index 000000000..68ed98942
--- /dev/null
+++ b/magpie-cron
@@ -0,0 +1 @@
+0 * * * * root python -c 'from magpie.helpers.sync_resources import main; main()' > ~/magpie_cron_status.log 2>&1
diff --git a/magpie/constants.py b/magpie/constants.py
index ef63cd3a2..3a8eca1ad 100644
--- a/magpie/constants.py
+++ b/magpie/constants.py
@@ -53,6 +53,7 @@
 MAGPIE_ANONYMOUS_EMAIL = '{}@mail.com'.format(MAGPIE_ANONYMOUS_USER)
 MAGPIE_ANONYMOUS_GROUP = MAGPIE_ANONYMOUS_USER
 MAGPIE_USERS_GROUP = os.getenv('MAGPIE_USERS_GROUP', 'users')
+MAGPIE_CRON_LOG = os.getenv('MAGPIE_CRON_LOG', '~/magpie-cron.log')
 PHOENIX_USER = os.getenv('PHOENIX_USER', 'phoenix')
 PHOENIX_PASSWORD = os.getenv('PHOENIX_PASSWORD', 'qwerty')
 PHOENIX_PORT = int(os.getenv('PHOENIX_PORT', 8443))
diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 1e7e892cb..8e51bac58 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -18,15 +18,6 @@
 from magpie.models import resource_tree_service
 
 LOGGER = logging.getLogger(__name__)
-log_path = constants.get_constant("MAGPIE_CRON_LOG")
-log_path = os.path.expandvars(log_path)
-log_path = os.path.expanduser(log_path)
-file_handler = logging.FileHandler(log_path)
-file_handler.setLevel(logging.INFO)
-formatter = logging.Formatter("%(asctime)s %(levelname)8s %(message)s")
-file_handler.setFormatter(formatter)
-LOGGER.addHandler(file_handler)
-LOGGER.setLevel(logging.INFO)
 
 SYNC_SERVICES_TYPES = defaultdict(lambda: sync_services._SyncServiceDefault)
 # noinspection PyTypeChecker
@@ -348,24 +339,34 @@ def housekeeping():
     session.close()
 
 
+def setup_cron_logger():
+    log_path = constants.get_constant("MAGPIE_CRON_LOG")
+    log_path = os.path.expandvars(log_path)
+    log_path = os.path.expanduser(log_path)
+    file_handler = logging.FileHandler(log_path)
+    file_handler.setLevel(logging.INFO)
+    formatter = logging.Formatter("%(asctime)s %(levelname)8s %(message)s")
+    file_handler.setFormatter(formatter)
+    LOGGER.addHandler(file_handler)
+    LOGGER.setLevel(logging.INFO)
+
+
 def main():
     """
     Main entry point for cron service.
     """
+    setup_cron_logger()
+
     LOGGER.info("Magpie cron started.")
-    db_ready = db.is_database_ready()
-    if not db_ready:
-        LOGGER.info("Database isn't ready")
-        return
 
     try:
+        db_ready = db.is_database_ready()
+        if not db_ready:
+            LOGGER.info("Database isn't ready")
+            return
         LOGGER.info("Starting to fetch data for all service types")
         fetch()
-    except Exception:
-        LOGGER.exception("There was an error when fetching the data from the remote services")
-        raise
 
-    try:
         LOGGER.info("Starting housekeeping script")
         housekeeping()
     except Exception:

From c386a328c72fe7a5cc1a92cc52777729ec241a53 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 17 Sep 2018 13:53:18 -0400
Subject: [PATCH 032/124] add note about accessing the database from the UI

---
 magpie/ui/management/views.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index a9287d3c5..3bff5a631 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -466,6 +466,10 @@ def edit_group(self):
 
         error_message = None
 
+        # Until the api is modified to make it possible to request from the RemoteResource table,
+        # we have to access the database directly here
+        session = self.request.db
+
         # move to service or edit requested group/permission changes
         if self.request.method == 'POST':
             res_id = self.request.POST.get('resource_id')
@@ -496,7 +500,7 @@ def edit_group(self):
                 self.edit_group_users(group_name)
             elif u'force_sync' in self.request.POST:
                 try:
-                    sync_resources.fetch_all_services_by_type(cur_svc_type, session=self.request.db)
+                    sync_resources.fetch_all_services_by_type(cur_svc_type, session=session)
                 except Exception as e:
                     error_message = "There was an error when trying to get remote resources. "
                     error_message += "({})".format(repr(e))
@@ -519,10 +523,10 @@ def edit_group(self):
             resources_for_service = sync_resources.merge_local_and_remote_resources(res_perms,
                                                                                     cur_svc_type,
                                                                                     service_name,
-                                                                                    self.request.db)
+                                                                                    session)
             res_perms[service_name] = resources_for_service[service_name]
 
-        last_sync_datetime = sync_resources.get_last_sync(cur_svc_type, self.request.db)
+        last_sync_datetime = sync_resources.get_last_sync(cur_svc_type, session)
         now = datetime.datetime.now()
         last_sync = humanize.naturaltime(now - last_sync_datetime) if last_sync_datetime else "Never"
 

From 614bfb4fae9be7ff887ec1e5a37c8af7cdfa48d6 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 17 Sep 2018 14:07:34 -0400
Subject: [PATCH 033/124] edit user view: show remote and local resources

---
 magpie/ui/management/views.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 3bff5a631..dc9c84a06 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -239,6 +239,12 @@ def edit_user(self):
         own_groups = self.get_user_groups(user_name)
         all_groups = self.get_all_groups(first_default_group=get_constant('MAGPIE_USERS_GROUP'))
 
+        error_message = None
+
+        # Until the api is modified to make it possible to request from the RemoteResource table,
+        # we have to access the database directly here
+        session = self.request.db
+
         user_resp = requests.get(user_url, cookies=self.request.cookies)
         check_response(user_resp)
         user_info = user_resp.json()['user']
@@ -312,13 +318,23 @@ def edit_user(self):
             svc_types, cur_svc_type, services = self.get_services(cur_svc_type)
             res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(
                 user_name, services, cur_svc_type, is_user=True, is_inherited_permissions=inherited_permissions)
-            user_info[u'cur_svc_type'] = cur_svc_type
-            user_info[u'svc_types'] = svc_types
-            user_info[u'resources'] = res_perms
-            user_info[u'permissions'] = res_perm_names
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
+        for service_name in services:
+            resources_for_service = sync_resources.merge_local_and_remote_resources(res_perms,
+                                                                                    cur_svc_type,
+                                                                                    service_name,
+                                                                                    session)
+            res_perms[service_name] = resources_for_service[service_name]
+
+        user_info[u'error_message'] = error_message
+        user_info[u'sync_implemented'] = cur_svc_type in sync_resources.SYNC_SERVICES_TYPES
+        user_info[u'cur_svc_type'] = cur_svc_type
+        user_info[u'svc_types'] = svc_types
+        user_info[u'resources'] = res_perms
+        user_info[u'permissions'] = res_perm_names
+
         return add_template_data(self.request, data=user_info)
 
     @view_config(route_name='view_groups', renderer='templates/view_groups.mako')

From 0002a9cb678b960890eacbe99900260db759539c Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 17 Sep 2018 14:12:36 -0400
Subject: [PATCH 034/124] edit user view: clean single resource

---
 magpie/ui/management/templates/edit_user.mako | 10 ++++++++++
 magpie/ui/management/views.py                 |  5 ++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/magpie/ui/management/templates/edit_user.mako b/magpie/ui/management/templates/edit_user.mako
index 82f7896aa..3067b0ebf 100644
--- a/magpie/ui/management/templates/edit_user.mako
+++ b/magpie/ui/management/templates/edit_user.mako
@@ -3,6 +3,7 @@
 <%namespace name="tree" file="ui.management:templates/tree_scripts.mako"/>
 
 <%def name="render_item(key, value, level)">
+    <input type="hidden" value="" name="edit_permissions">
     %for perm in permissions:
         % if perm in value['permission_names']:
             <div class="perm_checkbox">
@@ -24,6 +25,15 @@
             </div>
         % endif
     %endfor
+    % if not value.get('matches_remote', True):
+        <div class="tree_button">
+            <input type="submit" class="button warning" value="Clean" name="clean_resource">
+        </div>
+        <p class="tree_item_message">
+            <img title="This resource is absent from the remote server."
+                 src="${request.static_url('magpie.ui.home:static/warning_exclamation_orange.png')}" />
+        </p>
+    % endif
     % if level == 0:
         <div class="tree_button">
             <input type="submit" class="tree_button goto_service" value="Edit Service" name="goto_service">
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index dc9c84a06..81a1f34d6 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -268,7 +268,10 @@ def edit_user(self):
                 return HTTPFound(self.request.route_url('view_users'))
             elif u'goto_service' in self.request.POST:
                 return self.goto_service(res_id)
-            elif u'resource_id' in self.request.POST:
+            elif u'clean_resource' in self.request.POST:
+                # 'clean_resource' must be above 'edit_permissions' because they're in the same form.
+                self.delete_resource(res_id)
+            elif u'edit_permissions' in self.request.POST:
                 self.edit_user_or_group_resource_permissions(user_name, res_id, is_user=True)
             elif u'edit_group_membership' in self.request.POST:
                 is_edit_group_membership = True

From b9c367be0e62d0f52dde3e0c9c9ac65509f4c53d Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 17 Sep 2018 14:34:26 -0400
Subject: [PATCH 035/124] edit user view: add a resource that is stored in the
 remote table

---
 magpie/ui/management/views.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 81a1f34d6..dced637bb 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -272,6 +272,14 @@ def edit_user(self):
                 # 'clean_resource' must be above 'edit_permissions' because they're in the same form.
                 self.delete_resource(res_id)
             elif u'edit_permissions' in self.request.POST:
+                remote_path = self.request.POST.get('remote_path')
+                if remote_path:
+                    remote_type_path = self.request.POST.get('remote_type_path')
+                    res_id = self.add_remote_resource(cur_svc_type,
+                                                      user_name,
+                                                      remote_path,
+                                                      remote_type_path,
+                                                      is_user=True)
                 self.edit_user_or_group_resource_permissions(user_name, res_id, is_user=True)
             elif u'edit_group_membership' in self.request.POST:
                 is_edit_group_membership = True
@@ -583,12 +591,12 @@ def get_ids_to_clean(self, resources):
             ids += self.get_ids_to_clean(values['children'])
         return ids
 
-    def add_remote_resource(self, service_type, group_name, resource_path, remote_type_path):
+    def add_remote_resource(self, service_type, user_or_group, resource_path, remote_type_path, is_user=False):
         try:
-            res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(group_name,
+            res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(user_or_group,
                                                                                           services=[service_type],
                                                                                           service_type=service_type,
-                                                                                          is_user=False)
+                                                                                          is_user=is_user)
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 

From 4e46a2dbb7b7d67e025651610a9188571bc8c0ea Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 17 Sep 2018 14:38:56 -0400
Subject: [PATCH 036/124] edit user view: display sync information and add
 'Sync now' button

---
 magpie/ui/management/templates/edit_user.mako | 14 ++++++++++++++
 magpie/ui/management/views.py                 | 11 +++++++++++
 2 files changed, 25 insertions(+)

diff --git a/magpie/ui/management/templates/edit_user.mako b/magpie/ui/management/templates/edit_user.mako
index 3067b0ebf..851d3480a 100644
--- a/magpie/ui/management/templates/edit_user.mako
+++ b/magpie/ui/management/templates/edit_user.mako
@@ -163,6 +163,20 @@
 
     <div class="current_tab_panel">
         <div class="clear"/>
+        %if error_message:
+            <div class="alert danger visible">${error_message}</div>
+        %endif
+        <form id="sync_info" action="${request.path}" method="post">
+            <p class="panel_line">
+                <span class="panel_entry">Last synchronization with remote service: </span>
+                %if sync_implemented:
+                    <span class="panel_value">${last_sync} </span>
+                    <input type="submit" value="Sync now" name="force_sync">
+                %else:
+                    <span class="panel_value">Not implemented for this service type.</span>
+                %endif
+            </p>
+        </form>
         <div class="tree_header">
         <div class="tree_item">Resources</div>
         %for perm in permissions:
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index dced637bb..e056ca387 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -299,6 +299,12 @@ def edit_user(self):
             elif u'save_email' in self.request.POST:
                 user_info[u'email'] = self.request.POST.get(u'new_user_email')
                 is_save_user_info = True
+            elif u'force_sync' in self.request.POST:
+                try:
+                    sync_resources.fetch_all_services_by_type(cur_svc_type, session=session)
+                except Exception as e:
+                    error_message = "There was an error when trying to get remote resources. "
+                    error_message += "({})".format(repr(e))
 
             if is_save_user_info:
                 check_response(requests.put(user_url, data=user_info, cookies=self.request.cookies))
@@ -339,7 +345,12 @@ def edit_user(self):
                                                                                     session)
             res_perms[service_name] = resources_for_service[service_name]
 
+        last_sync_datetime = sync_resources.get_last_sync(cur_svc_type, session)
+        now = datetime.datetime.now()
+        last_sync = humanize.naturaltime(now - last_sync_datetime) if last_sync_datetime else "Never"
+
         user_info[u'error_message'] = error_message
+        user_info[u'last_sync'] = last_sync
         user_info[u'sync_implemented'] = cur_svc_type in sync_resources.SYNC_SERVICES_TYPES
         user_info[u'cur_svc_type'] = cur_svc_type
         user_info[u'svc_types'] = svc_types

From caa9f9f49d9a5965ae012e3fdc03e5eab8625861 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 17 Sep 2018 14:43:32 -0400
Subject: [PATCH 037/124] edit user view: add 'Clean all' button

---
 magpie/ui/management/templates/edit_user.mako | 8 ++++++++
 magpie/ui/management/views.py                 | 7 +++++++
 2 files changed, 15 insertions(+)

diff --git a/magpie/ui/management/templates/edit_user.mako b/magpie/ui/management/templates/edit_user.mako
index 851d3480a..3e752fb80 100644
--- a/magpie/ui/management/templates/edit_user.mako
+++ b/magpie/ui/management/templates/edit_user.mako
@@ -176,6 +176,14 @@
                     <span class="panel_value">Not implemented for this service type.</span>
                 %endif
             </p>
+            %if ids_to_clean:
+                <p class="panel_line">
+                    <span class="panel_entry">Note: </span>
+                    <span class="panel_value">Some resources are absent from the remote server </span>
+                    <input type="hidden" value="${ids_to_clean}" name="ids_to_clean">
+                    <input type="submit" class="button warning" value="Clean all" name="clean_all">
+                </p>
+            %endif
         </form>
         <div class="tree_header">
         <div class="tree_item">Resources</div>
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index e056ca387..084e1096a 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -305,6 +305,10 @@ def edit_user(self):
                 except Exception as e:
                     error_message = "There was an error when trying to get remote resources. "
                     error_message += "({})".format(repr(e))
+            elif u'clean_all' in self.request.POST:
+                ids_to_clean = self.request.POST.get('ids_to_clean').split(";")
+                for id_ in ids_to_clean:
+                    self.delete_resource(id_)
 
             if is_save_user_info:
                 check_response(requests.put(user_url, data=user_info, cookies=self.request.cookies))
@@ -349,7 +353,10 @@ def edit_user(self):
         now = datetime.datetime.now()
         last_sync = humanize.naturaltime(now - last_sync_datetime) if last_sync_datetime else "Never"
 
+        ids = self.get_ids_to_clean(res_perms)
+
         user_info[u'error_message'] = error_message
+        user_info[u'ids_to_clean'] = ";".join(ids)
         user_info[u'last_sync'] = last_sync
         user_info[u'sync_implemented'] = cur_svc_type in sync_resources.SYNC_SERVICES_TYPES
         user_info[u'cur_svc_type'] = cur_svc_type

From 0701285c6a423cc7a1b1151f3a1f1ba3f1487986 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 18 Sep 2018 14:25:29 -0400
Subject: [PATCH 038/124] start cron from makefile

---
 Dockerfile | 4 +---
 Makefile   | 7 ++++++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 34dbef357..52ff70780 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -28,6 +28,4 @@ ADD magpie-cron /etc/cron.d/magpie-cron
 RUN chmod 0644 /etc/cron.d/magpie-cron
 RUN touch ~/magpie_cron_status.log
 
-CMD env >> /etc/environment && \
-    cron && \
-    make start
+CMD make start
diff --git a/Makefile b/Makefile
index 9cd755ae3..17bf99d38 100644
--- a/Makefile
+++ b/Makefile
@@ -143,7 +143,12 @@ install: sysinstall
 	@echo "Installing Magpie..."
 	python setup.py install
 
+.PHONY: cron
+cron:
+	@echo "Starting Cron service..."
+	cron
+
 .PHONY: start
-start: install
+start: cron install
 	@echo "Starting Magpie..."
 	exec gunicorn -b 0.0.0.0:2001 --paste "$(CUR_DIR)/magpie/magpie.ini" --workers 10 --preload

From 0877e759e5fdabbeea7f48d6d6af2a6b62c9de6f Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 18 Sep 2018 14:25:44 -0400
Subject: [PATCH 039/124] mark todos

---
 magpie/ui/management/views.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 084e1096a..c8410011c 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -241,6 +241,7 @@ def edit_user(self):
 
         error_message = None
 
+        # Todo:
         # Until the api is modified to make it possible to request from the RemoteResource table,
         # we have to access the database directly here
         session = self.request.db
@@ -511,6 +512,7 @@ def edit_group(self):
 
         error_message = None
 
+        # Todo:
         # Until the api is modified to make it possible to request from the RemoteResource table,
         # we have to access the database directly here
         session = self.request.db

From a5c8033c27ef05515ac76b3553f111b4d3f03f03 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 18 Sep 2018 14:26:14 -0400
Subject: [PATCH 040/124] actually inherit abstract base class
 _SyncServiceInterface

---
 magpie/helpers/sync_services.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/magpie/helpers/sync_services.py b/magpie/helpers/sync_services.py
index 3dce6a7f6..fdde5a53c 100644
--- a/magpie/helpers/sync_services.py
+++ b/magpie/helpers/sync_services.py
@@ -44,8 +44,9 @@ def get_resources(self):
         pass
 
 
-class _SyncServiceGeoserver:
+class _SyncServiceGeoserver(_SyncServiceInterface):
     def __init__(self, service_name, geoserver_url):
+        super(_SyncServiceGeoserver, self).__init__()
         self.service_name = service_name
         self.geoserver_url = geoserver_url
 
@@ -65,8 +66,9 @@ def get_resources(self):
         return resources
 
 
-class _SyncServiceProjectAPI:
+class _SyncServiceProjectAPI(_SyncServiceInterface):
     def __init__(self, service_name, project_api_url):
+        super(_SyncServiceProjectAPI, self).__init__()
         self.service_name = service_name
         self.project_api_url = project_api_url
 
@@ -88,6 +90,7 @@ class _SyncServiceThreads(_SyncServiceInterface):
     DEPTH_DEFAULT = 3
 
     def __init__(self, service_name, thredds_url, depth=DEPTH_DEFAULT, **kwargs):
+        super(_SyncServiceThreads, self).__init__()
         self.service_name = service_name
         self.thredds_url = thredds_url
         self.depth = depth
@@ -118,6 +121,7 @@ def thredds_get_resources(url, depth, **kwargs):
 
 class _SyncServiceDefault(_SyncServiceInterface):
     def __init__(self, *_):
+        super(_SyncServiceDefault, self).__init__()
         pass
 
     def get_resources(self):

From 0f48f0287fb842599dbf9fb5eb87ecb65d576da9 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 18 Sep 2018 14:26:47 -0400
Subject: [PATCH 041/124] remove housekeeping script

---
 magpie/helpers/sync_resources.py | 27 ---------------------------
 1 file changed, 27 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 8e51bac58..aba073d38 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -315,30 +315,6 @@ def fetch():
     session.close()
 
 
-def housekeeping():
-    """
-    Clean resources that are in the Resource table but have no
-    group or user permissions associated to them.
-    """
-    url = db.get_db_url()
-    engine = create_engine(url)
-
-    session = Session(bind=engine)
-
-    # loop the resource tree by reversed ordering (starting from the leaves)
-    # if the resource doesn't have any children or permissions, delete it
-    for resource in session.query(models.Resource).order_by(models.Resource.ordering.desc()):
-        if resource.resource_type_name == 'service':
-            continue
-        n_children = resource_tree_service.count_children(resource.resource_id, session)
-        if n_children == 0:
-            if not resource.group_permissions and not resource.user_permissions:
-                session.delete(resource)
-
-    session.commit()
-    session.close()
-
-
 def setup_cron_logger():
     log_path = constants.get_constant("MAGPIE_CRON_LOG")
     log_path = os.path.expandvars(log_path)
@@ -366,9 +342,6 @@ def main():
             return
         LOGGER.info("Starting to fetch data for all service types")
         fetch()
-
-        LOGGER.info("Starting housekeeping script")
-        housekeeping()
     except Exception:
         LOGGER.exception("There was an error running the housekeeping script")
         raise

From 61ed2914d8c06452315bb405ef1be1b42a353d13 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 18 Sep 2018 14:27:23 -0400
Subject: [PATCH 042/124] handle single service failure in cron script

---
 magpie/helpers/sync_resources.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index aba073d38..c6614458f 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -19,6 +19,8 @@
 
 LOGGER = logging.getLogger(__name__)
 
+CRON_SERVICE = False
+
 SYNC_SERVICES_TYPES = defaultdict(lambda: sync_services._SyncServiceDefault)
 # noinspection PyTypeChecker
 SYNC_SERVICES_TYPES.update({
@@ -276,7 +278,15 @@ def fetch_all_services_by_type(service_type, session):
     :param session:
     """
     for service in session.query(models.Service).filter_by(type=service_type):
-        fetch_single_service(service, session)
+        # noinspection PyBroadException
+        try:
+            fetch_single_service(service, session)
+        except:
+            if CRON_SERVICE:
+                LOGGER.exception("There was an error when fetching data from the url: %s" % service.url)
+                pass
+            else:
+                raise
 
 
 def fetch_single_service(service, session):
@@ -331,6 +341,9 @@ def main():
     """
     Main entry point for cron service.
     """
+    global CRON_SERVICE
+    CRON_SERVICE = True
+
     setup_cron_logger()
 
     LOGGER.info("Magpie cron started.")
@@ -343,7 +356,7 @@ def main():
         LOGGER.info("Starting to fetch data for all service types")
         fetch()
     except Exception:
-        LOGGER.exception("There was an error running the housekeeping script")
+        LOGGER.exception("An error occured")
         raise
 
     LOGGER.info("Success, exiting.")

From f377717d150d5a51b2da91cef64f24c47536d63a Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 18 Sep 2018 14:54:29 -0400
Subject: [PATCH 043/124] refactoring

---
 magpie/helpers/sync_resources.py |  4 +--
 magpie/ui/management/views.py    | 56 +++++++++++++++++---------------
 2 files changed, 32 insertions(+), 28 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index c6614458f..03b78fef8 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -261,9 +261,9 @@ def _query_remote_resources_in_database(service_type, service_name, session):
     return {service_name: {'children': remote_resources, 'resource_type': 'directory'}}
 
 
-def get_last_sync(service_type, session):
+def get_last_sync(service_type, service_name, session):
     last_sync = None
-    service = session.query(models.Service).filter_by(type=service_type).first()
+    service = session.query(models.Service).filter_by(type=service_type, resource_name=service_name).first()
     _ensure_sync_info_exists(service.resource_id, session)
     sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
     if sync_info:
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index c8410011c..7fcd7537c 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -343,21 +343,13 @@ def edit_user(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        for service_name in services:
-            resources_for_service = sync_resources.merge_local_and_remote_resources(res_perms,
-                                                                                    cur_svc_type,
-                                                                                    service_name,
-                                                                                    session)
-            res_perms[service_name] = resources_for_service[service_name]
-
-        last_sync_datetime = sync_resources.get_last_sync(cur_svc_type, session)
-        now = datetime.datetime.now()
-        last_sync = humanize.naturaltime(now - last_sync_datetime) if last_sync_datetime else "Never"
-
-        ids = self.get_ids_to_clean(res_perms)
+        ids_to_clean, last_sync = self.merge_remote_resources(cur_svc_type,
+                                                              res_perms,
+                                                              services,
+                                                              session)
 
         user_info[u'error_message'] = error_message
-        user_info[u'ids_to_clean'] = ";".join(ids)
+        user_info[u'ids_to_clean'] = ";".join(ids_to_clean)
         user_info[u'last_sync'] = last_sync
         user_info[u'sync_implemented'] = cur_svc_type in sync_resources.SYNC_SERVICES_TYPES
         user_info[u'cur_svc_type'] = cur_svc_type
@@ -566,21 +558,13 @@ def edit_group(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        for service_name in services:
-            resources_for_service = sync_resources.merge_local_and_remote_resources(res_perms,
-                                                                                    cur_svc_type,
-                                                                                    service_name,
-                                                                                    session)
-            res_perms[service_name] = resources_for_service[service_name]
-
-        last_sync_datetime = sync_resources.get_last_sync(cur_svc_type, session)
-        now = datetime.datetime.now()
-        last_sync = humanize.naturaltime(now - last_sync_datetime) if last_sync_datetime else "Never"
-
-        ids = self.get_ids_to_clean(res_perms)
+        ids_to_clean, last_sync = self.merge_remote_resources(cur_svc_type,
+                                                              res_perms,
+                                                              services,
+                                                              session)
 
         group_info[u'error_message'] = error_message
-        group_info[u'ids_to_clean'] = ";".join(ids)
+        group_info[u'ids_to_clean'] = ";".join(ids_to_clean)
         group_info[u'last_sync'] = last_sync
         group_info[u'sync_implemented'] = cur_svc_type in sync_resources.SYNC_SERVICES_TYPES
         group_info[u'group_name'] = group_name
@@ -593,6 +577,26 @@ def edit_group(self):
         group_info[u'permissions'] = res_perm_names
         return add_template_data(self.request, data=group_info)
 
+    def merge_remote_resources(self, cur_svc_type, res_perms, services, session):
+        ids_to_clean = []
+        last_sync_datetimes = []
+        last_sync = "Never"
+        for service_name in services:
+            resources_for_service = sync_resources.merge_local_and_remote_resources(res_perms,
+                                                                                    cur_svc_type,
+                                                                                    service_name,
+                                                                                    session)
+            res_perms[service_name] = resources_for_service[service_name]
+
+            last_sync_service = sync_resources.get_last_sync(cur_svc_type, service_name, session)
+            last_sync_datetimes.append(last_sync_service)
+        if any(last_sync_datetimes):
+            last_sync_datetime = min(filter(bool, last_sync_datetimes))
+            now = datetime.datetime.now()
+            last_sync = humanize.naturaltime(now - last_sync_datetime)
+            ids_to_clean = self.get_ids_to_clean(res_perms)
+        return ids_to_clean, last_sync
+
     def delete_resource(self, res_id):
         url = '{url}/resources/{resource_id}'.format(url=self.magpie_url, resource_id=res_id)
         try:

From 3dfad47f42555ed40b2cfeeeba9f0d8f4e7c6ab5 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 18 Sep 2018 14:55:07 -0400
Subject: [PATCH 044/124] ensure there was a successful synchronization done
 before offering to clean

---
 magpie/helpers/sync_resources.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 03b78fef8..375baaf3c 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -32,6 +32,8 @@
 
 def merge_local_and_remote_resources(resources_local, service_type, service_name, session):
     """Main function to sync resources with remote server"""
+    if not get_last_sync(service_type, service_name, session):
+        return resources_local
     remote_resources = _query_remote_resources_in_database(service_type, service_name, session=session)
     merged_resources = _merge_resources(resources_local, remote_resources)
     _sort_resources(merged_resources)

From e3e5258a19849bcaa79af36eae322c68883cb2b3 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 18 Sep 2018 14:58:34 -0400
Subject: [PATCH 045/124] don't modify directly the array given as argument

---
 magpie/ui/management/views.py | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 7fcd7537c..e86d63485 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -343,10 +343,10 @@ def edit_user(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        ids_to_clean, last_sync = self.merge_remote_resources(cur_svc_type,
-                                                              res_perms,
-                                                              services,
-                                                              session)
+        res_perms, ids_to_clean, last_sync = self.merge_remote_resources(cur_svc_type,
+                                                                         res_perms,
+                                                                         services,
+                                                                         session)
 
         user_info[u'error_message'] = error_message
         user_info[u'ids_to_clean'] = ";".join(ids_to_clean)
@@ -558,10 +558,10 @@ def edit_group(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        ids_to_clean, last_sync = self.merge_remote_resources(cur_svc_type,
-                                                              res_perms,
-                                                              services,
-                                                              session)
+        res_perms, ids_to_clean, last_sync = self.merge_remote_resources(cur_svc_type,
+                                                                         res_perms,
+                                                                         services,
+                                                                         session)
 
         group_info[u'error_message'] = error_message
         group_info[u'ids_to_clean'] = ";".join(ids_to_clean)
@@ -581,12 +581,15 @@ def merge_remote_resources(self, cur_svc_type, res_perms, services, session):
         ids_to_clean = []
         last_sync_datetimes = []
         last_sync = "Never"
+
+        merged_resources = {}
+
         for service_name in services:
             resources_for_service = sync_resources.merge_local_and_remote_resources(res_perms,
                                                                                     cur_svc_type,
                                                                                     service_name,
                                                                                     session)
-            res_perms[service_name] = resources_for_service[service_name]
+            merged_resources[service_name] = resources_for_service[service_name]
 
             last_sync_service = sync_resources.get_last_sync(cur_svc_type, service_name, session)
             last_sync_datetimes.append(last_sync_service)
@@ -595,7 +598,7 @@ def merge_remote_resources(self, cur_svc_type, res_perms, services, session):
             now = datetime.datetime.now()
             last_sync = humanize.naturaltime(now - last_sync_datetime)
             ids_to_clean = self.get_ids_to_clean(res_perms)
-        return ids_to_clean, last_sync
+        return merged_resources, ids_to_clean, last_sync
 
     def delete_resource(self, res_id):
         url = '{url}/resources/{resource_id}'.format(url=self.magpie_url, resource_id=res_id)

From f5fe8624adf6f61c05d5b01fec8da1833657b7e2 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 18 Sep 2018 15:28:38 -0400
Subject: [PATCH 046/124] display error message if resources are 'out of sync'

If they have not been synchronized for more than 3 hours.
The cron service should synchronize every hour.
---
 magpie/helpers/sync_resources.py              |  2 +
 .../ui/management/templates/edit_group.mako   |  3 +-
 magpie/ui/management/templates/edit_user.mako |  2 +-
 magpie/ui/management/views.py                 | 42 ++++++++++++-------
 4 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 375baaf3c..a80a2acf0 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -21,6 +21,8 @@
 
 CRON_SERVICE = False
 
+OUT_OF_SYNC = datetime.timedelta(hours=3)
+
 SYNC_SERVICES_TYPES = defaultdict(lambda: sync_services._SyncServiceDefault)
 # noinspection PyTypeChecker
 SYNC_SERVICES_TYPES.update({
diff --git a/magpie/ui/management/templates/edit_group.mako b/magpie/ui/management/templates/edit_group.mako
index efbc58888..bde9613df 100644
--- a/magpie/ui/management/templates/edit_group.mako
+++ b/magpie/ui/management/templates/edit_group.mako
@@ -119,9 +119,8 @@
                 %else:
                     <span class="panel_value">Not implemented for this service type.</span>
                 %endif
-
             </p>
-            %if ids_to_clean:
+            %if ids_to_clean and not out_of_sync:
                 <p class="panel_line">
                     <span class="panel_entry">Note: </span>
                     <span class="panel_value">Some resources are absent from the remote server </span>
diff --git a/magpie/ui/management/templates/edit_user.mako b/magpie/ui/management/templates/edit_user.mako
index 3e752fb80..9ed173f79 100644
--- a/magpie/ui/management/templates/edit_user.mako
+++ b/magpie/ui/management/templates/edit_user.mako
@@ -176,7 +176,7 @@
                     <span class="panel_value">Not implemented for this service type.</span>
                 %endif
             </p>
-            %if ids_to_clean:
+            %if ids_to_clean and not out_of_sync:
                 <p class="panel_line">
                     <span class="panel_entry">Note: </span>
                     <span class="panel_value">Some resources are absent from the remote server </span>
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index e86d63485..b9e6d64c0 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -6,6 +6,7 @@
 from magpie.definitions.pyramid_definitions import *
 from magpie.constants import get_constant
 from magpie.common import str2bool
+from magpie.helpers.sync_resources import OUT_OF_SYNC
 from magpie.services import service_type_dict
 from magpie.models import resource_type_dict
 from magpie.ui.management import check_response
@@ -343,15 +344,19 @@ def edit_user(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        res_perms, ids_to_clean, last_sync = self.merge_remote_resources(cur_svc_type,
-                                                                         res_perms,
-                                                                         services,
-                                                                         session)
+        info = self.merge_remote_resources(cur_svc_type, res_perms, services, session)
+        res_perms, ids_to_clean, last_sync_humanized, last_sync_delta = info
+
+        out_of_sync = last_sync_delta > OUT_OF_SYNC if last_sync_delta else False
+
+        if out_of_sync:
+            error_message = "There seems to be an issue synchronizing resources from this service."
 
         user_info[u'error_message'] = error_message
         user_info[u'ids_to_clean'] = ";".join(ids_to_clean)
-        user_info[u'last_sync'] = last_sync
+        user_info[u'last_sync'] = last_sync_humanized
         user_info[u'sync_implemented'] = cur_svc_type in sync_resources.SYNC_SERVICES_TYPES
+        user_info[u'out_of_sync'] = out_of_sync
         user_info[u'cur_svc_type'] = cur_svc_type
         user_info[u'svc_types'] = svc_types
         user_info[u'resources'] = res_perms
@@ -558,15 +563,19 @@ def edit_group(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        res_perms, ids_to_clean, last_sync = self.merge_remote_resources(cur_svc_type,
-                                                                         res_perms,
-                                                                         services,
-                                                                         session)
+        info = self.merge_remote_resources(cur_svc_type, res_perms, services, session)
+        res_perms, ids_to_clean, last_sync_humanized, last_sync_delta = info
+
+        out_of_sync = last_sync_delta > datetime.timedelta(hours=3) if last_sync_delta else False
+
+        if out_of_sync:
+            error_message = "There seems to be an issue synchronizing resources from this service."
 
         group_info[u'error_message'] = error_message
         group_info[u'ids_to_clean'] = ";".join(ids_to_clean)
-        group_info[u'last_sync'] = last_sync
+        group_info[u'last_sync'] = last_sync_humanized
         group_info[u'sync_implemented'] = cur_svc_type in sync_resources.SYNC_SERVICES_TYPES
+        group_info[u'out_of_sync'] = out_of_sync
         group_info[u'group_name'] = group_name
         group_info[u'cur_svc_type'] = cur_svc_type
         group_info[u'users'] = self.get_user_names()
@@ -580,25 +589,28 @@ def edit_group(self):
     def merge_remote_resources(self, cur_svc_type, res_perms, services, session):
         ids_to_clean = []
         last_sync_datetimes = []
-        last_sync = "Never"
+        last_sync_humanized = "Never"
+        last_sync_delta = None
 
         merged_resources = {}
 
         for service_name in services:
+            last_sync = sync_resources.get_last_sync(cur_svc_type, service_name, session)
+            last_sync_datetimes.append(last_sync)
+
             resources_for_service = sync_resources.merge_local_and_remote_resources(res_perms,
                                                                                     cur_svc_type,
                                                                                     service_name,
                                                                                     session)
             merged_resources[service_name] = resources_for_service[service_name]
 
-            last_sync_service = sync_resources.get_last_sync(cur_svc_type, service_name, session)
-            last_sync_datetimes.append(last_sync_service)
         if any(last_sync_datetimes):
             last_sync_datetime = min(filter(bool, last_sync_datetimes))
             now = datetime.datetime.now()
-            last_sync = humanize.naturaltime(now - last_sync_datetime)
+            last_sync_humanized = humanize.naturaltime(now - last_sync_datetime)
+            last_sync_delta = now - last_sync_datetime
             ids_to_clean = self.get_ids_to_clean(res_perms)
-        return merged_resources, ids_to_clean, last_sync
+        return merged_resources, ids_to_clean, last_sync_humanized, last_sync_delta
 
     def delete_resource(self, res_id):
         url = '{url}/resources/{resource_id}'.format(url=self.magpie_url, resource_id=res_id)

From b64f7a622bd80bb23f1e4ef58d5da0bf25468a5f Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 18 Sep 2018 17:00:04 -0400
Subject: [PATCH 047/124] fix get_ids_to_clean

---
 magpie/ui/management/views.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index b9e6d64c0..5dd8d6e75 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -566,7 +566,7 @@ def edit_group(self):
         info = self.merge_remote_resources(cur_svc_type, res_perms, services, session)
         res_perms, ids_to_clean, last_sync_humanized, last_sync_delta = info
 
-        out_of_sync = last_sync_delta > datetime.timedelta(hours=3) if last_sync_delta else False
+        out_of_sync = last_sync_delta > OUT_OF_SYNC if last_sync_delta else False
 
         if out_of_sync:
             error_message = "There seems to be an issue synchronizing resources from this service."
@@ -607,9 +607,10 @@ def merge_remote_resources(self, cur_svc_type, res_perms, services, session):
         if any(last_sync_datetimes):
             last_sync_datetime = min(filter(bool, last_sync_datetimes))
             now = datetime.datetime.now()
-            last_sync_humanized = humanize.naturaltime(now - last_sync_datetime)
             last_sync_delta = now - last_sync_datetime
-            ids_to_clean = self.get_ids_to_clean(res_perms)
+            last_sync_humanized = humanize.naturaltime(last_sync_delta)
+            for service in merged_resources.values():
+                ids_to_clean += self.get_ids_to_clean(service['children'])
         return merged_resources, ids_to_clean, last_sync_humanized, last_sync_delta
 
     def delete_resource(self, res_id):

From 5b22b2967d3907190275962a970f1ca516851993 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 21 Sep 2018 11:52:24 -0400
Subject: [PATCH 048/124] identify resource by last id term instead of by name

In a thredds catalog, the name of the resource can be capitalized
and not its id. But in other cases, both the name and its id are
capitalized. We make sure to get the actual id to match the case of
the url.
---
 magpie/helpers/sync_services.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/magpie/helpers/sync_services.py b/magpie/helpers/sync_services.py
index fdde5a53c..f45da6f01 100644
--- a/magpie/helpers/sync_services.py
+++ b/magpie/helpers/sync_services.py
@@ -96,10 +96,16 @@ def __init__(self, service_name, thredds_url, depth=DEPTH_DEFAULT, **kwargs):
         self.depth = depth
         self.kwargs = kwargs  # kwargs is passed to the requests.get method.
 
+    def _resource_id(self, resource):
+        id_ = resource.name
+        if len(resource.datasets) > 0:
+            id_ = resource.datasets[0].ID.split("/")[-1]
+        return id_
+
     def get_resources(self):
         def thredds_get_resources(url, depth, **kwargs):
             cat = threddsclient.read_url(url, **kwargs)
-            name = cat.name
+            name = self._resource_id(cat)
             if depth == self.depth:
                 name = self.service_name
             resource_type = 'directory'

From de8419720458893bfe37360737d1b8bc5bb87425 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 21 Sep 2018 11:54:01 -0400
Subject: [PATCH 049/124] setup cron to use docker and docker-compose
 environment variables

---
 Dockerfile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index 52ff70780..8c92648c6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,8 +24,11 @@ COPY ./ $MAGPIE_DIR
 RUN make install -f $MAGPIE_DIR/Makefile
 RUN make docs -f $MAGPIE_DIR/Makefile
 
+# magpie cron service
 ADD magpie-cron /etc/cron.d/magpie-cron
 RUN chmod 0644 /etc/cron.d/magpie-cron
 RUN touch ~/magpie_cron_status.log
+# set /etc/environment so that cron runs using the environment variables set by docker
+RUN env >> /etc/environment
 
 CMD make start

From 5570b5a922df152e0ab8709b3e20dc522ebb306a Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 21 Sep 2018 15:11:47 -0400
Subject: [PATCH 050/124] load env files before running cron job

---
 magpie-cron | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/magpie-cron b/magpie-cron
index 68ed98942..fcb43530c 100644
--- a/magpie-cron
+++ b/magpie-cron
@@ -1 +1 @@
-0 * * * * root python -c 'from magpie.helpers.sync_resources import main; main()' > ~/magpie_cron_status.log 2>&1
+0 * * * * root /bin/bash -c "set -a ; . $MAGPIE_DIR/env/magpie.env ; . $MAGPIE_DIR/magpie/env/magpie.env ; . $MAGPIE_DIR/env/postgres.env ; . $MAGPIE_DIR/magpie/env/postgres.env ; set +a ; python -c 'from magpie.helpers.sync_resources import main; main()'" > ~/magpie_cron_status.log 2>&1

From 2936d3494069ff48cc2689ba006eea0161265d94 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 21 Sep 2018 15:57:25 -0400
Subject: [PATCH 051/124] don't flag a resource as absent from remote server if
 it's deeper...

than what was fetched
---
 magpie/helpers/sync_resources.py | 8 ++++++--
 magpie/helpers/sync_services.py  | 6 +++---
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index a80a2acf0..1466f820a 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -15,6 +15,7 @@
 
 from magpie import db, models, constants
 from magpie.helpers import sync_services
+from magpie.helpers.sync_services import THREDDS_DEPTH_DEFAULT
 from magpie.models import resource_tree_service
 
 LOGGER = logging.getLogger(__name__)
@@ -70,13 +71,16 @@ def recurse(_resources_local, _resources_remote, remote_path="", remote_type_pat
         for resource_name_local, values in _resources_local.items():
             current_path = "/".join([remote_path, str(resource_name_local)])
 
+            depth = current_path.count("/")
+            deeper_than_fetched = depth >= THREDDS_DEPTH_DEFAULT
+
             matches_remote = resource_name_local in _resources_remote
             resource_type = _resources_remote[resource_name_local]['resource_type'] if matches_remote else ""
             current_type_path = "/".join([remote_type_path, resource_type])
 
             values["remote_path"] = current_path if matches_remote else ""
             values["remote_type_path"] = current_type_path if matches_remote else ""
-            values["matches_remote"] = matches_remote
+            values["matches_remote"] = matches_remote or deeper_than_fetched
             values["resource_type"] = resource_type
 
             resource_remote_children = _resources_remote[resource_name_local]['children'] if matches_remote else {}
@@ -367,4 +371,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    fetch()
diff --git a/magpie/helpers/sync_services.py b/magpie/helpers/sync_services.py
index f45da6f01..bee8a5431 100644
--- a/magpie/helpers/sync_services.py
+++ b/magpie/helpers/sync_services.py
@@ -4,6 +4,8 @@
 import requests
 import threddsclient
 
+THREDDS_DEPTH_DEFAULT = 3
+
 
 def is_valid_resource_schema(resources, ignore_resource_type=False):
     """
@@ -87,9 +89,7 @@ def get_resources(self):
 
 
 class _SyncServiceThreads(_SyncServiceInterface):
-    DEPTH_DEFAULT = 3
-
-    def __init__(self, service_name, thredds_url, depth=DEPTH_DEFAULT, **kwargs):
+    def __init__(self, service_name, thredds_url, depth=THREDDS_DEPTH_DEFAULT, **kwargs):
         super(_SyncServiceThreads, self).__init__()
         self.service_name = service_name
         self.thredds_url = thredds_url

From 02d37d32079d1afa0b8dbdad0a1c382e95d4c991 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Fri, 21 Sep 2018 16:46:20 -0400
Subject: [PATCH 052/124] add max_depth property and simplify interface

---
 magpie/helpers/sync_resources.py | 19 +++++++---
 magpie/helpers/sync_services.py  | 59 +++++++++++++++++---------------
 2 files changed, 46 insertions(+), 32 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 1466f820a..f8e72b842 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -15,8 +15,6 @@
 
 from magpie import db, models, constants
 from magpie.helpers import sync_services
-from magpie.helpers.sync_services import THREDDS_DEPTH_DEFAULT
-from magpie.models import resource_tree_service
 
 LOGGER = logging.getLogger(__name__)
 
@@ -32,18 +30,24 @@
     "project-api": sync_services._SyncServiceProjectAPI,
 })
 
+# try to instantiate classes right away
+for sync_service_class in SYNC_SERVICES_TYPES.values():
+    name, url = "", ""
+    sync_service_class(name, url)
+
 
 def merge_local_and_remote_resources(resources_local, service_type, service_name, session):
     """Main function to sync resources with remote server"""
     if not get_last_sync(service_type, service_name, session):
         return resources_local
     remote_resources = _query_remote_resources_in_database(service_type, service_name, session=session)
-    merged_resources = _merge_resources(resources_local, remote_resources)
+    max_depth = _get_max_depth(service_type)
+    merged_resources = _merge_resources(resources_local, remote_resources, max_depth)
     _sort_resources(merged_resources)
     return merged_resources
 
 
-def _merge_resources(resources_local, resources_remote):
+def _merge_resources(resources_local, resources_remote, max_depth=None):
     """
     Merge resources_local and resources_remote, adding the following keys to the output:
 
@@ -72,7 +76,7 @@ def recurse(_resources_local, _resources_remote, remote_path="", remote_type_pat
             current_path = "/".join([remote_path, str(resource_name_local)])
 
             depth = current_path.count("/")
-            deeper_than_fetched = depth >= THREDDS_DEPTH_DEFAULT
+            deeper_than_fetched = depth >= max_depth if max_depth is not None else False
 
             matches_remote = resource_name_local in _resources_remote
             resource_type = _resources_remote[resource_name_local]['resource_type'] if matches_remote else ""
@@ -250,6 +254,11 @@ def _format_resource_tree(children):
     return fmt_res_tree
 
 
+def _get_max_depth(service_type):
+    name, url = "", ""
+    return SYNC_SERVICES_TYPES[service_type](name, url).max_depth
+
+
 def _query_remote_resources_in_database(service_type, service_name, session):
     """
     Reads remote resources from the RemoteResources table. No external request is made.
diff --git a/magpie/helpers/sync_services.py b/magpie/helpers/sync_services.py
index bee8a5431..b8ddaef9a 100644
--- a/magpie/helpers/sync_services.py
+++ b/magpie/helpers/sync_services.py
@@ -4,8 +4,6 @@
 import requests
 import threddsclient
 
-THREDDS_DEPTH_DEFAULT = 3
-
 
 def is_valid_resource_schema(resources, ignore_resource_type=False):
     """
@@ -35,6 +33,17 @@ def is_valid_resource_schema(resources, ignore_resource_type=False):
 class _SyncServiceInterface:
     __metaclass__ = abc.ABCMeta
 
+    def __init__(self, service_name, url):
+        self.service_name = service_name
+        self.url = url
+
+    @abc.abstractproperty
+    def max_depth(self):
+        """
+        The max depth at which remote resources are fetched
+        :return: (int)
+        """
+
     @abc.abstractmethod
     def get_resources(self):
         """
@@ -47,15 +56,14 @@ def get_resources(self):
 
 
 class _SyncServiceGeoserver(_SyncServiceInterface):
-    def __init__(self, service_name, geoserver_url):
-        super(_SyncServiceGeoserver, self).__init__()
-        self.service_name = service_name
-        self.geoserver_url = geoserver_url
+    @property
+    def max_depth(self):
+        return None
 
     def get_resources(self):
         # Only workspaces are fetched for now
         resource_type = "route"
-        workspaces_url = "{}/{}".format(self.geoserver_url, "workspaces")
+        workspaces_url = "{}/{}".format(self.url, "workspaces")
         resp = requests.get(workspaces_url, headers={"Accept": "application/json"})
         resp.raise_for_status()
         workspaces_list = resp.json().get("workspaces", {}).get("workspace", {})
@@ -69,15 +77,14 @@ def get_resources(self):
 
 
 class _SyncServiceProjectAPI(_SyncServiceInterface):
-    def __init__(self, service_name, project_api_url):
-        super(_SyncServiceProjectAPI, self).__init__()
-        self.service_name = service_name
-        self.project_api_url = project_api_url
+    @property
+    def max_depth(self):
+        return None
 
     def get_resources(self):
         # Only workspaces are fetched for now
         resource_type = "route"
-        projects_url = "/".join([self.project_api_url, "api", "Projects"])
+        projects_url = "/".join([self.url, "api", "Projects"])
         resp = requests.get(projects_url)
         resp.raise_for_status()
 
@@ -89,24 +96,22 @@ def get_resources(self):
 
 
 class _SyncServiceThreads(_SyncServiceInterface):
-    def __init__(self, service_name, thredds_url, depth=THREDDS_DEPTH_DEFAULT, **kwargs):
-        super(_SyncServiceThreads, self).__init__()
-        self.service_name = service_name
-        self.thredds_url = thredds_url
-        self.depth = depth
-        self.kwargs = kwargs  # kwargs is passed to the requests.get method.
+    @property
+    def max_depth(self):
+        return 3
 
-    def _resource_id(self, resource):
+    @staticmethod
+    def _resource_id(resource):
         id_ = resource.name
         if len(resource.datasets) > 0:
             id_ = resource.datasets[0].ID.split("/")[-1]
         return id_
 
     def get_resources(self):
-        def thredds_get_resources(url, depth, **kwargs):
-            cat = threddsclient.read_url(url, **kwargs)
+        def thredds_get_resources(url, depth):
+            cat = threddsclient.read_url(url)
             name = self._resource_id(cat)
-            if depth == self.depth:
+            if depth == self.max_depth:
                 name = self.service_name
             resource_type = 'directory'
             if cat.datasets and cat.datasets[0].content_type != "application/directory":
@@ -116,19 +121,19 @@ def thredds_get_resources(url, depth, **kwargs):
 
             if depth > 0:
                 for reference in cat.flat_references():
-                    tree_item[name]['children'].update(thredds_get_resources(reference.url, depth - 1, **kwargs))
+                    tree_item[name]['children'].update(thredds_get_resources(reference.url, depth - 1))
 
             return tree_item
 
-        resources = thredds_get_resources(self.thredds_url, self.depth, **self.kwargs)
+        resources = thredds_get_resources(self.url, self.max_depth)
         assert is_valid_resource_schema(resources), 'Error in Interface implementation'
         return resources
 
 
 class _SyncServiceDefault(_SyncServiceInterface):
-    def __init__(self, *_):
-        super(_SyncServiceDefault, self).__init__()
-        pass
+    @property
+    def max_depth(self):
+        return None
 
     def get_resources(self):
         return {}

From 344ace6c3d584eb7cdff191143588cccf689f131 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 24 Sep 2018 16:40:22 -0400
Subject: [PATCH 053/124] Only push remote resource id to UI.

Too much information was getting pushed to the ui (remote path, remote
resource type path)
This made it very complicated to implement custom display names
for resources, because they would have got to be pushed to the UI also.
---
 magpie/helpers/sync_resources.py              | 39 ++++-------
 magpie/helpers/sync_services.py               | 14 ++--
 .../ui/management/templates/edit_group.mako   |  4 +-
 magpie/ui/management/templates/edit_user.mako |  4 +-
 .../ui/management/templates/tree_scripts.mako |  5 +-
 magpie/ui/management/views.py                 | 65 +++++++++----------
 6 files changed, 54 insertions(+), 77 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index f8e72b842..b94e68e80 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -51,9 +51,8 @@ def _merge_resources(resources_local, resources_remote, max_depth=None):
     """
     Merge resources_local and resources_remote, adding the following keys to the output:
 
-        - remote_path: '/' separated string representing the remote path of the resource
+        - remote_id: id of the RemoteResource
         - matches_remote: True or False depending if the resource is present on the remote server
-        - id: set to the value of 'remote_path' if the resource if remote only
 
     returns a dictionary of the form validated by 'sync_services.is_valid_resource_schema'
 
@@ -61,7 +60,7 @@ def _merge_resources(resources_local, resources_remote, max_depth=None):
     if not resources_remote:
         return resources_local
 
-    assert sync_services.is_valid_resource_schema(resources_local, ignore_resource_type=True)
+    assert sync_services.is_valid_resource_schema(resources_local)
     assert sync_services.is_valid_resource_schema(resources_remote)
 
     if not resources_local:
@@ -70,41 +69,29 @@ def _merge_resources(resources_local, resources_remote, max_depth=None):
     # don't overwrite the input arguments
     merged_resources = copy.deepcopy(resources_local)
 
-    def recurse(_resources_local, _resources_remote, remote_path="", remote_type_path=""):
+    def recurse(_resources_local, _resources_remote, depth=0):
         # loop local resources, looking for matches in remote resources
         for resource_name_local, values in _resources_local.items():
-            current_path = "/".join([remote_path, str(resource_name_local)])
-
-            depth = current_path.count("/")
-            deeper_than_fetched = depth >= max_depth if max_depth is not None else False
-
             matches_remote = resource_name_local in _resources_remote
-            resource_type = _resources_remote[resource_name_local]['resource_type'] if matches_remote else ""
-            current_type_path = "/".join([remote_type_path, resource_type])
+            remote_id = _resources_remote.get(resource_name_local, {}).get('remote_id', '')
+            deeper_than_fetched = depth >= max_depth if max_depth is not None else False
 
-            values["remote_path"] = current_path if matches_remote else ""
-            values["remote_type_path"] = current_type_path if matches_remote else ""
-            values["matches_remote"] = matches_remote or deeper_than_fetched
-            values["resource_type"] = resource_type
+            values['remote_id'] = remote_id
+            values['matches_remote'] = matches_remote or deeper_than_fetched
 
             resource_remote_children = _resources_remote[resource_name_local]['children'] if matches_remote else {}
-
-            recurse(values['children'], resource_remote_children, current_path, current_type_path)
+            recurse(values['children'], resource_remote_children, depth + 1)
 
         # loop remote resources, looking for matches in local resources
         for resource_name_remote, values in _resources_remote.items():
             if resource_name_remote not in _resources_local:
-                current_path = "/".join([remote_path, str(resource_name_remote)])
-                current_type_path = "/".join([remote_type_path, values['resource_type']])
                 new_resource = {'permission_names': [],
                                 'children': {},
-                                'resource_type': values['resource_type'],
-                                'id': current_path,
-                                'remote_path': current_path,
-                                'remote_type_path': current_type_path,
+                                'id': None,
+                                'remote_id': values['remote_id'],
                                 'matches_remote': True}
                 _resources_local[resource_name_remote] = new_resource
-                recurse(new_resource['children'], values['children'], current_path, current_type_path)
+                recurse(new_resource['children'], values['children'], depth + 1)
 
     recurse(merged_resources, resources_remote)
 
@@ -249,7 +236,7 @@ def _format_resource_tree(children):
         resource = child_dict[u'node']
         new_children = child_dict[u'children']
         resource_dict = {'children': _format_resource_tree(new_children),
-                         'resource_type': resource.resource_type}
+                         'remote_id': resource.resource_id}
         fmt_res_tree[resource.resource_name] = resource_dict
     return fmt_res_tree
 
@@ -275,7 +262,7 @@ def _query_remote_resources_in_database(service_type, service_name, session):
     tree = _get_resource_children(main_resource, session)
 
     remote_resources = _format_resource_tree(tree)
-    return {service_name: {'children': remote_resources, 'resource_type': 'directory'}}
+    return {service_name: {'children': remote_resources, 'remote_id': main_resource.resource_id}}
 
 
 def get_last_sync(service_type, service_name, session):
diff --git a/magpie/helpers/sync_services.py b/magpie/helpers/sync_services.py
index b8ddaef9a..4ddd29885 100644
--- a/magpie/helpers/sync_services.py
+++ b/magpie/helpers/sync_services.py
@@ -5,28 +5,24 @@
 import threddsclient
 
 
-def is_valid_resource_schema(resources, ignore_resource_type=False):
+def is_valid_resource_schema(resources):
     """
     Returns True if the structure of the input dictionary is a tree of the form:
 
-    {'resource_name_1': {'children': {'resource_name_3': {'children': {}, 'resource_type': ...},
-                                      'resource_name_4': {'children': {}, 'resource_type': ...}
+    {'resource_name_1': {'children': {'resource_name_3': {'children': {}},
+                                      'resource_name_4': {'children': {}}
                                       },
-                         'resource_type': ...
                          },
-     'resource_name_2': {'children': {}, resource_type': ...}
+     'resource_name_2': {'children': {}}
      }
     :return: bool
     """
     for resource_name, values in resources.items():
         if 'children' not in values:
             return False
-        if not ignore_resource_type and 'resource_type' not in values:
-            return False
         if not isinstance(values['children'], (OrderedDict, dict)):
             return False
-        return is_valid_resource_schema(values['children'],
-                                        ignore_resource_type=ignore_resource_type)
+        return is_valid_resource_schema(values['children'])
     return True
 
 
diff --git a/magpie/ui/management/templates/edit_group.mako b/magpie/ui/management/templates/edit_group.mako
index bde9613df..3c1ba1de2 100644
--- a/magpie/ui/management/templates/edit_group.mako
+++ b/magpie/ui/management/templates/edit_group.mako
@@ -7,10 +7,10 @@
         <div class="perm_checkbox">
             % if perm in value['permission_names']:
             <input type="checkbox" value="${perm}" name="permission"
-                   onchange="document.getElementById('resource_${value['id']}').submit()" checked>
+                   onchange="document.getElementById('resource_${value['id']}_${value.get('remote_id', '')}').submit()" checked>
             % else:
             <input type="checkbox" value="${perm}" name="permission"
-                   onchange="document.getElementById('resource_${value['id']}').submit()">
+                   onchange="document.getElementById('resource_${value['id']}_${value.get('remote_id', '')}').submit()">
             % endif
         </div>
     %endfor
diff --git a/magpie/ui/management/templates/edit_user.mako b/magpie/ui/management/templates/edit_user.mako
index 9ed173f79..5f2a3583d 100644
--- a/magpie/ui/management/templates/edit_user.mako
+++ b/magpie/ui/management/templates/edit_user.mako
@@ -8,7 +8,7 @@
         % if perm in value['permission_names']:
             <div class="perm_checkbox">
                 <input type="checkbox" value="${perm}" name="permission"
-                       onchange="document.getElementById('resource_${value['id']}').submit()" checked
+                       onchange="document.getElementById('resource_${value['id']}_${value.get('remote_id', '')}').submit()" checked
                 %if inherited_permissions:
                     disabled
                 %endif
@@ -17,7 +17,7 @@
         % else:
             <div class="perm_checkbox">
                 <input type="checkbox" value="${perm}" name="permission"
-                       onchange="document.getElementById('resource_${value['id']}').submit()"
+                       onchange="document.getElementById('resource_${value['id']}_${value.get('remote_id', '')}').submit()"
                 %if inherited_permissions:
                     disabled
                 %endif
diff --git a/magpie/ui/management/templates/tree_scripts.mako b/magpie/ui/management/templates/tree_scripts.mako
index 6889da0c9..84377c906 100644
--- a/magpie/ui/management/templates/tree_scripts.mako
+++ b/magpie/ui/management/templates/tree_scripts.mako
@@ -42,7 +42,7 @@ li.Collapsed {
     <ul>
      %for key in tree:
         <div class="clear"/>
-        <form id="resource_${tree[key]['id']}" action="${request.path}" method="post">
+        <form id="resource_${tree[key]['id']}_${tree[key].get('remote_id', '')}" action="${request.path}" method="post">
             % if tree[key]['children']:
             <li class="Expanded">
             % else:
@@ -50,8 +50,7 @@ li.Collapsed {
             % endif
                 <div class="tree_item">${key}</div>
                 <input type="hidden" value=${tree[key]['id']} name="resource_id">
-                <input type="hidden" value=${tree[key].get('remote_path', '')} name="remote_path">
-                <input type="hidden" value=${tree[key].get('remote_type_path', '')} name="remote_type_path">
+                <input type="hidden" value=${tree[key].get('remote_id', '')} name="remote_id">
                 <input type="hidden" value=${tree[key].get('matches_remote', '')} name="matches_remote">
                 ${item_renderer(key, tree[key], level)}
             </li>
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 5dd8d6e75..1df465fc2 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -8,7 +8,7 @@
 from magpie.common import str2bool
 from magpie.helpers.sync_resources import OUT_OF_SYNC
 from magpie.services import service_type_dict
-from magpie.models import resource_type_dict
+from magpie.models import resource_type_dict, remote_resource_tree_service
 from magpie.ui.management import check_response
 from magpie.helpers import sync_resources
 from magpie.ui.home import add_template_data
@@ -274,14 +274,9 @@ def edit_user(self):
                 # 'clean_resource' must be above 'edit_permissions' because they're in the same form.
                 self.delete_resource(res_id)
             elif u'edit_permissions' in self.request.POST:
-                remote_path = self.request.POST.get('remote_path')
-                if remote_path:
-                    remote_type_path = self.request.POST.get('remote_type_path')
-                    res_id = self.add_remote_resource(cur_svc_type,
-                                                      user_name,
-                                                      remote_path,
-                                                      remote_type_path,
-                                                      is_user=True)
+                if not res_id or res_id == 'None':
+                    remote_id = int(self.request.POST.get('remote_id'))
+                    res_id = self.add_remote_resource(cur_svc_type, user_name, remote_id, is_user=True)
                 self.edit_user_or_group_resource_permissions(user_name, res_id, is_user=True)
             elif u'edit_group_membership' in self.request.POST:
                 is_edit_group_membership = True
@@ -535,10 +530,9 @@ def edit_group(self):
                 # 'clean_resource' must be above 'edit_permissions' because they're in the same form.
                 self.delete_resource(res_id)
             elif u'edit_permissions' in self.request.POST:
-                remote_path = self.request.POST.get('remote_path')
-                if remote_path:
-                    remote_type_path = self.request.POST.get('remote_type_path')
-                    res_id = self.add_remote_resource(cur_svc_type, group_name, remote_path, remote_type_path)
+                if not res_id or res_id == 'None':
+                    remote_id = int(self.request.POST.get('remote_id'))
+                    res_id = self.add_remote_resource(cur_svc_type, group_name, remote_id, is_user=False)
                 self.edit_user_or_group_resource_permissions(group_name, res_id, is_user=False)
             elif u'member' in self.request.POST:
                 self.edit_group_users(group_name)
@@ -631,7 +625,7 @@ def get_ids_to_clean(self, resources):
             ids += self.get_ids_to_clean(values['children'])
         return ids
 
-    def add_remote_resource(self, service_type, user_or_group, resource_path, remote_type_path, is_user=False):
+    def add_remote_resource(self, service_type, user_or_group, remote_id, is_user=False):
         try:
             res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(user_or_group,
                                                                                           services=[service_type],
@@ -640,32 +634,33 @@ def add_remote_resource(self, service_type, user_or_group, resource_path, remote
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        def traverse_path_and_post(resources, path, type_path, parent_id=None):
-            if not path:
-                return parent_id
-            current_name = path.pop(0)
-            current_type = type_path.pop(0)
-            if current_name in resources:
-                res_id = traverse_path_and_post(resources[current_name]['children'],
-                                                path,
-                                                type_path,
-                                                parent_id=resources[current_name]['id'])
+        # Todo:
+        # Until the api is modified to make it possible to request from the RemoteResource table,
+        # we have to access the database directly here
+        session = self.request.db
+
+        # get the parent resources for this remote_id
+        parents = remote_resource_tree_service.path_upper(remote_id, db_session=session)
+        parents = list(reversed(list(parents)))
+
+        parent_id = None
+        current_resources = res_perms
+        for remote_resource in parents:
+            name = remote_resource.resource_name
+            if name in current_resources:
+                parent_id = current_resources[name]['id']
+                current_resources = current_resources[name]['children']
             else:
-                resources_url = '{url}/resources'.format(url=self.magpie_url)
                 data = {
-                    'resource_name': current_name,
-                    'resource_type': current_type,
+                    'resource_name': name,
+                    'resource_type': remote_resource.resource_type,
                     'parent_id': parent_id,
                 }
+                resources_url = '{url}/resources'.format(url=self.magpie_url)
                 response = check_response(requests.post(resources_url, data=data, cookies=self.request.cookies))
-                res_id = response.json()['resource']['resource_id']
-                if path:
-                    res_id = traverse_path_and_post({}, path, type_path, parent_id=res_id)
-            return res_id
-
-        path_list = resource_path.split("/")[1:]
-        remote_type_path_list = remote_type_path.split("/")[1:]
-        return traverse_path_and_post(res_perms, path_list, remote_type_path_list)
+                parent_id = response.json()['resource']['resource_id']
+
+        return parent_id
 
     @view_config(route_name='view_services', renderer='templates/view_services.mako')
     def view_services(self):

From 311c506d63b2ff7daf2caf1ad8f292a18edd1e75 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 24 Sep 2018 11:50:54 -0400
Subject: [PATCH 054/124] alembic migration

---
 .../73b872478d87_add_resource_label.py        | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 magpie/alembic/versions/73b872478d87_add_resource_label.py

diff --git a/magpie/alembic/versions/73b872478d87_add_resource_label.py b/magpie/alembic/versions/73b872478d87_add_resource_label.py
new file mode 100644
index 000000000..94a58dff3
--- /dev/null
+++ b/magpie/alembic/versions/73b872478d87_add_resource_label.py
@@ -0,0 +1,24 @@
+"""empty message
+
+Revision ID: 73b872478d87
+Revises: d01af1f2e445
+Create Date: 2018-09-24 11:29:38.108819
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '73b872478d87'
+down_revision = 'd01af1f2e445'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('resources', sa.Column('display_name', sa.Unicode(100), nullable=True))
+
+
+def downgrade():
+    op.drop_column('resources', 'display_name')

From cd5d6d720925a867b791a31781e01514aac38cf2 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 24 Sep 2018 13:26:15 -0400
Subject: [PATCH 055/124] add display_name to ProjectAPI data

---
 magpie/helpers/sync_services.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/magpie/helpers/sync_services.py b/magpie/helpers/sync_services.py
index 4ddd29885..e3f5ceb46 100644
--- a/magpie/helpers/sync_services.py
+++ b/magpie/helpers/sync_services.py
@@ -84,7 +84,10 @@ def get_resources(self):
         resp = requests.get(projects_url)
         resp.raise_for_status()
 
-        projects = {p["id"]: {"children": {}, "resource_type": resource_type} for p in resp.json()}
+        projects = {p["id"]: {"children": {},
+                              "resource_type": resource_type,
+                              "display_name": p["name"]}
+                    for p in resp.json()}
 
         resources = {self.service_name: {"children": projects, "resource_type": resource_type}}
         assert is_valid_resource_schema(resources), "Error in Interface implementation"

From aba2c18725826fff6429dac8f1c8519457679858 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Mon, 24 Sep 2018 16:43:04 -0400
Subject: [PATCH 056/124] add display_name column to remote_resources table

---
 magpie/alembic/versions/73b872478d87_add_resource_label.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/magpie/alembic/versions/73b872478d87_add_resource_label.py b/magpie/alembic/versions/73b872478d87_add_resource_label.py
index 94a58dff3..1e69a5afc 100644
--- a/magpie/alembic/versions/73b872478d87_add_resource_label.py
+++ b/magpie/alembic/versions/73b872478d87_add_resource_label.py
@@ -18,7 +18,9 @@
 
 def upgrade():
     op.add_column('resources', sa.Column('display_name', sa.Unicode(100), nullable=True))
+    op.add_column('remote_resources', sa.Column('display_name', sa.Unicode(100), nullable=True))
 
 
 def downgrade():
     op.drop_column('resources', 'display_name')
+    op.drop_column('remote_resources', 'display_name')

From 329549b577c1a49168fca183233a3712f8b5feaf Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 25 Sep 2018 12:05:43 -0400
Subject: [PATCH 057/124] change display_name to resource_display_name

---
 .../alembic/versions/73b872478d87_add_resource_label.py   | 8 ++++----
 magpie/models.py                                          | 3 +++
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/magpie/alembic/versions/73b872478d87_add_resource_label.py b/magpie/alembic/versions/73b872478d87_add_resource_label.py
index 1e69a5afc..22ccd28fa 100644
--- a/magpie/alembic/versions/73b872478d87_add_resource_label.py
+++ b/magpie/alembic/versions/73b872478d87_add_resource_label.py
@@ -17,10 +17,10 @@
 
 
 def upgrade():
-    op.add_column('resources', sa.Column('display_name', sa.Unicode(100), nullable=True))
-    op.add_column('remote_resources', sa.Column('display_name', sa.Unicode(100), nullable=True))
+    op.add_column('resources', sa.Column('resource_display_name', sa.Unicode(100), nullable=True))
+    op.add_column('remote_resources', sa.Column('resource_display_name', sa.Unicode(100), nullable=True))
 
 
 def downgrade():
-    op.drop_column('resources', 'display_name')
-    op.drop_column('remote_resources', 'display_name')
+    op.drop_column('resources', 'resource_display_name')
+    op.drop_column('remote_resources', 'resource_display_name')
diff --git a/magpie/models.py b/magpie/models.py
index 2885b79e5..d1ca796d2 100644
--- a/magpie/models.py
+++ b/magpie/models.py
@@ -40,6 +40,8 @@ class Resource(ResourceMixin, Base):
     permission_names = []
     child_resource_allowed = True
 
+    resource_display_name = sa.Column(sa.Unicode(100), nullable=True)
+
     # reference to top-most service under which the resource is nested
     # if the resource is the service, id is None (NULL)
     @declared_attr
@@ -189,6 +191,7 @@ class RemoteResource(BaseModel, Base):
                           nullable=True)
     ordering = sa.Column(sa.Integer(), default=0, nullable=False)
     resource_name = sa.Column(sa.Unicode(100), nullable=False)
+    resource_display_name = sa.Column(sa.Unicode(100), nullable=True)
     resource_type = sa.Column(sa.Unicode(30), nullable=False)
 
     def __repr__(self):

From 8e39985db7c02120a7c514b61148b0de76e58db5 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 25 Sep 2018 12:07:48 -0400
Subject: [PATCH 058/124] Fix migration scripts

Running the migrations from scratch was broken because of the new
resource_display_name column. In previous searches like:
'SELECT * FROM resources', resource_display_name was included, but
wasn't yet added to the database.
---
 .../alembic/versions/a395ef9d3fe6_reference_root_service.py | 2 +-
 .../versions/c352a98d570e_project_api_route_resource.py     | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py b/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py
index 00b11a2ca..9d43944ad 100644
--- a/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py
+++ b/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py
@@ -52,7 +52,7 @@ def upgrade():
             op.add_column('resources', sa.Column('root_service_id', sa.Integer(), nullable=True))
 
         # add existing resource references to their root service, loop through reference tree chain
-        all_resources = session.query(models.Resource)
+        all_resources = session.query(models.Resource.parent_id)
         for resource in all_resources:
             service_resource = get_resource_root_service(resource, session)
             if service_resource.resource_id != resource.resource_id:
diff --git a/magpie/alembic/versions/c352a98d570e_project_api_route_resource.py b/magpie/alembic/versions/c352a98d570e_project_api_route_resource.py
index c4a2f048e..28833dd8d 100644
--- a/magpie/alembic/versions/c352a98d570e_project_api_route_resource.py
+++ b/magpie/alembic/versions/c352a98d570e_project_api_route_resource.py
@@ -32,12 +32,14 @@ def change_project_api_resource_type(new_type_name):
     if isinstance(context.connection.engine.dialect, PGDialect):
         # obtain service 'project-api'
         session = Session(bind=op.get_bind())
-        project_api_svc = models.Service.by_service_name('project-api', db_session=session)
+        project_api_svc = session.query(Service.resource_id)\
+            .filter(Resource.resource_name == 'project-api').first()
 
         # nothing to edit if it doesn't exist, otherwise change resource types to 'route'
         if project_api_svc:
             project_api_id = project_api_svc.resource_id
-            project_api_res = session.query(models.Resource).filter(models.Resource.root_service_id == project_api_id)
+            columns = models.Resource.resource_type, models.Resource.root_service_id
+            project_api_res = session.query(columns).filter(models.Resource.root_service_id == project_api_id)
 
             for res in project_api_res:
                 res.resource_type = 'route'

From 56577927f7706239f33b5ba4877d9798bcc44b64 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 25 Sep 2018 12:12:13 -0400
Subject: [PATCH 059/124] change api: add optional resource_display_name
 parameter ...

to /resources POST
---
 magpie/api/management/resource/resource_formats.py | 2 ++
 magpie/api/management/resource/resource_utils.py   | 3 ++-
 magpie/api/management/resource/resource_views.py   | 3 ++-
 magpie/api/management/service/service_views.py     | 4 +++-
 4 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/magpie/api/management/resource/resource_formats.py b/magpie/api/management/resource/resource_formats.py
index 1aec3aa36..c89c12786 100644
--- a/magpie/api/management/resource/resource_formats.py
+++ b/magpie/api/management/resource/resource_formats.py
@@ -9,11 +9,13 @@ def fmt_res(res, perms, info):
         if info:
             return {
                 u'resource_name': str(res.resource_name),
+                u'resource_display_name': res.resource_display_name.encode('utf-8'),
                 u'resource_type': str(res.resource_type),
                 u'resource_id': res.resource_id
             }
         return {
             u'resource_name': str(res.resource_name),
+            u'resource_display_name': res.resource_display_name.encode('utf-8'),
             u'resource_type': str(res.resource_type),
             u'resource_id': res.resource_id,
             u'parent_id': res.parent_id,
diff --git a/magpie/api/management/resource/resource_utils.py b/magpie/api/management/resource/resource_utils.py
index 70919a1b0..7486341a8 100644
--- a/magpie/api/management/resource/resource_utils.py
+++ b/magpie/api/management/resource/resource_utils.py
@@ -125,7 +125,7 @@ def get_resource_root_service(resource, db_session):
     return None
 
 
-def create_resource(resource_name, resource_type, parent_id, db_session):
+def create_resource(resource_name, resource_display_name, resource_type, parent_id, db_session):
     verify_param(resource_name, paramName=u'resource_name', notNone=True, notEmpty=True, httpError=HTTPBadRequest,
                  msgOnFail="Invalid `resource_name` specified for child resource creation.")
     verify_param(resource_type, paramName=u'resource_type', notNone=True, notEmpty=True, httpError=HTTPBadRequest,
@@ -142,6 +142,7 @@ def create_resource(resource_name, resource_type, parent_id, db_session):
     root_service = check_valid_service_resource(parent_resource, resource_type, db_session)
     new_resource = resource_factory(resource_type=resource_type,
                                     resource_name=resource_name,
+                                    resource_display_name=resource_display_name or resource_name,
                                     root_service_id=root_service.resource_id,
                                     parent_id=parent_resource.resource_id)
 
diff --git a/magpie/api/management/resource/resource_views.py b/magpie/api/management/resource/resource_views.py
index 4238c51aa..23f8dd3ac 100644
--- a/magpie/api/management/resource/resource_views.py
+++ b/magpie/api/management/resource/resource_views.py
@@ -42,9 +42,10 @@ def get_resource_view(request):
 def create_resource_view(request):
     """Register a new resource."""
     resource_name = get_value_multiformat_post_checked(request, 'resource_name')
+    resource_display_name = get_multiformat_any(request, 'resource_display_name', default=resource_name)
     resource_type = get_value_multiformat_post_checked(request, 'resource_type')
     parent_id = get_value_multiformat_post_checked(request, 'parent_id')
-    return create_resource(resource_name, resource_type, parent_id, request.db)
+    return create_resource(resource_name, resource_display_name, resource_type, parent_id, request.db)
 
 
 @ResourceAPI.delete(schema=Resource_DELETE_RequestSchema(), tags=[ResourcesTag],
diff --git a/magpie/api/management/service/service_views.py b/magpie/api/management/service/service_views.py
index ac6999447..da0960e08 100644
--- a/magpie/api/management/service/service_views.py
+++ b/magpie/api/management/service/service_views.py
@@ -195,11 +195,13 @@ def create_service_direct_resource(request):
     """Register a new resource directly under a service."""
     service = get_service_matchdict_checked(request)
     resource_name = get_multiformat_post(request, 'resource_name')
+    resource_display_name = get_multiformat_post(request, 'resource_display_name')
     resource_type = get_multiformat_post(request, 'resource_type')
     parent_id = get_multiformat_post(request, 'parent_id')  # no check because None/empty is allowed
     if not parent_id:
         parent_id = service.resource_id
-    return create_resource(resource_name, resource_type, parent_id=parent_id, db_session=request.db)
+    return create_resource(resource_name, resource_display_name, resource_type, parent_id=parent_id,
+                           db_session=request.db)
 
 
 @ServiceResourceTypesAPI.get(tags=[ServicesTag], response_schemas=ServiceResource_GET_responses)

From 3798187a6c5527aac80da216d491dad624583f1e Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 25 Sep 2018 12:13:19 -0400
Subject: [PATCH 060/124] add resource_display_name parameter when fetching and
 merging remote ...

resources
---
 magpie/helpers/sync_resources.py | 6 +++++-
 magpie/helpers/sync_services.py  | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index b94e68e80..5732bfeff 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -89,6 +89,7 @@ def recurse(_resources_local, _resources_remote, depth=0):
                                 'children': {},
                                 'id': None,
                                 'remote_id': values['remote_id'],
+                                'resource_display_name': values.get('resource_display_name', resource_name_remote),
                                 'matches_remote': True}
                 _resources_local[resource_name_remote] = new_resource
                 recurse(new_resource['children'], values['children'], depth + 1)
@@ -180,8 +181,10 @@ def _update_db(remote_resources, service_id, session):
 
     def add_children(resources, parent_id, position=0):
         for resource_name, values in resources.items():
+            resource_display_name = unicode(values.get('resource_display_name', resource_name))
             new_resource = models.RemoteResource(service_id=sync_info.service_id,
                                                  resource_name=unicode(resource_name),
+                                                 resource_display_name=resource_display_name,
                                                  resource_type=values['resource_type'],
                                                  parent_id=parent_id,
                                                  ordering=position)
@@ -236,7 +239,8 @@ def _format_resource_tree(children):
         resource = child_dict[u'node']
         new_children = child_dict[u'children']
         resource_dict = {'children': _format_resource_tree(new_children),
-                         'remote_id': resource.resource_id}
+                         'remote_id': resource.resource_id,
+                         'resource_display_name': resource.resource_display_name}
         fmt_res_tree[resource.resource_name] = resource_dict
     return fmt_res_tree
 
diff --git a/magpie/helpers/sync_services.py b/magpie/helpers/sync_services.py
index e3f5ceb46..5a10221ab 100644
--- a/magpie/helpers/sync_services.py
+++ b/magpie/helpers/sync_services.py
@@ -86,7 +86,7 @@ def get_resources(self):
 
         projects = {p["id"]: {"children": {},
                               "resource_type": resource_type,
-                              "display_name": p["name"]}
+                              "resource_display_name": p["name"]}
                     for p in resp.json()}
 
         resources = {self.service_name: {"children": projects, "resource_type": resource_type}}

From bfb48829a369b816addd1c802f9a325f902f3d52 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 25 Sep 2018 12:14:04 -0400
Subject: [PATCH 061/124] display resource_display_name instead of
 resource_name when available

---
 magpie/ui/management/templates/tree_scripts.mako | 2 +-
 magpie/ui/management/views.py                    | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/magpie/ui/management/templates/tree_scripts.mako b/magpie/ui/management/templates/tree_scripts.mako
index 84377c906..5476f7f12 100644
--- a/magpie/ui/management/templates/tree_scripts.mako
+++ b/magpie/ui/management/templates/tree_scripts.mako
@@ -48,7 +48,7 @@ li.Collapsed {
             % else:
             <li class="NoChild">
             % endif
-                <div class="tree_item">${key}</div>
+                <div class="tree_item">${tree[key].get('resource_display_name', key)}</div>
                 <input type="hidden" value=${tree[key]['id']} name="resource_id">
                 <input type="hidden" value=${tree[key].get('remote_id', '')} name="remote_id">
                 <input type="hidden" value=${tree[key].get('matches_remote', '')} name="matches_remote">
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 1df465fc2..3a30c7273 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -404,7 +404,10 @@ def resource_tree_parser(self, raw_resources_tree, permission):
             perm_names = self.default_get(permission, r_id, [])
             children = self.resource_tree_parser(resource['children'], permission)
             children = OrderedDict(sorted(children.items()))
-            resources_tree[resource['resource_name']] = dict(id=r_id, permission_names=perm_names, children=children)
+            resources_tree[resource['resource_name']] = dict(id=r_id,
+                                                             permission_names=perm_names,
+                                                             resource_display_name=resource['resource_display_name'],
+                                                             children=children)
         return resources_tree
 
     def perm_tree_parser(self, raw_perm_tree):
@@ -653,6 +656,7 @@ def add_remote_resource(self, service_type, user_or_group, remote_id, is_user=Fa
             else:
                 data = {
                     'resource_name': name,
+                    'resource_display_name': remote_resource.resource_display_name,
                     'resource_type': remote_resource.resource_type,
                     'parent_id': parent_id,
                 }

From bf9143918ff7f3957d131736570ac8653d24de07 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 25 Sep 2018 12:15:09 -0400
Subject: [PATCH 062/124] fix unit tests to include new resource_display_name
 parameter

---
 tests/interfaces.py | 8 ++++++--
 tests/utils.py      | 7 ++++++-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/tests/interfaces.py b/tests/interfaces.py
index 014689908..0c9f3f757 100644
--- a/tests/interfaces.py
+++ b/tests/interfaces.py
@@ -527,13 +527,15 @@ def test_PostResources_DirectServiceResource(self):
 
         data = {
             "resource_name": self.test_resource_name,
+            "resource_display_name": self.test_resource_name,
             "resource_type": self.test_resource_type,
             "parent_id": service_resource_id
         }
         resp = utils.test_request(self.url, 'POST', '/resources',
                                   headers=self.json_headers, cookies=self.cookies, data=data)
         json_body = utils.check_response_basic_info(resp, 201)
-        utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type, self.version)
+        utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type,
+                                            self.test_resource_name, self.version)
 
     @pytest.mark.resources
     @unittest.skipUnless(runner.MAGPIE_TEST_RESOURCES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('resources'))
@@ -546,13 +548,15 @@ def test_PostResources_ChildrenResource(self):
 
         data = {
             "resource_name": self.test_resource_name,
+            "resource_display_name": self.test_resource_name,
             "resource_type": self.test_resource_type,
             "parent_id": direct_resource_id
         }
         resp = utils.test_request(self.url, 'POST', '/resources',
                                   headers=self.json_headers, cookies=self.cookies, data=data)
         json_body = utils.check_response_basic_info(resp, 201)
-        utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type, self.version)
+        utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type,
+                                            self.test_resource_name, self.version)
 
     @pytest.mark.resources
     @unittest.skipUnless(runner.MAGPIE_TEST_RESOURCES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('resources'))
diff --git a/tests/utils.py b/tests/utils.py
index a9d509451..5e82b337a 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -288,12 +288,13 @@ def check_error_param_structure(json_body, paramValue=Null, paramName=Null, para
             check_val_equal(json_body['paramCompare'], paramCompare)
 
 
-def check_post_resource_structure(json_body, resource_name, resource_type, version=None):
+def check_post_resource_structure(json_body, resource_name, resource_type, resource_display_name, version=None):
     """
     Validates POST /resource response information based on different Magpie version formats.
     :param json_body: json body of the response to validate.
     :param resource_name: name of the resource to validate.
     :param resource_type: type of the resource to validate.
+    :param resource_display_name: display name of the resource to validate.
     :param version: version of application/remote server to use for format validation, use local Magpie version if None.
     :raise failing condition
     """
@@ -302,9 +303,11 @@ def check_post_resource_structure(json_body, resource_name, resource_type, versi
         check_val_is_in('resource', json_body)
         check_val_type(json_body['resource'], dict)
         check_val_is_in('resource_name', json_body['resource'])
+        check_val_is_in('resource_display_name', json_body['resource'])
         check_val_is_in('resource_type', json_body['resource'])
         check_val_is_in('resource_id', json_body['resource'])
         check_val_equal(json_body['resource']['resource_name'], resource_name)
+        check_val_equal(json_body['resource']['resource_display_name'], resource_display_name)
         check_val_equal(json_body['resource']['resource_type'], resource_type)
         check_val_type(json_body['resource']['resource_id'], int)
     else:
@@ -340,6 +343,8 @@ def check_resource_children(resource_dict, parent_resource_id, root_service_id):
         check_val_equal(resource_info['parent_id'], parent_resource_id)
         check_val_is_in('resource_name', resource_info)
         check_val_type(resource_info['resource_name'], six.string_types)
+        check_val_is_in('resource_display_name', resource_info)
+        check_val_type(resource_info['resource_display_name'], six.string_types)
         check_val_is_in('permission_names', resource_info)
         check_val_type(resource_info['permission_names'], list)
         check_val_is_in('children', resource_info)

From 0a52b543f8a094d668c3ca2a1ce5ec24fde03790 Mon Sep 17 00:00:00 2001
From: davidcaron <dcaron05@gmail.com>
Date: Tue, 25 Sep 2018 14:00:10 -0400
Subject: [PATCH 063/124] Fixes for when resource_display_name is None

---
 .../management/resource/resource_formats.py   | 32 ++++++++++---------
 magpie/helpers/sync_resources.py              |  3 +-
 2 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/magpie/api/management/resource/resource_formats.py b/magpie/api/management/resource/resource_formats.py
index c89c12786..013cabeae 100644
--- a/magpie/api/management/resource/resource_formats.py
+++ b/magpie/api/management/resource/resource_formats.py
@@ -6,23 +6,25 @@
 
 def format_resource(resource, permissions=None, basic_info=False):
     def fmt_res(res, perms, info):
-        if info:
-            return {
-                u'resource_name': str(res.resource_name),
-                u'resource_display_name': res.resource_display_name.encode('utf-8'),
-                u'resource_type': str(res.resource_type),
-                u'resource_id': res.resource_id
-            }
-        return {
-            u'resource_name': str(res.resource_name),
-            u'resource_display_name': res.resource_display_name.encode('utf-8'),
+        resource_name = str(res.resource_name)
+        resource_display_name = resource_name
+        if res.resource_display_name:
+            resource_display_name = res.resource_display_name.encode('utf-8')
+
+        result = {
+            u'resource_name': resource_name,
+            u'resource_display_name': resource_display_name,
             u'resource_type': str(res.resource_type),
-            u'resource_id': res.resource_id,
-            u'parent_id': res.parent_id,
-            u'root_service_id': res.root_service_id,
-            u'children': {},
-            u'permission_names': list() if perms is None else sorted(perms)
+            u'resource_id': res.resource_id
         }
+        if not info:
+            result.update({
+                u'parent_id': res.parent_id,
+                u'root_service_id': res.root_service_id,
+                u'children': {},
+                u'permission_names': list() if perms is None else sorted(perms)
+            })
+        return result
 
     return evaluate_call(
         lambda: fmt_res(resource, permissions, basic_info),
diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index 5732bfeff..dc6b4bc15 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -238,9 +238,10 @@ def _format_resource_tree(children):
     for child_id, child_dict in children.items():
         resource = child_dict[u'node']
         new_children = child_dict[u'children']
+        resource_display_name = resource.resource_display_name or resource.resource_name
         resource_dict = {'children': _format_resource_tree(new_children),
                          'remote_id': resource.resource_id,
-                         'resource_display_name': resource.resource_display_name}
+                         'resource_display_name': resource_display_name}
         fmt_res_tree[resource.resource_name] = resource_dict
     return fmt_res_tree
 

From bd30ee1da5519ec71aa75e6627abd81f409cd0b7 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 27 Sep 2018 16:10:08 -0400
Subject: [PATCH 064/124] ignore generated docs + single service api

---
 docs/.gitignore    |  2 ++
 magpie/services.py | 21 +--------------------
 2 files changed, 3 insertions(+), 20 deletions(-)
 create mode 100644 docs/.gitignore

diff --git a/docs/.gitignore b/docs/.gitignore
new file mode 100644
index 000000000..d8b9e5175
--- /dev/null
+++ b/docs/.gitignore
@@ -0,0 +1,2 @@
+magpie*.rst
+modules.rst
diff --git a/magpie/services.py b/magpie/services.py
index db32e7dcf..3c5b4b44f 100644
--- a/magpie/services.py
+++ b/magpie/services.py
@@ -287,24 +287,6 @@ def permission_requested(self):
         return u'write'
 
 
-class ServiceGeoserverAPI(ServiceAPI):
-    def __init__(self, service, request):
-        super(ServiceGeoserverAPI, self).__init__(service, request)
-
-    @property
-    def __acl__(self):
-        return ServiceAPI.route_acl.fget(self)
-
-
-class ServiceProjectAPI(ServiceAPI):
-    def __init__(self, service, request):
-        super(ServiceProjectAPI, self).__init__(service, request)
-
-    @property
-    def __acl__(self):
-        return ServiceAPI.route_acl.fget(self, sub_api_route='api')
-
-
 class ServiceWFS(ServiceI):
 
     permission_names = [
@@ -411,10 +393,9 @@ def permission_requested(self):
 
 service_type_dict = {
     u'access':          ServiceAccess,
-    u'geoserver-api':   ServiceGeoserverAPI,
+    u'api':             ServiceAPI,
     u'geoserverwms':    ServiceGeoserver,
     u'ncwms':           ServiceNCWMS2,
-    u'project-api':     ServiceProjectAPI,
     u'thredds':         ServiceTHREDDS,
     u'wfs':             ServiceWFS,
     u'wps':             ServiceWPS,

From e32719a64dba8914efc32a53ae40ee673c812df5 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 27 Sep 2018 16:40:28 -0400
Subject: [PATCH 065/124] alembic upgrade db service types

---
 .../73639c63c4fc_unified_api_service.py       | 57 +++++++++++++++++++
 magpie/services.py                            |  2 +-
 2 files changed, 58 insertions(+), 1 deletion(-)
 create mode 100644 magpie/alembic/versions/73639c63c4fc_unified_api_service.py

diff --git a/magpie/alembic/versions/73639c63c4fc_unified_api_service.py b/magpie/alembic/versions/73639c63c4fc_unified_api_service.py
new file mode 100644
index 000000000..51351bceb
--- /dev/null
+++ b/magpie/alembic/versions/73639c63c4fc_unified_api_service.py
@@ -0,0 +1,57 @@
+"""unified api service
+
+Revision ID: 73639c63c4fc
+Revises: d01af1f2e445
+Create Date: 2018-09-27 16:12:02.282830
+
+"""
+import os, sys
+cur_file = os.path.abspath(__file__)
+root_dir = os.path.dirname(cur_file)    # version
+root_dir = os.path.dirname(root_dir)    # alembic
+root_dir = os.path.dirname(root_dir)    # magpie
+root_dir = os.path.dirname(root_dir)    # root
+sys.path.insert(0, root_dir)
+
+from alembic import op
+from alembic.context import get_context
+from magpie.definitions.sqlalchemy_definitions import *
+from magpie import models
+from magpie.api.management.resource.resource_utils import get_resource_root_service
+
+Session = sessionmaker()
+
+
+# revision identifiers, used by Alembic.
+revision = '73639c63c4fc'
+down_revision = 'd01af1f2e445'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    context = get_context()
+    session = Session(bind=op.get_bind())
+    if isinstance(context.connection.engine.dialect, PGDialect):
+        for service in models.Service.all(db_session=session):
+            if service.type == 'project-api':
+                service.type = 'api'
+                service.url = service.url + '/api'
+            elif service.type == 'geoserver-api':
+                service.type = 'api'
+                service.url = service.url.rstrip('/')
+    session.commit()
+
+
+def downgrade():
+    context = get_context()
+    session = Session(bind=op.get_bind())
+    if isinstance(context.connection.engine.dialect, PGDialect):
+        for service in models.Service.all(db_session=session):
+            if service.type == 'api':
+                if 'geoserver/rest' in service.url:
+                    service.type = 'geoserver-api'
+                else:
+                    service.type = 'project-api'
+                    service.url = service.url.rstrip('/api')
+    session.commit()
diff --git a/magpie/services.py b/magpie/services.py
index 3c5b4b44f..6ca8a954e 100644
--- a/magpie/services.py
+++ b/magpie/services.py
@@ -282,7 +282,7 @@ def route_acl(self, sub_api_route=None):
         return self.acl
 
     def permission_requested(self):
-        if self.request.method == 'GET':
+        if self.request.method.upper() in ['GET', 'HEAD']:
             return u'read'
         return u'write'
 

From ef7d59631bb531ea1de00dc9993d1fd5f7a1af10 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Fri, 28 Sep 2018 12:05:57 -0400
Subject: [PATCH 066/124] fix empty strings in mako templates

---
 magpie/ui/management/templates/tree_scripts.mako | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/magpie/ui/management/templates/tree_scripts.mako b/magpie/ui/management/templates/tree_scripts.mako
index 84377c906..58b81d77e 100644
--- a/magpie/ui/management/templates/tree_scripts.mako
+++ b/magpie/ui/management/templates/tree_scripts.mako
@@ -49,9 +49,9 @@ li.Collapsed {
             <li class="NoChild">
             % endif
                 <div class="tree_item">${key}</div>
-                <input type="hidden" value=${tree[key]['id']} name="resource_id">
-                <input type="hidden" value=${tree[key].get('remote_id', '')} name="remote_id">
-                <input type="hidden" value=${tree[key].get('matches_remote', '')} name="matches_remote">
+                <input type="hidden" value="${tree[key]['id']}" name="resource_id">
+                <input type="hidden" value="${tree[key].get('remote_id', '')}" name="remote_id">
+                <input type="hidden" value="${tree[key].get('matches_remote', '')}" name="matches_remote">
                 ${item_renderer(key, tree[key], level)}
             </li>
         </form>

From be920d4c6013eb574d1f8cef30b09640c399d5fc Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Fri, 28 Sep 2018 12:50:14 -0400
Subject: [PATCH 067/124] iteritems -> items

---
 magpie/helpers/sync_resources.py | 2 +-
 magpie/ui/management/views.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index b94e68e80..af1cf581f 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -107,7 +107,7 @@ def _sort_resources(resources):
     :return: None
     """
     for resource_name, values in resources.items():
-        values['children'] = OrderedDict(sorted(values['children'].iteritems()))
+        values['children'] = OrderedDict(sorted(values['children'].items()))
         return _sort_resources(values['children'])
 
 
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 1df465fc2..c5ffdbbd9 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -619,7 +619,7 @@ def delete_resource(self, res_id):
 
     def get_ids_to_clean(self, resources):
         ids = []
-        for resource_name, values in resources.iteritems():
+        for resource_name, values in resources.items():
             if "matches_remote" in values and not values["matches_remote"]:
                 ids.append(values['id'])
             ids += self.get_ids_to_clean(values['children'])

From 92fd2155c4efb8f9f5f62ac5b49fdc4f6ab0563c Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Fri, 28 Sep 2018 13:38:31 -0400
Subject: [PATCH 068/124] adjust api providers

---
 magpie/services.py | 3 +--
 providers.cfg      | 8 ++++----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/magpie/services.py b/magpie/services.py
index 6ca8a954e..0ff3ecad1 100644
--- a/magpie/services.py
+++ b/magpie/services.py
@@ -258,9 +258,8 @@ def __init__(self, service, request):
 
     @property
     def __acl__(self):
-        raise NotImplementedError
+        return self.route_acl()
 
-    @property
     def route_acl(self, sub_api_route=None):
         self.expand_acl(self.service, self.request.user)
 
diff --git a/providers.cfg b/providers.cfg
index 523f95813..fc61386b1 100644
--- a/providers.cfg
+++ b/providers.cfg
@@ -56,15 +56,15 @@ providers:
     type: access
 
   geoserver-api:
-    url: http://${HOSTNAME}:8087/geoserver/rest/
+    url: http://${HOSTNAME}:8087/geoserver/rest
     title: geoserver-api
     public: true
     c4i: false
-    type: geoserver-api
+    type: api
 
   project-api:
-    url: http://${HOSTNAME}:3005
+    url: http://${HOSTNAME}:3005/api
     title: project-api
     public: true
     c4i: false
-    type: project-api
+    type: api

From f52ffb0a7a137334bd306f942a9028c71580ba34 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Fri, 28 Sep 2018 15:57:04 -0400
Subject: [PATCH 069/124] identify a service by its id instead of its name and
 type

---
 magpie/helpers/sync_resources.py | 23 ++++++-----
 magpie/ui/management/views.py    | 67 ++++++++++++++++++--------------
 2 files changed, 48 insertions(+), 42 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index af1cf581f..bfdc20aac 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -36,11 +36,11 @@
     sync_service_class(name, url)
 
 
-def merge_local_and_remote_resources(resources_local, service_type, service_name, session):
+def merge_local_and_remote_resources(resources_local, service_type, service_id, session):
     """Main function to sync resources with remote server"""
-    if not get_last_sync(service_type, service_name, session):
+    if not get_last_sync(service_id, session):
         return resources_local
-    remote_resources = _query_remote_resources_in_database(service_type, service_name, session=session)
+    remote_resources = _query_remote_resources_in_database(service_id, session=session)
     max_depth = _get_max_depth(service_type)
     merged_resources = _merge_resources(resources_local, remote_resources, max_depth)
     _sort_resources(merged_resources)
@@ -246,30 +246,29 @@ def _get_max_depth(service_type):
     return SYNC_SERVICES_TYPES[service_type](name, url).max_depth
 
 
-def _query_remote_resources_in_database(service_type, service_name, session):
+def _query_remote_resources_in_database(service_id, session):
     """
     Reads remote resources from the RemoteResources table. No external request is made.
     :param service_type:
     :param session:
     :return: a dictionary of the form defined in 'sync_services.is_valid_resource_schema'
     """
-    service = session.query(models.Service).filter_by(type=service_type, resource_name=service_name).first()
-    _ensure_sync_info_exists(service.resource_id, session)
+    service = session.query(models.Service).filter_by(resource_id=service_id).first()
+    _ensure_sync_info_exists(service_id, session)
 
-    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
+    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session)
     main_resource = session.query(models.RemoteResource).filter_by(
         resource_id=sync_info.remote_resource_id).first()
     tree = _get_resource_children(main_resource, session)
 
     remote_resources = _format_resource_tree(tree)
-    return {service_name: {'children': remote_resources, 'remote_id': main_resource.resource_id}}
+    return {service.resource_name: {'children': remote_resources, 'remote_id': main_resource.resource_id}}
 
 
-def get_last_sync(service_type, service_name, session):
+def get_last_sync(service_id, session):
     last_sync = None
-    service = session.query(models.Service).filter_by(type=service_type, resource_name=service_name).first()
-    _ensure_sync_info_exists(service.resource_id, session)
-    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service.resource_id, session)
+    _ensure_sync_info_exists(service_id, session)
+    sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session)
     if sync_info:
         last_sync = sync_info.last_sync
     return last_sync
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index c5ffdbbd9..fa39f9767 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -339,13 +339,11 @@ def edit_user(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        info = self.merge_remote_resources(cur_svc_type, res_perms, services, session)
-        res_perms, ids_to_clean, last_sync_humanized, last_sync_delta = info
-
-        out_of_sync = last_sync_delta > OUT_OF_SYNC if last_sync_delta else False
+        info = self.get_remote_resources_info(cur_svc_type, res_perms, services, session)
+        res_perms, ids_to_clean, last_sync_humanized, out_of_sync = info
 
         if out_of_sync:
-            error_message = "There seems to be an issue synchronizing resources from this service."
+            error_message = self.make_out_of_sync_message(out_of_sync)
 
         user_info[u'error_message'] = error_message
         user_info[u'ids_to_clean'] = ";".join(ids_to_clean)
@@ -557,13 +555,11 @@ def edit_group(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        info = self.merge_remote_resources(cur_svc_type, res_perms, services, session)
-        res_perms, ids_to_clean, last_sync_humanized, last_sync_delta = info
-
-        out_of_sync = last_sync_delta > OUT_OF_SYNC if last_sync_delta else False
+        info = self.get_remote_resources_info(cur_svc_type, res_perms, services, session)
+        res_perms, ids_to_clean, last_sync_humanized, out_of_sync = info
 
         if out_of_sync:
-            error_message = "There seems to be an issue synchronizing resources from this service."
+            error_message = self.make_out_of_sync_message(out_of_sync)
 
         group_info[u'error_message'] = error_message
         group_info[u'ids_to_clean'] = ";".join(ids_to_clean)
@@ -580,32 +576,43 @@ def edit_group(self):
         group_info[u'permissions'] = res_perm_names
         return add_template_data(self.request, data=group_info)
 
-    def merge_remote_resources(self, cur_svc_type, res_perms, services, session):
-        ids_to_clean = []
-        last_sync_datetimes = []
+    def make_out_of_sync_message(self, out_of_sync):
+        this = "this service" if len(out_of_sync) == 1 else "these services"
+        error_message = ("There seems to be an issue synchronizing resources from "
+                         "{}: {}".format(this, ",".join(out_of_sync)))
+        return error_message
+
+    def get_remote_resources_info(self, cur_svc_type, res_perms, services, session):
         last_sync_humanized = "Never"
-        last_sync_delta = None
+        ids_to_clean, out_of_sync = [], []
+        now = datetime.datetime.now()
 
-        merged_resources = {}
+        service_ids = [s["resource_id"] for s in services.values()]
+        last_sync_datetimes = self.get_last_sync_datetimes(service_ids, session)
+
+        if any(last_sync_datetimes):
+            last_sync_datetime = min(filter(bool, last_sync_datetimes))
+            last_sync_humanized = humanize.naturaltime(now - last_sync_datetime)
+            res_perms = self.merge_remote_resources(cur_svc_type, res_perms, services, session)
 
-        for service_name in services:
-            last_sync = sync_resources.get_last_sync(cur_svc_type, service_name, session)
-            last_sync_datetimes.append(last_sync)
+        for last_sync, service_name in zip(last_sync_datetimes, services):
+            if last_sync:
+                ids_to_clean += self.get_ids_to_clean(res_perms[service_name]["children"])
+                if now - last_sync > OUT_OF_SYNC:
+                    out_of_sync.append(service_name)
+        return res_perms, ids_to_clean, last_sync_humanized, out_of_sync
 
-            resources_for_service = sync_resources.merge_local_and_remote_resources(res_perms,
-                                                                                    cur_svc_type,
-                                                                                    service_name,
-                                                                                    session)
+    def merge_remote_resources(self, cur_svc_type, res_perms, services, session):
+        merged_resources = {}
+        for service_name, service_values in services.items():
+            service_id = service_values["resource_id"]
+            merge = sync_resources.merge_local_and_remote_resources
+            resources_for_service = merge(res_perms, cur_svc_type, service_id, session)
             merged_resources[service_name] = resources_for_service[service_name]
+        return merged_resources
 
-        if any(last_sync_datetimes):
-            last_sync_datetime = min(filter(bool, last_sync_datetimes))
-            now = datetime.datetime.now()
-            last_sync_delta = now - last_sync_datetime
-            last_sync_humanized = humanize.naturaltime(last_sync_delta)
-            for service in merged_resources.values():
-                ids_to_clean += self.get_ids_to_clean(service['children'])
-        return merged_resources, ids_to_clean, last_sync_humanized, last_sync_delta
+    def get_last_sync_datetimes(self, service_ids, session):
+        return [sync_resources.get_last_sync(s, session) for s in service_ids]
 
     def delete_resource(self, res_id):
         url = '{url}/resources/{resource_id}'.format(url=self.magpie_url, resource_id=res_id)

From 3f71d52e93d2c7aa9c4f1c5898e56c27df2effcb Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Fri, 28 Sep 2018 15:57:54 -0400
Subject: [PATCH 070/124] get services sooner so that the cur_svc_type is not
 "default"

---
 magpie/ui/management/views.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index fa39f9767..e5ee2c844 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -247,6 +247,12 @@ def edit_user(self):
         # we have to access the database directly here
         session = self.request.db
 
+        try:
+            # The service type is 'default'. This function replaces cur_svc_type with the first service type.
+            svc_types, cur_svc_type, services = self.get_services(cur_svc_type)
+        except Exception as e:
+            raise HTTPBadRequest(detail=repr(e))
+
         user_resp = requests.get(user_url, cookies=self.request.cookies)
         check_response(user_resp)
         user_info = user_resp.json()['user']
@@ -333,7 +339,6 @@ def edit_user(self):
 
         # display resources permissions per service type tab
         try:
-            svc_types, cur_svc_type, services = self.get_services(cur_svc_type)
             res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(
                 user_name, services, cur_svc_type, is_user=True, is_inherited_permissions=inherited_permissions)
         except Exception as e:
@@ -507,6 +512,12 @@ def edit_group(self):
         # we have to access the database directly here
         session = self.request.db
 
+        try:
+            # The service type is 'default'. This function replaces cur_svc_type with the first service type.
+            svc_types, cur_svc_type, services = self.get_services(cur_svc_type)
+        except Exception as e:
+            raise HTTPBadRequest(detail=repr(e))
+
         # move to service or edit requested group/permission changes
         if self.request.method == 'POST':
             res_id = self.request.POST.get('resource_id')
@@ -549,7 +560,6 @@ def edit_group(self):
 
         # display resources permissions per service type tab
         try:
-            svc_types, cur_svc_type, services = self.get_services(cur_svc_type)
             res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(group_name, services,
                                                                                           cur_svc_type, is_user=False)
         except Exception as e:

From ea05900bb9fa57cd213ecf58c5668306ceb3f734 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Fri, 28 Sep 2018 15:58:33 -0400
Subject: [PATCH 071/124] ui syntax

---
 magpie/ui/management/templates/edit_group.mako | 2 +-
 magpie/ui/management/templates/edit_user.mako  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/magpie/ui/management/templates/edit_group.mako b/magpie/ui/management/templates/edit_group.mako
index 3c1ba1de2..e0c215ffa 100644
--- a/magpie/ui/management/templates/edit_group.mako
+++ b/magpie/ui/management/templates/edit_group.mako
@@ -112,7 +112,7 @@
         %endif
         <form id="sync_info" action="${request.path}" method="post">
             <p class="panel_line">
-                <span class="panel_entry">Last synchronization with remote service: </span>
+                <span class="panel_entry">Last synchronization with remote services: </span>
                 %if sync_implemented:
                     <span class="panel_value">${last_sync} </span>
                     <input type="submit" value="Sync now" name="force_sync">
diff --git a/magpie/ui/management/templates/edit_user.mako b/magpie/ui/management/templates/edit_user.mako
index 5f2a3583d..4a7315804 100644
--- a/magpie/ui/management/templates/edit_user.mako
+++ b/magpie/ui/management/templates/edit_user.mako
@@ -168,7 +168,7 @@
         %endif
         <form id="sync_info" action="${request.path}" method="post">
             <p class="panel_line">
-                <span class="panel_entry">Last synchronization with remote service: </span>
+                <span class="panel_entry">Last synchronization with remote services: </span>
                 %if sync_implemented:
                     <span class="panel_value">${last_sync} </span>
                     <input type="submit" value="Sync now" name="force_sync">

From 73580a9e8acbec5638bd98b3568a7b08c2de59a4 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Fri, 28 Sep 2018 16:05:30 -0400
Subject: [PATCH 072/124] related to #107, prepare for new table column

---
 magpie/helpers/sync_resources.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index bfdc20aac..c1106f58b 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -135,7 +135,7 @@ def _get_remote_resources(service):
     if service_url.endswith("/"):  # remove trailing slash
         service_url = service_url[:-1]
 
-    sync_service_class = SYNC_SERVICES_TYPES.get(service.type.lower(), sync_services._SyncServiceDefault)
+    sync_service_class = SYNC_SERVICES_TYPES.get(service.sync_type.lower(), sync_services._SyncServiceDefault)
     sync_service = sync_service_class(service.resource_name, service_url)
     return sync_service.get_resources()
 

From c1685e1786a03c23db39b082c3b1ffe74b3c08cc Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Fri, 28 Sep 2018 16:49:45 -0400
Subject: [PATCH 073/124] update swagger

---
 magpie/api/api_rest_schemas.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py
index 87952e146..2fa8a96d7 100644
--- a/magpie/api/api_rest_schemas.py
+++ b/magpie/api/api_rest_schemas.py
@@ -477,6 +477,11 @@ class ResourceBodySchema(colander.MappingSchema):
         description="Name of the resource",
         example="thredds"
     )
+    resource_display_name = colander.SchemaNode(
+        colander.String(),
+        description="Display name of the resource",
+        example="Birdhouse Thredds Data Server"
+    )
     resource_type = colander.SchemaNode(
         colander.String(),
         description="Type of the resource",
@@ -666,6 +671,11 @@ class Resources_POST_BodySchema(colander.MappingSchema):
         colander.String(),
         description="Name of the resource to create"
     )
+    resource_display_name = colander.SchemaNode(
+        colander.String(),
+        description="Display name of the resource to create, defaults to resource_name.",
+        missing=colander.drop
+    )
     resource_type = colander.SchemaNode(
         colander.String(),
         description="Type of the resource to create"

From da15173949fe0d1405647f78ed474de7efd94200 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Fri, 28 Sep 2018 16:57:44 -0400
Subject: [PATCH 074/124] make resource_display_name optional

---
 magpie/api/management/service/service_views.py |  2 +-
 tests/interfaces.py                            | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/magpie/api/management/service/service_views.py b/magpie/api/management/service/service_views.py
index da0960e08..d110f57aa 100644
--- a/magpie/api/management/service/service_views.py
+++ b/magpie/api/management/service/service_views.py
@@ -195,7 +195,7 @@ def create_service_direct_resource(request):
     """Register a new resource directly under a service."""
     service = get_service_matchdict_checked(request)
     resource_name = get_multiformat_post(request, 'resource_name')
-    resource_display_name = get_multiformat_post(request, 'resource_display_name')
+    resource_display_name = get_multiformat_post(request, 'resource_display_name', default=resource_name)
     resource_type = get_multiformat_post(request, 'resource_type')
     parent_id = get_multiformat_post(request, 'parent_id')  # no check because None/empty is allowed
     if not parent_id:
diff --git a/tests/interfaces.py b/tests/interfaces.py
index 0c9f3f757..9e42d29f8 100644
--- a/tests/interfaces.py
+++ b/tests/interfaces.py
@@ -537,6 +537,24 @@ def test_PostResources_DirectServiceResource(self):
         utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type,
                                             self.test_resource_name, self.version)
 
+    @pytest.mark.resources
+    @unittest.skipUnless(runner.MAGPIE_TEST_RESOURCES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('resources'))
+    def test_PostResources_DirectServiceResource(self):
+        service_info = utils.TestSetup.get_ExistingTestServiceInfo(self)
+        service_resource_id = service_info['resource_id']
+
+        data = {
+            "resource_name": self.test_resource_name,
+            # resource_display_name should default to self.test_resource_name,
+            "resource_type": self.test_resource_type,
+            "parent_id": service_resource_id
+        }
+        resp = utils.test_request(self.url, 'POST', '/resources',
+                                  headers=self.json_headers, cookies=self.cookies, data=data)
+        json_body = utils.check_response_basic_info(resp, 201)
+        utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type,
+                                            self.test_resource_name, self.version)
+
     @pytest.mark.resources
     @unittest.skipUnless(runner.MAGPIE_TEST_RESOURCES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('resources'))
     def test_PostResources_ChildrenResource(self):

From 6e799e1438dc8ab2486a6fd2f25150960dfb2b53 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Fri, 28 Sep 2018 17:32:05 -0400
Subject: [PATCH 075/124] update 'api' in db + add sync_type

---
 magpie/alembic/__init__.py                    |  0
 magpie/alembic/utils.py                       |  9 +++
 .../73639c63c4fc_unified_api_service.py       | 61 ++++++++++++-------
 .../a395ef9d3fe6_reference_root_service.py    | 15 ++---
 magpie/models.py                              | 18 +++++-
 5 files changed, 69 insertions(+), 34 deletions(-)
 create mode 100644 magpie/alembic/__init__.py
 create mode 100644 magpie/alembic/utils.py

diff --git a/magpie/alembic/__init__.py b/magpie/alembic/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/magpie/alembic/utils.py b/magpie/alembic/utils.py
new file mode 100644
index 000000000..53e68b199
--- /dev/null
+++ b/magpie/alembic/utils.py
@@ -0,0 +1,9 @@
+from magpie.definitions.sqlalchemy_definitions import *
+
+
+def has_column(context, table_name, column_name):
+    inspector = reflection.Inspector.from_engine(context.connection.engine)
+    for column in inspector.get_columns(table_name=table_name):
+        if column_name in column['name']:
+            return True
+    return False
diff --git a/magpie/alembic/versions/73639c63c4fc_unified_api_service.py b/magpie/alembic/versions/73639c63c4fc_unified_api_service.py
index 51351bceb..6a30c29d6 100644
--- a/magpie/alembic/versions/73639c63c4fc_unified_api_service.py
+++ b/magpie/alembic/versions/73639c63c4fc_unified_api_service.py
@@ -6,22 +6,22 @@
 
 """
 import os, sys
+
 cur_file = os.path.abspath(__file__)
-root_dir = os.path.dirname(cur_file)    # version
-root_dir = os.path.dirname(root_dir)    # alembic
-root_dir = os.path.dirname(root_dir)    # magpie
-root_dir = os.path.dirname(root_dir)    # root
+root_dir = os.path.dirname(cur_file)  # version
+root_dir = os.path.dirname(root_dir)  # alembic
+root_dir = os.path.dirname(root_dir)  # magpie
+root_dir = os.path.dirname(root_dir)  # root
 sys.path.insert(0, root_dir)
 
 from alembic import op
 from alembic.context import get_context
 from magpie.definitions.sqlalchemy_definitions import *
-from magpie import models
-from magpie.api.management.resource.resource_utils import get_resource_root_service
+from magpie.models import Service
+from magpie.alembic.utils import has_column
 
 Session = sessionmaker()
 
-
 # revision identifiers, used by Alembic.
 revision = '73639c63c4fc'
 down_revision = 'd01af1f2e445'
@@ -33,25 +33,40 @@ def upgrade():
     context = get_context()
     session = Session(bind=op.get_bind())
     if isinstance(context.connection.engine.dialect, PGDialect):
-        for service in models.Service.all(db_session=session):
-            if service.type == 'project-api':
-                service.type = 'api'
-                service.url = service.url + '/api'
-            elif service.type == 'geoserver-api':
-                service.type = 'api'
-                service.url = service.url.rstrip('/')
-    session.commit()
+        # add 'sync_type' column if missing
+        if not has_column(context, 'services', 'sync_type'):
+            op.add_column('services', sa.Column('sync_type', sa.UnicodeText(), nullable=True))
+
+        # transfer 'api' service types
+        session.query(Service). \
+            filter(Service.type == 'project-api'). \
+            update({Service.type: 'api',
+                    Service.url: Service.url + '/api',
+                    Service.sync_type: 'project-api'}, synchronize_session=False)
+        session.query(Service). \
+            filter(Service.type == 'geoserver-api'). \
+            update({Service.type: 'api',
+                    Service.sync_type: 'geoserver-api'}, synchronize_session=False)
+        session.commit()
 
 
 def downgrade():
     context = get_context()
     session = Session(bind=op.get_bind())
     if isinstance(context.connection.engine.dialect, PGDialect):
-        for service in models.Service.all(db_session=session):
-            if service.type == 'api':
-                if 'geoserver/rest' in service.url:
-                    service.type = 'geoserver-api'
-                else:
-                    service.type = 'project-api'
-                    service.url = service.url.rstrip('/api')
-    session.commit()
+        # transfer 'api' service types
+        services_project_api = session.query(Service).filter(Service.sync_type == 'project-api')
+        for svc in services_project_api:
+            svc_url = svc.url.rstrip('/api')
+            session.query(Service). \
+                filter(Service.resource_id == svc.resource_id). \
+                update({Service.type: 'project-api', Service.url: svc_url}, synchronize_session=False)
+        session.flush()
+        session.query(Service). \
+            filter(Service.sync_type == 'geoserver-api'). \
+            update({Service.type: 'geoserver-api'}, synchronize_session=False)
+        session.flush()
+        # drop 'sync_type' column
+        if has_column(context, 'services', 'sync_type'):
+            op.drop_column('services', 'sync_type')
+        session.commit()
diff --git a/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py b/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py
index 00b11a2ca..b1eb00e13 100644
--- a/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py
+++ b/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py
@@ -18,6 +18,7 @@
 from magpie.definitions.sqlalchemy_definitions import *
 from magpie import models
 from magpie.api.management.resource.resource_utils import get_resource_root_service
+from magpie.alembic.utils import has_column
 
 Session = sessionmaker()
 
@@ -41,14 +42,7 @@ def upgrade():
     if isinstance(context.connection.engine.dialect, PGDialect):
 
         # check if column exists, add it otherwise
-        inspector = reflection.Inspector.from_engine(context.connection.engine)
-        has_root_service_column = False
-        for column in inspector.get_columns(table_name='resources'):
-            if 'root_service_id' in column['name']:
-                has_root_service_column = True
-                break
-
-        if not has_root_service_column:
+        if has_column(context, 'resources', 'root_service_id'):
             op.add_column('resources', sa.Column('root_service_id', sa.Integer(), nullable=True))
 
         # add existing resource references to their root service, loop through reference tree chain
@@ -61,4 +55,7 @@ def upgrade():
 
 
 def downgrade():
-    pass
+    context = get_context()
+    if isinstance(context.connection.engine.dialect, PGDialect):
+        if has_column(context, 'resources', 'root_service_id'):
+            op.drop_column('resources', 'root_service_id')
diff --git a/magpie/models.py b/magpie/models.py
index 2885b79e5..cd2b5ca15 100644
--- a/magpie/models.py
+++ b/magpie/models.py
@@ -106,8 +106,22 @@ class Service(Resource):
     __mapper_args__ = {u'polymorphic_identity': u'service', u'inherit_condition': resource_id == Resource.resource_id}
 
     # ... your own properties....
-    url = sa.Column(sa.UnicodeText(), unique=True)  # http://localhost:8083
-    type = sa.Column(sa.UnicodeText())  # wps, wms, thredds, ...
+
+    @declared_attr
+    def url(self):
+        # http://localhost:8083
+        return sa.Column(sa.UnicodeText(), unique=True)
+
+    @declared_attr
+    def type(self):
+        # wps, wms, thredds, ...
+        return sa.Column(sa.UnicodeText())
+
+    @declared_attr
+    def sync_type(self):
+        # project-api, geoserver-api, ...
+        return sa.Column(sa.UnicodeText(), nullable=True)
+
     resource_type_name = u'service'
 
     @staticmethod

From 1f68d5abe75cfdead51c76de6eae2a2c78646fb0 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Fri, 28 Sep 2018 17:44:48 -0400
Subject: [PATCH 076/124] remove init that messes up import alembic in magpie

---
 magpie/alembic/__init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 magpie/alembic/__init__.py

diff --git a/magpie/alembic/__init__.py b/magpie/alembic/__init__.py
deleted file mode 100644
index e69de29bb..000000000

From 70117d624295687734c977c07d8fa4673f43ee36 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Mon, 1 Oct 2018 15:10:52 -0400
Subject: [PATCH 077/124] fix migrations

---
 .../73639c63c4fc_unified_api_service.py       | 79 +++++++++++--------
 .../a395ef9d3fe6_reference_root_service.py    |  9 +--
 ...c352a98d570e_project_api_route_resource.py |  2 +-
 providers.cfg                                 | 16 +++-
 setup.py                                      |  2 +-
 tests/test_magpie_api.py                      |  2 +-
 tests/test_magpie_ui.py                       |  4 +-
 7 files changed, 65 insertions(+), 49 deletions(-)

diff --git a/magpie/alembic/versions/73639c63c4fc_unified_api_service.py b/magpie/alembic/versions/73639c63c4fc_unified_api_service.py
index 6a30c29d6..e164dd90e 100644
--- a/magpie/alembic/versions/73639c63c4fc_unified_api_service.py
+++ b/magpie/alembic/versions/73639c63c4fc_unified_api_service.py
@@ -17,8 +17,8 @@
 from alembic import op
 from alembic.context import get_context
 from magpie.definitions.sqlalchemy_definitions import *
-from magpie.models import Service
-from magpie.alembic.utils import has_column
+# from magpie.models import Service
+from sqlalchemy.sql import table
 
 Session = sessionmaker()
 
@@ -31,42 +31,53 @@
 
 def upgrade():
     context = get_context()
-    session = Session(bind=op.get_bind())
     if isinstance(context.connection.engine.dialect, PGDialect):
         # add 'sync_type' column if missing
-        if not has_column(context, 'services', 'sync_type'):
-            op.add_column('services', sa.Column('sync_type', sa.UnicodeText(), nullable=True))
+        op.add_column('services', sa.Column('sync_type', sa.UnicodeText(), nullable=True))
+
+        services = table('services',
+                         sa.Column('url', sa.UnicodeText()),
+                         sa.Column('type', sa.UnicodeText()),
+                         sa.Column('sync_type', sa.UnicodeText()),
+                         )
 
         # transfer 'api' service types
-        session.query(Service). \
-            filter(Service.type == 'project-api'). \
-            update({Service.type: 'api',
-                    Service.url: Service.url + '/api',
-                    Service.sync_type: 'project-api'}, synchronize_session=False)
-        session.query(Service). \
-            filter(Service.type == 'geoserver-api'). \
-            update({Service.type: 'api',
-                    Service.sync_type: 'geoserver-api'}, synchronize_session=False)
-        session.commit()
+        op.execute(services.
+                   update().
+                   where(services.c.type == op.inline_literal('project-api')).
+                   values({'type': op.inline_literal('api'),
+                           'url': op.inline_literal(str(services.c.url) + '/api'),
+                           'sync_type': op.inline_literal('project-api')
+                           })
+                   )
+        op.execute(services.
+                   update().
+                   where(services.c.type == op.inline_literal('geoserver-api')).
+                   values({'type': op.inline_literal('api'),
+                           'sync_type': op.inline_literal('geoserver-api')
+                           })
+                   )
 
 
 def downgrade():
-    context = get_context()
-    session = Session(bind=op.get_bind())
-    if isinstance(context.connection.engine.dialect, PGDialect):
-        # transfer 'api' service types
-        services_project_api = session.query(Service).filter(Service.sync_type == 'project-api')
-        for svc in services_project_api:
-            svc_url = svc.url.rstrip('/api')
-            session.query(Service). \
-                filter(Service.resource_id == svc.resource_id). \
-                update({Service.type: 'project-api', Service.url: svc_url}, synchronize_session=False)
-        session.flush()
-        session.query(Service). \
-            filter(Service.sync_type == 'geoserver-api'). \
-            update({Service.type: 'geoserver-api'}, synchronize_session=False)
-        session.flush()
-        # drop 'sync_type' column
-        if has_column(context, 'services', 'sync_type'):
-            op.drop_column('services', 'sync_type')
-        session.commit()
+    op.drop_column('services', 'sync_type')
+
+    service = table('old_service',
+                    sa.Column('url', sa.UnicodeText()),
+                    sa.Column('type', sa.UnicodeText()),
+                    )
+
+    # transfer 'api' service types
+    op.execute(service.
+               update().
+               where(service.c.type == op.inline_literal('project-api')).
+               values({'type': op.inline_literal('project-api'),
+                       'url': op.inline_literal(str(service.c.url).rstrip('/api')),
+                       })
+               )
+    op.execute(service.
+               update().
+               where(service.c.type == op.inline_literal('geoserver-api')).
+               values({'type': op.inline_literal('geoserver-api'),
+                       })
+               )
diff --git a/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py b/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py
index b1eb00e13..134d3c3db 100644
--- a/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py
+++ b/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py
@@ -18,7 +18,6 @@
 from magpie.definitions.sqlalchemy_definitions import *
 from magpie import models
 from magpie.api.management.resource.resource_utils import get_resource_root_service
-from magpie.alembic.utils import has_column
 
 Session = sessionmaker()
 
@@ -40,10 +39,7 @@ def upgrade():
     context.connection.engine.dialect.supports_sane_multi_rowcount = False
 
     if isinstance(context.connection.engine.dialect, PGDialect):
-
-        # check if column exists, add it otherwise
-        if has_column(context, 'resources', 'root_service_id'):
-            op.add_column('resources', sa.Column('root_service_id', sa.Integer(), nullable=True))
+        op.add_column('resources', sa.Column('root_service_id', sa.Integer(), nullable=True))
 
         # add existing resource references to their root service, loop through reference tree chain
         all_resources = session.query(models.Resource)
@@ -57,5 +53,4 @@ def upgrade():
 def downgrade():
     context = get_context()
     if isinstance(context.connection.engine.dialect, PGDialect):
-        if has_column(context, 'resources', 'root_service_id'):
-            op.drop_column('resources', 'root_service_id')
+        op.drop_column('resources', 'root_service_id')
diff --git a/magpie/alembic/versions/c352a98d570e_project_api_route_resource.py b/magpie/alembic/versions/c352a98d570e_project_api_route_resource.py
index c4a2f048e..b86680930 100644
--- a/magpie/alembic/versions/c352a98d570e_project_api_route_resource.py
+++ b/magpie/alembic/versions/c352a98d570e_project_api_route_resource.py
@@ -32,7 +32,7 @@ def change_project_api_resource_type(new_type_name):
     if isinstance(context.connection.engine.dialect, PGDialect):
         # obtain service 'project-api'
         session = Session(bind=op.get_bind())
-        project_api_svc = models.Service.by_service_name('project-api', db_session=session)
+        project_api_svc = session.query(models.Service.resource_id).filter_by(resource_name='project-api').first()
 
         # nothing to edit if it doesn't exist, otherwise change resource types to 'route'
         if project_api_svc:
diff --git a/providers.cfg b/providers.cfg
index fc61386b1..56b54cd9e 100644
--- a/providers.cfg
+++ b/providers.cfg
@@ -5,6 +5,7 @@ providers:
     public: true
     c4i: false
     type: wps
+    sync_type: wps
 
   malleefowl:
     url: http://${HOSTNAME}:8091/wps
@@ -12,6 +13,7 @@ providers:
     public: true
     c4i: false
     type: wps
+    sync_type: wps
 
   lb_flyingpigeon:
     url: http://${HOSTNAME}:58093/wps
@@ -19,13 +21,15 @@ providers:
     public: true
     c4i: false
     type: wps
+    sync_type: wps
 
   thredds:
-    url: http://${HOSTNAME}:8083/thredds
+    url: http://colibri.crim.ca:8083/thredds
     title: Thredds
     public: true
     c4i: false
     type: thredds
+    sync_type: thredds
 
   ncWMS2:
     url: http://${HOSTNAME}:8080/ncWMS2
@@ -33,6 +37,7 @@ providers:
     public: true
     c4i: false
     type: ncwms
+    sync_type: ncwms
 
   geoserverwms:
     url: http://${HOSTNAME}:8087/geoserver
@@ -40,6 +45,7 @@ providers:
     public: true
     c4i: false
     type: geoserverwms
+    sync_type: geoserverwms
 
   geoserver:
     url: http://${HOSTNAME}:8087/geoserver
@@ -47,6 +53,7 @@ providers:
     public: true
     c4i: false
     type: wfs
+    sync_type: wfs
 
   geoserver-web:
     url: http://${HOSTNAME}:8087/geoserver/web/
@@ -54,17 +61,20 @@ providers:
     public: true
     c4i: false
     type: access
+    sync_type: access
 
   geoserver-api:
-    url: http://${HOSTNAME}:8087/geoserver/rest
+    url: http://colibri.crim.ca:8087/geoserver/rest
     title: geoserver-api
     public: true
     c4i: false
     type: api
+    sync_type: geoserver-api
 
   project-api:
-    url: http://${HOSTNAME}:3005/api
+    url: http://hirondelle.crim.ca:3005/api
     title: project-api
     public: true
     c4i: false
     type: api
+    sync_type: project-api
diff --git a/setup.py b/setup.py
index e48d9810d..cf6b04f12 100644
--- a/setup.py
+++ b/setup.py
@@ -97,7 +97,7 @@
     # -- script entry points -----------------------------------------------
     entry_points="""\
           [paste.app_factory]
-          main = magpiectl:main
+          main = magpie.magpiectl:main
           [console_scripts]
           """,
 )
diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py
index 489bfcc99..6f74365ed 100644
--- a/tests/test_magpie_api.py
+++ b/tests/test_magpie_api.py
@@ -52,7 +52,7 @@ class TestMagpieAPI_UsersAuth_Local(ti.TestMagpieAPI_UsersAuth_Interface):
     def setUpClass(cls):
         cls.app = utils.get_test_magpie_app()
 
-
+@unittest.skip
 @pytest.mark.api
 @pytest.mark.local
 @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api'))
diff --git a/tests/test_magpie_ui.py b/tests/test_magpie_ui.py
index cdef1acf5..48d2f6ec0 100644
--- a/tests/test_magpie_ui.py
+++ b/tests/test_magpie_ui.py
@@ -16,7 +16,7 @@
 # NOTE: must be imported without 'from', otherwise the interface's test cases are also executed
 import tests.interfaces as ti
 
-
+@unittest.skip
 @pytest.mark.ui
 @pytest.mark.local
 @unittest.skipUnless(runner.MAGPIE_TEST_UI, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('ui'))
@@ -40,7 +40,7 @@ def setUpClass(cls):
         cls.test_service_type = 'wps'
         cls.test_service_name = 'flyingpigeon'
 
-
+@unittest.skip
 @pytest.mark.ui
 @pytest.mark.local
 @unittest.skipUnless(runner.MAGPIE_TEST_UI, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('ui'))

From 0a07acdcd208da033fb707e453c30c307f249316 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Mon, 1 Oct 2018 15:14:18 -0400
Subject: [PATCH 078/124] [api change] add service_sync_type to services

---
 magpie/api/api_rest_schemas.py                   | 10 ++++++++++
 magpie/api/management/service/service_formats.py |  1 +
 tests/interfaces.py                              |  4 ++++
 3 files changed, 15 insertions(+)

diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py
index ea1c1a5e2..525fe636f 100644
--- a/magpie/api/api_rest_schemas.py
+++ b/magpie/api/api_rest_schemas.py
@@ -455,6 +455,11 @@ class ServiceBodySchema(colander.MappingSchema):
         description="Type of the service",
         example="thredds"
     )
+    service_sync_type = colander.SchemaNode(
+        colander.String(),
+        description="Type of resource synchronization implementation.",
+        example="thredds"
+    )
     public_url = colander.SchemaNode(
         colander.String(),
         description="Proxy URL available for public access with permissions",
@@ -857,6 +862,11 @@ class Services_POST_BodySchema(colander.MappingSchema):
         description="Type of the service to create",
         example="wps"
     )
+    service_sync_type = colander.SchemaNode(
+        colander.String(),
+        description="Type of the service to create",
+        example="wps"
+    )
     service_url = colander.SchemaNode(
         colander.String(),
         description="Private URL of the service to create",
diff --git a/magpie/api/management/service/service_formats.py b/magpie/api/management/service/service_formats.py
index 2a0b3d350..163675306 100644
--- a/magpie/api/management/service/service_formats.py
+++ b/magpie/api/management/service/service_formats.py
@@ -13,6 +13,7 @@ def fmt_svc(svc, perms):
             u'service_url': str(svc.url),
             u'service_name': str(svc.resource_name),
             u'service_type': str(svc.type),
+            u'service_sync_type': str(svc.sync_type),
             u'resource_id': svc.resource_id,
             u'permission_names': sorted(service_type_dict[svc.type].permission_names if perms is None else perms)
         }
diff --git a/tests/interfaces.py b/tests/interfaces.py
index 7feb5bad0..72e5a6556 100644
--- a/tests/interfaces.py
+++ b/tests/interfaces.py
@@ -248,6 +248,7 @@ def test_GetUserInheritedResources(self):
                 utils.check_val_is_in('resource_id', svc_dict)
                 utils.check_val_is_in('service_name', svc_dict)
                 utils.check_val_is_in('service_type', svc_dict)
+                utils.check_val_is_in('service_sync_type', svc_dict)
                 utils.check_val_is_in('service_url', svc_dict)
                 utils.check_val_is_in('public_url', svc_dict)
                 utils.check_val_is_in('permission_names', svc_dict)
@@ -256,6 +257,7 @@ def test_GetUserInheritedResources(self):
                 utils.check_val_type(svc_dict['service_name'], six.string_types)
                 utils.check_val_type(svc_dict['service_url'], six.string_types)
                 utils.check_val_type(svc_dict['service_type'], six.string_types)
+                utils.check_val_type(svc_dict['service_sync_type'], six.string_types)
                 utils.check_val_type(svc_dict['public_url'], six.string_types)
                 utils.check_val_type(svc_dict['permission_names'], list)
                 utils.check_val_type(svc_dict['resources'], dict)
@@ -381,6 +383,7 @@ def test_GetServiceResources(self):
         utils.check_val_is_in('resource_id', svc_dict)
         utils.check_val_is_in('service_name', svc_dict)
         utils.check_val_is_in('service_type', svc_dict)
+        utils.check_val_is_in('service_sync_type', svc_dict)
         utils.check_val_is_in('service_url', svc_dict)
         utils.check_val_is_in('public_url', svc_dict)
         utils.check_val_is_in('permission_names', svc_dict)
@@ -389,6 +392,7 @@ def test_GetServiceResources(self):
         utils.check_val_type(svc_dict['service_name'], six.string_types)
         utils.check_val_type(svc_dict['service_url'], six.string_types)
         utils.check_val_type(svc_dict['service_type'], six.string_types)
+        utils.check_val_type(svc_dict['service_sync_type'], six.string_types)
         utils.check_val_type(svc_dict['public_url'], six.string_types)
         utils.check_val_type(svc_dict['permission_names'], list)
         utils.check_resource_children(svc_dict['resources'], svc_dict['resource_id'], svc_dict['resource_id'])

From 19ff48f0609858f1e8e2cdab349baf3167ad34e1 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Mon, 1 Oct 2018 15:16:33 -0400
Subject: [PATCH 079/124] edit user and edit group views handle multiple
 services of same type

---
 magpie/helpers/sync_resources.py |  8 ++--
 magpie/helpers/sync_services.py  |  2 +-
 magpie/ui/management/views.py    | 71 +++++++++++++++++++-------------
 3 files changed, 49 insertions(+), 32 deletions(-)

diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py
index c1106f58b..d89bfb517 100644
--- a/magpie/helpers/sync_resources.py
+++ b/magpie/helpers/sync_resources.py
@@ -36,12 +36,12 @@
     sync_service_class(name, url)
 
 
-def merge_local_and_remote_resources(resources_local, service_type, service_id, session):
+def merge_local_and_remote_resources(resources_local, service_sync_type, service_id, session):
     """Main function to sync resources with remote server"""
     if not get_last_sync(service_id, session):
         return resources_local
     remote_resources = _query_remote_resources_in_database(service_id, session=session)
-    max_depth = _get_max_depth(service_type)
+    max_depth = _get_max_depth(service_sync_type)
     merged_resources = _merge_resources(resources_local, remote_resources, max_depth)
     _sort_resources(merged_resources)
     return merged_resources
@@ -295,9 +295,11 @@ def fetch_all_services_by_type(service_type, session):
 def fetch_single_service(service, session):
     """
     Get remote resources for a single service.
-    :param service: (models.Service)
+    :param service: (models.Service) or service_id
     :param session:
     """
+    if isinstance(service, int):
+        service = session.query(models.Service).filter_by(resource_id=service).first()
     LOGGER.info("Requesting remote resources")
     remote_resources = _get_remote_resources(service)
     service_id = service.resource_id
diff --git a/magpie/helpers/sync_services.py b/magpie/helpers/sync_services.py
index 4ddd29885..8e2b89a81 100644
--- a/magpie/helpers/sync_services.py
+++ b/magpie/helpers/sync_services.py
@@ -80,7 +80,7 @@ def max_depth(self):
     def get_resources(self):
         # Only workspaces are fetched for now
         resource_type = "route"
-        projects_url = "/".join([self.url, "api", "Projects"])
+        projects_url = "/".join([self.url, "Projects"])
         resp = requests.get(projects_url)
         resp.raise_for_status()
 
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 073b8189d..adffc8083 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -282,7 +282,8 @@ def edit_user(self):
             elif u'edit_permissions' in self.request.POST:
                 if not res_id or res_id == 'None':
                     remote_id = int(self.request.POST.get('remote_id'))
-                    res_id = self.add_remote_resource(cur_svc_type, user_name, remote_id, is_user=True)
+                    services_names = [s['service_name'] for s in services.values()]
+                    res_id = self.add_remote_resource(cur_svc_type, services_names, user_name, remote_id, is_user=True)
                 self.edit_user_or_group_resource_permissions(user_name, res_id, is_user=True)
             elif u'edit_group_membership' in self.request.POST:
                 is_edit_group_membership = True
@@ -303,11 +304,14 @@ def edit_user(self):
                 user_info[u'email'] = self.request.POST.get(u'new_user_email')
                 is_save_user_info = True
             elif u'force_sync' in self.request.POST:
-                try:
-                    sync_resources.fetch_all_services_by_type(cur_svc_type, session=session)
-                except Exception as e:
-                    error_message = "There was an error when trying to get remote resources. "
-                    error_message += "({})".format(repr(e))
+                errors = []
+                for service_info in services.values():
+                    try:
+                        sync_resources.fetch_single_service(service_info['resource_id'], session)
+                    except Exception:
+                        errors.append(service_info['service_name'])
+                if errors:
+                    error_message += self.make_sync_error_message(errors)
             elif u'clean_all' in self.request.POST:
                 ids_to_clean = self.request.POST.get('ids_to_clean').split(";")
                 for id_ in ids_to_clean:
@@ -346,16 +350,19 @@ def edit_user(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        info = self.get_remote_resources_info(cur_svc_type, res_perms, services, session)
+        sync_types = [s["service_sync_type"] for s in services.values()]
+        sync_implemented = any(s in sync_resources.SYNC_SERVICES_TYPES for s in sync_types)
+
+        info = self.get_remote_resources_info(res_perms, services, session)
         res_perms, ids_to_clean, last_sync_humanized, out_of_sync = info
 
         if out_of_sync:
-            error_message = self.make_out_of_sync_message(out_of_sync)
+            error_message = self.make_sync_error_message(out_of_sync)
 
         user_info[u'error_message'] = error_message
         user_info[u'ids_to_clean'] = ";".join(ids_to_clean)
         user_info[u'last_sync'] = last_sync_humanized
-        user_info[u'sync_implemented'] = cur_svc_type in sync_resources.SYNC_SERVICES_TYPES
+        user_info[u'sync_implemented'] = sync_implemented
         user_info[u'out_of_sync'] = out_of_sync
         user_info[u'cur_svc_type'] = cur_svc_type
         user_info[u'svc_types'] = svc_types
@@ -509,7 +516,7 @@ def edit_group(self):
         cur_svc_type = self.request.matchdict['cur_svc_type']
         group_info = {u'edit_mode': u'no_edit', u'group_name': group_name, u'cur_svc_type': cur_svc_type}
 
-        error_message = None
+        error_message = ""
 
         # Todo:
         # Until the api is modified to make it possible to request from the RemoteResource table,
@@ -545,16 +552,21 @@ def edit_group(self):
             elif u'edit_permissions' in self.request.POST:
                 if not res_id or res_id == 'None':
                     remote_id = int(self.request.POST.get('remote_id'))
-                    res_id = self.add_remote_resource(cur_svc_type, group_name, remote_id, is_user=False)
+                    services_names = [s['service_name'] for s in services.values()]
+                    res_id = self.add_remote_resource(cur_svc_type, services_names, group_name, remote_id, is_user=False)
                 self.edit_user_or_group_resource_permissions(group_name, res_id, is_user=False)
             elif u'member' in self.request.POST:
                 self.edit_group_users(group_name)
             elif u'force_sync' in self.request.POST:
-                try:
-                    sync_resources.fetch_all_services_by_type(cur_svc_type, session=session)
-                except Exception as e:
-                    error_message = "There was an error when trying to get remote resources. "
-                    error_message += "({})".format(repr(e))
+                errors = []
+                for service_info in services.values():
+                    try:
+                        sync_resources.fetch_single_service(service_info['resource_id'], session)
+                    except Exception:
+                        errors.append(service_info['service_name'])
+                if errors:
+                    error_message += self.make_sync_error_message(errors)
+
             elif u'clean_all' in self.request.POST:
                 ids_to_clean = self.request.POST.get('ids_to_clean').split(";")
                 for id_ in ids_to_clean:
@@ -569,16 +581,19 @@ def edit_group(self):
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
-        info = self.get_remote_resources_info(cur_svc_type, res_perms, services, session)
+        sync_types = [s["service_sync_type"] for s in services.values()]
+        sync_implemented = any(s in sync_resources.SYNC_SERVICES_TYPES for s in sync_types)
+
+        info = self.get_remote_resources_info(res_perms, services, session)
         res_perms, ids_to_clean, last_sync_humanized, out_of_sync = info
 
         if out_of_sync:
-            error_message = self.make_out_of_sync_message(out_of_sync)
+            error_message = self.make_sync_error_message(out_of_sync)
 
         group_info[u'error_message'] = error_message
         group_info[u'ids_to_clean'] = ";".join(ids_to_clean)
         group_info[u'last_sync'] = last_sync_humanized
-        group_info[u'sync_implemented'] = cur_svc_type in sync_resources.SYNC_SERVICES_TYPES
+        group_info[u'sync_implemented'] = sync_implemented
         group_info[u'out_of_sync'] = out_of_sync
         group_info[u'group_name'] = group_name
         group_info[u'cur_svc_type'] = cur_svc_type
@@ -590,13 +605,13 @@ def edit_group(self):
         group_info[u'permissions'] = res_perm_names
         return add_template_data(self.request, data=group_info)
 
-    def make_out_of_sync_message(self, out_of_sync):
-        this = "this service" if len(out_of_sync) == 1 else "these services"
+    def make_sync_error_message(self, service_names):
+        this = "this service" if len(service_names) == 1 else "these services"
         error_message = ("There seems to be an issue synchronizing resources from "
-                         "{}: {}".format(this, ",".join(out_of_sync)))
+                         "{}: {}".format(this, ", ".join(service_names)))
         return error_message
 
-    def get_remote_resources_info(self, cur_svc_type, res_perms, services, session):
+    def get_remote_resources_info(self, res_perms, services, session):
         last_sync_humanized = "Never"
         ids_to_clean, out_of_sync = [], []
         now = datetime.datetime.now()
@@ -607,7 +622,7 @@ def get_remote_resources_info(self, cur_svc_type, res_perms, services, session):
         if any(last_sync_datetimes):
             last_sync_datetime = min(filter(bool, last_sync_datetimes))
             last_sync_humanized = humanize.naturaltime(now - last_sync_datetime)
-            res_perms = self.merge_remote_resources(cur_svc_type, res_perms, services, session)
+            res_perms = self.merge_remote_resources(res_perms, services, session)
 
         for last_sync, service_name in zip(last_sync_datetimes, services):
             if last_sync:
@@ -616,12 +631,12 @@ def get_remote_resources_info(self, cur_svc_type, res_perms, services, session):
                     out_of_sync.append(service_name)
         return res_perms, ids_to_clean, last_sync_humanized, out_of_sync
 
-    def merge_remote_resources(self, cur_svc_type, res_perms, services, session):
+    def merge_remote_resources(self, res_perms, services, session):
         merged_resources = {}
         for service_name, service_values in services.items():
             service_id = service_values["resource_id"]
             merge = sync_resources.merge_local_and_remote_resources
-            resources_for_service = merge(res_perms, cur_svc_type, service_id, session)
+            resources_for_service = merge(res_perms, service_values["service_sync_type"], service_id, session)
             merged_resources[service_name] = resources_for_service[service_name]
         return merged_resources
 
@@ -646,10 +661,10 @@ def get_ids_to_clean(self, resources):
             ids += self.get_ids_to_clean(values['children'])
         return ids
 
-    def add_remote_resource(self, service_type, user_or_group, remote_id, is_user=False):
+    def add_remote_resource(self, service_type, services_names, user_or_group, remote_id, is_user=False):
         try:
             res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(user_or_group,
-                                                                                          services=[service_type],
+                                                                                          services=services_names,
                                                                                           service_type=service_type,
                                                                                           is_user=is_user)
         except Exception as e:

From f881f44f84c5ff1aaf0fdaa6f4efa6436ee93843 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Mon, 1 Oct 2018 15:17:37 -0400
Subject: [PATCH 080/124] fix register_services function to be nice with
 migrations and ...

write svc_sync_type to db
---
 magpie/register.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/magpie/register.py b/magpie/register.py
index d789c7f93..fd05b64d4 100644
--- a/magpie/register.py
+++ b/magpie/register.py
@@ -352,14 +352,14 @@ def magpie_register_services(services_dict, push_to_phoenix, user, password, pro
 
 def magpie_register_services_with_db_session(services_dict, db_session, push_to_phoenix=False,
                                              force_update=False, update_getcapabilities_permissions=False):
-    existing_services = models.Service.all(db_session=db_session)
-    existing_services_names = [svc.resource_name for svc in existing_services]
+    existing_services_names = [n[0] for n in db_session.query(models.Service.resource_name)]
     magpie_anonymous_user = get_constant('MAGPIE_ANONYMOUS_USER')
     anonymous_user = models.User.by_user_name(magpie_anonymous_user, db_session=db_session)
 
-    for svc_name in services_dict:
-        svc_new_url = os.path.expandvars(services_dict[svc_name]['url'])
-        svc_type = services_dict[svc_name]['type']
+    for svc_name, svc_values in services_dict.items():
+        svc_new_url = os.path.expandvars(svc_values['url'])
+        svc_type = svc_values['type']
+        svc_sync_type = svc_values['sync_type']
         if force_update and svc_name in existing_services_names:
             svc = models.Service.by_service_name(svc_name, db_session=db_session)
             if svc.url == svc_new_url:
@@ -372,13 +372,17 @@ def magpie_register_services_with_db_session(services_dict, db_session, push_to_
             print_log("Skipping service [{svc}] (conflict)" .format(svc=svc_name))
         else:
             print_log("Adding service [{svc}]".format(svc=svc_name))
-            svc = models.Service(resource_name=svc_name, resource_type=u'service', url=svc_new_url, type=svc_type)
+            svc = models.Service(resource_name=svc_name,
+                                 resource_type=u'service',
+                                 url=svc_new_url,
+                                 type=svc_type,
+                                 sync_type=svc_sync_type)
             db_session.add(svc)
 
         if update_getcapabilities_permissions and anonymous_user is None:
             print_log("Cannot update 'getcapabilities' permission of non existing anonymous user", level=logging.WARN)
         elif update_getcapabilities_permissions and 'getcapabilities' in service_type_dict[svc_type].permission_names:
-            svc = models.Service.by_service_name(svc_name, db_session=db_session)
+            svc = db_session.query(models.Service.resource_id).filter_by(resource_name=svc_name).first()
             svc_perm_getcapabilities = models.UserResourcePermissionService.by_resource_user_and_perm(
                 user_id=anonymous_user.id, perm_name='getcapabilities',
                 resource_id=svc.resource_id, db_session=db_session

From f60344a05f960374d69350c2461e035936ae9bfe Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Mon, 1 Oct 2018 16:40:27 -0400
Subject: [PATCH 081/124] Revert: these files shouldn't have been commited

---
 providers.cfg            | 16 +++-------------
 setup.py                 |  2 +-
 tests/test_magpie_api.py |  2 +-
 tests/test_magpie_ui.py  |  4 ++--
 4 files changed, 7 insertions(+), 17 deletions(-)

diff --git a/providers.cfg b/providers.cfg
index 56b54cd9e..fc61386b1 100644
--- a/providers.cfg
+++ b/providers.cfg
@@ -5,7 +5,6 @@ providers:
     public: true
     c4i: false
     type: wps
-    sync_type: wps
 
   malleefowl:
     url: http://${HOSTNAME}:8091/wps
@@ -13,7 +12,6 @@ providers:
     public: true
     c4i: false
     type: wps
-    sync_type: wps
 
   lb_flyingpigeon:
     url: http://${HOSTNAME}:58093/wps
@@ -21,15 +19,13 @@ providers:
     public: true
     c4i: false
     type: wps
-    sync_type: wps
 
   thredds:
-    url: http://colibri.crim.ca:8083/thredds
+    url: http://${HOSTNAME}:8083/thredds
     title: Thredds
     public: true
     c4i: false
     type: thredds
-    sync_type: thredds
 
   ncWMS2:
     url: http://${HOSTNAME}:8080/ncWMS2
@@ -37,7 +33,6 @@ providers:
     public: true
     c4i: false
     type: ncwms
-    sync_type: ncwms
 
   geoserverwms:
     url: http://${HOSTNAME}:8087/geoserver
@@ -45,7 +40,6 @@ providers:
     public: true
     c4i: false
     type: geoserverwms
-    sync_type: geoserverwms
 
   geoserver:
     url: http://${HOSTNAME}:8087/geoserver
@@ -53,7 +47,6 @@ providers:
     public: true
     c4i: false
     type: wfs
-    sync_type: wfs
 
   geoserver-web:
     url: http://${HOSTNAME}:8087/geoserver/web/
@@ -61,20 +54,17 @@ providers:
     public: true
     c4i: false
     type: access
-    sync_type: access
 
   geoserver-api:
-    url: http://colibri.crim.ca:8087/geoserver/rest
+    url: http://${HOSTNAME}:8087/geoserver/rest
     title: geoserver-api
     public: true
     c4i: false
     type: api
-    sync_type: geoserver-api
 
   project-api:
-    url: http://hirondelle.crim.ca:3005/api
+    url: http://${HOSTNAME}:3005/api
     title: project-api
     public: true
     c4i: false
     type: api
-    sync_type: project-api
diff --git a/setup.py b/setup.py
index cf6b04f12..e48d9810d 100644
--- a/setup.py
+++ b/setup.py
@@ -97,7 +97,7 @@
     # -- script entry points -----------------------------------------------
     entry_points="""\
           [paste.app_factory]
-          main = magpie.magpiectl:main
+          main = magpiectl:main
           [console_scripts]
           """,
 )
diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py
index 6f74365ed..489bfcc99 100644
--- a/tests/test_magpie_api.py
+++ b/tests/test_magpie_api.py
@@ -52,7 +52,7 @@ class TestMagpieAPI_UsersAuth_Local(ti.TestMagpieAPI_UsersAuth_Interface):
     def setUpClass(cls):
         cls.app = utils.get_test_magpie_app()
 
-@unittest.skip
+
 @pytest.mark.api
 @pytest.mark.local
 @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api'))
diff --git a/tests/test_magpie_ui.py b/tests/test_magpie_ui.py
index 48d2f6ec0..cdef1acf5 100644
--- a/tests/test_magpie_ui.py
+++ b/tests/test_magpie_ui.py
@@ -16,7 +16,7 @@
 # NOTE: must be imported without 'from', otherwise the interface's test cases are also executed
 import tests.interfaces as ti
 
-@unittest.skip
+
 @pytest.mark.ui
 @pytest.mark.local
 @unittest.skipUnless(runner.MAGPIE_TEST_UI, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('ui'))
@@ -40,7 +40,7 @@ def setUpClass(cls):
         cls.test_service_type = 'wps'
         cls.test_service_name = 'flyingpigeon'
 
-@unittest.skip
+
 @pytest.mark.ui
 @pytest.mark.local
 @unittest.skipUnless(runner.MAGPIE_TEST_UI, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('ui'))

From 7e5bc28a1f9e551432f7bf8f309c86ee2353e75c Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Mon, 1 Oct 2018 16:41:32 -0400
Subject: [PATCH 082/124] add sync_type to providers

---
 providers.cfg | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/providers.cfg b/providers.cfg
index fc61386b1..95c96dfa0 100644
--- a/providers.cfg
+++ b/providers.cfg
@@ -5,6 +5,7 @@ providers:
     public: true
     c4i: false
     type: wps
+    sync_type: wps
 
   malleefowl:
     url: http://${HOSTNAME}:8091/wps
@@ -12,6 +13,7 @@ providers:
     public: true
     c4i: false
     type: wps
+    sync_type: wps
 
   lb_flyingpigeon:
     url: http://${HOSTNAME}:58093/wps
@@ -19,6 +21,7 @@ providers:
     public: true
     c4i: false
     type: wps
+    sync_type: wps
 
   thredds:
     url: http://${HOSTNAME}:8083/thredds
@@ -26,6 +29,7 @@ providers:
     public: true
     c4i: false
     type: thredds
+    sync_type: thredds
 
   ncWMS2:
     url: http://${HOSTNAME}:8080/ncWMS2
@@ -33,6 +37,7 @@ providers:
     public: true
     c4i: false
     type: ncwms
+    sync_type: ncwms
 
   geoserverwms:
     url: http://${HOSTNAME}:8087/geoserver
@@ -40,6 +45,7 @@ providers:
     public: true
     c4i: false
     type: geoserverwms
+    sync_type: geoserverwms
 
   geoserver:
     url: http://${HOSTNAME}:8087/geoserver
@@ -47,6 +53,7 @@ providers:
     public: true
     c4i: false
     type: wfs
+    sync_type: wfs
 
   geoserver-web:
     url: http://${HOSTNAME}:8087/geoserver/web/
@@ -54,6 +61,7 @@ providers:
     public: true
     c4i: false
     type: access
+    sync_type: access
 
   geoserver-api:
     url: http://${HOSTNAME}:8087/geoserver/rest
@@ -61,6 +69,7 @@ providers:
     public: true
     c4i: false
     type: api
+    sync_type: geoserver-api
 
   project-api:
     url: http://${HOSTNAME}:3005/api
@@ -68,3 +77,4 @@ providers:
     public: true
     c4i: false
     type: api
+    sync_type: project-api

From 8023f4c7b4dbab916295266582c6c3e64f6a732d Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Mon, 1 Oct 2018 17:02:22 -0400
Subject: [PATCH 083/124] don't crash when sync_type is absent from
 providers.cfg

---
 magpie/register.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/magpie/register.py b/magpie/register.py
index fd05b64d4..58ab68c1c 100644
--- a/magpie/register.py
+++ b/magpie/register.py
@@ -359,7 +359,8 @@ def magpie_register_services_with_db_session(services_dict, db_session, push_to_
     for svc_name, svc_values in services_dict.items():
         svc_new_url = os.path.expandvars(svc_values['url'])
         svc_type = svc_values['type']
-        svc_sync_type = svc_values['sync_type']
+
+        svc_sync_type = svc_values.get('sync_type', svc_values['title'])
         if force_update and svc_name in existing_services_names:
             svc = models.Service.by_service_name(svc_name, db_session=db_session)
             if svc.url == svc_new_url:

From 5161e8e4eeed9d15476023bb653eaa05fef34935 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Mon, 1 Oct 2018 16:45:55 -0400
Subject: [PATCH 084/124] read/write-match API route permissions

---
 magpie/adapter/magpieservice.py          |  2 +-
 magpie/api/management/user/user_utils.py |  9 +++++----
 magpie/api/management/user/user_views.py |  1 +
 magpie/models.py                         |  4 +++-
 magpie/services.py                       | 14 ++++++++++++++
 magpie/ui/management/views.py            | 18 +++++++++++-------
 6 files changed, 35 insertions(+), 13 deletions(-)

diff --git a/magpie/adapter/magpieservice.py b/magpie/adapter/magpieservice.py
index 8b61fcbb4..4897b6a94 100644
--- a/magpie/adapter/magpieservice.py
+++ b/magpie/adapter/magpieservice.py
@@ -48,7 +48,7 @@ def list_services(self, request=None):
         Lists all services registered in magpie.
         """
         my_services = []
-        response = requests.get('{url}/users/current/services'.format(url=self.magpie_url),
+        response = requests.get('{url}/users/current/inherited_services'.format(url=self.magpie_url),
                                 cookies=request.cookies)
         if response.status_code != 200:
             raise response.raise_for_status()
diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py
index 3250fc845..4e11a59bf 100644
--- a/magpie/api/management/user/user_utils.py
+++ b/magpie/api/management/user/user_utils.py
@@ -5,6 +5,7 @@
 from magpie.definitions.ziggurat_definitions import *
 from magpie.services import service_type_dict
 from magpie import models
+from collections import OrderedDict
 
 
 def create_user(user_name, password, email, group_name, db_session):
@@ -79,7 +80,7 @@ def get_user_resource_permissions(user, resource, db_session, inherited_permissi
         if not inherited_permissions:
             res_perm_tuple_list = filter_user_permission(res_perm_tuple_list, user)
         permission_names = [permission.perm_name for permission in res_perm_tuple_list]
-    return list(set(permission_names))  # remove any duplicates that could be incorporated by multiple groups
+    return sorted(set(permission_names))  # remove any duplicates that could be incorporated by multiple groups
 
 
 def get_user_service_permissions(user, service, db_session, inherited_permissions=True):
@@ -90,7 +91,7 @@ def get_user_service_permissions(user, service, db_session, inherited_permission
         if not inherited_permissions:
             svc_perm_tuple_list = filter_user_permission(svc_perm_tuple_list, user)
         permission_names = [permission.perm_name for permission in svc_perm_tuple_list]
-    return list(set(permission_names))  # remove any duplicates that could be incorporated by multiple groups
+    return sorted(set(permission_names))  # remove any duplicates that could be incorporated by multiple groups
 
 
 def get_user_resources_permissions_dict(user, db_session, resource_types=None,
@@ -101,7 +102,7 @@ def get_user_resources_permissions_dict(user, db_session, resource_types=None,
                                                              resource_types=resource_types, db_session=db_session)
     if not inherited_permissions:
         res_perm_tuple_list = filter_user_permission(res_perm_tuple_list, user)
-    resources_permissions_dict = {}
+    resources_permissions_dict = OrderedDict()
     for res_perm in res_perm_tuple_list:
         if res_perm.resource.resource_id not in resources_permissions_dict:
             resources_permissions_dict[res_perm.resource.resource_id] = [res_perm.perm_name]
@@ -110,7 +111,7 @@ def get_user_resources_permissions_dict(user, db_session, resource_types=None,
 
     # remove any duplicates that could be incorporated by multiple groups
     for res_id in resources_permissions_dict:
-        resources_permissions_dict[res_id] = list(set(resources_permissions_dict[res_id]))
+        resources_permissions_dict[res_id] = sorted(set(resources_permissions_dict[res_id]))
 
     return resources_permissions_dict
 
diff --git a/magpie/api/management/user/user_views.py b/magpie/api/management/user/user_views.py
index a992d9f64..f68d18c41 100644
--- a/magpie/api/management/user/user_views.py
+++ b/magpie/api/management/user/user_views.py
@@ -5,6 +5,7 @@
 from magpie.api.management.user.user_formats import *
 from magpie.api.management.user.user_utils import *
 from magpie.api.management.group.group_utils import *
+from magpie.api.management.service.service_utils import get_services_by_type
 from magpie.api.management.service.service_formats import format_service, format_service_resources
 
 
diff --git a/magpie/models.py b/magpie/models.py
index cd2b5ca15..06de24c73 100644
--- a/magpie/models.py
+++ b/magpie/models.py
@@ -179,7 +179,9 @@ class Workspace(Resource):
 class Route(Resource):
     __mapper_args__ = {u'polymorphic_identity': u'route'}
     permission_names = [u'read',
-                        u'write']
+                        u'write',
+                        u'read-match',
+                        u'write-match']
     resource_type_name = u'route'
 
 
diff --git a/magpie/services.py b/magpie/services.py
index 0ff3ecad1..78008910c 100644
--- a/magpie/services.py
+++ b/magpie/services.py
@@ -263,6 +263,7 @@ def __acl__(self):
     def route_acl(self, sub_api_route=None):
         self.expand_acl(self.service, self.request.user)
 
+        match_index = 0
         route_parts = self.request.path.split('/')
         route_api_base = self.service.resource_name if sub_api_route is None else sub_api_route
 
@@ -272,15 +273,28 @@ def route_acl(self, sub_api_route=None):
             if len(route_parts) - 1 > api_idx:
                 route_parts = route_parts[api_idx + 1::]
                 route_child = self.service
+
+                # process read/write inheritance permission access
                 while route_child and route_parts:
                     part_name = route_parts.pop(0)
                     route_res_id = route_child.resource_id
                     route_child = models.find_children_by_name(part_name, parent_id=route_res_id,
                                                                db_session=self.request.db)
+                    match_index = len(self.acl)
                     self.expand_acl(route_child, self.request.user)
+
+        # process read/write-match specific permission access
+        # (convert exact route '-match' to read/write only if matching last item's permissions)
+        for i in range(match_index, len(self.acl)):
+            if self.acl[i][2] == 'read-match':
+                self.acl[i] = (self.acl[i][0], self.acl[i][1], 'read')
+            if self.acl[i][2] == 'write-match':
+                self.acl[i] = (self.acl[i][0], self.acl[i][1], 'write')
+
         return self.acl
 
     def permission_requested(self):
+        # only read/write are used for 'real' access control, '-match' permissions must be updated accordingly
         if self.request.method.upper() in ['GET', 'HEAD']:
             return u'read'
         return u'write'
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index adffc8083..878f4b13b 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -483,17 +483,20 @@ def get_user_or_group_resources_permissions_dict(self, user_or_group_name, servi
         svc_perm_url = '{url}/services/types/{svc_type}'.format(url=self.magpie_url, svc_type=service_type)
         resp_svc_type = check_response(requests.get(svc_perm_url, cookies=self.request.cookies))
         resp_available_svc_types = resp_svc_type.json()['services'][service_type]
+
+        # remove possible duplicate permissions from different services
         resources_permission_names = set()
         for svc in resp_available_svc_types:
             resources_permission_names.update(set(resp_available_svc_types[svc]['permission_names']))
-        resources_permission_names = list(resources_permission_names)
+        # inverse sort so that displayed permissions are sorted, since added from right to left in tree view
+        resources_permission_names = sorted(resources_permission_names, reverse=True)
 
-        resources = {}
-        for service in services:
+        resources = OrderedDict()
+        for service in sorted(services):
             if not service:
                 continue
 
-            permission = {}
+            permission = OrderedDict()
             try:
                 raw_perms = resp_group_perms_json['resources'][service_type][service]
                 permission[raw_perms['resource_id']] = raw_perms['permission_names']
@@ -505,9 +508,10 @@ def get_user_or_group_resources_permissions_dict(self, user_or_group_name, servi
                                                          .format(url=self.magpie_url, svc=service),
                                                          cookies=self.request.cookies))
             raw_resources = resp_resources.json()[service]
-            resources[service] = dict(id=raw_resources['resource_id'],
-                                      permission_names=self.default_get(permission, raw_resources['resource_id'], []),
-                                      children=self.resource_tree_parser(raw_resources['resources'], permission))
+            resources[service] = OrderedDict(
+                id=raw_resources['resource_id'],
+                permission_names=self.default_get(permission, raw_resources['resource_id'], []),
+                children=self.resource_tree_parser(raw_resources['resources'], permission))
         return resources_permission_names, resources
 
     @view_config(route_name='edit_group', renderer='templates/edit_group.mako')

From b277171a640cc56b0eb0e2bdf03d63a08ad37609 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Mon, 1 Oct 2018 20:01:25 -0400
Subject: [PATCH 085/124] use query strings inheritance, remove
 inherited_services

---
 AUTHORS.rst                              |   5 +-
 HISTORY.rst                              |  10 +
 magpie/__meta__.py                       |   2 +-
 magpie/adapter/__init__.py               |   2 +-
 magpie/adapter/magpieservice.py          |   5 +-
 magpie/api/api_requests.py               |  17 ++
 magpie/api/api_rest_schemas.py           | 243 +++++++++++++----------
 magpie/api/management/user/__init__.py   |   2 -
 magpie/api/management/user/user_utils.py |  60 +++++-
 magpie/api/management/user/user_views.py |  77 +++----
 magpie/magpiectl.py                      |   1 -
 requirements.txt                         |   1 +
 12 files changed, 260 insertions(+), 165 deletions(-)

diff --git a/AUTHORS.rst b/AUTHORS.rst
index bc6dfc048..07096adf9 100644
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@ -4,9 +4,12 @@ Credits
 Development Lead
 ----------------
 
-* Francois-Xavier <francois-xavier.derue@crim.ca>
+* Francis Charette Migneault <francis.charette-migneault@crim.ca>
 
 Contributors
 ------------
 
+* David Byrns <David.Byrns@crim.ca>
+* David Caron <david.caron@crim.ca>
 * Francis Charette Migneault <francis.charette-migneault@crim.ca>
+* Francois-Xavier Derue <francois-xavier.derue@crim.ca>
diff --git a/HISTORY.rst b/HISTORY.rst
index 0b964e4f9..36870a596 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,6 +3,15 @@
 History
 =======
 
+0.7.x
+---------------------
+
+`Magpie REST API latest documentation`_
+
+* add service resource auto-sync feature
+* return user/group services if any sub-resource has permissions
+* [WIP] add inherited resource permission with querystring (deprecate `inherited_<>` routes)
+
 0.6.x
 ---------------------
 
@@ -97,3 +106,4 @@ History
 .. _Magpie REST API 0.4.x documentation: magpie_api_0.4.x_
 .. _Magpie REST API 0.5.x documentation: magpie_api_0.5.x_
 .. _Magpie REST API 0.6.x documentation: magpie_api_0.6.x_
+.. _Magpie REST API latest documentation: _magpie_api_latest
diff --git a/magpie/__meta__.py b/magpie/__meta__.py
index b50c30ad7..786edc8af 100644
--- a/magpie/__meta__.py
+++ b/magpie/__meta__.py
@@ -2,7 +2,7 @@
 General meta information on the magpie package.
 """
 
-__version__ = '0.6.5-dev'
+__version__ = '0.7.0'
 __author__ = "Francois-Xavier Derue, Francis Charette-Migneault"
 __maintainer__ = "Francis Charette-Migneault"
 __email__ = 'francis.charette-migneault@crim.ca'
diff --git a/magpie/adapter/__init__.py b/magpie/adapter/__init__.py
index 505461691..0a246b05f 100644
--- a/magpie/adapter/__init__.py
+++ b/magpie/adapter/__init__.py
@@ -8,7 +8,7 @@
 from magpie.db import *
 from magpie import __meta__
 import logging
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("TWITCHER")
 
 
 class MagpieAdapter(AdapterInterface):
diff --git a/magpie/adapter/magpieservice.py b/magpie/adapter/magpieservice.py
index 4897b6a94..c68e43627 100644
--- a/magpie/adapter/magpieservice.py
+++ b/magpie/adapter/magpieservice.py
@@ -6,7 +6,7 @@
 import logging
 import requests
 import json
-LOGGER = logging.getLogger(__name__)
+LOGGER = logging.getLogger("TWITCHER")
 
 from magpie.definitions.twitcher_definitions import *
 from magpie.definitions.pyramid_definitions import ConfigurationError
@@ -48,7 +48,8 @@ def list_services(self, request=None):
         Lists all services registered in magpie.
         """
         my_services = []
-        response = requests.get('{url}/users/current/inherited_services'.format(url=self.magpie_url),
+        path = '/users/current/services?groups=inherit&resources=inherit'
+        response = requests.get('{url}{path}'.format(url=self.magpie_url, path=path),
                                 cookies=request.cookies)
         if response.status_code != 200:
             raise response.raise_for_status()
diff --git a/magpie/api/api_requests.py b/magpie/api/api_requests.py
index db93289ab..81f869049 100644
--- a/magpie/api/api_requests.py
+++ b/magpie/api/api_requests.py
@@ -152,3 +152,20 @@ def get_value_matchdict_checked(request, key):
     verify_param(val, notNone=True, notEmpty=True, httpError=HTTPUnprocessableEntity,
                  paramName=key, msgOnFail=UnprocessableEntityResponseSchema.description)
     return val
+
+
+def get_query_param(request, case_insensitive_key, default=None):
+    for p in request.params:
+        if p.lower() == case_insensitive_key:
+            value = request.params.get(p)
+            if isinstance(value, six.string_types):
+                return value.lower()
+            return value
+    return default
+
+
+def get_query_inherit_checked(request, case_insensitive_key):
+    inherit = get_query_param(request, case_insensitive_key) or 'inherit'
+    verify_param(inherit, isIn=True, paramName=case_insensitive_key, paramCompare=['inherit', 'direct'],
+                 httpError=HTTPBadRequest, msgOnFail="Invalid query parameter is not one of allowed values.")
+    return inherit == 'inherit'
diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py
index 525fe636f..ef3034a34 100644
--- a/magpie/api/api_rest_schemas.py
+++ b/magpie/api/api_rest_schemas.py
@@ -91,9 +91,6 @@ def service_api_route_info(service_api):
 UserResourceTypesAPI = Service(
     path='/users/{user_name}/resources/types/{resource_type}',
     name='UserResourceTypes')
-UserInheritedServicesAPI = Service(
-    path='/users/{user_name}/inherited_services',
-    name='UserInheritedServices')
 UserServicesAPI = Service(
     path='/users/{user_name}/services',
     name='UserServices')
@@ -142,9 +139,6 @@ def service_api_route_info(service_api):
 LoggedUserResourceTypesAPI = Service(
     path=LoggedUserBase + '/resources/types/{resource_type}',
     name='LoggedUserResourceTypes')
-LoggedUserInheritedServicesAPI = Service(
-    path=LoggedUserBase + '/inherited_services',
-    name='LoggedUserInheritedServices')
 LoggedUserServicesAPI = Service(
     path=LoggedUserBase + '/services',
     name='LoggedUserServices')
@@ -254,12 +248,21 @@ class HeaderRequestSchema(colander.MappingSchema):
     content_type.name = 'Content-Type'
 
 
-class BaseBodySchema(colander.MappingSchema):
+QueryInheritGroups = colander.SchemaNode(
+    colander.String(), default='inherit', validator=colander.OneOf(['inherit', 'direct']), missing=colander.drop,
+    description='User groups memberships inheritance to resolve service resource permissions.')
+QueryInheritResources = colander.SchemaNode(
+    colander.String(), default='inherit', validator=colander.OneOf(['inherit', 'direct']), missing=colander.drop,
+    description='Display any service that has at least one sub-resource user permission, '
+                'or only services that have user permissions directly set on them.', )
+
+
+class BaseResponseBodySchema(colander.MappingSchema):
     __code = None
     __desc = None
 
     def __init__(self, code, description):
-        super(BaseBodySchema, self).__init__()
+        super(BaseResponseBodySchema, self).__init__()
         assert isinstance(code, int)
         assert isinstance(description, six.string_types)
         self.__code = code
@@ -301,7 +304,7 @@ class ErrorVerifyParamBodySchema(colander.MappingSchema):
         missing=colander.drop)
 
 
-class ErrorRequestInfoBodySchema(BaseBodySchema):
+class ErrorRequestInfoBodySchema(BaseResponseBodySchema):
     def __init__(self, **kw):
         super(ErrorRequestInfoBodySchema, self).__init__(**kw)
         assert kw.get('code') >= 400
@@ -326,7 +329,7 @@ def __init__(self, **kw):
         super(InternalServerErrorResponseBodySchema, self).__init__(**kw)
 
 
-class UnauthorizedResponseBodySchema(BaseBodySchema):
+class UnauthorizedResponseBodySchema(BaseResponseBodySchema):
     def __init__(self, **kw):
         kw['code'] = HTTPUnauthorized.code
         super(UnauthorizedResponseBodySchema, self).__init__(**kw)
@@ -356,7 +359,7 @@ class MethodNotAllowedResponseSchema(colander.MappingSchema):
 class UnprocessableEntityResponseSchema(colander.MappingSchema):
     description = "Invalid value specified."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPUnprocessableEntity.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPUnprocessableEntity.code, description=description)
 
 
 class InternalServerErrorResponseSchema(colander.MappingSchema):
@@ -547,29 +550,29 @@ class ResourcesSchemaNode(colander.MappingSchema):
     thredds = Resource_ServiceType_thredds_SchemaNode()
 
 
-class Resources_ResponseBodySchema(BaseBodySchema):
+class Resources_ResponseBodySchema(BaseResponseBodySchema):
     resources = ResourcesSchemaNode()
 
 
 class Resource_MatchDictCheck_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Resource query by id refused by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class Resource_MatchDictCheck_NotFoundResponseSchema(colander.MappingSchema):
     description = "Resource ID not found in db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotFound.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
 class Resource_MatchDictCheck_NotAcceptableResponseSchema(colander.MappingSchema):
     description = "Resource ID is an invalid literal for `int` type."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotAcceptable.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description)
 
 
-class Resource_GET_ResponseBodySchema(BaseBodySchema):
+class Resource_GET_ResponseBodySchema(BaseResponseBodySchema):
     resource_id = Resource_ChildResourceWithChildrenContainerBodySchema()
     resource_id.name = '{resource_id}'
 
@@ -603,7 +606,7 @@ class Resource_PUT_RequestSchema(colander.MappingSchema):
     body = Resource_PUT_RequestBodySchema()
 
 
-class Resource_PUT_ResponseBodySchema(BaseBodySchema):
+class Resource_PUT_ResponseBodySchema(BaseResponseBodySchema):
     resource_id = colander.SchemaNode(
         colander.String(),
         description="Updated resource identification number."
@@ -631,7 +634,7 @@ class Resource_PUT_OkResponseSchema(colander.MappingSchema):
 class Resource_PUT_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Failed to update resource with new name."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class Resource_DELETE_RequestBodySchema(colander.MappingSchema):
@@ -651,13 +654,13 @@ class Resource_DELETE_RequestSchema(colander.MappingSchema):
 class Resource_DELETE_OkResponseSchema(colander.MappingSchema):
     description = "Delete resource successful."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPOk.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPOk.code, description=description)
 
 
 class Resource_DELETE_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Delete resource from db failed."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class Resources_GET_OkResponseSchema(colander.MappingSchema):
@@ -687,7 +690,7 @@ class Resources_POST_RequestBodySchema(colander.MappingSchema):
     body = Resources_POST_BodySchema()
 
 
-class Resource_POST_ResponseBodySchema(BaseBodySchema):
+class Resource_POST_ResponseBodySchema(BaseResponseBodySchema):
     resource_id = Resource_ChildResourceWithChildrenContainerBodySchema()
     resource_id.name = '{resource_id}'
 
@@ -701,28 +704,28 @@ class Resources_POST_CreatedResponseSchema(colander.MappingSchema):
 class Resources_POST_BadRequestResponseSchema(colander.MappingSchema):
     description = "Invalid [`resource_name`|`resource_type`|`parent_id`] specified for child resource creation."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPBadRequest.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPBadRequest.code, description=description)
 
 
 class Resources_POST_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Failed to insert new resource in service tree using parent id."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class Resources_POST_NotFoundResponseSchema(colander.MappingSchema):
     description = "Could not find specified resource parent id."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotFound.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
 class Resources_POST_ConflictResponseSchema(colander.MappingSchema):
     description = "Resource name already exists at requested tree level for creation."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPConflict.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPConflict.code, description=description)
 
 
-class ResourcePermissions_GET_ResponseBodySchema(BaseBodySchema):
+class ResourcePermissions_GET_ResponseBodySchema(BaseResponseBodySchema):
     permission_names = PermissionListSchema()
 
 
@@ -735,7 +738,7 @@ class ResourcePermissions_GET_OkResponseSchema(colander.MappingSchema):
 class ResourcePermissions_GET_NotAcceptableResponseSchema(colander.MappingSchema):
     description = "Invalid resource type to extract permissions."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotAcceptable.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description)
 
 
 class ServiceResourcesBodySchema(ServiceBodySchema):
@@ -798,7 +801,7 @@ class ServicesSchemaNode(colander.MappingSchema):
     wps = ServiceType_wps_SchemaNode(missing=colander.drop)
 
 
-class Service_FailureBodyResponseSchema(BaseBodySchema):
+class Service_FailureBodyResponseSchema(BaseResponseBodySchema):
     service_name = colander.SchemaNode(
         colander.String(),
         description="Service name extracted from path"
@@ -808,7 +811,7 @@ class Service_FailureBodyResponseSchema(BaseBodySchema):
 class Service_MatchDictCheck_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Service query by name refused by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class Service_MatchDictCheck_NotFoundResponseSchema(colander.MappingSchema):
@@ -817,7 +820,7 @@ class Service_MatchDictCheck_NotFoundResponseSchema(colander.MappingSchema):
     body = Service_FailureBodyResponseSchema(code=HTTPNotFound.code, description=description)
 
 
-class Service_GET_ResponseBodySchema(BaseBodySchema):
+class Service_GET_ResponseBodySchema(BaseResponseBodySchema):
     service_name = ServiceBodySchema()
     service_name.name = '{service_name}'
 
@@ -828,7 +831,7 @@ class Service_GET_OkResponseSchema(colander.MappingSchema):
     body = Service_GET_ResponseBodySchema(code=HTTPOk.code, description=description)
 
 
-class Services_GET_ResponseBodySchema(BaseBodySchema):
+class Services_GET_ResponseBodySchema(BaseResponseBodySchema):
     services = ServicesSchemaNode()
 
 
@@ -838,7 +841,7 @@ class Services_GET_OkResponseSchema(colander.MappingSchema):
     body = Services_GET_ResponseBodySchema(code=HTTPOk.code, description=description)
 
 
-class Services_GET_NotAcceptableResponseBodySchema(BaseBodySchema):
+class Services_GET_NotAcceptableResponseBodySchema(BaseResponseBodySchema):
     service_type = colander.SchemaNode(
         colander.String(),
         description="Name of the service type filter employed when applicable",
@@ -882,25 +885,25 @@ class Services_POST_RequestBodySchema(colander.MappingSchema):
 class Services_POST_CreatedResponseSchema(colander.MappingSchema):
     description = "Service registration to db successful."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPOk.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPOk.code, description=description)
 
 
 class Services_POST_BadRequestResponseSchema(colander.MappingSchema):
     description = "Invalid `service_type` value does not correspond to any of the existing service types."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPBadRequest.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPBadRequest.code, description=description)
 
 
 class Services_POST_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Service registration forbidden by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class Services_POST_ConflictResponseSchema(colander.MappingSchema):
     description = "Specified `service_name` value already exists."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPConflict.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPConflict.code, description=description)
 
 
 class Service_PUT_ResponseBodySchema(colander.MappingSchema):
@@ -931,7 +934,7 @@ class Service_PUT_RequestBodySchema(colander.MappingSchema):
     body = Service_PUT_ResponseBodySchema()
 
 
-class Service_SuccessBodyResponseSchema(BaseBodySchema):
+class Service_SuccessBodyResponseSchema(BaseResponseBodySchema):
     service = ServiceBodySchema()
 
 
@@ -975,7 +978,7 @@ class Service_DELETE_ForbiddenResponseSchema(colander.MappingSchema):
     body = Service_FailureBodyResponseSchema(code=HTTPForbidden.code, description=description)
 
 
-class ServicePermissions_ResponseBodySchema(BaseBodySchema):
+class ServicePermissions_ResponseBodySchema(BaseResponseBodySchema):
     permission_names = PermissionListSchema()
 
 
@@ -985,7 +988,7 @@ class ServicePermissions_GET_OkResponseSchema(colander.MappingSchema):
     body = ServicePermissions_ResponseBodySchema(code=HTTPOk.code, description=description)
 
 
-class ServicePermissions_GET_NotAcceptableResponseBodySchema(BaseBodySchema):
+class ServicePermissions_GET_NotAcceptableResponseBodySchema(BaseResponseBodySchema):
     service = ServiceBodySchema()
 
 
@@ -1011,7 +1014,7 @@ class ServicePermissions_GET_NotAcceptableResponseSchema(colander.MappingSchema)
 ServiceResource_DELETE_OkResponseSchema = Resource_DELETE_OkResponseSchema
 
 
-class ServiceResources_GET_ResponseBodySchema(BaseBodySchema):
+class ServiceResources_GET_ResponseBodySchema(BaseResponseBodySchema):
     service_name = Resource_ServiceWithChildrenResourcesContainerBodySchema()
     service_name.name = '{service_name}'
 
@@ -1022,7 +1025,7 @@ class ServiceResources_GET_OkResponseSchema(colander.MappingSchema):
     body = ServiceResources_GET_ResponseBodySchema(code=HTTPOk.code, description=description)
 
 
-class ServiceResourceTypes_GET_ResponseBodySchema(BaseBodySchema):
+class ServiceResourceTypes_GET_ResponseBodySchema(BaseResponseBodySchema):
     resource_types = ResourceTypesListSchema()
 
 
@@ -1032,7 +1035,7 @@ class ServiceResourceTypes_GET_OkResponseSchema(colander.MappingSchema):
     body = ServiceResourceTypes_GET_ResponseBodySchema(code=HTTPOk.code, description=description)
 
 
-class ServiceResourceTypes_GET_FailureBodyResponseSchema(BaseBodySchema):
+class ServiceResourceTypes_GET_FailureBodyResponseSchema(BaseResponseBodySchema):
     service_type = colander.SchemaNode(
         colander.String(),
         description="Service type retrieved from route path."
@@ -1051,7 +1054,7 @@ class ServiceResourceTypes_GET_NotFoundResponseSchema(colander.MappingSchema):
     body = ServiceResourceTypes_GET_FailureBodyResponseSchema(code=HTTPNotFound.code, description=description)
 
 
-class Users_GET_ResponseBodySchema(BaseBodySchema):
+class Users_GET_ResponseBodySchema(BaseResponseBodySchema):
     user_names = UserNamesListSchema()
 
 
@@ -1064,10 +1067,10 @@ class Users_GET_OkResponseSchema(colander.MappingSchema):
 class Users_GET_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Get users query refused by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
-class Users_CheckInfo_ResponseBodySchema(BaseBodySchema):
+class Users_CheckInfo_ResponseBodySchema(BaseResponseBodySchema):
     param = ErrorVerifyParamBodySchema()
 
 
@@ -1111,13 +1114,13 @@ class Users_CheckInfo_Login_ConflictResponseSchema(colander.MappingSchema):
 class User_Check_ForbiddenResponseSchema(colander.MappingSchema):
     description = "User check query was refused by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class User_Check_ConflictResponseSchema(colander.MappingSchema):
     description = "User name matches an already existing user name."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class User_POST_RequestBodySchema(colander.MappingSchema):
@@ -1148,7 +1151,7 @@ class Users_POST_RequestSchema(colander.MappingSchema):
     body = User_POST_RequestBodySchema()
 
 
-class Users_POST_ResponseBodySchema(BaseBodySchema):
+class Users_POST_ResponseBodySchema(BaseResponseBodySchema):
     user = UserBodySchema()
 
 
@@ -1161,13 +1164,13 @@ class Users_POST_CreatedResponseSchema(colander.MappingSchema):
 class Users_POST_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Failed to add user to db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class UserNew_POST_ForbiddenResponseSchema(colander.MappingSchema):
     description = "New user query was refused by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class User_PUT_RequestBodySchema(colander.MappingSchema):
@@ -1199,7 +1202,7 @@ class User_PUT_RequestSchema(colander.MappingSchema):
 class Users_PUT_OkResponseSchema(colander.MappingSchema):
     description = "Update user successful."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPOk.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPOk.code, description=description)
 
 
 # PUT method uses same sub-function as POST method (same responses)
@@ -1209,10 +1212,10 @@ class Users_PUT_OkResponseSchema(colander.MappingSchema):
 class User_PUT_ConflictResponseSchema(colander.MappingSchema):
     description = "New name user already exists."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPConflict.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPConflict.code, description=description)
 
 
-class User_GET_ResponseBodySchema(BaseBodySchema):
+class User_GET_ResponseBodySchema(BaseResponseBodySchema):
     user = UserBodySchema()
 
 
@@ -1225,25 +1228,25 @@ class User_GET_OkResponseSchema(colander.MappingSchema):
 class User_CheckAnonymous_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Anonymous user query refused by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class User_CheckAnonymous_NotFoundResponseSchema(colander.MappingSchema):
     description = "Anonymous user not found in db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotFound.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
 class User_GET_ForbiddenResponseSchema(colander.MappingSchema):
     description = "User name query refused by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class User_GET_NotFoundResponseSchema(colander.MappingSchema):
     description = "User name not found in db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotFound.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
 class User_DELETE_RequestSchema(colander.MappingSchema):
@@ -1254,34 +1257,34 @@ class User_DELETE_RequestSchema(colander.MappingSchema):
 class User_DELETE_OkResponseSchema(colander.MappingSchema):
     description = "Delete user successful."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class User_DELETE_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Delete user by name refused by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class UserGroup_GET_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Group query was refused by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class UserGroup_GET_NotAcceptableResponseSchema(colander.MappingSchema):
     description = "Group for new user doesn't exist."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotAcceptable.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description)
 
 
 class UserGroup_Check_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Failed to add user-group to db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
-class UserGroups_GET_ResponseBodySchema(BaseBodySchema):
+class UserGroups_GET_ResponseBodySchema(BaseResponseBodySchema):
     group_names = GroupNamesListSchema()
 
 
@@ -1309,7 +1312,7 @@ class UserGroups_POST_RequestSchema(colander.MappingSchema):
     body = UserGroups_POST_RequestBodySchema()
 
 
-class UserGroups_POST_ResponseBodySchema(BaseBodySchema):
+class UserGroups_POST_ResponseBodySchema(BaseResponseBodySchema):
     user_name = colander.SchemaNode(
         colander.String(),
         description="Name of the user in the user-group relationship",
@@ -1343,7 +1346,7 @@ class UserGroups_POST_ForbiddenResponseSchema(colander.MappingSchema):
 class UserGroups_POST_ConflictResponseSchema(colander.MappingSchema):
     description = "User already belongs to this group."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPConflict.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPConflict.code, description=description)
 
 
 class UserGroup_DELETE_RequestSchema(colander.MappingSchema):
@@ -1354,16 +1357,16 @@ class UserGroup_DELETE_RequestSchema(colander.MappingSchema):
 class UserGroup_DELETE_OkResponseSchema(colander.MappingSchema):
     description = "Delete user-group successful."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPOk.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPOk.code, description=description)
 
 
 class UserGroup_DELETE_NotFoundResponseSchema(colander.MappingSchema):
     description = "Invalid user-group combination for delete."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotFound.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
-class UserResources_GET_ResponseBodySchema(BaseBodySchema):
+class UserResources_GET_ResponseBodySchema(BaseResponseBodySchema):
     resources = ResourcesSchemaNode()
 
 
@@ -1373,7 +1376,7 @@ class UserResources_GET_OkResponseSchema(colander.MappingSchema):
     body = UserResources_GET_ResponseBodySchema(code=HTTPOk.code, description=description)
 
 
-class UserResources_GET_NotFoundResponseBodySchema(BaseBodySchema):
+class UserResources_GET_NotFoundResponseBodySchema(BaseResponseBodySchema):
     user_name = colander.SchemaNode(colander.String(), description="User name value read from path")
     resource_types = ResourceTypesListSchema(description="Resource types searched for")
 
@@ -1384,7 +1387,7 @@ class UserResources_GET_NotFoundResponseSchema(colander.MappingSchema):
     body = UserResources_GET_NotFoundResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
-class UserResourcePermissions_GET_ResponseBodySchema(BaseBodySchema):
+class UserResourcePermissions_GET_ResponseBodySchema(BaseResponseBodySchema):
     permission_names = PermissionListSchema()
 
 
@@ -1429,7 +1432,7 @@ class UserResourcePermissions_GET_NotAcceptableResourceTypeResponseSchema(coland
 class UserResourcePermissions_GET_NotFoundResponseSchema(colander.MappingSchema):
     description = "Specified user not found to obtain resource permissions."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotFound.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
 class UserResourcePermissions_POST_RequestBodySchema(colander.MappingSchema):
@@ -1449,7 +1452,7 @@ class UserResourcePermissions_POST_RequestSchema(colander.MappingSchema):
     body = UserResourcePermissions_POST_RequestBodySchema()
 
 
-class UserResourcePermissions_POST_ResponseBodySchema(BaseBodySchema):
+class UserResourcePermissions_POST_ResponseBodySchema(BaseResponseBodySchema):
     resource_id = colander.SchemaNode(
         colander.Integer(),
         description="resource_id of the created user-resource-permission reference.")
@@ -1466,7 +1469,7 @@ class UserResourcePermissions_POST_ParamResponseBodySchema(colander.MappingSchem
     value = colander.SchemaNode(colander.String(), description="Specified parameter value.")
 
 
-class UserResourcePermissions_POST_BadResponseBodySchema(BaseBodySchema):
+class UserResourcePermissions_POST_BadResponseBodySchema(BaseResponseBodySchema):
     resource_type = colander.SchemaNode(colander.String(), description="Specified resource_type.")
     resource_name = colander.SchemaNode(colander.String(), description="Specified resource_name.")
     param = UserResourcePermissions_POST_ParamResponseBodySchema()
@@ -1508,7 +1511,7 @@ class UserResourcePermission_DELETE_RequestSchema(colander.MappingSchema):
 class UserResourcePermissions_DELETE_OkResponseSchema(colander.MappingSchema):
     description = "Delete user resource permission successful."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPOk.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPOk.code, description=description)
 
 
 class UserResourcePermissions_DELETE_NotFoundResponseSchema(colander.MappingSchema):
@@ -1517,7 +1520,7 @@ class UserResourcePermissions_DELETE_NotFoundResponseSchema(colander.MappingSche
     body = UserResourcePermissions_DELETE_BadResponseBodySchema(code=HTTPOk.code, description=description)
 
 
-class UserServiceResources_GET_ResponseBodySchema(BaseBodySchema):
+class UserServiceResources_GET_ResponseBodySchema(BaseResponseBodySchema):
     service = ServiceResourcesBodySchema()
 
 
@@ -1541,7 +1544,20 @@ class UserServicePermission_DELETE_RequestSchema(colander.MappingSchema):
     body = colander.MappingSchema(default={})
 
 
-class UserServices_GET_ResponseBodySchema(BaseBodySchema):
+class UserServices_GET_QuerySchema(colander.MappingSchema):
+    resources = QueryInheritResources
+    groups = QueryInheritGroups
+    list = colander.SchemaNode(
+        colander.Boolean(), default=False, missing=colander.drop,
+        description='Return services as a list of dicts. Default is a dict by service type, and by service name.')
+
+
+class UserServices_GET_RequestSchema(colander.MappingSchema):
+    header = HeaderRequestSchema()
+    querystring = UserServices_GET_QuerySchema()
+
+
+class UserServices_GET_ResponseBodySchema(BaseResponseBodySchema):
     services = ServicesSchemaNode()
 
 
@@ -1551,7 +1567,18 @@ class UserServices_GET_OkResponseSchema(colander.MappingSchema):
     body = UserServices_GET_ResponseBodySchema
 
 
-class UserServicePermissions_GET_ResponseBodySchema(BaseBodySchema):
+class UserServicePermissions_GET_QuerySchema(colander.MappingSchema):
+    groups = colander.SchemaNode(
+        colander.String(), default='inherit', validator=colander.OneOf(['inherit', 'direct']), missing=colander.drop,
+        description='User groups memberships inheritance to resolve service resource permissions.')
+
+
+class UserServicePermissions_GET_RequestSchema(colander.MappingSchema):
+    header = HeaderRequestSchema()
+    querystring = UserServicePermissions_GET_QuerySchema()
+
+
+class UserServicePermissions_GET_ResponseBodySchema(BaseResponseBodySchema):
     permission_names = PermissionListSchema()
 
 
@@ -1564,34 +1591,34 @@ class UserServicePermissions_GET_OkResponseSchema(colander.MappingSchema):
 class UserServicePermissions_GET_NotFoundResponseSchema(colander.MappingSchema):
     description = "Could not find permissions using specified `service_name` and `user_name`."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotFound.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
 class Group_MatchDictCheck_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Group query by name refused by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class Group_MatchDictCheck_NotFoundResponseSchema(colander.MappingSchema):
     description = "Group name not found in db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotFound.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
 class Groups_CheckInfo_NotFoundResponseSchema(colander.MappingSchema):
     description = "User name not found in db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotFound.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
 class Groups_CheckInfo_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Failed to obtain groups of user."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
-class Groups_GET_ResponseBodySchema(BaseBodySchema):
+class Groups_GET_ResponseBodySchema(BaseResponseBodySchema):
     group_names = GroupNamesListSchema()
 
 
@@ -1604,14 +1631,14 @@ class Groups_GET_OkResponseSchema(colander.MappingSchema):
 class Groups_GET_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Obtain group names refused by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class Groups_POST_RequestSchema(colander.MappingSchema):
     group_name = colander.SchemaNode(colander.String(), description="Name of the group to create.")
 
 
-class Groups_POST_ResponseBodySchema(BaseBodySchema):
+class Groups_POST_ResponseBodySchema(BaseResponseBodySchema):
     group = GroupBodySchema()
 
 
@@ -1636,10 +1663,10 @@ class Groups_POST_ForbiddenAddResponseSchema(colander.MappingSchema):
 class Groups_POST_ConflictResponseSchema(colander.MappingSchema):
     description = "Group name matches an already existing group name."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPConflict.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPConflict.code, description=description)
 
 
-class Group_GET_ResponseBodySchema(BaseBodySchema):
+class Group_GET_ResponseBodySchema(BaseResponseBodySchema):
     group = GroupBodySchema()
 
 
@@ -1652,7 +1679,7 @@ class Group_GET_OkResponseSchema(colander.MappingSchema):
 class Group_GET_NotFoundResponseSchema(colander.MappingSchema):
     description = "Group name was not found."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotFound.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
 class Group_PUT_RequestSchema(colander.MappingSchema):
@@ -1662,32 +1689,32 @@ class Group_PUT_RequestSchema(colander.MappingSchema):
 class Group_PUT_OkResponseSchema(colander.MappingSchema):
     description = "Update group successful."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPOk.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPOk.code, description=description)
 
 
 class Group_PUT_Name_NotAcceptableResponseSchema(colander.MappingSchema):
     description = "Invalid `group_name` value specified."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotAcceptable.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description)
 
 
 class Group_PUT_Size_NotAcceptableResponseSchema(colander.MappingSchema):
     description = "Invalid `group_name` length specified (>{length} characters)." \
         .format(length=MAGPIE_USER_NAME_MAX_LENGTH)
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotAcceptable.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description)
 
 
 class Group_PUT_Same_NotAcceptableResponseSchema(colander.MappingSchema):
     description = "Invalid `group_name` must be different than current name."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotAcceptable.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description)
 
 
 class Group_PUT_ConflictResponseSchema(colander.MappingSchema):
     description = "Group name already exists."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPConflict.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPConflict.code, description=description)
 
 
 class Group_DELETE_RequestSchema(colander.MappingSchema):
@@ -1698,28 +1725,28 @@ class Group_DELETE_RequestSchema(colander.MappingSchema):
 class Group_DELETE_OkResponseSchema(colander.MappingSchema):
     description = "Delete group successful."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPOk.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPOk.code, description=description)
 
 
 class Group_DELETE_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Delete group forbidden by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPOk.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPOk.code, description=description)
 
 
 class GroupUsers_GET_OkResponseSchema(colander.MappingSchema):
     description = "Get group users successful."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPOk.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPOk.code, description=description)
 
 
 class GroupUsers_GET_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Failed to obtain group user names from db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
-class GroupServices_GET_ResponseBodySchema(BaseBodySchema):
+class GroupServices_GET_ResponseBodySchema(BaseResponseBodySchema):
     services = ServicesSchemaNode()
 
 
@@ -1740,7 +1767,7 @@ class GroupServices_InternalServerErrorResponseSchema(colander.MappingSchema):
         code=HTTPInternalServerError.code, description=description)
 
 
-class GroupServicePermissions_GET_ResponseBodySchema(BaseBodySchema):
+class GroupServicePermissions_GET_ResponseBodySchema(BaseResponseBodySchema):
     permission_names = PermissionListSchema()
 
 
@@ -1769,7 +1796,7 @@ class GroupServicePermissions_POST_RequestSchema(colander.MappingSchema):
 GroupResourcePermissions_POST_RequestSchema = GroupServicePermissions_POST_RequestSchema
 
 
-class GroupResourcePermissions_POST_ResponseBodySchema(BaseBodySchema):
+class GroupResourcePermissions_POST_ResponseBodySchema(BaseResponseBodySchema):
     permission_name = colander.SchemaNode(colander.String(), description="Name of the permission requested.")
     resource = ResourceBodySchema()
     group = GroupBodySchema()
@@ -1835,7 +1862,7 @@ class GroupResourcePermissions_InternalServerErrorResponseSchema(colander.Mappin
         code=HTTPInternalServerError.code, description=description)
 
 
-class GroupResources_GET_ResponseBodySchema(BaseBodySchema):
+class GroupResources_GET_ResponseBodySchema(BaseResponseBodySchema):
     resources = ResourcesSchemaNode()
 
 
@@ -1856,7 +1883,7 @@ class GroupResources_GET_InternalServerErrorResponseSchema(colander.MappingSchem
         code=HTTPInternalServerError.code, description=description)
 
 
-class GroupResourcePermissions_GET_ResponseBodySchema(BaseBodySchema):
+class GroupResourcePermissions_GET_ResponseBodySchema(BaseResponseBodySchema):
     permissions_names = PermissionListSchema()
 
 
@@ -1866,7 +1893,7 @@ class GroupResourcePermissions_GET_OkResponseSchema(colander.MappingSchema):
     body = GroupResourcePermissions_GET_ResponseBodySchema(code=HTTPOk.code, description=description)
 
 
-class GroupServiceResources_GET_ResponseBodySchema(BaseBodySchema):
+class GroupServiceResources_GET_ResponseBodySchema(BaseResponseBodySchema):
     service = ServiceResourcesBodySchema()
 
 
@@ -1880,7 +1907,7 @@ class GroupServicePermission_DELETE_RequestSchema(colander.MappingSchema):
     permission_name = colander.SchemaNode(colander.String(), description="Name of the permission to delete.")
 
 
-class GroupServicePermission_DELETE_ResponseBodySchema(BaseBodySchema):
+class GroupServicePermission_DELETE_ResponseBodySchema(BaseResponseBodySchema):
     permission_name = colander.SchemaNode(colander.String(), description="Name of the permission requested.")
     resource = ResourceBodySchema()
     group = GroupBodySchema()
@@ -1910,7 +1937,7 @@ class GroupServicePermission_DELETE_NotFoundResponseSchema(colander.MappingSchem
     body = GroupServicePermission_DELETE_ResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
-class Session_GET_ResponseBodySchema(BaseBodySchema):
+class Session_GET_ResponseBodySchema(BaseResponseBodySchema):
     user = UserBodySchema(missing=colander.drop)
     authenticated = colander.SchemaNode(
         colander.Boolean(),
@@ -1929,19 +1956,19 @@ class Session_GET_InternalServerErrorResponseSchema(colander.MappingSchema):
     body = InternalServerErrorResponseSchema()
 
 
-class Providers_GET_ResponseBodySchema(BaseBodySchema):
+class Providers_GET_ResponseBodySchema(BaseResponseBodySchema):
     provider_names = ProvidersListSchema()
     internal_providers = ProvidersListSchema()
     external_providers = ProvidersListSchema()
 
 
-class Providers_GET_OkResponseSchema(BaseBodySchema):
+class Providers_GET_OkResponseSchema(BaseResponseBodySchema):
     description = "Get providers successful."
     header = HeaderResponseSchema()
     body = Providers_GET_ResponseBodySchema(code=HTTPOk.code, description=description)
 
 
-class Version_GET_ResponseBodySchema(BaseBodySchema):
+class Version_GET_ResponseBodySchema(BaseResponseBodySchema):
     version = colander.SchemaNode(
         colander.String(),
         description="Magpie version string",
diff --git a/magpie/api/management/user/__init__.py b/magpie/api/management/user/__init__.py
index dc2bcf24b..ef8cb90ff 100644
--- a/magpie/api/management/user/__init__.py
+++ b/magpie/api/management/user/__init__.py
@@ -12,7 +12,6 @@ def includeme(config):
     config.add_route(**service_api_route_info(UserGroupsAPI))
     config.add_route(**service_api_route_info(UserGroupAPI))
     config.add_route(**service_api_route_info(UserServicesAPI))
-    config.add_route(**service_api_route_info(UserInheritedServicesAPI))
     config.add_route(**service_api_route_info(UserServicePermissionsAPI))
     config.add_route(**service_api_route_info(UserServicePermissionAPI))
     config.add_route(**service_api_route_info(UserServiceInheritedPermissionsAPI))
@@ -29,7 +28,6 @@ def includeme(config):
     config.add_route(**service_api_route_info(LoggedUserGroupsAPI))
     config.add_route(**service_api_route_info(LoggedUserGroupAPI))
     config.add_route(**service_api_route_info(LoggedUserServicesAPI))
-    config.add_route(**service_api_route_info(LoggedUserInheritedServicesAPI))
     config.add_route(**service_api_route_info(LoggedUserServicePermissionsAPI))
     config.add_route(**service_api_route_info(LoggedUserServicePermissionAPI))
     config.add_route(**service_api_route_info(LoggedUserServiceInheritedPermissionsAPI))
diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py
index 4e11a59bf..940e55fbe 100644
--- a/magpie/api/management/user/user_utils.py
+++ b/magpie/api/management/user/user_utils.py
@@ -1,11 +1,11 @@
 from magpie.api.api_except import *
 from magpie.api.api_rest_schemas import *
+from magpie.api.management.service.service_formats import format_service
 from magpie.api.management.resource.resource_utils import check_valid_service_resource_permission
 from magpie.api.management.user.user_formats import *
 from magpie.definitions.ziggurat_definitions import *
 from magpie.services import service_type_dict
 from magpie import models
-from collections import OrderedDict
 
 
 def create_user(user_name, password, email, group_name, db_session):
@@ -83,6 +83,50 @@ def get_user_resource_permissions(user, resource, db_session, inherited_permissi
     return sorted(set(permission_names))  # remove any duplicates that could be incorporated by multiple groups
 
 
+def get_user_services(user, db_session, inherit_resources=False, inherit_groups=False, format_as_list=False):
+    """
+    Returns services by type with corresponding services by name containing sub-dict information.
+
+    :param user: user for which to find services
+    :param db_session: database session connection
+    :param inherit_resources:
+        If `False`, return only services with *Direct* user permissions on their corresponding service-resource.
+        Otherwise, return every service that has at least one sub-resource with user permissions.
+    :param inherit_groups:
+        If `False`, return only user-specific service/sub-resources permissions.
+        Otherwise, resolve inherited permissions using all groups the user is member of.
+    :param format_as_list:
+        returns as list of service dict information (not grouped by type and by name)
+    :return: only services which the user as *Direct* or *Inherited* permissions, according to `inherit_from_resources`
+    :rtype:
+        dict of services by type with corresponding services by name containing sub-dict information,
+        unless `format_as_dict` is `True`
+    """
+    resource_type = None if inherit_resources else ['service']
+    res_perm_dict = get_user_resources_permissions_dict(user, resource_types=resource_type, db_session=db_session,
+                                                        inherited_permissions=inherit_groups)
+
+    services = {}
+    for resource_id, perms in res_perm_dict.items():
+        svc = models.Service.by_resource_id(resource_id=resource_id, db_session=db_session)
+        if svc.resource_type != 'service' and inherit_resources:
+            svc = models.Service.by_resource_id(resource_id=svc.root_service_id, db_session=db_session)
+            perms = service_type_dict[svc.type].permission_names
+        if svc.type not in services:
+            services[svc.type] = {}
+        if svc.resource_name not in services[svc.type]:
+            services[svc.type][svc.resource_name] = format_service(svc, perms)
+
+    if not format_as_list:
+        return services
+
+    services_list = list()
+    for svc_type in services:
+        for svc_name in services[svc_type]:
+            services_list.append(services[svc_type][svc_name])
+    return services_list
+
+
 def get_user_service_permissions(user, service, db_session, inherited_permissions=True):
     if service.owner_user_id == user.id:
         permission_names = service_type_dict[service.type].permission_names
@@ -96,13 +140,25 @@ def get_user_service_permissions(user, service, db_session, inherited_permission
 
 def get_user_resources_permissions_dict(user, db_session, resource_types=None,
                                         resource_ids=None, inherited_permissions=True):
+    """
+    Creates a dictionary of resources by id with corresponding permissions of the user.
+
+    :param user: user for which to find services
+    :param db_session: database session connection
+    :param resource_types: (list) filter the search query with specified resource types
+    :param resource_ids: (list) filter the search query with specified resource ids
+    :param inherited_permissions:
+        If `False`, return only user-specific resource permissions.
+        Otherwise, resolve inherited permissions using all groups the user is member of.
+    :return: only services which the user as *Direct* or *Inherited* permissions, according to `inherit_from_resources`
+    """
     verify_param(user, notNone=True, httpError=HTTPNotFound,
                  msgOnFail=UserResourcePermissions_GET_NotFoundResponseSchema.description)
     res_perm_tuple_list = user.resources_with_possible_perms(resource_ids=resource_ids,
                                                              resource_types=resource_types, db_session=db_session)
     if not inherited_permissions:
         res_perm_tuple_list = filter_user_permission(res_perm_tuple_list, user)
-    resources_permissions_dict = OrderedDict()
+    resources_permissions_dict = {}
     for res_perm in res_perm_tuple_list:
         if res_perm.resource.resource_id not in resources_permissions_dict:
             resources_permissions_dict[res_perm.resource.resource_id] = [res_perm.perm_name]
diff --git a/magpie/api/management/user/user_views.py b/magpie/api/management/user/user_views.py
index f68d18c41..15d88d91e 100644
--- a/magpie/api/management/user/user_views.py
+++ b/magpie/api/management/user/user_views.py
@@ -7,6 +7,7 @@
 from magpie.api.management.group.group_utils import *
 from magpie.api.management.service.service_utils import get_services_by_type
 from magpie.api.management.service.service_formats import format_service, format_service_resources
+from magpie.common import str2bool
 
 
 @UsersAPI.get(tags=[UsersTag], response_schemas=Users_GET_responses)
@@ -248,44 +249,47 @@ def delete_user_resource_permission_view(request):
 
 def get_user_services_runner(request, inherited_group_services_permissions):
     user = get_user_matchdict_checked_or_logged(request)
-    res_perm_dict = get_user_resources_permissions_dict(user, resource_types=['service'], db_session=request.db,
-                                                        inherited_permissions=inherited_group_services_permissions)
-
-    svc_json = {}
-    for resource_id, perms in res_perm_dict.items():
-        svc = models.Service.by_resource_id(resource_id=resource_id, db_session=request.db)
-        if svc.type not in svc_json:
-            svc_json[svc.type] = {}
-        svc_json[svc.type][svc.resource_name] = format_service(svc, perms)
 
+    svc_json = get_user_services(user, db_session=request.db, inherit_resources=False,
+                                 inherit_groups=inherited_group_services_permissions)
     return valid_http(httpSuccess=HTTPOk, detail=UserServices_GET_OkResponseSchema.description,
                       content={u'services': svc_json})
 
 
-@UserServicesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, response_schemas=UserServices_GET_responses)
+@UserServicesAPI.get(tags=[UsersTag], schema=UserServices_GET_RequestSchema,
+                     api_security=SecurityEveryoneAPI, response_schemas=UserServices_GET_responses)
 @LoggedUserServicesAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
                            response_schemas=LoggedUserServices_GET_responses)
 @view_config(route_name=UserServicesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
 def get_user_services_view(request):
-    """List all services a user has direct permission on (not including his groups permissions)."""
-    return get_user_services_runner(request, inherited_group_services_permissions=False)
-
-
-@UserInheritedServicesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI,
-                              response_schemas=UserServices_GET_responses)
-@LoggedUserInheritedServicesAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
-                                    response_schemas=LoggedUserServices_GET_responses)
-@view_config(route_name=UserInheritedServicesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
-def get_user_inherited_services_view(request):
-    """List all services a user has permission on with his inherited user and groups permissions."""
-    return get_user_services_runner(request, inherited_group_services_permissions=True)
+    """List all services a user has permission on."""
+    user = get_user_matchdict_checked_or_logged(request)
+    inherit_resources = get_query_inherit_checked(request, 'resources')
+    inherit_groups = get_query_inherit_checked(request, 'groups')
+    format_as_list = str2bool(request.params.get('list', False))
+
+    svc_json = get_user_services(user, db_session=request.db,
+                                 inherit_resources=inherit_resources,
+                                 inherit_groups=inherit_groups,
+                                 format_as_list=format_as_list)
+    return valid_http(httpSuccess=HTTPOk, detail=UserServices_GET_OkResponseSchema.description,
+                      content={u'services': svc_json})
 
 
-def get_user_service_permissions_runner(request, inherited_permissions):
+@UserServicePermissionsAPI.get(schema=UserServicePermissions_GET_RequestSchema,
+                               tags=[UsersTag], api_security=SecurityEveryoneAPI,
+                               response_schemas=UserServicePermissions_GET_responses)
+@LoggedUserServicePermissionsAPI.get(schema=UserServicePermissions_GET_RequestSchema,
+                                     tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
+                                     response_schemas=LoggedUserServicePermissions_GET_responses)
+@view_config(route_name=UserServicePermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
+def get_user_service_permissions_view(request):
+    """List all permissions a user has on a service."""
     user = get_user_matchdict_checked_or_logged(request)
     service = get_service_matchdict_checked(request)
+    inherit_groups = get_query_inherit_checked(request, 'groups')
     perms = evaluate_call(lambda: get_user_service_permissions(service=service, user=user, db_session=request.db,
-                                                               inherited_permissions=inherited_permissions),
+                                                               inherited_permissions=inherit_groups),
                           fallback=lambda: request.db.rollback(), httpError=HTTPNotFound,
                           msgOnFail=UserServicePermissions_GET_NotFoundResponseSchema.description,
                           content={u'service_name': str(service.resource_name), u'user_name': str(user.user_name)})
@@ -293,30 +297,9 @@ def get_user_service_permissions_runner(request, inherited_permissions):
                       content={u'permission_names': sorted(perms)})
 
 
-@UserServicePermissionsAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI,
-                               response_schemas=UserServicePermissions_GET_responses)
-@LoggedUserServicePermissionsAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
-                                     response_schemas=LoggedUserServicePermissions_GET_responses)
-@view_config(route_name=UserServicePermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
-def get_user_service_permissions_view(request):
-    """List all direct permissions a user has on a service (not including his groups permissions)."""
-    return get_user_service_permissions_runner(request, inherited_permissions=False)
-
-
-@UserServiceInheritedPermissionsAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI,
-                                        response_schemas=UserServicePermissions_GET_responses)
-@LoggedUserServiceInheritedPermissionsAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
-                                              response_schemas=LoggedUserServicePermissions_GET_responses)
-@view_config(route_name=UserServiceInheritedPermissionsAPI.name, request_method='GET',
-             permission=NO_PERMISSION_REQUIRED)
-def get_user_service_inherited_permissions_view(request):
-    """List all permissions a user has on a service using all his inherited user and groups permissions."""
-    return get_user_service_permissions_runner(request, inherited_permissions=True)
-
-
-@UserServicePermissionsAPI.post(schema=UserServicePermissions_POST_RequestBodySchema, tags=[UsersTag],
+@UserServicePermissionsAPI.post(schema=UserServicePermissions_POST_RequestSchema, tags=[UsersTag],
                                 response_schemas=UserServicePermissions_POST_responses)
-@LoggedUserServicePermissionsAPI.post(schema=UserServicePermissions_POST_RequestBodySchema, tags=[LoggedUserTag],
+@LoggedUserServicePermissionsAPI.post(schema=UserServicePermissions_POST_RequestSchema, tags=[LoggedUserTag],
                                       response_schemas=LoggedUserServicePermissions_POST_responses)
 @view_config(route_name=UserServicePermissionsAPI.name, request_method='POST')
 def create_user_service_permission(request):
diff --git a/magpie/magpiectl.py b/magpie/magpiectl.py
index b58e3b15c..dcdfcbc6e 100644
--- a/magpie/magpiectl.py
+++ b/magpie/magpiectl.py
@@ -6,7 +6,6 @@
 """
 
 # -- Standard library --403------------------------------------------------------
-import logging.config
 import argparse
 import time
 import warnings
diff --git a/requirements.txt b/requirements.txt
index aac827c00..8424ec2bd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,6 +6,7 @@ pluggy
 flake8==3.5.0
 coverage==4.0
 Sphinx==1.3.1
+logging
 #cryptography==1.9
 PyYAML>=3.11
 pyramid==1.8.3

From cf4aa451bb67fae024352baf0aab50500572aed3 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Mon, 1 Oct 2018 20:06:59 -0400
Subject: [PATCH 086/124] rename response swagger class

---
 magpie/api/api_rest_schemas.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py
index ef3034a34..cfe11204a 100644
--- a/magpie/api/api_rest_schemas.py
+++ b/magpie/api/api_rest_schemas.py
@@ -1334,13 +1334,13 @@ class UserGroups_POST_CreatedResponseSchema(colander.MappingSchema):
 class UserGroups_POST_GroupNotFoundResponseSchema(colander.MappingSchema):
     description = "Can't find the group to assign to."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPNotFound.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description)
 
 
 class UserGroups_POST_ForbiddenResponseSchema(colander.MappingSchema):
     description = "Group query by name refused by db."
     header = HeaderResponseSchema()
-    body = BaseBodySchema(code=HTTPForbidden.code, description=description)
+    body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description)
 
 
 class UserGroups_POST_ConflictResponseSchema(colander.MappingSchema):

From f864a5f5d89e6d0d596646522975eea44f17bb5d Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 2 Oct 2018 11:30:39 -0400
Subject: [PATCH 087/124] add query param inherit/cascade perms + inherit
 routes deprecate warning

---
 magpie/adapter/magpieservice.py               |   2 +-
 magpie/api/api_requests.py                    |   7 -
 magpie/api/api_rest_schemas.py                |  39 +++--
 magpie/api/management/user/__init__.py        |   2 +
 magpie/api/management/user/user_utils.py      |  31 ++--
 magpie/api/management/user/user_views.py      | 140 ++++++++++++------
 magpie/magpiectl.py                           |   2 +-
 magpie/ui/management/templates/edit_user.mako |  12 +-
 magpie/ui/management/views.py                 |  16 +-
 9 files changed, 157 insertions(+), 94 deletions(-)

diff --git a/magpie/adapter/magpieservice.py b/magpie/adapter/magpieservice.py
index c68e43627..27349084c 100644
--- a/magpie/adapter/magpieservice.py
+++ b/magpie/adapter/magpieservice.py
@@ -48,7 +48,7 @@ def list_services(self, request=None):
         Lists all services registered in magpie.
         """
         my_services = []
-        path = '/users/current/services?groups=inherit&resources=inherit'
+        path = '/users/current/services?inherit=True&cascade=True'
         response = requests.get('{url}{path}'.format(url=self.magpie_url, path=path),
                                 cookies=request.cookies)
         if response.status_code != 200:
diff --git a/magpie/api/api_requests.py b/magpie/api/api_requests.py
index 81f869049..ae96735aa 100644
--- a/magpie/api/api_requests.py
+++ b/magpie/api/api_requests.py
@@ -162,10 +162,3 @@ def get_query_param(request, case_insensitive_key, default=None):
                 return value.lower()
             return value
     return default
-
-
-def get_query_inherit_checked(request, case_insensitive_key):
-    inherit = get_query_param(request, case_insensitive_key) or 'inherit'
-    verify_param(inherit, isIn=True, paramName=case_insensitive_key, paramCompare=['inherit', 'direct'],
-                 httpError=HTTPBadRequest, msgOnFail="Invalid query parameter is not one of allowed values.")
-    return inherit == 'inherit'
diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py
index cfe11204a..f4b4418d9 100644
--- a/magpie/api/api_rest_schemas.py
+++ b/magpie/api/api_rest_schemas.py
@@ -80,7 +80,7 @@ def service_api_route_info(service_api):
     path='/users/{user_name}/resources',
     name='UserResources')
 UserResourceInheritedPermissionsAPI = Service(
-    path='/users/{user_name}/resources/{resource_id}/inherited_permissions',
+    path='/users/{user_name}/resources/{resource_id}/inherit_groups_permissions',
     name='UserResourceInheritedPermissions')
 UserResourcePermissionAPI = Service(
     path='/users/{user_name}/resources/{resource_id}/permissions/{permission_name}',
@@ -91,6 +91,9 @@ def service_api_route_info(service_api):
 UserResourceTypesAPI = Service(
     path='/users/{user_name}/resources/types/{resource_type}',
     name='UserResourceTypes')
+UserInheritedServicesAPI = Service(
+    path='/users/{user_name}/inherited_services',
+    name='UserInheritedServices')
 UserServicesAPI = Service(
     path='/users/{user_name}/services',
     name='UserServices')
@@ -104,7 +107,7 @@ def service_api_route_info(service_api):
     path='/users/{user_name}/services/{service_name}/resources',
     name='UserServiceResources')
 UserServiceInheritedPermissionsAPI = Service(
-    path='/users/{user_name}/services/{service_name}/inherited_permissions',
+    path='/users/{user_name}/services/{service_name}/inherit_groups_permissions',
     name='UserServiceInheritedPermissions')
 UserServicePermissionsAPI = Service(
     path='/users/{user_name}/services/{service_name}/permissions',
@@ -128,7 +131,7 @@ def service_api_route_info(service_api):
     path=LoggedUserBase + '/resources',
     name='LoggedUserResources')
 LoggedUserResourceInheritedPermissionsAPI = Service(
-    path=LoggedUserBase + '/resources/{resource_id}/inherited_permissions',
+    path=LoggedUserBase + '/resources/{resource_id}/inherit_groups_permissions',
     name='LoggedUserResourceInheritedPermissions')
 LoggedUserResourcePermissionAPI = Service(
     path=LoggedUserBase + '/resources/{resource_id}/permissions/{permission_name}',
@@ -139,6 +142,9 @@ def service_api_route_info(service_api):
 LoggedUserResourceTypesAPI = Service(
     path=LoggedUserBase + '/resources/types/{resource_type}',
     name='LoggedUserResourceTypes')
+LoggedUserInheritedServicesAPI = Service(
+    path=LoggedUserBase + '/inherited_services',
+    name='LoggedUserInheritedServices')
 LoggedUserServicesAPI = Service(
     path=LoggedUserBase + '/services',
     name='LoggedUserServices')
@@ -149,7 +155,7 @@ def service_api_route_info(service_api):
     path=LoggedUserBase + '/services/{service_name}/resources',
     name='LoggedUserServiceResources')
 LoggedUserServiceInheritedPermissionsAPI = Service(
-    path=LoggedUserBase + '/services/{service_name}/inherited_permissions',
+    path=LoggedUserBase + '/services/{service_name}/inherit_groups_permissions',
     name='LoggedUserServiceInheritedPermissions')
 LoggedUserServicePermissionsAPI = Service(
     path=LoggedUserBase + '/services/{service_name}/permissions',
@@ -248,11 +254,11 @@ class HeaderRequestSchema(colander.MappingSchema):
     content_type.name = 'Content-Type'
 
 
-QueryInheritGroups = colander.SchemaNode(
-    colander.String(), default='inherit', validator=colander.OneOf(['inherit', 'direct']), missing=colander.drop,
+QueryInheritGroupsPermissions = colander.SchemaNode(
+    colander.Boolean(), default=False, missing=colander.drop,
     description='User groups memberships inheritance to resolve service resource permissions.')
-QueryInheritResources = colander.SchemaNode(
-    colander.String(), default='inherit', validator=colander.OneOf(['inherit', 'direct']), missing=colander.drop,
+QueryCascadeResourcesPermissions = colander.SchemaNode(
+    colander.Boolean(), default=False, missing=colander.drop,
     description='Display any service that has at least one sub-resource user permission, '
                 'or only services that have user permissions directly set on them.', )
 
@@ -1530,6 +1536,15 @@ class UserServiceResources_GET_OkResponseSchema(colander.MappingSchema):
     body = UserServiceResources_GET_ResponseBodySchema(code=HTTPOk.code, description=description)
 
 
+class UserServicePermissions_GET_QuerySchema(colander.MappingSchema):
+    inherit = QueryInheritGroupsPermissions
+
+
+class UserServiceResources_GET_RequestSchema(colander.MappingSchema):
+    header = HeaderRequestSchema()
+    querystring = UserServicePermissions_GET_QuerySchema()
+
+
 class UserServicePermissions_POST_RequestBodySchema(colander.MappingSchema):
     permission_name = colander.SchemaNode(colander.String(), description="Name of the permission to create.")
 
@@ -1545,8 +1560,8 @@ class UserServicePermission_DELETE_RequestSchema(colander.MappingSchema):
 
 
 class UserServices_GET_QuerySchema(colander.MappingSchema):
-    resources = QueryInheritResources
-    groups = QueryInheritGroups
+    cascade = QueryCascadeResourcesPermissions
+    inherit = QueryInheritGroupsPermissions
     list = colander.SchemaNode(
         colander.Boolean(), default=False, missing=colander.drop,
         description='Return services as a list of dicts. Default is a dict by service type, and by service name.')
@@ -1568,9 +1583,7 @@ class UserServices_GET_OkResponseSchema(colander.MappingSchema):
 
 
 class UserServicePermissions_GET_QuerySchema(colander.MappingSchema):
-    groups = colander.SchemaNode(
-        colander.String(), default='inherit', validator=colander.OneOf(['inherit', 'direct']), missing=colander.drop,
-        description='User groups memberships inheritance to resolve service resource permissions.')
+    inherit = QueryInheritGroupsPermissions
 
 
 class UserServicePermissions_GET_RequestSchema(colander.MappingSchema):
diff --git a/magpie/api/management/user/__init__.py b/magpie/api/management/user/__init__.py
index ef8cb90ff..dc2bcf24b 100644
--- a/magpie/api/management/user/__init__.py
+++ b/magpie/api/management/user/__init__.py
@@ -12,6 +12,7 @@ def includeme(config):
     config.add_route(**service_api_route_info(UserGroupsAPI))
     config.add_route(**service_api_route_info(UserGroupAPI))
     config.add_route(**service_api_route_info(UserServicesAPI))
+    config.add_route(**service_api_route_info(UserInheritedServicesAPI))
     config.add_route(**service_api_route_info(UserServicePermissionsAPI))
     config.add_route(**service_api_route_info(UserServicePermissionAPI))
     config.add_route(**service_api_route_info(UserServiceInheritedPermissionsAPI))
@@ -28,6 +29,7 @@ def includeme(config):
     config.add_route(**service_api_route_info(LoggedUserGroupsAPI))
     config.add_route(**service_api_route_info(LoggedUserGroupAPI))
     config.add_route(**service_api_route_info(LoggedUserServicesAPI))
+    config.add_route(**service_api_route_info(LoggedUserInheritedServicesAPI))
     config.add_route(**service_api_route_info(LoggedUserServicePermissionsAPI))
     config.add_route(**service_api_route_info(LoggedUserServicePermissionAPI))
     config.add_route(**service_api_route_info(LoggedUserServiceInheritedPermissionsAPI))
diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py
index 940e55fbe..8caeebd72 100644
--- a/magpie/api/management/user/user_utils.py
+++ b/magpie/api/management/user/user_utils.py
@@ -72,27 +72,28 @@ def filter_user_permission(resource_permission_tuple_list, user):
                   resource_permission_tuple_list)
 
 
-def get_user_resource_permissions(user, resource, db_session, inherited_permissions=True):
+def get_user_resource_permissions(user, resource, db_session, inherit_groups_permissions=True):
     if resource.owner_user_id == user.id:
         permission_names = models.resource_type_dict[resource.type].permission_names
     else:
         res_perm_tuple_list = resource.perms_for_user(user, db_session=db_session)
-        if not inherited_permissions:
+        if not inherit_groups_permissions:
             res_perm_tuple_list = filter_user_permission(res_perm_tuple_list, user)
         permission_names = [permission.perm_name for permission in res_perm_tuple_list]
     return sorted(set(permission_names))  # remove any duplicates that could be incorporated by multiple groups
 
 
-def get_user_services(user, db_session, inherit_resources=False, inherit_groups=False, format_as_list=False):
+def get_user_services(user, db_session, cascade_resources=False,
+                      inherit_groups_permissions=False, format_as_list=False):
     """
     Returns services by type with corresponding services by name containing sub-dict information.
 
     :param user: user for which to find services
     :param db_session: database session connection
-    :param inherit_resources:
+    :param cascade_resources:
         If `False`, return only services with *Direct* user permissions on their corresponding service-resource.
         Otherwise, return every service that has at least one sub-resource with user permissions.
-    :param inherit_groups:
+    :param inherit_groups_permissions:
         If `False`, return only user-specific service/sub-resources permissions.
         Otherwise, resolve inherited permissions using all groups the user is member of.
     :param format_as_list:
@@ -102,14 +103,14 @@ def get_user_services(user, db_session, inherit_resources=False, inherit_groups=
         dict of services by type with corresponding services by name containing sub-dict information,
         unless `format_as_dict` is `True`
     """
-    resource_type = None if inherit_resources else ['service']
+    resource_type = None if cascade_resources else ['service']
     res_perm_dict = get_user_resources_permissions_dict(user, resource_types=resource_type, db_session=db_session,
-                                                        inherited_permissions=inherit_groups)
+                                                        inherit_groups_permissions=inherit_groups_permissions)
 
     services = {}
     for resource_id, perms in res_perm_dict.items():
         svc = models.Service.by_resource_id(resource_id=resource_id, db_session=db_session)
-        if svc.resource_type != 'service' and inherit_resources:
+        if svc.resource_type != 'service' and cascade_resources:
             svc = models.Service.by_resource_id(resource_id=svc.root_service_id, db_session=db_session)
             perms = service_type_dict[svc.type].permission_names
         if svc.type not in services:
@@ -127,19 +128,19 @@ def get_user_services(user, db_session, inherit_resources=False, inherit_groups=
     return services_list
 
 
-def get_user_service_permissions(user, service, db_session, inherited_permissions=True):
+def get_user_service_permissions(user, service, db_session, inherit_groups_permissions=True):
     if service.owner_user_id == user.id:
         permission_names = service_type_dict[service.type].permission_names
     else:
         svc_perm_tuple_list = service.perms_for_user(user, db_session=db_session)
-        if not inherited_permissions:
+        if not inherit_groups_permissions:
             svc_perm_tuple_list = filter_user_permission(svc_perm_tuple_list, user)
         permission_names = [permission.perm_name for permission in svc_perm_tuple_list]
     return sorted(set(permission_names))  # remove any duplicates that could be incorporated by multiple groups
 
 
 def get_user_resources_permissions_dict(user, db_session, resource_types=None,
-                                        resource_ids=None, inherited_permissions=True):
+                                        resource_ids=None, inherit_groups_permissions=True):
     """
     Creates a dictionary of resources by id with corresponding permissions of the user.
 
@@ -147,7 +148,7 @@ def get_user_resources_permissions_dict(user, db_session, resource_types=None,
     :param db_session: database session connection
     :param resource_types: (list) filter the search query with specified resource types
     :param resource_ids: (list) filter the search query with specified resource ids
-    :param inherited_permissions:
+    :param inherit_groups_permissions:
         If `False`, return only user-specific resource permissions.
         Otherwise, resolve inherited permissions using all groups the user is member of.
     :return: only services which the user as *Direct* or *Inherited* permissions, according to `inherit_from_resources`
@@ -156,7 +157,7 @@ def get_user_resources_permissions_dict(user, db_session, resource_types=None,
                  msgOnFail=UserResourcePermissions_GET_NotFoundResponseSchema.description)
     res_perm_tuple_list = user.resources_with_possible_perms(resource_ids=resource_ids,
                                                              resource_types=resource_types, db_session=db_session)
-    if not inherited_permissions:
+    if not inherit_groups_permissions:
         res_perm_tuple_list = filter_user_permission(res_perm_tuple_list, user)
     resources_permissions_dict = {}
     for res_perm in res_perm_tuple_list:
@@ -172,12 +173,12 @@ def get_user_resources_permissions_dict(user, db_session, resource_types=None,
     return resources_permissions_dict
 
 
-def get_user_service_resources_permissions_dict(user, service, db_session, inherited_permissions=True):
+def get_user_service_resources_permissions_dict(user, service, db_session, inherit_groups_permissions=True):
     resources_under_service = models.resource_tree_service.from_parent_deeper(parent_id=service.resource_id,
                                                                               db_session=db_session)
     resource_ids = [resource.Resource.resource_id for resource in resources_under_service]
     return get_user_resources_permissions_dict(user, db_session, resource_types=None, resource_ids=resource_ids,
-                                               inherited_permissions=inherited_permissions)
+                                               inherit_groups_permissions=inherit_groups_permissions)
 
 
 def check_user_info(user_name, email, password, group_name):
diff --git a/magpie/api/management/user/user_views.py b/magpie/api/management/user/user_views.py
index 15d88d91e..6b5bb8bdb 100644
--- a/magpie/api/management/user/user_views.py
+++ b/magpie/api/management/user/user_views.py
@@ -8,6 +8,9 @@
 from magpie.api.management.service.service_utils import get_services_by_type
 from magpie.api.management.service.service_formats import format_service, format_service_resources
 from magpie.common import str2bool
+from zope.deprecation import deprecate
+import logging
+LOGGER = logging.getLogger(__name__)
 
 
 @UsersAPI.get(tags=[UsersTag], response_schemas=Users_GET_responses)
@@ -150,11 +153,11 @@ def build_json_user_resource_tree(usr):
         json_res = {}
         for svc in models.Service.all(db_session=db):
             svc_perms = get_user_service_permissions(user=usr, service=svc, db_session=db,
-                                                     inherited_permissions=inherit_perms)
+                                                     inherit_groups_permissions=inherit_perms)
             if svc.type not in json_res:
                 json_res[svc.type] = {}
             res_perms_dict = get_user_service_resources_permissions_dict(user=usr, service=svc, db_session=db,
-                                                                         inherited_permissions=inherit_perms)
+                                                                         inherit_groups_permissions=inherit_perms)
             json_res[svc.type][svc.resource_name] = format_service_resources(
                 svc,
                 db_session=db,
@@ -178,9 +181,12 @@ def build_json_user_resource_tree(usr):
 @view_config(route_name=UserResourcesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
 def get_user_resources_view(request):
     """List all resources a user has direct permission on (not including his groups permissions)."""
-    return get_user_resources_runner(request, inherited_group_resources_permissions=False)
+    inherit_groups_perms = str2bool(get_query_param(request, 'inherit'))
+    return get_user_resources_runner(request, inherited_group_resources_permissions=inherit_groups_perms)
 
 
+@deprecate("Route deprecated: [{0}], Instead Use: [{1}]"
+           .format(UserInheritedResourcesAPI.path, UserResourcesAPI.path + "?inherit=true"))
 @UserInheritedResourcesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI,
                                response_schemas=UserResources_GET_responses)
 @LoggedUserInheritedResourcesAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
@@ -188,18 +194,11 @@ def get_user_resources_view(request):
 @view_config(route_name=UserInheritedResourcesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
 def get_user_inherited_resources_view(request):
     """List all resources a user has permission on with his inherited user and groups permissions."""
+    LOGGER.warn("Route deprecated: [{0}], Instead Use: [{1}]"
+                .format(UserInheritedResourcesAPI.path, UserResourcesAPI.path + "?inherit=true"))
     return get_user_resources_runner(request, inherited_group_resources_permissions=True)
 
 
-def get_user_resource_permissions_runner(request, inherited_permissions=True):
-    user = get_user_matchdict_checked_or_logged(request)
-    resource = get_resource_matchdict_checked(request, 'resource_id')
-    perm_names = get_user_resource_permissions(resource=resource, user=user, db_session=request.db,
-                                               inherited_permissions=inherited_permissions)
-    return valid_http(httpSuccess=HTTPOk, detail=UserResourcePermissions_GET_OkResponseSchema.description,
-                      content={u'permission_names': sorted(perm_names)})
-
-
 @UserResourcePermissionsAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI,
                                 response_schemas=UserResourcePermissions_GET_responses)
 @LoggedUserResourcePermissionsAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
@@ -207,18 +206,34 @@ def get_user_resource_permissions_runner(request, inherited_permissions=True):
 @view_config(route_name=UserResourcePermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
 def get_user_resource_permissions_view(request):
     """List all direct permissions a user has on a specific resource (not including his groups permissions)."""
-    return get_user_resource_permissions_runner(request, inherited_permissions=False)
+    user = get_user_matchdict_checked_or_logged(request)
+    resource = get_resource_matchdict_checked(request, 'resource_id')
+    inherit_groups_perms = str2bool(get_query_param(request, 'inherit'))
+    perm_names = get_user_resource_permissions(resource=resource, user=user, db_session=request.db,
+                                               inherit_groups_permissions=inherit_groups_perms)
+    return valid_http(httpSuccess=HTTPOk, detail=UserResourcePermissions_GET_OkResponseSchema.description,
+                      content={u'permission_names': sorted(perm_names)})
 
 
+@deprecate("Route deprecated: [{0}], Instead Use: [{1}]"
+           .format(UserResourceInheritedPermissionsAPI.path, UserResourcePermissionsAPI.path + "?inherit=true"))
 @UserResourceInheritedPermissionsAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI,
                                          response_schemas=UserResourcePermissions_GET_responses)
 @LoggedUserResourceInheritedPermissionsAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
                                                response_schemas=LoggedUserResourcePermissions_GET_responses)
 @view_config(route_name=UserResourceInheritedPermissionsAPI.name, request_method='GET',
              permission=NO_PERMISSION_REQUIRED)
-def get_user_resource_inherited_permissions_view(request):
+def get_user_resource_inherit_groups_permissions_view(request):
     """List all permissions a user has on a specific resource with his inherited user and groups permissions."""
-    return get_user_resource_permissions_runner(request, inherited_permissions=True)
+    LOGGER.warn("Route deprecated: [{0}], Instead Use: [{1}]"
+                .format(UserResourceInheritedPermissionsAPI.path, UserResourcePermissionsAPI.path + "?inherit=true"))
+
+    user = get_user_matchdict_checked_or_logged(request)
+    resource = get_resource_matchdict_checked(request, 'resource_id')
+    perm_names = get_user_resource_permissions(resource=resource, user=user, db_session=request.db,
+                                               inherit_groups_permissions=True)
+    return valid_http(httpSuccess=HTTPOk, detail=UserResourcePermissions_GET_OkResponseSchema.description,
+                      content={u'permission_names': sorted(perm_names)})
 
 
 @UserResourcePermissionsAPI.post(schema=UserResourcePermissions_POST_RequestSchema(), tags=[UsersTag],
@@ -247,15 +262,6 @@ def delete_user_resource_permission_view(request):
     return delete_user_resource_permission(perm_name, resource, user.id, request.db)
 
 
-def get_user_services_runner(request, inherited_group_services_permissions):
-    user = get_user_matchdict_checked_or_logged(request)
-
-    svc_json = get_user_services(user, db_session=request.db, inherit_resources=False,
-                                 inherit_groups=inherited_group_services_permissions)
-    return valid_http(httpSuccess=HTTPOk, detail=UserServices_GET_OkResponseSchema.description,
-                      content={u'services': svc_json})
-
-
 @UserServicesAPI.get(tags=[UsersTag], schema=UserServices_GET_RequestSchema,
                      api_security=SecurityEveryoneAPI, response_schemas=UserServices_GET_responses)
 @LoggedUserServicesAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
@@ -264,18 +270,59 @@ def get_user_services_runner(request, inherited_group_services_permissions):
 def get_user_services_view(request):
     """List all services a user has permission on."""
     user = get_user_matchdict_checked_or_logged(request)
-    inherit_resources = get_query_inherit_checked(request, 'resources')
-    inherit_groups = get_query_inherit_checked(request, 'groups')
-    format_as_list = str2bool(request.params.get('list', False))
+    cascade_resources = str2bool(get_query_param(request, 'cascade'))
+    inherit_groups_perms = str2bool(get_query_param(request, 'inherit'))
+    format_as_list = str2bool(get_query_param(request, 'list'))
 
     svc_json = get_user_services(user, db_session=request.db,
-                                 inherit_resources=inherit_resources,
-                                 inherit_groups=inherit_groups,
+                                 cascade_resources=cascade_resources,
+                                 inherit_groups_permissions=inherit_groups_perms,
                                  format_as_list=format_as_list)
     return valid_http(httpSuccess=HTTPOk, detail=UserServices_GET_OkResponseSchema.description,
                       content={u'services': svc_json})
 
 
+@deprecate("Route deprecated: [{0}], Instead Use: [{1}]"
+           .format(LoggedUserInheritedServicesAPI.path, LoggedUserServicesAPI.path + "?inherit=true"))
+@UserInheritedServicesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI,
+                              response_schemas=UserServices_GET_responses)
+@LoggedUserInheritedServicesAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
+                                    response_schemas=LoggedUserServices_GET_responses)
+@view_config(route_name=UserInheritedServicesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
+def get_user_inherited_services_view(request):
+    """List all services a user has permission on with his inherited user and groups permissions."""
+    LOGGER.warn("Route deprecated: [{0}], Instead Use: [{1}]"
+                .format(LoggedUserInheritedServicesAPI.path, LoggedUserServicesAPI.path + "?inherit=true"))
+    user = get_user_matchdict_checked_or_logged(request)
+    svc_json = get_user_services(user, db_session=request.db, cascade_resources=False, inherit_groups_permissions=True)
+    return valid_http(httpSuccess=HTTPOk, detail=UserServices_GET_OkResponseSchema.description,
+                      content={u'services': svc_json})
+
+
+@deprecate("Route deprecated: [{0}], Instead Use: [{1}]"
+           .format(UserServiceInheritedPermissionsAPI.path, UserServicePermissionsAPI.path + "?inherit=true"))
+@UserServiceInheritedPermissionsAPI.get(schema=UserServicePermissions_GET_RequestSchema,
+                                        tags=[UsersTag], api_security=SecurityEveryoneAPI,
+                                        response_schemas=UserServicePermissions_GET_responses)
+@LoggedUserServiceInheritedPermissionsAPI.get(schema=UserServicePermissions_GET_RequestSchema,
+                                              tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
+                                              response_schemas=LoggedUserServicePermissions_GET_responses)
+@view_config(route_name=UserServicePermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
+def get_user_service_inherited_permissions_view(request):
+    """List all permissions a user has on a service using all his inherited user and groups permissions."""
+    LOGGER.warn("Route deprecated: [{0}], Instead Use: [{1}]"
+                .format(UserServiceInheritedPermissionsAPI.path, UserServicePermissionsAPI.path + "?inherit=true"))
+    user = get_user_matchdict_checked_or_logged(request)
+    service = get_service_matchdict_checked(request)
+    perms = evaluate_call(lambda: get_user_service_permissions(service=service, user=user, db_session=request.db,
+                                                               inherit_groups_permissions=True),
+                          fallback=lambda: request.db.rollback(), httpError=HTTPNotFound,
+                          msgOnFail=UserServicePermissions_GET_NotFoundResponseSchema.description,
+                          content={u'service_name': str(service.resource_name), u'user_name': str(user.user_name)})
+    return valid_http(httpSuccess=HTTPOk, detail=UserServicePermissions_GET_OkResponseSchema.description,
+                      content={u'permission_names': sorted(perms)})
+
+
 @UserServicePermissionsAPI.get(schema=UserServicePermissions_GET_RequestSchema,
                                tags=[UsersTag], api_security=SecurityEveryoneAPI,
                                response_schemas=UserServicePermissions_GET_responses)
@@ -287,9 +334,9 @@ def get_user_service_permissions_view(request):
     """List all permissions a user has on a service."""
     user = get_user_matchdict_checked_or_logged(request)
     service = get_service_matchdict_checked(request)
-    inherit_groups = get_query_inherit_checked(request, 'groups')
+    inherit_groups_perms = str2bool(get_query_param(request, 'inherit'))
     perms = evaluate_call(lambda: get_user_service_permissions(service=service, user=user, db_session=request.db,
-                                                               inherited_permissions=inherit_groups),
+                                                               inherit_groups_permissions=inherit_groups_perms),
                           fallback=lambda: request.db.rollback(), httpError=HTTPNotFound,
                           msgOnFail=UserServicePermissions_GET_NotFoundResponseSchema.description,
                           content={u'service_name': str(service.resource_name), u'user_name': str(user.user_name)})
@@ -316,27 +363,27 @@ def create_user_service_permission(request):
                                        response_schemas=LoggedUserServicePermission_DELETE_responses)
 @view_config(route_name=UserServicePermissionAPI.name, request_method='DELETE')
 def delete_user_service_permission(request):
-    """Delete a permission on a service for a user (not including his groups permissions)."""
+    """Delete a direct permission on a service for a user (not including his groups permissions)."""
     user = get_user_matchdict_checked_or_logged(request)
     service = get_service_matchdict_checked(request)
     perm_name = get_permission_multiformat_post_checked(request, service)
     return delete_user_resource_permission(perm_name, service, user.id, request.db)
 
 
-def get_user_service_resource_permissions_runner(request, inherited_permissions):
+def get_user_service_resource_permissions_runner(request, inherit_groups_permissions):
     """
     Resource permissions a user as on a specific service
 
     :param request:
-    :param inherited_permissions: only direct permissions if False, else resolve permissions with user and his groups.
+    :param inherit_groups_permissions: only direct permissions if False, else resolve permissions with user and his groups.
     :return:
     """
     user = get_user_matchdict_checked_or_logged(request)
     service = get_service_matchdict_checked(request)
-    service_perms = get_user_service_permissions(user, service, db_session=request.db,
-                                                 inherited_permissions=inherited_permissions)
-    resources_perms_dict = get_user_service_resources_permissions_dict(user, service, db_session=request.db,
-                                                                       inherited_permissions=inherited_permissions)
+    service_perms = get_user_service_permissions(
+        user, service, db_session=request.db, inherit_groups_permissions=inherit_groups_permissions)
+    resources_perms_dict = get_user_service_resources_permissions_dict(
+        user, service, db_session=request.db, inherit_groups_permissions=inherit_groups_permissions)
     user_svc_res_json = format_service_resources(
         service=service,
         db_session=request.db,
@@ -348,16 +395,21 @@ def get_user_service_resource_permissions_runner(request, inherited_permissions)
                       content={u'service': user_svc_res_json})
 
 
-@UserServiceResourcesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI,
+@UserServiceResourcesAPI.get(schema=UserServiceResources_GET_RequestSchema,
+                             tags=[UsersTag], api_security=SecurityEveryoneAPI,
                              response_schemas=UserServiceResources_GET_responses)
-@LoggedUserServiceResourcesAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
+@LoggedUserServiceResourcesAPI.get(schema=UserServiceResources_GET_RequestSchema,
+                                   tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
                                    response_schemas=LoggedUserServiceResources_GET_responses)
 @view_config(route_name=UserServiceResourcesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
 def get_user_service_resources_view(request):
-    """List all resources under a service a user has direct permission on (not including his groups permissions)."""
-    return get_user_service_resource_permissions_runner(request, inherited_permissions=False)
+    """List all resources under a service a user has permission on."""
+    inherit_groups_perms = str2bool(get_query_param(request, 'inherit'))
+    return get_user_service_resource_permissions_runner(request, inherit_groups_permissions=inherit_groups_perms)
 
 
+@deprecate("Route deprecated: [{0}], Instead Use: [{1}]"
+           .format(UserServiceInheritedResourcesAPI.path, UserServiceResourcesAPI.path + "?inherit=true"))
 @UserServiceInheritedResourcesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI,
                                       response_schemas=UserServiceResources_GET_responses)
 @LoggedUserServiceInheritedResourcesAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
@@ -366,4 +418,6 @@ def get_user_service_resources_view(request):
 def get_user_service_inherited_resources_view(request):
     """List all resources under a service a user has permission on using all his inherited user and groups
     permissions."""
-    return get_user_service_resource_permissions_runner(request, inherited_permissions=True)
+    LOGGER.warn("Route deprecated: [{0}], Instead Use: [{1}]"
+                .format(UserServiceInheritedResourcesAPI.path, UserServiceResourcesAPI.path + "?inherit=true"))
+    return get_user_service_resource_permissions_runner(request, inherit_groups_permissions=True)
diff --git a/magpie/magpiectl.py b/magpie/magpiectl.py
index dcdfcbc6e..3f4bec8ea 100644
--- a/magpie/magpiectl.py
+++ b/magpie/magpiectl.py
@@ -5,7 +5,7 @@
 Magpie is a service for AuthN and AuthZ based on Ziggurat-Foundations
 """
 
-# -- Standard library --403------------------------------------------------------
+# -- Standard library --------------------------------------------------------
 import argparse
 import time
 import warnings
diff --git a/magpie/ui/management/templates/edit_user.mako b/magpie/ui/management/templates/edit_user.mako
index 4a7315804..c19430fb1 100644
--- a/magpie/ui/management/templates/edit_user.mako
+++ b/magpie/ui/management/templates/edit_user.mako
@@ -9,7 +9,7 @@
             <div class="perm_checkbox">
                 <input type="checkbox" value="${perm}" name="permission"
                        onchange="document.getElementById('resource_${value['id']}_${value.get('remote_id', '')}').submit()" checked
-                %if inherited_permissions:
+                %if inherit_groups_permissions:
                     disabled
                 %endif
                 >
@@ -18,7 +18,7 @@
             <div class="perm_checkbox">
                 <input type="checkbox" value="${perm}" name="permission"
                        onchange="document.getElementById('resource_${value['id']}_${value.get('remote_id', '')}').submit()"
-                %if inherited_permissions:
+                %if inherit_groups_permissions:
                     disabled
                 %endif
                 >
@@ -139,14 +139,14 @@
 <h3>Permissions</h3>
 
 <form id="toggle_visible_perms" action="${request.path}" method="post">
-    <input type="checkbox" value="${inherited_permissions}" name="toggle_inherited_permissions"
+    <input type="checkbox" value="${inherit_groups_permissions}" name="toggle_inherit_groups_permissions"
            onchange="document.getElementById('toggle_visible_perms').submit()"
-    %if inherited_permissions:
+    %if inherit_groups_permissions:
         checked>
-        <input type="hidden" value="False" name="inherited_permissions"/>
+        <input type="hidden" value="False" name="inherit_groups_permissions"/>
     %else:
         >
-        <input type="hidden" value="True" name="inherited_permissions"/>
+        <input type="hidden" value="True" name="inherit_groups_permissions"/>
     %endif
     View inherited group permissions
 </form>
diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index 878f4b13b..d8a862b6c 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -234,7 +234,7 @@ def add_user(self):
     def edit_user(self):
         user_name = self.request.matchdict['user_name']
         cur_svc_type = self.request.matchdict['cur_svc_type']
-        inherited_permissions = self.request.matchdict.get('inherited_permissions', False)
+        inherit_groups_permissions = self.request.matchdict.get('inherit_groups_permissions', False)
 
         user_url = '{url}/users/{usr}'.format(url=self.magpie_url, usr=user_name)
         own_groups = self.get_user_groups(user_name)
@@ -259,7 +259,7 @@ def edit_user(self):
         user_info[u'edit_mode'] = u'no_edit'
         user_info[u'own_groups'] = own_groups
         user_info[u'groups'] = all_groups
-        user_info[u'inherited_permissions'] = inherited_permissions
+        user_info[u'inherit_groups_permissions'] = inherit_groups_permissions
 
         if self.request.method == 'POST':
             res_id = self.request.POST.get(u'resource_id')
@@ -267,9 +267,9 @@ def edit_user(self):
             is_save_user_info = False
             requires_update_name = False
 
-            if u'inherited_permissions' in self.request.POST:
-                inherited_permissions = str2bool(self.request.POST[u'inherited_permissions'])
-                user_info[u'inherited_permissions'] = inherited_permissions
+            if u'inherit_groups_permissions' in self.request.POST:
+                inherit_groups_permissions = str2bool(self.request.POST[u'inherit_groups_permissions'])
+                user_info[u'inherit_groups_permissions'] = inherit_groups_permissions
 
             if u'delete' in self.request.POST:
                 check_response(requests.delete(user_url, cookies=self.request.cookies))
@@ -346,7 +346,7 @@ def edit_user(self):
         # display resources permissions per service type tab
         try:
             res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(
-                user_name, services, cur_svc_type, is_user=True, is_inherited_permissions=inherited_permissions)
+                user_name, services, cur_svc_type, is_user=True, is_inherit_groups_permissions=inherit_groups_permissions)
         except Exception as e:
             raise HTTPBadRequest(detail=repr(e))
 
@@ -470,9 +470,9 @@ def edit_user_or_group_resource_permissions(self, user_or_group_name, resource_i
             check_response(requests.post(res_perms_url, data=data, cookies=self.request.cookies))
 
     def get_user_or_group_resources_permissions_dict(self, user_or_group_name, services, service_type,
-                                                     is_user=False, is_inherited_permissions=False):
+                                                     is_user=False, is_inherit_groups_permissions=False):
         user_or_group_type = 'users' if is_user else 'groups'
-        inherit_type = 'inherited_' if is_inherited_permissions and is_user else ''
+        inherit_type = 'inherited_' if is_inherit_groups_permissions and is_user else ''
 
         group_perms_url = '{url}/{usr_grp_type}/{usr_grp}/{inherit}resources' \
             .format(url=self.magpie_url, usr_grp_type=user_or_group_type,

From f93d97a902baacd45bbc0aebe2b4fe52fb795956 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Tue, 2 Oct 2018 15:00:50 -0400
Subject: [PATCH 088/124] fix update sync_type when force updating providers

---
 magpie/register.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/magpie/register.py b/magpie/register.py
index 58ab68c1c..f96f2331e 100644
--- a/magpie/register.py
+++ b/magpie/register.py
@@ -360,7 +360,7 @@ def magpie_register_services_with_db_session(services_dict, db_session, push_to_
         svc_new_url = os.path.expandvars(svc_values['url'])
         svc_type = svc_values['type']
 
-        svc_sync_type = svc_values.get('sync_type', svc_values['title'])
+        svc_sync_type = svc_values.get('sync_type')
         if force_update and svc_name in existing_services_names:
             svc = models.Service.by_service_name(svc_name, db_session=db_session)
             if svc.url == svc_new_url:
@@ -369,6 +369,7 @@ def magpie_register_services_with_db_session(services_dict, db_session, push_to_
                 print_log("Service URL update [{url_old}] => [{url_new}] ({svc})"
                           .format(url_old=svc.url, url_new=svc_new_url, svc=svc_name))
                 svc.url = svc_new_url
+            svc.sync_type = svc_sync_type
         elif not force_update and svc_name in existing_services_names:
             print_log("Skipping service [{svc}] (conflict)" .format(svc=svc_name))
         else:

From 06ba6b03e2f28bd6ddcc415cf01d08c63b87ddb5 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Tue, 2 Oct 2018 15:40:20 -0400
Subject: [PATCH 089/124] let sync_type be None when passed to json renderer

---
 magpie/api/management/service/service_formats.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/magpie/api/management/service/service_formats.py b/magpie/api/management/service/service_formats.py
index 163675306..6b0beee15 100644
--- a/magpie/api/management/service/service_formats.py
+++ b/magpie/api/management/service/service_formats.py
@@ -13,7 +13,7 @@ def fmt_svc(svc, perms):
             u'service_url': str(svc.url),
             u'service_name': str(svc.resource_name),
             u'service_type': str(svc.type),
-            u'service_sync_type': str(svc.sync_type),
+            u'service_sync_type': svc.sync_type,
             u'resource_id': svc.resource_id,
             u'permission_names': sorted(service_type_dict[svc.type].permission_names if perms is None else perms)
         }

From 5b7b79a6e37ede80b74dbb3906985f40b19451bc Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 2 Oct 2018 15:17:01 -0400
Subject: [PATCH 090/124] fix error search replace

---
 magpie/adapter/magpieprocess.py |  0
 magpie/api/api_rest_schemas.py  | 12 ++++++------
 2 files changed, 6 insertions(+), 6 deletions(-)
 create mode 100644 magpie/adapter/magpieprocess.py

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py
index f4b4418d9..5d88d6778 100644
--- a/magpie/api/api_rest_schemas.py
+++ b/magpie/api/api_rest_schemas.py
@@ -80,7 +80,7 @@ def service_api_route_info(service_api):
     path='/users/{user_name}/resources',
     name='UserResources')
 UserResourceInheritedPermissionsAPI = Service(
-    path='/users/{user_name}/resources/{resource_id}/inherit_groups_permissions',
+    path='/users/{user_name}/resources/{resource_id}/inherited_permissions',
     name='UserResourceInheritedPermissions')
 UserResourcePermissionAPI = Service(
     path='/users/{user_name}/resources/{resource_id}/permissions/{permission_name}',
@@ -107,7 +107,7 @@ def service_api_route_info(service_api):
     path='/users/{user_name}/services/{service_name}/resources',
     name='UserServiceResources')
 UserServiceInheritedPermissionsAPI = Service(
-    path='/users/{user_name}/services/{service_name}/inherit_groups_permissions',
+    path='/users/{user_name}/services/{service_name}/inherited_permissions',
     name='UserServiceInheritedPermissions')
 UserServicePermissionsAPI = Service(
     path='/users/{user_name}/services/{service_name}/permissions',
@@ -131,7 +131,7 @@ def service_api_route_info(service_api):
     path=LoggedUserBase + '/resources',
     name='LoggedUserResources')
 LoggedUserResourceInheritedPermissionsAPI = Service(
-    path=LoggedUserBase + '/resources/{resource_id}/inherit_groups_permissions',
+    path=LoggedUserBase + '/resources/{resource_id}/inherited_permissions',
     name='LoggedUserResourceInheritedPermissions')
 LoggedUserResourcePermissionAPI = Service(
     path=LoggedUserBase + '/resources/{resource_id}/permissions/{permission_name}',
@@ -155,7 +155,7 @@ def service_api_route_info(service_api):
     path=LoggedUserBase + '/services/{service_name}/resources',
     name='LoggedUserServiceResources')
 LoggedUserServiceInheritedPermissionsAPI = Service(
-    path=LoggedUserBase + '/services/{service_name}/inherit_groups_permissions',
+    path=LoggedUserBase + '/services/{service_name}/inherited_permissions',
     name='LoggedUserServiceInheritedPermissions')
 LoggedUserServicePermissionsAPI = Service(
     path=LoggedUserBase + '/services/{service_name}/permissions',
@@ -1536,13 +1536,13 @@ class UserServiceResources_GET_OkResponseSchema(colander.MappingSchema):
     body = UserServiceResources_GET_ResponseBodySchema(code=HTTPOk.code, description=description)
 
 
-class UserServicePermissions_GET_QuerySchema(colander.MappingSchema):
+class UserServiceResources_GET_QuerySchema(colander.MappingSchema):
     inherit = QueryInheritGroupsPermissions
 
 
 class UserServiceResources_GET_RequestSchema(colander.MappingSchema):
     header = HeaderRequestSchema()
-    querystring = UserServicePermissions_GET_QuerySchema()
+    querystring = UserServiceResources_GET_QuerySchema()
 
 
 class UserServicePermissions_POST_RequestBodySchema(colander.MappingSchema):

From 33d576b16ee4937584a4b6aa7a636474def7bc68 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Tue, 2 Oct 2018 17:40:44 -0400
Subject: [PATCH 091/124] fix migrations

---
 .../versions/73639c63c4fc_unified_api_service.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/magpie/alembic/versions/73639c63c4fc_unified_api_service.py b/magpie/alembic/versions/73639c63c4fc_unified_api_service.py
index e164dd90e..7096e2b52 100644
--- a/magpie/alembic/versions/73639c63c4fc_unified_api_service.py
+++ b/magpie/alembic/versions/73639c63c4fc_unified_api_service.py
@@ -19,6 +19,7 @@
 from magpie.definitions.sqlalchemy_definitions import *
 # from magpie.models import Service
 from sqlalchemy.sql import table
+from sqlalchemy import func
 
 Session = sessionmaker()
 
@@ -46,7 +47,7 @@ def upgrade():
                    update().
                    where(services.c.type == op.inline_literal('project-api')).
                    values({'type': op.inline_literal('api'),
-                           'url': op.inline_literal(str(services.c.url) + '/api'),
+                           'url': services.c.url + "/api",
                            'sync_type': op.inline_literal('project-api')
                            })
                    )
@@ -60,24 +61,25 @@ def upgrade():
 
 
 def downgrade():
-    op.drop_column('services', 'sync_type')
-
-    service = table('old_service',
+    service = table('services',
                     sa.Column('url', sa.UnicodeText()),
                     sa.Column('type', sa.UnicodeText()),
+                    sa.Column('sync_type', sa.UnicodeText()),
                     )
 
     # transfer 'api' service types
     op.execute(service.
                update().
-               where(service.c.type == op.inline_literal('project-api')).
+               where(service.c.sync_type == op.inline_literal('project-api')).
                values({'type': op.inline_literal('project-api'),
-                       'url': op.inline_literal(str(service.c.url).rstrip('/api')),
+                       'url': func.replace(service.c.url, '/api', ''),
                        })
                )
     op.execute(service.
                update().
-               where(service.c.type == op.inline_literal('geoserver-api')).
+               where(service.c.sync_type == op.inline_literal('geoserver-api')).
                values({'type': op.inline_literal('geoserver-api'),
                        })
                )
+
+    op.drop_column('services', 'sync_type')

From 3bcfa224f843d6ff485519341e31549ac89bc515 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Tue, 2 Oct 2018 17:41:03 -0400
Subject: [PATCH 092/124] fix error message issue

---
 magpie/ui/management/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py
index adffc8083..2a7db2aa5 100644
--- a/magpie/ui/management/views.py
+++ b/magpie/ui/management/views.py
@@ -240,7 +240,7 @@ def edit_user(self):
         own_groups = self.get_user_groups(user_name)
         all_groups = self.get_all_groups(first_default_group=get_constant('MAGPIE_USERS_GROUP'))
 
-        error_message = None
+        error_message = ""
 
         # Todo:
         # Until the api is modified to make it possible to request from the RemoteResource table,

From 293010e575872c2d2856e059595ccdcd89b27653 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 2 Oct 2018 18:01:54 -0400
Subject: [PATCH 093/124] remove deprecation decorators that make view not
 found

---
 magpie/api/management/user/user_views.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/magpie/api/management/user/user_views.py b/magpie/api/management/user/user_views.py
index 6b5bb8bdb..02a74ff24 100644
--- a/magpie/api/management/user/user_views.py
+++ b/magpie/api/management/user/user_views.py
@@ -8,7 +8,6 @@
 from magpie.api.management.service.service_utils import get_services_by_type
 from magpie.api.management.service.service_formats import format_service, format_service_resources
 from magpie.common import str2bool
-from zope.deprecation import deprecate
 import logging
 LOGGER = logging.getLogger(__name__)
 
@@ -185,8 +184,6 @@ def get_user_resources_view(request):
     return get_user_resources_runner(request, inherited_group_resources_permissions=inherit_groups_perms)
 
 
-@deprecate("Route deprecated: [{0}], Instead Use: [{1}]"
-           .format(UserInheritedResourcesAPI.path, UserResourcesAPI.path + "?inherit=true"))
 @UserInheritedResourcesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI,
                                response_schemas=UserResources_GET_responses)
 @LoggedUserInheritedResourcesAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
@@ -215,8 +212,6 @@ def get_user_resource_permissions_view(request):
                       content={u'permission_names': sorted(perm_names)})
 
 
-@deprecate("Route deprecated: [{0}], Instead Use: [{1}]"
-           .format(UserResourceInheritedPermissionsAPI.path, UserResourcePermissionsAPI.path + "?inherit=true"))
 @UserResourceInheritedPermissionsAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI,
                                          response_schemas=UserResourcePermissions_GET_responses)
 @LoggedUserResourceInheritedPermissionsAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
@@ -282,8 +277,6 @@ def get_user_services_view(request):
                       content={u'services': svc_json})
 
 
-@deprecate("Route deprecated: [{0}], Instead Use: [{1}]"
-           .format(LoggedUserInheritedServicesAPI.path, LoggedUserServicesAPI.path + "?inherit=true"))
 @UserInheritedServicesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI,
                               response_schemas=UserServices_GET_responses)
 @LoggedUserInheritedServicesAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
@@ -299,8 +292,6 @@ def get_user_inherited_services_view(request):
                       content={u'services': svc_json})
 
 
-@deprecate("Route deprecated: [{0}], Instead Use: [{1}]"
-           .format(UserServiceInheritedPermissionsAPI.path, UserServicePermissionsAPI.path + "?inherit=true"))
 @UserServiceInheritedPermissionsAPI.get(schema=UserServicePermissions_GET_RequestSchema,
                                         tags=[UsersTag], api_security=SecurityEveryoneAPI,
                                         response_schemas=UserServicePermissions_GET_responses)
@@ -375,7 +366,8 @@ def get_user_service_resource_permissions_runner(request, inherit_groups_permiss
     Resource permissions a user as on a specific service
 
     :param request:
-    :param inherit_groups_permissions: only direct permissions if False, else resolve permissions with user and his groups.
+    :param inherit_groups_permissions:
+        only direct permissions if False, otherwise resolve permissions with user and his groups.
     :return:
     """
     user = get_user_matchdict_checked_or_logged(request)
@@ -408,8 +400,6 @@ def get_user_service_resources_view(request):
     return get_user_service_resource_permissions_runner(request, inherit_groups_permissions=inherit_groups_perms)
 
 
-@deprecate("Route deprecated: [{0}], Instead Use: [{1}]"
-           .format(UserServiceInheritedResourcesAPI.path, UserServiceResourcesAPI.path + "?inherit=true"))
 @UserServiceInheritedResourcesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI,
                                       response_schemas=UserServiceResources_GET_responses)
 @LoggedUserServiceInheritedResourcesAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,

From 3edeadf2668346d2850cdc2154dc44b8765f00f1 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Wed, 3 Oct 2018 09:50:46 -0400
Subject: [PATCH 094/124] chain alembic migrations instead of branching

---
 magpie/alembic/versions/73b872478d87_add_resource_label.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/magpie/alembic/versions/73b872478d87_add_resource_label.py b/magpie/alembic/versions/73b872478d87_add_resource_label.py
index 22ccd28fa..b13ebd4ef 100644
--- a/magpie/alembic/versions/73b872478d87_add_resource_label.py
+++ b/magpie/alembic/versions/73b872478d87_add_resource_label.py
@@ -11,7 +11,7 @@
 
 # revision identifiers, used by Alembic.
 revision = '73b872478d87'
-down_revision = 'd01af1f2e445'
+down_revision = '73639c63c4fc'
 branch_labels = None
 depends_on = None
 

From 3d43169313c36ff03a7ed7a998685d82102ca096 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Wed, 3 Oct 2018 09:54:21 -0400
Subject: [PATCH 095/124] add package name to entry point in setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index e48d9810d..cf6b04f12 100644
--- a/setup.py
+++ b/setup.py
@@ -97,7 +97,7 @@
     # -- script entry points -----------------------------------------------
     entry_points="""\
           [paste.app_factory]
-          main = magpiectl:main
+          main = magpie.magpiectl:main
           [console_scripts]
           """,
 )

From 33eea56121c40f5a24a48afc138c3f710cfa0385 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Wed, 3 Oct 2018 10:08:20 -0400
Subject: [PATCH 096/124] fix route conflict for deprecated
 /inherited_permissions

---
 magpie/api/management/user/user_views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/magpie/api/management/user/user_views.py b/magpie/api/management/user/user_views.py
index 02a74ff24..0d9346220 100644
--- a/magpie/api/management/user/user_views.py
+++ b/magpie/api/management/user/user_views.py
@@ -298,7 +298,7 @@ def get_user_inherited_services_view(request):
 @LoggedUserServiceInheritedPermissionsAPI.get(schema=UserServicePermissions_GET_RequestSchema,
                                               tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
                                               response_schemas=LoggedUserServicePermissions_GET_responses)
-@view_config(route_name=UserServicePermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
+@view_config(route_name=UserServiceInheritedPermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
 def get_user_service_inherited_permissions_view(request):
     """List all permissions a user has on a service using all his inherited user and groups permissions."""
     LOGGER.warn("Route deprecated: [{0}], Instead Use: [{1}]"

From e723b626b5dea07bb4db8bdc861e0708b05a7073 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Wed, 3 Oct 2018 11:52:57 -0400
Subject: [PATCH 097/124] fix conflict routes

---
 magpie/api/management/user/user_views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/magpie/api/management/user/user_views.py b/magpie/api/management/user/user_views.py
index 02a74ff24..0d9346220 100644
--- a/magpie/api/management/user/user_views.py
+++ b/magpie/api/management/user/user_views.py
@@ -298,7 +298,7 @@ def get_user_inherited_services_view(request):
 @LoggedUserServiceInheritedPermissionsAPI.get(schema=UserServicePermissions_GET_RequestSchema,
                                               tags=[LoggedUserTag], api_security=SecurityEveryoneAPI,
                                               response_schemas=LoggedUserServicePermissions_GET_responses)
-@view_config(route_name=UserServicePermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
+@view_config(route_name=UserServiceInheritedPermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED)
 def get_user_service_inherited_permissions_view(request):
     """List all permissions a user has on a service using all his inherited user and groups permissions."""
     LOGGER.warn("Route deprecated: [{0}], Instead Use: [{1}]"

From 0ce632314a493805f4f4935367a0378ebc267a84 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 2 Oct 2018 15:37:54 -0400
Subject: [PATCH 098/124] initial setup magpie adapter for process visibility

---
 magpie/adapter/__init__.py      |   4 +-
 magpie/adapter/magpieprocess.py | 121 ++++++++++++++++++++++++++++++++
 magpie/adapter/magpieservice.py |   4 +-
 3 files changed, 126 insertions(+), 3 deletions(-)

diff --git a/magpie/adapter/__init__.py b/magpie/adapter/__init__.py
index 0a246b05f..2bfcf0504 100644
--- a/magpie/adapter/__init__.py
+++ b/magpie/adapter/__init__.py
@@ -19,8 +19,10 @@ def servicestore_factory(self, registry, headers=None):
         return MagpieServiceStore(registry=registry)
 
     def processstore_factory(self, registry):
-        # no reimplementation of processes on magpie side
+        # no reimplementation of processes on magpie side if 'default'
         # simply return the default twitcher process store
+
+
         return DefaultAdapter().processstore_factory(registry)
 
     def jobstore_factory(self, registry):
diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index e69de29bb..bbeb5f54f 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -0,0 +1,121 @@
+"""
+Store adapters to read data from magpie.
+"""
+
+from six.moves.urllib.parse import urlparse
+import logging
+import requests
+import json
+LOGGER = logging.getLogger("TWITCHER")
+
+from magpie.definitions.twitcher_definitions import *
+from magpie.definitions.pyramid_definitions import ConfigurationError, HTTPOk, HTTPCreated, HTTPNotFound
+
+# import 'process' elements separately than 'twitcher_definitions' because not defined in master
+from twitcher.config import get_twitcher_configuration, TWITCHER_CONFIGURATION_EMS
+from twitcher.exceptions import ProcessNotFound
+from twitcher.store import processstore_defaultfactory
+from twitcher.store.base import ProcessStore
+from twitcher.visibility import VISIBILITY_PUBLIC, VISIBILITY_PRIVATE
+
+
+class MagpieProcessStore(ProcessStore):
+    """
+    Registry for OWS processes.
+    Uses default process store for most operations.
+    Uses magpie to update process access and visibility.
+    """
+
+    def __init__(self, registry):
+        try:
+            # add 'http' scheme to url if omitted from config since further 'requests' calls fail without it
+            # mostly for testing when only 'localhost' is specified
+            # otherwise twitcher config should explicitly define it in MAGPIE_URL
+            url_parsed = urlparse(registry.settings.get('magpie.url').strip('/'))
+            if url_parsed.scheme in ['http', 'https']:
+                self.magpie_url = url_parsed.geturl()
+            else:
+                self.magpie_url = 'http://{}'.format(url_parsed.geturl())
+                LOGGER.warn("Missing scheme from MagpieServiceStore url, new value: '{}'".format(self.magpie_url))
+
+            self.twitcher_config = get_twitcher_configuration(registry.settings)
+        except AttributeError:
+            #If magpie.url does not exist, calling strip fct over None will raise this issue
+            raise ConfigurationError('magpie.url config cannot be found')
+
+    def save_process(self, process, overwrite=True, request=None):
+        """Delegate execution to default twitcher process store."""
+        return processstore_defaultfactory(request.registry).save_process(process, overwrite, request)
+
+    def delete_process(self, process_id, request=None):
+        """Delegate execution to default twitcher process store."""
+        return processstore_defaultfactory(request.registry).delete_process(process_id, request)
+
+    def list_processes(self, request=None):
+        """Delegate execution to default twitcher process store."""
+        return processstore_defaultfactory(request.registry).list_processes(request)
+
+    def fetch_by_id(self, process_id, request=None):
+        """Delegate execution to default twitcher process store."""
+        return processstore_defaultfactory(request.registry).fetch_by_id(process_id, request)
+
+    def _get_process_resource_id(self, process_id, request):
+        resp = requests.get('{host}/groups/users/resources'.format(host=self.magpie_url), cookies=request.cookies)
+        if resp.status_code != HTTPOk.code:
+            raise resp.raise_for_status()
+        try:
+            ems_resources = resp.json()['resources']['api']['ems']['resources']
+            ems_processes = None
+            for res_id in ems_resources:
+                if ems_resources[res_id]['resource_name'] == 'processes':
+                    ems_processes = ems_resources[res_id]['children']
+                    break
+            if not ems_processes:
+                raise ProcessNotFound("Could not find processes resource endpoint for visibility retrieval.")
+            for process_res_id in ems_processes:
+                if ems_processes[process_res_id]['resource_name'] == process_id:
+                    return ems_processes[process_res_id]['resource_id']
+        except KeyError:
+            raise ProcessNotFound('Could not find process `{}` resource for visibility retrieval.'.format(process_id))
+        return None
+
+    def get_visibility(self, process_id, request=None):
+        """
+        Get visibility of a process.
+
+        If twitcher is not in EMS mode, simply delegate execution to default twitcher process store.
+        If twitcher is in EMS mode, return the magpie visibility status according to user permissions.
+        """
+        if self.twitcher_config != TWITCHER_CONFIGURATION_EMS:
+            return processstore_defaultfactory(request.registry).get_visibility(process_id, request)
+
+        process_res_id = self._get_process_resource_id(process_id, request)
+        return VISIBILITY_PUBLIC if process_res_id is not None else VISIBILITY_PRIVATE
+
+    def set_visibility(self, process_id, visibility, request=None):
+        """
+        Set visibility of a process.
+
+        Delegate change of process visibility to default twitcher process store.
+        If twitcher is in EMS mode, also modify magpie permissions of corresponding process access point.
+        """
+        # write visibility to store to remain consistent in processes structures even if using magpie permissions
+        processstore_defaultfactory(request.registry).set_visibility(process_id, request)
+
+        if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
+            process_res_id = self._get_process_resource_id(process_id, request)
+            if not process_res_id:
+                raise ProcessNotFound('Could not find process `{}` resource to change visibility.'.format(process_id))
+
+            if visibility == VISIBILITY_PRIVATE:
+                path = '{host}/groups/users/resources/{id}/permissions/{perm}' \
+                       .format(host=self.magpie_url, id=process_res_id, perm='read')
+                reps = requests.delete(path, cookies=request.cookies)
+                if reps.status_code not in (HTTPOk.code, HTTPNotFound.code):
+                    raise reps.raise_for_status()
+
+            elif visibility == VISIBILITY_PUBLIC:
+                path = '{host}/groups/users/resources/{id}/permissions'.format(host=self.magpie_url, id=process_res_id)
+                reps = requests.post(path, cookies=request.cookies, data={u'permission_name': u'read'})
+                if reps.status_code not in (HTTPOk.code, HTTPCreated.code):
+                    raise reps.raise_for_status()
diff --git a/magpie/adapter/magpieservice.py b/magpie/adapter/magpieservice.py
index 27349084c..d9acbc87c 100644
--- a/magpie/adapter/magpieservice.py
+++ b/magpie/adapter/magpieservice.py
@@ -9,7 +9,7 @@
 LOGGER = logging.getLogger("TWITCHER")
 
 from magpie.definitions.twitcher_definitions import *
-from magpie.definitions.pyramid_definitions import ConfigurationError
+from magpie.definitions.pyramid_definitions import ConfigurationError, HTTPOk
 
 
 class MagpieServiceStore(ServiceStore):
@@ -51,7 +51,7 @@ def list_services(self, request=None):
         path = '/users/current/services?inherit=True&cascade=True'
         response = requests.get('{url}{path}'.format(url=self.magpie_url, path=path),
                                 cookies=request.cookies)
-        if response.status_code != 200:
+        if response.status_code != HTTPOk.code:
             raise response.raise_for_status()
         services = json.loads(response.text)
         for service_type in services['services']:

From efa4df441eff9f2d5486076369fc10066de2121d Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 2 Oct 2018 17:00:39 -0400
Subject: [PATCH 099/124] instantiate magpie adapter process store

---
 magpie/adapter/__init__.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/magpie/adapter/__init__.py b/magpie/adapter/__init__.py
index 2bfcf0504..5b9d47ce7 100644
--- a/magpie/adapter/__init__.py
+++ b/magpie/adapter/__init__.py
@@ -3,6 +3,7 @@
 from magpie.definitions.twitcher_definitions import *
 from magpie.adapter.magpieowssecurity import *
 from magpie.adapter.magpieservice import *
+from magpie.adapter.magpieprocess import *
 from magpie.models import get_user
 from magpie.security import auth_config_from_settings
 from magpie.db import *
@@ -19,11 +20,7 @@ def servicestore_factory(self, registry, headers=None):
         return MagpieServiceStore(registry=registry)
 
     def processstore_factory(self, registry):
-        # no reimplementation of processes on magpie side if 'default'
-        # simply return the default twitcher process store
-
-
-        return DefaultAdapter().processstore_factory(registry)
+        return MagpieProcessStore(registry=registry)
 
     def jobstore_factory(self, registry):
         # no reimplementation of jobs on magpie side

From 146e794ca65311ade1a9d81045b5709c8634fd62 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 2 Oct 2018 17:03:28 -0400
Subject: [PATCH 100/124] add input visibility processstore

---
 magpie/adapter/magpieprocess.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index bbeb5f54f..74509125d 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -100,7 +100,7 @@ def set_visibility(self, process_id, visibility, request=None):
         If twitcher is in EMS mode, also modify magpie permissions of corresponding process access point.
         """
         # write visibility to store to remain consistent in processes structures even if using magpie permissions
-        processstore_defaultfactory(request.registry).set_visibility(process_id, request)
+        processstore_defaultfactory(request.registry).set_visibility(process_id, visibility, request)
 
         if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
             process_res_id = self._get_process_resource_id(process_id, request)

From 8bdc29b3c8bab57f0ce1fdc5ff708677ad9db0e1 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 2 Oct 2018 17:13:35 -0400
Subject: [PATCH 101/124] adjust permissions verification

---
 magpie/adapter/magpieprocess.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index 74509125d..d1049a6e4 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -9,7 +9,7 @@
 LOGGER = logging.getLogger("TWITCHER")
 
 from magpie.definitions.twitcher_definitions import *
-from magpie.definitions.pyramid_definitions import ConfigurationError, HTTPOk, HTTPCreated, HTTPNotFound
+from magpie.definitions.pyramid_definitions import ConfigurationError, HTTPOk, HTTPCreated, HTTPNotFound, HTTPConflict
 
 # import 'process' elements separately than 'twitcher_definitions' because not defined in master
 from twitcher.config import get_twitcher_configuration, TWITCHER_CONFIGURATION_EMS
@@ -111,11 +111,13 @@ def set_visibility(self, process_id, visibility, request=None):
                 path = '{host}/groups/users/resources/{id}/permissions/{perm}' \
                        .format(host=self.magpie_url, id=process_res_id, perm='read')
                 reps = requests.delete(path, cookies=request.cookies)
+                # permission is not set if deleted or non existing
                 if reps.status_code not in (HTTPOk.code, HTTPNotFound.code):
                     raise reps.raise_for_status()
 
             elif visibility == VISIBILITY_PUBLIC:
                 path = '{host}/groups/users/resources/{id}/permissions'.format(host=self.magpie_url, id=process_res_id)
                 reps = requests.post(path, cookies=request.cookies, data={u'permission_name': u'read'})
-                if reps.status_code not in (HTTPOk.code, HTTPCreated.code):
+                # permission is set if created or already exists
+                if reps.status_code not in (HTTPCreated.code, HTTPConflict.code):
                     raise reps.raise_for_status()

From 31a92d19fc2a63e392658c8e52e0048d409d3105 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 2 Oct 2018 17:59:19 -0400
Subject: [PATCH 102/124] filter processes by visibility

---
 magpie/adapter/magpieprocess.py | 96 ++++++++++++++++++++++++---------
 1 file changed, 71 insertions(+), 25 deletions(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index d1049a6e4..29a762149 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -43,42 +43,86 @@ def __init__(self, registry):
             #If magpie.url does not exist, calling strip fct over None will raise this issue
             raise ConfigurationError('magpie.url config cannot be found')
 
-    def save_process(self, process, overwrite=True, request=None):
-        """Delegate execution to default twitcher process store."""
-        return processstore_defaultfactory(request.registry).save_process(process, overwrite, request)
-
-    def delete_process(self, process_id, request=None):
-        """Delegate execution to default twitcher process store."""
-        return processstore_defaultfactory(request.registry).delete_process(process_id, request)
-
-    def list_processes(self, request=None):
-        """Delegate execution to default twitcher process store."""
-        return processstore_defaultfactory(request.registry).list_processes(request)
-
-    def fetch_by_id(self, process_id, request=None):
-        """Delegate execution to default twitcher process store."""
-        return processstore_defaultfactory(request.registry).fetch_by_id(process_id, request)
+    def _get_process_resources(self, request):
+        """
+        Gets all 'process' resources corresponding to results under 'ems/processes/{id}'.
+        Only visible processes (resources with 'read' permissions of group 'users') are returned.
 
-    def _get_process_resource_id(self, process_id, request):
+        :return: list of twitcher 'process' instances filtered by relevant magpie resources with permissions set.
+        """
         resp = requests.get('{host}/groups/users/resources'.format(host=self.magpie_url), cookies=request.cookies)
         if resp.status_code != HTTPOk.code:
             raise resp.raise_for_status()
         try:
             ems_resources = resp.json()['resources']['api']['ems']['resources']
-            ems_processes = None
             for res_id in ems_resources:
                 if ems_resources[res_id]['resource_name'] == 'processes':
                     ems_processes = ems_resources[res_id]['children']
-                    break
-            if not ems_processes:
-                raise ProcessNotFound("Could not find processes resource endpoint for visibility retrieval.")
-            for process_res_id in ems_processes:
-                if ems_processes[process_res_id]['resource_name'] == process_id:
-                    return ems_processes[process_res_id]['resource_id']
+                    return ems_processes
+        except KeyError:
+            raise ProcessNotFound("Could not find processes resource endpoint for visibility retrieval.")
+        return list()
+
+    def _get_process_resource_id(self, process_id, ems_processes_resources):
+        """
+        Searches for a 'process' resource corresponding to 'ems/processes/{id}'.
+        Only visible processes (resources with 'read' permissions of group 'users') are returned.
+
+        :returns: id of the found 'process' resource, or None.
+        """
+        if not ems_processes_resources:
+            raise ProcessNotFound("Could not find processes resource endpoint for visibility retrieval.")
+        try:
+            for process_res_id in ems_processes_resources:
+                if ems_processes_resources[process_res_id]['resource_name'] == process_id:
+                    return ems_processes_resources[process_res_id]['resource_id']
         except KeyError:
             raise ProcessNotFound('Could not find process `{}` resource for visibility retrieval.'.format(process_id))
         return None
 
+    def save_process(self, process, overwrite=True, request=None):
+        """Delegate execution to default twitcher process store."""
+        return processstore_defaultfactory(request.registry).save_process(process, overwrite, request)
+
+    def delete_process(self, process_id, request=None):
+        """
+        Delete a process.
+
+        If twitcher is not in EMS mode, simply delegate execution to default twitcher process store.
+        If twitcher is in EMS mode, user requesting deletion must have sufficient permissions in magpie to do so.
+        """
+        try:
+
+
+            resp = requests.delete('{host}/resources'.format(host=self.magpie_url), cookies=request.cookies)
+            if resp.status_code != HTTPOk.code:
+                raise resp.raise_for_status()
+
+        return processstore_defaultfactory(request.registry).delete_process(process_id, request)
+
+    def list_processes(self, request=None):
+        """
+        List processes.
+
+        If twitcher is not in EMS mode, simply delegate execution to default twitcher process store.
+        If twitcher is in EMS mode, filter by corresponding resources with read permissions.
+        """
+        process_list = processstore_defaultfactory(request.registry).list_processes(request)
+        if self.twitcher_config != TWITCHER_CONFIGURATION_EMS:
+            return process_list
+
+        ems_processes = self._get_process_resources(request)
+        ems_processes_visible = list()
+        for process in process_list:
+            # if id is returned, filtered group resource permission was set, therefore visibility is permitted
+            if self._get_process_resource_id(process.id, ems_processes):
+                ems_processes_visible.append(process)
+        return ems_processes_visible
+
+    def fetch_by_id(self, process_id, request=None):
+        """Delegate execution to default twitcher process store."""
+        return processstore_defaultfactory(request.registry).fetch_by_id(process_id, request)
+
     def get_visibility(self, process_id, request=None):
         """
         Get visibility of a process.
@@ -89,7 +133,8 @@ def get_visibility(self, process_id, request=None):
         if self.twitcher_config != TWITCHER_CONFIGURATION_EMS:
             return processstore_defaultfactory(request.registry).get_visibility(process_id, request)
 
-        process_res_id = self._get_process_resource_id(process_id, request)
+        ems_processes = self._get_process_resources(request)
+        process_res_id = self._get_process_resource_id(process_id, ems_processes)
         return VISIBILITY_PUBLIC if process_res_id is not None else VISIBILITY_PRIVATE
 
     def set_visibility(self, process_id, visibility, request=None):
@@ -103,7 +148,8 @@ def set_visibility(self, process_id, visibility, request=None):
         processstore_defaultfactory(request.registry).set_visibility(process_id, visibility, request)
 
         if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
-            process_res_id = self._get_process_resource_id(process_id, request)
+            ems_processes = self._get_process_resources(request)
+            process_res_id = self._get_process_resource_id(process_id, ems_processes)
             if not process_res_id:
                 raise ProcessNotFound('Could not find process `{}` resource to change visibility.'.format(process_id))
 

From e0645a78749d168f8c12cd49d7db190d6df6402d Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 2 Oct 2018 19:17:44 -0400
Subject: [PATCH 103/124] more magpie adapter process handling

---
 magpie/adapter/__init__.py                 |  2 +
 magpie/adapter/magpieprocess.py            | 83 +++++++++++++++++++---
 magpie/definitions/twitcher_definitions.py |  1 +
 3 files changed, 78 insertions(+), 8 deletions(-)

diff --git a/magpie/adapter/__init__.py b/magpie/adapter/__init__.py
index 5b9d47ce7..e98242958 100644
--- a/magpie/adapter/__init__.py
+++ b/magpie/adapter/__init__.py
@@ -20,6 +20,8 @@ def servicestore_factory(self, registry, headers=None):
         return MagpieServiceStore(registry=registry)
 
     def processstore_factory(self, registry):
+        if get_twitcher_configuration(registry.settings) == TWITCHER_CONFIGURATION_DEFAULT:
+            return DefaultAdapter().processstore_factory(registry)
         return MagpieProcessStore(registry=registry)
 
     def jobstore_factory(self, registry):
diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index 29a762149..2424a679f 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -13,7 +13,7 @@
 
 # import 'process' elements separately than 'twitcher_definitions' because not defined in master
 from twitcher.config import get_twitcher_configuration, TWITCHER_CONFIGURATION_EMS
-from twitcher.exceptions import ProcessNotFound
+from twitcher.exceptions import ProcessNotFound, ProcessRegistrationError
 from twitcher.store import processstore_defaultfactory
 from twitcher.store.base import ProcessStore
 from twitcher.visibility import VISIBILITY_PUBLIC, VISIBILITY_PRIVATE
@@ -80,8 +80,71 @@ def _get_process_resource_id(self, process_id, ems_processes_resources):
             raise ProcessNotFound('Could not find process `{}` resource for visibility retrieval.'.format(process_id))
         return None
 
+    def _create_resource(self, resource_name, resource_parent_id, request):
+        """
+        Creates a resource under another parent resource, and sets basic user read permission on it.
+
+        :returns: id of the created resource
+        """
+        try:
+            data = {u'parent_id': resource_parent_id, u'resource_name': resource_name, u'resource_type': u'route'}
+            path = '{host}/resources'.format(host=self.magpie_url)
+            resp = requests.post(path, cookies=request.cookies, data=data)
+            if resp.status_code != HTTPCreated.code:
+                raise resp.raise_for_status()
+            res_id = resp.json()['resource']['resource_id']
+
+            data = {u'permission_name': u'read-match'}
+            path = '{host}/users/current/resources/{id}/permissions'.format(host=self.magpie_url, id=res_id)
+            resp = requests.post(path, cookies=request.cookies, data=data)
+            if resp.status_code != HTTPCreated.code:
+                raise resp.raise_for_status()
+
+            return res_id
+        except KeyError:
+            raise ProcessRegistrationError('Failed adding process resource route `{}`.'.format(resource_name))
+
     def save_process(self, process, overwrite=True, request=None):
-        """Delegate execution to default twitcher process store."""
+        """
+        Save a new process.
+
+        If twitcher is not in EMS mode, simply delegate execution to default twitcher process store.
+        If twitcher is in EMS mode, user requesting creation must have sufficient permissions in magpie to do so.
+        """
+        if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
+            try:
+                # get resource id of ems service
+                resp = requests.get('{host}/services/ems'.format(host=self.magpie_url), cookies=request.cookies)
+                if resp.status_code != HTTPOk.code:
+                    raise resp.raise_for_status()
+                ems_res_id = resp.json()['ems']['resource_id']
+            except KeyError:
+                raise ProcessRegistrationError('Failed retrieving EMS service resource.')
+
+            try:
+                # get resource id of route '/processes', create it as necessary
+                path = '{host}/resources/{id}'.format(host=self.magpie_url, id=ems_res_id)
+                resp = requests.get(path, cookies=request.cookies)
+                if resp.status_code != HTTPOk.code:
+                    raise resp.raise_for_status()
+                processes_res_id = None
+                ems_resources = resp.json()[str(ems_res_id)]['children']
+                for child_resource in ems_resources:
+                    if ems_resources[child_resource]['resource_name'] == 'processes':
+                        processes_res_id = ems_resources[child_resource]['resource_id']
+                        break
+                if not processes_res_id:
+                    processes_res_id = self._create_resource(u'processes', ems_res_id, request)
+            except KeyError:
+                raise ProcessRegistrationError('Failed retrieving EMS processes resource.')
+
+            # create resource id of route '/processes/{process_id}' and set minimal permissions
+            process_res_id = self._create_resource(process.id, processes_res_id, request)
+            path = '{host}/users/current/{id}'.format(host=self.magpie_url, id=ems_res_id)
+            resp = requests.get(path, cookies=request.cookies)
+            if resp.status_code != HTTPOk.code:
+                raise resp.raise_for_status()
+
         return processstore_defaultfactory(request.registry).save_process(process, overwrite, request)
 
     def delete_process(self, process_id, request=None):
@@ -91,10 +154,14 @@ def delete_process(self, process_id, request=None):
         If twitcher is not in EMS mode, simply delegate execution to default twitcher process store.
         If twitcher is in EMS mode, user requesting deletion must have sufficient permissions in magpie to do so.
         """
-        try:
-
+        if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
+            resources = self._get_process_resources(request)
+            resource_id = self._get_process_resource_id(process_id, resources)
+            if not resource_id:
+                raise ProcessNotFound('Could not find process `{}` resource for deletion.'.format(process_id))
 
-            resp = requests.delete('{host}/resources'.format(host=self.magpie_url), cookies=request.cookies)
+            path = '{host}/resources/{id}'.format(host=self.magpie_url, id=resource_id)
+            resp = requests.delete(path, cookies=request.cookies)
             if resp.status_code != HTTPOk.code:
                 raise resp.raise_for_status()
 
@@ -144,9 +211,6 @@ def set_visibility(self, process_id, visibility, request=None):
         Delegate change of process visibility to default twitcher process store.
         If twitcher is in EMS mode, also modify magpie permissions of corresponding process access point.
         """
-        # write visibility to store to remain consistent in processes structures even if using magpie permissions
-        processstore_defaultfactory(request.registry).set_visibility(process_id, visibility, request)
-
         if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
             ems_processes = self._get_process_resources(request)
             process_res_id = self._get_process_resource_id(process_id, ems_processes)
@@ -167,3 +231,6 @@ def set_visibility(self, process_id, visibility, request=None):
                 # permission is set if created or already exists
                 if reps.status_code not in (HTTPCreated.code, HTTPConflict.code):
                     raise reps.raise_for_status()
+
+        # write visibility to store to remain consistent in processes structures even if using magpie permissions
+        processstore_defaultfactory(request.registry).set_visibility(process_id, visibility, request)
diff --git a/magpie/definitions/twitcher_definitions.py b/magpie/definitions/twitcher_definitions.py
index d532808cc..279ec5f15 100644
--- a/magpie/definitions/twitcher_definitions.py
+++ b/magpie/definitions/twitcher_definitions.py
@@ -3,6 +3,7 @@
 from twitcher.owsproxy import owsproxy
 from twitcher.owssecurity import OWSSecurityInterface
 from twitcher.owsexceptions import OWSAccessForbidden
+from twitcher.config import get_twitcher_configuration, TWITCHER_CONFIGURATION_DEFAULT
 from twitcher.utils import parse_service_name
 from twitcher.esgf import fetch_certificate, ESGF_CREDENTIALS
 from twitcher.datatype import Service

From 898fbb4adf6a92e7168e7c25620279cfd4c85477 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Tue, 2 Oct 2018 20:06:21 -0400
Subject: [PATCH 104/124] create/list/delete processes with magpie resources

---
 magpie/adapter/__init__.py      |  6 ++--
 magpie/adapter/magpieprocess.py | 64 ++++++++++++++++++++++++---------
 2 files changed, 49 insertions(+), 21 deletions(-)

diff --git a/magpie/adapter/__init__.py b/magpie/adapter/__init__.py
index e98242958..0c9966503 100644
--- a/magpie/adapter/__init__.py
+++ b/magpie/adapter/__init__.py
@@ -2,8 +2,8 @@
 from magpie.definitions.ziggurat_definitions import *
 from magpie.definitions.twitcher_definitions import *
 from magpie.adapter.magpieowssecurity import *
-from magpie.adapter.magpieservice import *
-from magpie.adapter.magpieprocess import *
+from magpie.adapter.magpieprocess import MagpieProcessStore
+from magpie.adapter.magpieservice import MagpieServiceStore
 from magpie.models import get_user
 from magpie.security import auth_config_from_settings
 from magpie.db import *
@@ -20,8 +20,6 @@ def servicestore_factory(self, registry, headers=None):
         return MagpieServiceStore(registry=registry)
 
     def processstore_factory(self, registry):
-        if get_twitcher_configuration(registry.settings) == TWITCHER_CONFIGURATION_DEFAULT:
-            return DefaultAdapter().processstore_factory(registry)
         return MagpieProcessStore(registry=registry)
 
     def jobstore_factory(self, registry):
diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index 2424a679f..f0027217a 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -6,10 +6,18 @@
 import logging
 import requests
 import json
+import six
 LOGGER = logging.getLogger("TWITCHER")
 
 from magpie.definitions.twitcher_definitions import *
-from magpie.definitions.pyramid_definitions import ConfigurationError, HTTPOk, HTTPCreated, HTTPNotFound, HTTPConflict
+from magpie.definitions.pyramid_definitions import (
+    ConfigurationError,
+    HTTPOk,
+    HTTPCreated,
+    HTTPNotFound,
+    HTTPConflict,
+    HTTPUnauthorized,
+)
 
 # import 'process' elements separately than 'twitcher_definitions' because not defined in master
 from twitcher.config import get_twitcher_configuration, TWITCHER_CONFIGURATION_EMS
@@ -80,10 +88,14 @@ def _get_process_resource_id(self, process_id, ems_processes_resources):
             raise ProcessNotFound('Could not find process `{}` resource for visibility retrieval.'.format(process_id))
         return None
 
-    def _create_resource(self, resource_name, resource_parent_id, request):
+    def _create_resource(self, resource_name, resource_parent_id, permission_names, request):
         """
-        Creates a resource under another parent resource, and sets basic user read permission on it.
+        Creates a resource under another parent resource, and sets basic user permissions on it.
 
+        :param resource_name: (str) name of the resource to create.
+        :param resource_parent_id: (int) id of the parent resource under which to create `resource_name`.
+        :param permission_names: (None, str, iterator) permissions to apply to the created resource, if any.
+        :param request: calling request for headers and credentials
         :returns: id of the created resource
         """
         try:
@@ -94,11 +106,17 @@ def _create_resource(self, resource_name, resource_parent_id, request):
                 raise resp.raise_for_status()
             res_id = resp.json()['resource']['resource_id']
 
-            data = {u'permission_name': u'read-match'}
-            path = '{host}/users/current/resources/{id}/permissions'.format(host=self.magpie_url, id=res_id)
-            resp = requests.post(path, cookies=request.cookies, data=data)
-            if resp.status_code != HTTPCreated.code:
-                raise resp.raise_for_status()
+            if permission_names is None:
+                permission_names = []
+            if isinstance(permission_names, six.string_types):
+                permission_names = [permission_names]
+
+            for perm in permission_names:
+                data = {u'permission_name': perm}
+                path = '{host}/users/current/resources/{id}/permissions'.format(host=self.magpie_url, id=res_id)
+                resp = requests.post(path, cookies=request.cookies, data=data)
+                if resp.status_code not in (HTTPCreated.code, HTTPConflict.code):
+                    raise resp.raise_for_status()
 
             return res_id
         except KeyError:
@@ -134,16 +152,20 @@ def save_process(self, process, overwrite=True, request=None):
                         processes_res_id = ems_resources[child_resource]['resource_id']
                         break
                 if not processes_res_id:
-                    processes_res_id = self._create_resource(u'processes', ems_res_id, request)
+                    # all members of 'users' group can query '/ems/processes' (read exact route match),
+                    # but visibility of each process will be filtered by specific '/ems/processes/{id}' permissions
+                    processes_res_id = self._create_resource(u'processes', ems_res_id, u'read-match', request)
+                    data = {u'permission_name': u'read-match'}
+                    path = '{host}/groups/users/resources/{id}'.format(host=self.magpie_url, id=processes_res_id)
+                    resp = requests.post(path, cookies=request.cookies, data=data)
+                    if resp.status_code not in (HTTPCreated.code, HTTPConflict.code):
+                        raise resp.raise_for_status()
             except KeyError:
                 raise ProcessRegistrationError('Failed retrieving EMS processes resource.')
 
-            # create resource id of route '/processes/{process_id}' and set minimal permissions
-            process_res_id = self._create_resource(process.id, processes_res_id, request)
-            path = '{host}/users/current/{id}'.format(host=self.magpie_url, id=ems_res_id)
-            resp = requests.get(path, cookies=request.cookies)
-            if resp.status_code != HTTPOk.code:
-                raise resp.raise_for_status()
+            # create resource id of route '/ems/processes/{id}' and set minimal permissions
+            # use (read/write) permissions so that user creating the process can execute any sub-route request on it
+            self._create_resource(process.id, processes_res_id, [u'read', u'write'], request)
 
         return processstore_defaultfactory(request.registry).save_process(process, overwrite, request)
 
@@ -187,7 +209,15 @@ def list_processes(self, request=None):
         return ems_processes_visible
 
     def fetch_by_id(self, process_id, request=None):
-        """Delegate execution to default twitcher process store."""
+        """
+        Get a process if visible for user.
+
+        If twitcher is not in EMS mode, simply delegate execution to default twitcher process store.
+        If twitcher is in EMS mode, return the process if visible based on magpie user permissions.
+        """
+        if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
+            if self.get_visibility(process_id, request) != VISIBILITY_PUBLIC:
+                raise HTTPUnauthorized()
         return processstore_defaultfactory(request.registry).fetch_by_id(process_id, request)
 
     def get_visibility(self, process_id, request=None):
@@ -195,7 +225,7 @@ def get_visibility(self, process_id, request=None):
         Get visibility of a process.
 
         If twitcher is not in EMS mode, simply delegate execution to default twitcher process store.
-        If twitcher is in EMS mode, return the magpie visibility status according to user permissions.
+        If twitcher is in EMS mode, return the process visibility based on magpie user permissions.
         """
         if self.twitcher_config != TWITCHER_CONFIGURATION_EMS:
             return processstore_defaultfactory(request.registry).get_visibility(process_id, request)

From 20fffeb05d8307b74156b1081684b1a8382c652d Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Wed, 3 Oct 2018 13:15:18 -0400
Subject: [PATCH 105/124] reapply logging.config import

---
 magpie/magpiectl.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/magpie/magpiectl.py b/magpie/magpiectl.py
index 3f4bec8ea..0657afb1c 100644
--- a/magpie/magpiectl.py
+++ b/magpie/magpiectl.py
@@ -10,6 +10,7 @@
 import time
 import warnings
 import logging
+import logging.config
 LOGGER = logging.getLogger(__name__)
 
 # -- Definitions

From 3d1bc2396acbc837c8e897140dfb9d992d29193b Mon Sep 17 00:00:00 2001
From: Francis Chrette Migneault <francis.charette.migneault@gmail.com>
Date: Wed, 3 Oct 2018 17:24:24 +0000
Subject: [PATCH 106/124] remove logging req breaking stuff (?)

---
 requirements.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 8424ec2bd..aac827c00 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,6 @@ pluggy
 flake8==3.5.0
 coverage==4.0
 Sphinx==1.3.1
-logging
 #cryptography==1.9
 PyYAML>=3.11
 pyramid==1.8.3

From c5b8db913dd5369b41d3b0a97d6243a7c32af102 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Wed, 3 Oct 2018 13:31:56 -0400
Subject: [PATCH 107/124] test function name conflict

---
 tests/interfaces.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/interfaces.py b/tests/interfaces.py
index b22b0fff9..ce728334e 100644
--- a/tests/interfaces.py
+++ b/tests/interfaces.py
@@ -567,7 +567,7 @@ def test_PostResources_DirectServiceResource(self):
 
     @pytest.mark.resources
     @unittest.skipUnless(runner.MAGPIE_TEST_RESOURCES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('resources'))
-    def test_PostResources_DirectServiceResource(self):
+    def test_PostResources_DirectServiceResourceOptional(self):
         service_info = utils.TestSetup.get_ExistingTestServiceInfo(self)
         service_resource_id = service_info['resource_id']
 

From d259550776fb7d63198d5873c5efe0bffd5eb8b0 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Wed, 3 Oct 2018 13:51:31 -0400
Subject: [PATCH 108/124] enforce specific gunicorn version

---
 requirements.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 8424ec2bd..ec558560f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,6 @@ pluggy
 flake8==3.5.0
 coverage==4.0
 Sphinx==1.3.1
-logging
 #cryptography==1.9
 PyYAML>=3.11
 pyramid==1.8.3
@@ -22,7 +21,7 @@ lxml>=3.7
 bcrypt==3.1.3
 futures==3.1.1
 zope.sqlalchemy
-gunicorn
+gunicorn==19.8.1
 alembic==0.9.6
 paste
 python-dotenv

From bcbd05b040675de80ec4755492d85f98ed989335 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Wed, 3 Oct 2018 14:59:45 -0400
Subject: [PATCH 109/124] add debug info

---
 magpie/adapter/magpieprocess.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index f0027217a..9ee082699 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -58,7 +58,9 @@ def _get_process_resources(self, request):
 
         :return: list of twitcher 'process' instances filtered by relevant magpie resources with permissions set.
         """
-        resp = requests.get('{host}/groups/users/resources'.format(host=self.magpie_url), cookies=request.cookies)
+        path = '{host}/groups/users/resources'.format(host=self.magpie_url)
+        resp = requests.get(path, cookies=request.cookies)
+        LOGGER.debug('Looking for resources on: `{}`.'.format(path))
         if resp.status_code != HTTPOk.code:
             raise resp.raise_for_status()
         try:
@@ -69,6 +71,7 @@ def _get_process_resources(self, request):
                     return ems_processes
         except KeyError:
             raise ProcessNotFound("Could not find processes resource endpoint for visibility retrieval.")
+        LOGGER.debug('Could not find resource: `processes`.')
         return list()
 
     def _get_process_resource_id(self, process_id, ems_processes_resources):
@@ -79,13 +82,14 @@ def _get_process_resource_id(self, process_id, ems_processes_resources):
         :returns: id of the found 'process' resource, or None.
         """
         if not ems_processes_resources:
-            raise ProcessNotFound("Could not find processes resource endpoint for visibility retrieval.")
+            raise ProcessNotFound("Could not parse undefined processes resource endpoint for visibility retrieval.")
         try:
             for process_res_id in ems_processes_resources:
                 if ems_processes_resources[process_res_id]['resource_name'] == process_id:
                     return ems_processes_resources[process_res_id]['resource_id']
         except KeyError:
             raise ProcessNotFound('Could not find process `{}` resource for visibility retrieval.'.format(process_id))
+        LOGGER.debug('Could not find resource: `{}`.'.format(process_id))
         return None
 
     def _create_resource(self, resource_name, resource_parent_id, permission_names, request):
@@ -197,6 +201,7 @@ def list_processes(self, request=None):
         If twitcher is in EMS mode, filter by corresponding resources with read permissions.
         """
         process_list = processstore_defaultfactory(request.registry).list_processes(request)
+        LOGGER.debug('Found processes: {!r}.'.format(process_list))
         if self.twitcher_config != TWITCHER_CONFIGURATION_EMS:
             return process_list
 

From 46a02d81db4b13ac2c222056f22077bcc96b23be Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Wed, 3 Oct 2018 15:26:15 -0400
Subject: [PATCH 110/124] add request headers + more debug

---
 magpie/adapter/magpieprocess.py | 34 ++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index 9ee082699..4b27be2e0 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -27,6 +27,9 @@
 from twitcher.visibility import VISIBILITY_PUBLIC, VISIBILITY_PRIVATE
 
 
+json_headers = {'Accept': 'application/json'}
+
+
 class MagpieProcessStore(ProcessStore):
     """
     Registry for OWS processes.
@@ -59,10 +62,11 @@ def _get_process_resources(self, request):
         :return: list of twitcher 'process' instances filtered by relevant magpie resources with permissions set.
         """
         path = '{host}/groups/users/resources'.format(host=self.magpie_url)
-        resp = requests.get(path, cookies=request.cookies)
+        resp = requests.get(path, cookies=request.cookies, headers=json_headers)
         LOGGER.debug('Looking for resources on: `{}`.'.format(path))
         if resp.status_code != HTTPOk.code:
             raise resp.raise_for_status()
+        ems_resources = None
         try:
             ems_resources = resp.json()['resources']['api']['ems']['resources']
             for res_id in ems_resources:
@@ -70,8 +74,9 @@ def _get_process_resources(self, request):
                     ems_processes = ems_resources[res_id]['children']
                     return ems_processes
         except KeyError:
-            raise ProcessNotFound("Could not find processes resource endpoint for visibility retrieval.")
-        LOGGER.debug('Could not find resource: `processes`.')
+            LOGGER.debug("Content of ems service resources: `{!r}`.".format(ems_resources))
+            raise ProcessNotFound("Could not find resource `processes` endpoint for visibility retrieval.")
+        LOGGER.debug("Could not find resource: `processes`.")
         return list()
 
     def _get_process_resource_id(self, process_id, ems_processes_resources):
@@ -88,8 +93,9 @@ def _get_process_resource_id(self, process_id, ems_processes_resources):
                 if ems_processes_resources[process_res_id]['resource_name'] == process_id:
                     return ems_processes_resources[process_res_id]['resource_id']
         except KeyError:
-            raise ProcessNotFound('Could not find process `{}` resource for visibility retrieval.'.format(process_id))
-        LOGGER.debug('Could not find resource: `{}`.'.format(process_id))
+            LOGGER.debug("Content of ems processes resources: `{!r}`.".format(ems_processes_resources))
+            raise ProcessNotFound("Could not find process `{}` resource for visibility retrieval.".format(process_id))
+        LOGGER.debug("Could not find resource: `{}`.".format(process_id))
         return None
 
     def _create_resource(self, resource_name, resource_parent_id, permission_names, request):
@@ -105,7 +111,7 @@ def _create_resource(self, resource_name, resource_parent_id, permission_names,
         try:
             data = {u'parent_id': resource_parent_id, u'resource_name': resource_name, u'resource_type': u'route'}
             path = '{host}/resources'.format(host=self.magpie_url)
-            resp = requests.post(path, cookies=request.cookies, data=data)
+            resp = requests.post(path, data=data, cookies=request.cookies, headers=json_headers)
             if resp.status_code != HTTPCreated.code:
                 raise resp.raise_for_status()
             res_id = resp.json()['resource']['resource_id']
@@ -118,7 +124,7 @@ def _create_resource(self, resource_name, resource_parent_id, permission_names,
             for perm in permission_names:
                 data = {u'permission_name': perm}
                 path = '{host}/users/current/resources/{id}/permissions'.format(host=self.magpie_url, id=res_id)
-                resp = requests.post(path, cookies=request.cookies, data=data)
+                resp = requests.post(path, data=data, cookies=request.cookies, headers=json_headers)
                 if resp.status_code not in (HTTPCreated.code, HTTPConflict.code):
                     raise resp.raise_for_status()
 
@@ -136,7 +142,8 @@ def save_process(self, process, overwrite=True, request=None):
         if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
             try:
                 # get resource id of ems service
-                resp = requests.get('{host}/services/ems'.format(host=self.magpie_url), cookies=request.cookies)
+                path = '{host}/services/ems'.format(host=self.magpie_url)
+                resp = requests.get(path, cookies=request.cookies, headers=json_headers)
                 if resp.status_code != HTTPOk.code:
                     raise resp.raise_for_status()
                 ems_res_id = resp.json()['ems']['resource_id']
@@ -146,7 +153,7 @@ def save_process(self, process, overwrite=True, request=None):
             try:
                 # get resource id of route '/processes', create it as necessary
                 path = '{host}/resources/{id}'.format(host=self.magpie_url, id=ems_res_id)
-                resp = requests.get(path, cookies=request.cookies)
+                resp = requests.get(path, cookies=request.cookies, headers=json_headers)
                 if resp.status_code != HTTPOk.code:
                     raise resp.raise_for_status()
                 processes_res_id = None
@@ -161,7 +168,7 @@ def save_process(self, process, overwrite=True, request=None):
                     processes_res_id = self._create_resource(u'processes', ems_res_id, u'read-match', request)
                     data = {u'permission_name': u'read-match'}
                     path = '{host}/groups/users/resources/{id}'.format(host=self.magpie_url, id=processes_res_id)
-                    resp = requests.post(path, cookies=request.cookies, data=data)
+                    resp = requests.post(path, data=data, cookies=request.cookies, headers=json_headers)
                     if resp.status_code not in (HTTPCreated.code, HTTPConflict.code):
                         raise resp.raise_for_status()
             except KeyError:
@@ -187,7 +194,7 @@ def delete_process(self, process_id, request=None):
                 raise ProcessNotFound('Could not find process `{}` resource for deletion.'.format(process_id))
 
             path = '{host}/resources/{id}'.format(host=self.magpie_url, id=resource_id)
-            resp = requests.delete(path, cookies=request.cookies)
+            resp = requests.delete(path, cookies=request.cookies, headers=json_headers)
             if resp.status_code != HTTPOk.code:
                 raise resp.raise_for_status()
 
@@ -255,14 +262,15 @@ def set_visibility(self, process_id, visibility, request=None):
             if visibility == VISIBILITY_PRIVATE:
                 path = '{host}/groups/users/resources/{id}/permissions/{perm}' \
                        .format(host=self.magpie_url, id=process_res_id, perm='read')
-                reps = requests.delete(path, cookies=request.cookies)
+                reps = requests.delete(path, cookies=request.cookies, headers=json_headers)
                 # permission is not set if deleted or non existing
                 if reps.status_code not in (HTTPOk.code, HTTPNotFound.code):
                     raise reps.raise_for_status()
 
             elif visibility == VISIBILITY_PUBLIC:
                 path = '{host}/groups/users/resources/{id}/permissions'.format(host=self.magpie_url, id=process_res_id)
-                reps = requests.post(path, cookies=request.cookies, data={u'permission_name': u'read'})
+                data = {u'permission_name': u'read'}
+                reps = requests.post(path, data=data, cookies=request.cookies, headers=json_headers)
                 # permission is set if created or already exists
                 if reps.status_code not in (HTTPCreated.code, HTTPConflict.code):
                     raise reps.raise_for_status()

From 4d40df0b61138fcaecf2a8a245e00c174ac645a6 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Wed, 3 Oct 2018 13:51:31 -0400
Subject: [PATCH 111/124] enforce specific gunicorn version

---
 requirements.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 8424ec2bd..ec558560f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,6 @@ pluggy
 flake8==3.5.0
 coverage==4.0
 Sphinx==1.3.1
-logging
 #cryptography==1.9
 PyYAML>=3.11
 pyramid==1.8.3
@@ -22,7 +21,7 @@ lxml>=3.7
 bcrypt==3.1.3
 futures==3.1.1
 zope.sqlalchemy
-gunicorn
+gunicorn==19.8.1
 alembic==0.9.6
 paste
 python-dotenv

From 91878b4c3c2185f241fd234bbe4b49ad99e0b6e4 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Wed, 3 Oct 2018 17:15:42 -0400
Subject: [PATCH 112/124] revise permissions handling for processstore adapter

---
 magpie/adapter/magpieprocess.py | 47 +++++++++++++++++----------------
 1 file changed, 24 insertions(+), 23 deletions(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index 4b27be2e0..4078d18fd 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -57,11 +57,11 @@ def __init__(self, registry):
     def _get_process_resources(self, request):
         """
         Gets all 'process' resources corresponding to results under 'ems/processes/{id}'.
-        Only visible processes (resources with 'read' permissions of group 'users') are returned.
+        Only visible processes filtered by inherited user/group 'read' permissions are returned.
 
         :return: list of twitcher 'process' instances filtered by relevant magpie resources with permissions set.
         """
-        path = '{host}/groups/users/resources'.format(host=self.magpie_url)
+        path = '{host}/users/current/resources?inherit=true'.format(host=self.magpie_url)
         resp = requests.get(path, cookies=request.cookies, headers=json_headers)
         LOGGER.debug('Looking for resources on: `{}`.'.format(path))
         if resp.status_code != HTTPOk.code:
@@ -98,13 +98,14 @@ def _get_process_resource_id(self, process_id, ems_processes_resources):
         LOGGER.debug("Could not find resource: `{}`.".format(process_id))
         return None
 
-    def _create_resource(self, resource_name, resource_parent_id, permission_names, request):
+    def _create_resource(self, resource_name, resource_parent_id, group_name, permission_names, request):
         """
         Creates a resource under another parent resource, and sets basic user permissions on it.
 
         :param resource_name: (str) name of the resource to create.
         :param resource_parent_id: (int) id of the parent resource under which to create `resource_name`.
-        :param permission_names: (None, str, iterator) permissions to apply to the created resource, if any.
+        :param group_name: (str) name of the group for which to apply permissions to the created resource, if any.
+        :param permission_names: (None, str, iterator) group permissions to apply to the created resource, if any.
         :param request: calling request for headers and credentials
         :returns: id of the created resource
         """
@@ -116,17 +117,19 @@ def _create_resource(self, resource_name, resource_parent_id, permission_names,
                 raise resp.raise_for_status()
             res_id = resp.json()['resource']['resource_id']
 
-            if permission_names is None:
-                permission_names = []
-            if isinstance(permission_names, six.string_types):
-                permission_names = [permission_names]
+            if isinstance(group_name, six.string_types):
+                if permission_names is None:
+                    permission_names = []
+                if isinstance(permission_names, six.string_types):
+                    permission_names = [permission_names]
 
-            for perm in permission_names:
-                data = {u'permission_name': perm}
-                path = '{host}/users/current/resources/{id}/permissions'.format(host=self.magpie_url, id=res_id)
-                resp = requests.post(path, data=data, cookies=request.cookies, headers=json_headers)
-                if resp.status_code not in (HTTPCreated.code, HTTPConflict.code):
-                    raise resp.raise_for_status()
+                for perm in permission_names:
+                    data = {u'permission_name': perm}
+                    path = '{host}/groups/{grp}/resources/{id}/permissions' \
+                           .format(host=self.magpie_url, grp=group_name, id=res_id)
+                    resp = requests.post(path, data=data, cookies=request.cookies, headers=json_headers)
+                    if resp.status_code not in (HTTPCreated.code, HTTPConflict.code):
+                        raise resp.raise_for_status()
 
             return res_id
         except KeyError:
@@ -151,7 +154,7 @@ def save_process(self, process, overwrite=True, request=None):
                 raise ProcessRegistrationError('Failed retrieving EMS service resource.')
 
             try:
-                # get resource id of route '/processes', create it as necessary
+                # get resource id of route '/ems/processes', create it as necessary
                 path = '{host}/resources/{id}'.format(host=self.magpie_url, id=ems_res_id)
                 resp = requests.get(path, cookies=request.cookies, headers=json_headers)
                 if resp.status_code != HTTPOk.code:
@@ -165,18 +168,16 @@ def save_process(self, process, overwrite=True, request=None):
                 if not processes_res_id:
                     # all members of 'users' group can query '/ems/processes' (read exact route match),
                     # but visibility of each process will be filtered by specific '/ems/processes/{id}' permissions
-                    processes_res_id = self._create_resource(u'processes', ems_res_id, u'read-match', request)
-                    data = {u'permission_name': u'read-match'}
-                    path = '{host}/groups/users/resources/{id}'.format(host=self.magpie_url, id=processes_res_id)
-                    resp = requests.post(path, data=data, cookies=request.cookies, headers=json_headers)
-                    if resp.status_code not in (HTTPCreated.code, HTTPConflict.code):
-                        raise resp.raise_for_status()
+                    processes_res_id = self._create_resource(u'processes', ems_res_id, u'users', u'read-match', request)
             except KeyError:
                 raise ProcessRegistrationError('Failed retrieving EMS processes resource.')
 
             # create resource id of route '/ems/processes/{id}' and set minimal permissions
-            # use (read/write) permissions so that user creating the process can execute any sub-route request on it
-            self._create_resource(process.id, processes_res_id, [u'read', u'write'], request)
+            # use read permission so that users can execute any sub-route GET request on it
+            process_res_id = self._create_resource(process.id, processes_res_id, u'users', u'read', request)
+            # create resource id of route '/ems/processes/{id}/jobs' and set minimal permissions
+            # use write-match permission so that users can ONLY execute a job (cannot DELETE process, job, etc.)
+            self._create_resource(u'jobs', process_res_id, u'users', u'write-match', request)
 
         return processstore_defaultfactory(request.registry).save_process(process, overwrite, request)
 

From 4ae8b137b3138d115244140e42a85b9249265ef1 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Wed, 3 Oct 2018 17:50:00 -0400
Subject: [PATCH 113/124] change process resource retrieval to reflect desired
 visibility results

---
 magpie/adapter/magpieprocess.py | 36 ++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index 4078d18fd..69783bca5 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -56,12 +56,10 @@ def __init__(self, registry):
 
     def _get_process_resources(self, request):
         """
-        Gets all 'process' resources corresponding to results under 'ems/processes/{id}'.
-        Only visible processes filtered by inherited user/group 'read' permissions are returned.
-
-        :return: list of twitcher 'process' instances filtered by relevant magpie resources with permissions set.
+        Gets all 'process' resources corresponding to results under '/ems/processes'.
+        Resources are not filtered by user permissions.
         """
-        path = '{host}/users/current/resources?inherit=true'.format(host=self.magpie_url)
+        path = '{host}/resources'.format(host=self.magpie_url)
         resp = requests.get(path, cookies=request.cookies, headers=json_headers)
         LOGGER.debug('Looking for resources on: `{}`.'.format(path))
         if resp.status_code != HTTPOk.code:
@@ -79,19 +77,33 @@ def _get_process_resources(self, request):
         LOGGER.debug("Could not find resource: `processes`.")
         return list()
 
-    def _get_process_resource_id(self, process_id, ems_processes_resources):
+    def _get_process_resource_id(self, process_id, ems_processes_resources, request):
         """
         Searches for a 'process' resource corresponding to 'ems/processes/{id}'.
-        Only visible processes (resources with 'read' permissions of group 'users') are returned.
+        Only visible processes are returned (resources with reading permission assigned to request user).
 
         :returns: id of the found 'process' resource, or None.
+        :raises:
+            HTTPException if not matching Ok or Unauthorized statuses.
+            ProcessNotFound if some response parsing error occurred.
         """
         if not ems_processes_resources:
             raise ProcessNotFound("Could not parse undefined processes resource endpoint for visibility retrieval.")
         try:
             for process_res_id in ems_processes_resources:
+                # find the requested process resource by matching ids
                 if ems_processes_resources[process_res_id]['resource_name'] == process_id:
-                    return ems_processes_resources[process_res_id]['resource_id']
+                    LOGGER.debug("Found process resource: `{}`.".format(process_id))
+
+                    # if GET is permitted (200) on corresponding magpie resource route, twitcher process is visible
+                    path = '{host}/users/current/resources/{id}'.format(host=self.magpie_url, id=process_res_id)
+                    resp = requests.get(path, cookies=request.cookies, headers=json_headers)
+                    if resp.status_code == HTTPOk.code:
+                        return ems_processes_resources[process_res_id]['resource_id']
+                    elif resp.status_code == HTTPUnauthorized.code:
+                        return None
+                    else:
+                        raise resp.raise_for_status()
         except KeyError:
             LOGGER.debug("Content of ems processes resources: `{!r}`.".format(ems_processes_resources))
             raise ProcessNotFound("Could not find process `{}` resource for visibility retrieval.".format(process_id))
@@ -190,7 +202,7 @@ def delete_process(self, process_id, request=None):
         """
         if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
             resources = self._get_process_resources(request)
-            resource_id = self._get_process_resource_id(process_id, resources)
+            resource_id = self._get_process_resource_id(process_id, resources, request)
             if not resource_id:
                 raise ProcessNotFound('Could not find process `{}` resource for deletion.'.format(process_id))
 
@@ -217,7 +229,7 @@ def list_processes(self, request=None):
         ems_processes_visible = list()
         for process in process_list:
             # if id is returned, filtered group resource permission was set, therefore visibility is permitted
-            if self._get_process_resource_id(process.id, ems_processes):
+            if self._get_process_resource_id(process.id, ems_processes, request):
                 ems_processes_visible.append(process)
         return ems_processes_visible
 
@@ -244,7 +256,7 @@ def get_visibility(self, process_id, request=None):
             return processstore_defaultfactory(request.registry).get_visibility(process_id, request)
 
         ems_processes = self._get_process_resources(request)
-        process_res_id = self._get_process_resource_id(process_id, ems_processes)
+        process_res_id = self._get_process_resource_id(process_id, ems_processes, request)
         return VISIBILITY_PUBLIC if process_res_id is not None else VISIBILITY_PRIVATE
 
     def set_visibility(self, process_id, visibility, request=None):
@@ -256,7 +268,7 @@ def set_visibility(self, process_id, visibility, request=None):
         """
         if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
             ems_processes = self._get_process_resources(request)
-            process_res_id = self._get_process_resource_id(process_id, ems_processes)
+            process_res_id = self._get_process_resource_id(process_id, ems_processes, request)
             if not process_res_id:
                 raise ProcessNotFound('Could not find process `{}` resource to change visibility.'.format(process_id))
 

From 0ef0d5cdf453603684dfb0fbd6fea8de5e37bbe0 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Wed, 3 Oct 2018 18:36:16 -0400
Subject: [PATCH 114/124] get visible processes by GET request on proxy
 /processes

---
 magpie/adapter/magpieprocess.py            | 22 ++++++++++++++++++++--
 magpie/definitions/twitcher_definitions.py |  2 +-
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index 69783bca5..8570076de 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -50,10 +50,27 @@ def __init__(self, registry):
                 LOGGER.warn("Missing scheme from MagpieServiceStore url, new value: '{}'".format(self.magpie_url))
 
             self.twitcher_config = get_twitcher_configuration(registry.settings)
+            self.twitcher_service_url = None
         except AttributeError:
             #If magpie.url does not exist, calling strip fct over None will raise this issue
             raise ConfigurationError('magpie.url config cannot be found')
 
+    def _get_service_public_url(self, request):
+        if not self.twitcher_service_url and self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
+            # use generic 'current' user route to fetch service URL to ensure that even
+            # a user with minimal privileges will still return a match
+            path = '{host}/users/current/services?inherit=true&cascade=true'.format(host=self.magpie_url)
+            resp = requests.get(path, cookies=request.cookies, headers=json_headers)
+            if resp.status_code != HTTPOk.code:
+                raise resp.raise_for_status()
+            try:
+                self.twitcher_service_url = resp.json()['services']['api']['ems']['public_url']
+            except KeyError:
+                raise ProcessNotFound("Could not find resource `processes` endpoint for visibility retrieval.")
+            LOGGER.debug("Could not find resource: `processes`.")
+        return self.twitcher_service_url
+
+
     def _get_process_resources(self, request):
         """
         Gets all 'process' resources corresponding to results under '/ems/processes'.
@@ -95,8 +112,9 @@ def _get_process_resource_id(self, process_id, ems_processes_resources, request)
                 if ems_processes_resources[process_res_id]['resource_name'] == process_id:
                     LOGGER.debug("Found process resource: `{}`.".format(process_id))
 
-                    # if GET is permitted (200) on corresponding magpie resource route, twitcher process is visible
-                    path = '{host}/users/current/resources/{id}'.format(host=self.magpie_url, id=process_res_id)
+                    # if read permission is granted on corresponding magpie resource route, twitcher
+                    # '/ems/process/{process_id}' will be accessible, otherwise unauthorized is a private process
+                    path = '{host}/processes/{id}'.format(host=self._get_service_public_url(request), id=process_id)
                     resp = requests.get(path, cookies=request.cookies, headers=json_headers)
                     if resp.status_code == HTTPOk.code:
                         return ems_processes_resources[process_res_id]['resource_id']
diff --git a/magpie/definitions/twitcher_definitions.py b/magpie/definitions/twitcher_definitions.py
index 279ec5f15..0d95ee6e9 100644
--- a/magpie/definitions/twitcher_definitions.py
+++ b/magpie/definitions/twitcher_definitions.py
@@ -4,7 +4,7 @@
 from twitcher.owssecurity import OWSSecurityInterface
 from twitcher.owsexceptions import OWSAccessForbidden
 from twitcher.config import get_twitcher_configuration, TWITCHER_CONFIGURATION_DEFAULT
-from twitcher.utils import parse_service_name
+from twitcher.utils import parse_service_name, get_twitcher_url
 from twitcher.esgf import fetch_certificate, ESGF_CREDENTIALS
 from twitcher.datatype import Service
 from twitcher.store.base import ServiceStore

From ff63f044cbfd8f7ef238e2bdd5a60c2504d9fedb Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 4 Oct 2018 12:23:17 -0400
Subject: [PATCH 115/124] move imports to processstore specific usage

---
 magpie/adapter/__init__.py                 | 3 ++-
 magpie/definitions/twitcher_definitions.py | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/magpie/adapter/__init__.py b/magpie/adapter/__init__.py
index 0c9966503..4c6d730b6 100644
--- a/magpie/adapter/__init__.py
+++ b/magpie/adapter/__init__.py
@@ -2,7 +2,6 @@
 from magpie.definitions.ziggurat_definitions import *
 from magpie.definitions.twitcher_definitions import *
 from magpie.adapter.magpieowssecurity import *
-from magpie.adapter.magpieprocess import MagpieProcessStore
 from magpie.adapter.magpieservice import MagpieServiceStore
 from magpie.models import get_user
 from magpie.security import auth_config_from_settings
@@ -20,6 +19,8 @@ def servicestore_factory(self, registry, headers=None):
         return MagpieServiceStore(registry=registry)
 
     def processstore_factory(self, registry):
+        # import here to avoid import errors on default twitcher not implementing processes
+        from magpie.adapter.magpieprocess import MagpieProcessStore
         return MagpieProcessStore(registry=registry)
 
     def jobstore_factory(self, registry):
diff --git a/magpie/definitions/twitcher_definitions.py b/magpie/definitions/twitcher_definitions.py
index 0d95ee6e9..3f1bf0024 100644
--- a/magpie/definitions/twitcher_definitions.py
+++ b/magpie/definitions/twitcher_definitions.py
@@ -3,7 +3,6 @@
 from twitcher.owsproxy import owsproxy
 from twitcher.owssecurity import OWSSecurityInterface
 from twitcher.owsexceptions import OWSAccessForbidden
-from twitcher.config import get_twitcher_configuration, TWITCHER_CONFIGURATION_DEFAULT
 from twitcher.utils import parse_service_name, get_twitcher_url
 from twitcher.esgf import fetch_certificate, ESGF_CREDENTIALS
 from twitcher.datatype import Service

From cfbcae11267871206d6cc421cfdbcf5bf9885b17 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 4 Oct 2018 14:14:33 -0400
Subject: [PATCH 116/124] add ssl verify option use from twitcher

---
 magpie/adapter/magpieprocess.py | 36 ++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index 8570076de..52c77913e 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -27,9 +27,6 @@
 from twitcher.visibility import VISIBILITY_PUBLIC, VISIBILITY_PRIVATE
 
 
-json_headers = {'Accept': 'application/json'}
-
-
 class MagpieProcessStore(ProcessStore):
     """
     Registry for OWS processes.
@@ -50,6 +47,8 @@ def __init__(self, registry):
                 LOGGER.warn("Missing scheme from MagpieServiceStore url, new value: '{}'".format(self.magpie_url))
 
             self.twitcher_config = get_twitcher_configuration(registry.settings)
+            self.twitcher_ssl_verify = registry.settings.get('twitcher.ows_proxy_ssl_verify', True)
+            self.json_headers = {'Accept': 'application/json'}
             self.twitcher_service_url = None
         except AttributeError:
             #If magpie.url does not exist, calling strip fct over None will raise this issue
@@ -60,7 +59,8 @@ def _get_service_public_url(self, request):
             # use generic 'current' user route to fetch service URL to ensure that even
             # a user with minimal privileges will still return a match
             path = '{host}/users/current/services?inherit=true&cascade=true'.format(host=self.magpie_url)
-            resp = requests.get(path, cookies=request.cookies, headers=json_headers)
+            resp = requests.get(path, cookies=request.cookies,
+                                headers=self.json_headers, verify=self.twitcher_ssl_verify)
             if resp.status_code != HTTPOk.code:
                 raise resp.raise_for_status()
             try:
@@ -70,14 +70,14 @@ def _get_service_public_url(self, request):
             LOGGER.debug("Could not find resource: `processes`.")
         return self.twitcher_service_url
 
-
     def _get_process_resources(self, request):
         """
         Gets all 'process' resources corresponding to results under '/ems/processes'.
         Resources are not filtered by user permissions.
         """
         path = '{host}/resources'.format(host=self.magpie_url)
-        resp = requests.get(path, cookies=request.cookies, headers=json_headers)
+        resp = requests.get(path, cookies=request.cookies,
+                            headers=self.json_headers, verify=self.twitcher_ssl_verify)
         LOGGER.debug('Looking for resources on: `{}`.'.format(path))
         if resp.status_code != HTTPOk.code:
             raise resp.raise_for_status()
@@ -115,7 +115,8 @@ def _get_process_resource_id(self, process_id, ems_processes_resources, request)
                     # if read permission is granted on corresponding magpie resource route, twitcher
                     # '/ems/process/{process_id}' will be accessible, otherwise unauthorized is a private process
                     path = '{host}/processes/{id}'.format(host=self._get_service_public_url(request), id=process_id)
-                    resp = requests.get(path, cookies=request.cookies, headers=json_headers)
+                    resp = requests.get(path, cookies=request.cookies,
+                                        headers=self.json_headers, verify=self.twitcher_ssl_verify)
                     if resp.status_code == HTTPOk.code:
                         return ems_processes_resources[process_res_id]['resource_id']
                     elif resp.status_code == HTTPUnauthorized.code:
@@ -142,7 +143,8 @@ def _create_resource(self, resource_name, resource_parent_id, group_name, permis
         try:
             data = {u'parent_id': resource_parent_id, u'resource_name': resource_name, u'resource_type': u'route'}
             path = '{host}/resources'.format(host=self.magpie_url)
-            resp = requests.post(path, data=data, cookies=request.cookies, headers=json_headers)
+            resp = requests.post(path, data=data, cookies=request.cookies,
+                                 headers=self.json_headers, verify=self.twitcher_ssl_verify)
             if resp.status_code != HTTPCreated.code:
                 raise resp.raise_for_status()
             res_id = resp.json()['resource']['resource_id']
@@ -157,7 +159,8 @@ def _create_resource(self, resource_name, resource_parent_id, group_name, permis
                     data = {u'permission_name': perm}
                     path = '{host}/groups/{grp}/resources/{id}/permissions' \
                            .format(host=self.magpie_url, grp=group_name, id=res_id)
-                    resp = requests.post(path, data=data, cookies=request.cookies, headers=json_headers)
+                    resp = requests.post(path, data=data, cookies=request.cookies,
+                                         headers=self.json_headers, verify=self.twitcher_ssl_verify)
                     if resp.status_code not in (HTTPCreated.code, HTTPConflict.code):
                         raise resp.raise_for_status()
 
@@ -176,7 +179,8 @@ def save_process(self, process, overwrite=True, request=None):
             try:
                 # get resource id of ems service
                 path = '{host}/services/ems'.format(host=self.magpie_url)
-                resp = requests.get(path, cookies=request.cookies, headers=json_headers)
+                resp = requests.get(path, cookies=request.cookies,
+                                    headers=self.json_headers, verify=self.twitcher_ssl_verify)
                 if resp.status_code != HTTPOk.code:
                     raise resp.raise_for_status()
                 ems_res_id = resp.json()['ems']['resource_id']
@@ -186,7 +190,8 @@ def save_process(self, process, overwrite=True, request=None):
             try:
                 # get resource id of route '/ems/processes', create it as necessary
                 path = '{host}/resources/{id}'.format(host=self.magpie_url, id=ems_res_id)
-                resp = requests.get(path, cookies=request.cookies, headers=json_headers)
+                resp = requests.get(path, cookies=request.cookies,
+                                    headers=self.json_headers, verify=self.twitcher_ssl_verify)
                 if resp.status_code != HTTPOk.code:
                     raise resp.raise_for_status()
                 processes_res_id = None
@@ -225,7 +230,8 @@ def delete_process(self, process_id, request=None):
                 raise ProcessNotFound('Could not find process `{}` resource for deletion.'.format(process_id))
 
             path = '{host}/resources/{id}'.format(host=self.magpie_url, id=resource_id)
-            resp = requests.delete(path, cookies=request.cookies, headers=json_headers)
+            resp = requests.delete(path, cookies=request.cookies,
+                                   headers=self.json_headers, verify=self.twitcher_ssl_verify)
             if resp.status_code != HTTPOk.code:
                 raise resp.raise_for_status()
 
@@ -293,7 +299,8 @@ def set_visibility(self, process_id, visibility, request=None):
             if visibility == VISIBILITY_PRIVATE:
                 path = '{host}/groups/users/resources/{id}/permissions/{perm}' \
                        .format(host=self.magpie_url, id=process_res_id, perm='read')
-                reps = requests.delete(path, cookies=request.cookies, headers=json_headers)
+                reps = requests.delete(path, cookies=request.cookies,
+                                       headers=self.json_headers, verify=self.twitcher_ssl_verify)
                 # permission is not set if deleted or non existing
                 if reps.status_code not in (HTTPOk.code, HTTPNotFound.code):
                     raise reps.raise_for_status()
@@ -301,7 +308,8 @@ def set_visibility(self, process_id, visibility, request=None):
             elif visibility == VISIBILITY_PUBLIC:
                 path = '{host}/groups/users/resources/{id}/permissions'.format(host=self.magpie_url, id=process_res_id)
                 data = {u'permission_name': u'read'}
-                reps = requests.post(path, data=data, cookies=request.cookies, headers=json_headers)
+                reps = requests.post(path, data=data, cookies=request.cookies,
+                                     headers=self.json_headers, verify=self.twitcher_ssl_verify)
                 # permission is set if created or already exists
                 if reps.status_code not in (HTTPCreated.code, HTTPConflict.code):
                     raise reps.raise_for_status()

From b7ab9a634afdef90c8e4493dab3c510b68b21179 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 4 Oct 2018 15:13:14 -0400
Subject: [PATCH 117/124] add service name at end of public url if full url env
 var is defined

---
 magpie/register.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/magpie/register.py b/magpie/register.py
index d789c7f93..21473f669 100644
--- a/magpie/register.py
+++ b/magpie/register.py
@@ -163,18 +163,19 @@ def get_magpie_url():
 
 
 def get_twitcher_protected_service_url(magpie_service_name, hostname=None):
-    hostname = hostname or get_constant('HOSTNAME')
     twitcher_proxy_url = get_constant('TWITCHER_PROTECTED_URL', raise_not_set=False)
-    if twitcher_proxy_url:
-        return twitcher_proxy_url
-    twitcher_proxy = get_constant('TWITCHER_PROTECTED_PATH', raise_not_set=False)
-    if not twitcher_proxy.endswith('/'):
-        twitcher_proxy = twitcher_proxy + '/'
-    if not twitcher_proxy.startswith('/'):
-        twitcher_proxy = '/' + twitcher_proxy
-    if not twitcher_proxy.startswith('/twitcher'):
-        twitcher_proxy = '/twitcher' + twitcher_proxy
-    return "https://{0}{1}{2}".format(hostname, twitcher_proxy, magpie_service_name)
+    if not twitcher_proxy_url:
+        twitcher_proxy = get_constant('TWITCHER_PROTECTED_PATH', raise_not_set=False)
+        if not twitcher_proxy.endswith('/'):
+            twitcher_proxy = twitcher_proxy + '/'
+        if not twitcher_proxy.startswith('/'):
+            twitcher_proxy = '/' + twitcher_proxy
+        if not twitcher_proxy.startswith('/twitcher'):
+            twitcher_proxy = '/twitcher' + twitcher_proxy
+        hostname = hostname or get_constant('HOSTNAME')
+        twitcher_proxy_url = "https://{0}{1}".format(hostname, twitcher_proxy)
+    twitcher_proxy_url.rstrip('/')
+    return "{0}/{1}".format(twitcher_proxy_url, magpie_service_name)
 
 
 def register_services(register_service_url, services_dict, cookies,

From 0f59998b6d7c163814440dbe5abad27b6a3027c3 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 4 Oct 2018 15:43:45 -0400
Subject: [PATCH 118/124] change adapter process read permission method to
 avoid circular calls

---
 magpie/adapter/magpieprocess.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index 52c77913e..f128155c3 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -113,16 +113,20 @@ def _get_process_resource_id(self, process_id, ems_processes_resources, request)
                     LOGGER.debug("Found process resource: `{}`.".format(process_id))
 
                     # if read permission is granted on corresponding magpie resource route, twitcher
-                    # '/ems/process/{process_id}' will be accessible, otherwise unauthorized is a private process
-                    path = '{host}/processes/{id}'.format(host=self._get_service_public_url(request), id=process_id)
+                    # '/ems/process/{process_id}' will be accessible, otherwise unauthorized on private process
+                    # NB:
+                    #   - cannot test directly GET '/ems/process/{process_id}' for 401 because it causes circular calls
+                    #   - must test with current user (not '/resources') because he might not have administrator access
+                    path = '{host}/users/current/resources/{id}/permissions?inherit=true' \
+                           .format(host=self.magpie_url, id=process_res_id)
                     resp = requests.get(path, cookies=request.cookies,
                                         headers=self.json_headers, verify=self.twitcher_ssl_verify)
-                    if resp.status_code == HTTPOk.code:
-                        return ems_processes_resources[process_res_id]['resource_id']
-                    elif resp.status_code == HTTPUnauthorized.code:
-                        return None
-                    else:
+                    if resp.status_code != HTTPOk.code:
                         raise resp.raise_for_status()
+                    user_process_permissions = resp.json()['permission_names']
+                    if 'read' in user_process_permissions or 'read-match' in user_process_permissions:
+                        return ems_processes_resources[process_res_id]['resource_id']
+                    return None
         except KeyError:
             LOGGER.debug("Content of ems processes resources: `{!r}`.".format(ems_processes_resources))
             raise ProcessNotFound("Could not find process `{}` resource for visibility retrieval.".format(process_id))

From 350bc0aa715aabe2195c0c0cca3c98ff98ec5878 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 4 Oct 2018 18:17:48 -0400
Subject: [PATCH 119/124] another attempt at process visibility

---
 magpie/adapter/magpieprocess.py | 81 +++++++++++++++++++++++----------
 magpie/api/api_rest_schemas.py  | 33 ++++++++------
 2 files changed, 76 insertions(+), 38 deletions(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index f128155c3..b297d9e61 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -17,6 +17,7 @@
     HTTPNotFound,
     HTTPConflict,
     HTTPUnauthorized,
+    HTTPInternalServerError,
 )
 
 # import 'process' elements separately than 'twitcher_definitions' because not defined in master
@@ -66,16 +67,20 @@ def _get_service_public_url(self, request):
             try:
                 self.twitcher_service_url = resp.json()['services']['api']['ems']['public_url']
             except KeyError:
-                raise ProcessNotFound("Could not find resource `processes` endpoint for visibility retrieval.")
-            LOGGER.debug("Could not find resource: `processes`.")
+                raise ProcessNotFound("Could not find service `ems` endpoint.")
+            except Exception as ex:
+                LOGGER.debug("Exception during ems service url retrieval: [{}]".format(repr(ex)))
+                raise
         return self.twitcher_service_url
 
     def _get_process_resources(self, request):
         """
-        Gets all 'process' resources corresponding to results under '/ems/processes'.
-        Resources are not filtered by user permissions.
+        Gets all 'process' magpie resources corresponding to results under '/ems/processes'.
+        Resources are filtered by user/groups permissions (only processes visible by this user).
+
+        :returns: list of magpie resources.
         """
-        path = '{host}/resources'.format(host=self.magpie_url)
+        path = '{host}/users/current/resources?inherit=true'.format(host=self.magpie_url)
         resp = requests.get(path, cookies=request.cookies,
                             headers=self.json_headers, verify=self.twitcher_ssl_verify)
         LOGGER.debug('Looking for resources on: `{}`.'.format(path))
@@ -91,13 +96,16 @@ def _get_process_resources(self, request):
         except KeyError:
             LOGGER.debug("Content of ems service resources: `{!r}`.".format(ems_resources))
             raise ProcessNotFound("Could not find resource `processes` endpoint for visibility retrieval.")
+        except Exception as ex:
+            LOGGER.debug("Exception during ems resources retrieval: [{}]".format(repr(ex)))
+            raise
         LOGGER.debug("Could not find resource: `processes`.")
         return list()
 
     def _get_process_resource_id(self, process_id, ems_processes_resources, request):
         """
-        Searches for a 'process' resource corresponding to 'ems/processes/{id}'.
-        Only visible processes are returned (resources with reading permission assigned to request user).
+        Requests for a 'process' resource corresponding to '/ems/processes/{id}' and returns its corresponding
+        magpie resource id if reading permission was granted to requesting user.
 
         :returns: id of the found 'process' resource, or None.
         :raises:
@@ -114,22 +122,20 @@ def _get_process_resource_id(self, process_id, ems_processes_resources, request)
 
                     # if read permission is granted on corresponding magpie resource route, twitcher
                     # '/ems/process/{process_id}' will be accessible, otherwise unauthorized on private process
-                    # NB:
-                    #   - cannot test directly GET '/ems/process/{process_id}' for 401 because it causes circular calls
-                    #   - must test with current user (not '/resources') because he might not have administrator access
-                    path = '{host}/users/current/resources/{id}/permissions?inherit=true' \
-                           .format(host=self.magpie_url, id=process_res_id)
+                    path = '{host}/processes/{id}'.format(host=self._get_service_public_url(request), id=process_id)
                     resp = requests.get(path, cookies=request.cookies,
                                         headers=self.json_headers, verify=self.twitcher_ssl_verify)
-                    if resp.status_code != HTTPOk.code:
-                        raise resp.raise_for_status()
-                    user_process_permissions = resp.json()['permission_names']
-                    if 'read' in user_process_permissions or 'read-match' in user_process_permissions:
+                    if resp.status_code == HTTPUnauthorized:
+                        return None
+                    elif resp.status_code == HTTPOk.code:
                         return ems_processes_resources[process_res_id]['resource_id']
-                    return None
+                    raise resp.raise_for_status()
         except KeyError:
             LOGGER.debug("Content of ems processes resources: `{!r}`.".format(ems_processes_resources))
             raise ProcessNotFound("Could not find process `{}` resource for visibility retrieval.".format(process_id))
+        except Exception as ex:
+            LOGGER.debug("Exception during process resource retrieval: [{}]".format(repr(ex)))
+            raise
         LOGGER.debug("Could not find resource: `{}`.".format(process_id))
         return None
 
@@ -171,6 +177,9 @@ def _create_resource(self, resource_name, resource_parent_id, group_name, permis
             return res_id
         except KeyError:
             raise ProcessRegistrationError('Failed adding process resource route `{}`.'.format(resource_name))
+        except Exception as ex:
+            LOGGER.debug("Exception during process resource creation: [{}]".format(repr(ex)))
+            raise
 
     def save_process(self, process, overwrite=True, request=None):
         """
@@ -190,6 +199,9 @@ def save_process(self, process, overwrite=True, request=None):
                 ems_res_id = resp.json()['ems']['resource_id']
             except KeyError:
                 raise ProcessRegistrationError('Failed retrieving EMS service resource.')
+            except Exception as ex:
+                LOGGER.debug("Exception during `ems` resource retrieval: [{}]".format(repr(ex)))
+                raise
 
             try:
                 # get resource id of route '/ems/processes', create it as necessary
@@ -210,6 +222,9 @@ def save_process(self, process, overwrite=True, request=None):
                     processes_res_id = self._create_resource(u'processes', ems_res_id, u'users', u'read-match', request)
             except KeyError:
                 raise ProcessRegistrationError('Failed retrieving EMS processes resource.')
+            except Exception as ex:
+                LOGGER.debug("Exception during `processes` resource retrieval: [{}]".format(repr(ex)))
+                raise
 
             # create resource id of route '/ems/processes/{id}' and set minimal permissions
             # use read permission so that users can execute any sub-route GET request on it
@@ -265,27 +280,43 @@ def fetch_by_id(self, process_id, request=None):
         """
         Get a process if visible for user.
 
-        If twitcher is not in EMS mode, simply delegate execution to default twitcher process store.
-        If twitcher is in EMS mode, return the process if visible based on magpie user permissions.
+        Delegate operation to default twitcher process store.
+        If using twitcher proxy, magpie user/group permissions on corresponding resource (/ems/processes/{process_id})
+        will automatically handle Ok/Unauthorized responses using the API route's read access.
         """
-        if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
-            if self.get_visibility(process_id, request) != VISIBILITY_PUBLIC:
-                raise HTTPUnauthorized()
         return processstore_defaultfactory(request.registry).fetch_by_id(process_id, request)
 
     def get_visibility(self, process_id, request=None):
         """
         Get visibility of a process.
 
-        If twitcher is not in EMS mode, simply delegate execution to default twitcher process store.
-        If twitcher is in EMS mode, return the process visibility based on magpie user permissions.
+        If twitcher is not in EMS mode, delegate operation to default twitcher process store.
+        If twitcher is in EMS mode, process visibility is checked using corresponding magpie resource permissions.
         """
         if self.twitcher_config != TWITCHER_CONFIGURATION_EMS:
             return processstore_defaultfactory(request.registry).get_visibility(process_id, request)
 
+        # process visibility depends on permissions granted on corresponding magpie resource for group 'users'
+        # administrator MUST have read permissions on corresponding resource to retrieve its permissions
+        # non-admin users should not be able to even reach this code, since proxy will block them before
         ems_processes = self._get_process_resources(request)
         process_res_id = self._get_process_resource_id(process_id, ems_processes, request)
-        return VISIBILITY_PUBLIC if process_res_id is not None else VISIBILITY_PRIVATE
+        try:
+            path = '{host}/groups/users/resources/{id}/permissions'.format(host=self.magpie_url, id=process_res_id)
+            resp = requests.get(path, cookies=request.cookies,
+                                headers=self.json_headers, verify=self.twitcher_ssl_verify)
+            if resp.status_code != HTTPOk.code:
+                raise resp.raise_for_status()
+            process_res_perms = resp.json()['permission_names']
+            has_permissions = 'read' in process_res_perms or 'read-match' in process_res_perms
+            return VISIBILITY_PUBLIC if has_permissions else VISIBILITY_PRIVATE
+        except KeyError:
+            LOGGER.debug("Content of EMS processes resources: `{!r}`.".format(ems_processes))
+            LOGGER.debug("Value of process resource: `{!r}`.".format(process_res_id))
+            raise HTTPInternalServerError("Could not retrieve process `{}` visibility value.".format(process_id))
+        except Exception as ex:
+            LOGGER.debug("Exception during EMS process visibility permission retrieval: [{}]".format(repr(ex)))
+            raise
 
     def set_visibility(self, process_id, visibility, request=None):
         """
diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py
index 5d88d6778..2e2618e71 100644
--- a/magpie/api/api_rest_schemas.py
+++ b/magpie/api/api_rest_schemas.py
@@ -375,42 +375,42 @@ class InternalServerErrorResponseSchema(colander.MappingSchema):
 
 
 class ProvidersListSchema(colander.SequenceSchema):
-    item = colander.SchemaNode(
+    provider_name = colander.SchemaNode(
         colander.String(),
         description="Available login providers.",
-        example=["ziggurat", "openid"],
+        example="openid",
     )
 
 
 class ResourceTypesListSchema(colander.SequenceSchema):
-    item = colander.SchemaNode(
+    resource_type = colander.SchemaNode(
         colander.String(),
         description="Available resource type under root service.",
-        example=["file", "dictionary"],
+        example="file",
     )
 
 
 class GroupNamesListSchema(colander.SequenceSchema):
-    item = colander.SchemaNode(
+    group_name = colander.SchemaNode(
         colander.String(),
         description="List of groups depending on context.",
-        example=["anonymous"]
+        example="administrators"
     )
 
 
 class UserNamesListSchema(colander.SequenceSchema):
-    item = colander.SchemaNode(
+    user_name = colander.SchemaNode(
         colander.String(),
         description="Users registered in the db",
-        example=["anonymous", "admin", "toto"]
+        example="bob"
     )
 
 
 class PermissionListSchema(colander.SequenceSchema):
-    item = colander.SchemaNode(
+    permission_name = colander.SchemaNode(
         colander.String(),
         description="Permissions applicable to the service/resource",
-        example=["read", "write"]
+        example="read"
     )
 
 
@@ -423,7 +423,9 @@ class UserBodySchema(colander.MappingSchema):
         colander.String(),
         description="Email of the user.",
         example="toto@mail.com")
-    group_names = GroupNamesListSchema()
+    group_names = GroupNamesListSchema(
+        example=['administrators', 'users']
+    )
 
 
 class GroupBodySchema(colander.MappingSchema):
@@ -445,7 +447,10 @@ class GroupBodySchema(colander.MappingSchema):
         description="Number of users member of the group.",
         example=2,
         missing=colander.drop)
-    user_names = UserNamesListSchema(missing=colander.drop)
+    user_names = UserNamesListSchema(
+        example=['alice', 'bob'],
+        missing=colander.drop
+    )
 
 
 class ServiceBodySchema(colander.MappingSchema):
@@ -453,7 +458,9 @@ class ServiceBodySchema(colander.MappingSchema):
         colander.Integer(),
         description="Resource identification number",
     )
-    permission_names = PermissionListSchema()
+    permission_names = PermissionListSchema(
+        example=['read', 'write']
+    )
     service_name = colander.SchemaNode(
         colander.String(),
         description="Name of the service",

From 42b4188560a6dd77447de2ed2cbcbfbbc92db1b3 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Thu, 4 Oct 2018 18:35:24 -0400
Subject: [PATCH 120/124] proper ssl verification off for localhost

---
 magpie/adapter/magpieprocess.py           | 3 ++-
 magpie/definitions/pyramid_definitions.py | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index b297d9e61..f2a1cc833 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -18,6 +18,7 @@
     HTTPConflict,
     HTTPUnauthorized,
     HTTPInternalServerError,
+    asbool
 )
 
 # import 'process' elements separately than 'twitcher_definitions' because not defined in master
@@ -48,7 +49,7 @@ def __init__(self, registry):
                 LOGGER.warn("Missing scheme from MagpieServiceStore url, new value: '{}'".format(self.magpie_url))
 
             self.twitcher_config = get_twitcher_configuration(registry.settings)
-            self.twitcher_ssl_verify = registry.settings.get('twitcher.ows_proxy_ssl_verify', True)
+            self.twitcher_ssl_verify = asbool(registry.settings.get('twitcher.ows_proxy_ssl_verify', True))
             self.json_headers = {'Accept': 'application/json'}
             self.twitcher_service_url = None
         except AttributeError:
diff --git a/magpie/definitions/pyramid_definitions.py b/magpie/definitions/pyramid_definitions.py
index 3871c1085..1ce3cffc2 100644
--- a/magpie/definitions/pyramid_definitions.py
+++ b/magpie/definitions/pyramid_definitions.py
@@ -16,6 +16,7 @@
     HTTPUnprocessableEntity,
     HTTPInternalServerError,
 )
+from pyramid.settings import asbool
 from pyramid.interfaces import IAuthenticationPolicy, IAuthorizationPolicy
 from pyramid.response import Response, FileResponse
 from pyramid.view import (

From a1951aff889e52f84b71f29cf91fde0dd2cebbba Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Fri, 5 Oct 2018 11:50:13 -0400
Subject: [PATCH 121/124] another method to control adapter process visibility

---
 magpie/adapter/magpieprocess.py | 232 ++++++++++++++++++--------------
 1 file changed, 131 insertions(+), 101 deletions(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index f2a1cc833..f9a6a9723 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -5,10 +5,10 @@
 from six.moves.urllib.parse import urlparse
 import logging
 import requests
-import json
 import six
 LOGGER = logging.getLogger("TWITCHER")
 
+from magpie.constants import get_constant
 from magpie.definitions.twitcher_definitions import *
 from magpie.definitions.pyramid_definitions import (
     ConfigurationError,
@@ -26,7 +26,7 @@
 from twitcher.exceptions import ProcessNotFound, ProcessRegistrationError
 from twitcher.store import processstore_defaultfactory
 from twitcher.store.base import ProcessStore
-from twitcher.visibility import VISIBILITY_PUBLIC, VISIBILITY_PRIVATE
+from twitcher.visibility import VISIBILITY_PUBLIC, VISIBILITY_PRIVATE, visibility_values
 
 
 class MagpieProcessStore(ProcessStore):
@@ -47,11 +47,14 @@ def __init__(self, registry):
             else:
                 self.magpie_url = 'http://{}'.format(url_parsed.geturl())
                 LOGGER.warn("Missing scheme from MagpieServiceStore url, new value: '{}'".format(self.magpie_url))
-
+            self.magpie_users = get_constant('MAGPIE_USERS_GROUP')
+            self.magpie_admin = get_constant('MAGPIE_ADMIN_GROUP')
+            self.magpie_current = get_constant('MAGPIE_LOGGED_USER')
+            self.magpie_service = 'ems'
             self.twitcher_config = get_twitcher_configuration(registry.settings)
             self.twitcher_ssl_verify = asbool(registry.settings.get('twitcher.ows_proxy_ssl_verify', True))
-            self.json_headers = {'Accept': 'application/json'}
             self.twitcher_service_url = None
+            self.json_headers = {'Accept': 'application/json'}
         except AttributeError:
             #If magpie.url does not exist, calling strip fct over None will raise this issue
             raise ConfigurationError('magpie.url config cannot be found')
@@ -60,15 +63,17 @@ def _get_service_public_url(self, request):
         if not self.twitcher_service_url and self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
             # use generic 'current' user route to fetch service URL to ensure that even
             # a user with minimal privileges will still return a match
-            path = '{host}/users/current/services?inherit=true&cascade=true'.format(host=self.magpie_url)
+            path = '{host}/users/{usr}/services?inherit=true&cascade=true' \
+                   .format(host=self.magpie_url, usr=self.magpie_current)
             resp = requests.get(path, cookies=request.cookies,
                                 headers=self.json_headers, verify=self.twitcher_ssl_verify)
             if resp.status_code != HTTPOk.code:
                 raise resp.raise_for_status()
             try:
-                self.twitcher_service_url = resp.json()['services']['api']['ems']['public_url']
+                self.twitcher_service_url = resp.json()['services']['api'][self.magpie_service]['public_url']
+                LOGGER.debug("Found service proxy url: {}".format(self.twitcher_service_url))
             except KeyError:
-                raise ProcessNotFound("Could not find service `ems` endpoint.")
+                raise ProcessNotFound("Could not find service `{}` endpoint.".format(self.magpie_service))
             except Exception as ex:
                 LOGGER.debug("Exception during ems service url retrieval: [{}]".format(repr(ex)))
                 raise
@@ -81,15 +86,15 @@ def _get_process_resources(self, request):
 
         :returns: list of magpie resources.
         """
-        path = '{host}/users/current/resources?inherit=true'.format(host=self.magpie_url)
+        path = '{host}/users/{usr}/resources?inherit=true'.format(host=self.magpie_url, usr=self.magpie_current)
         resp = requests.get(path, cookies=request.cookies,
                             headers=self.json_headers, verify=self.twitcher_ssl_verify)
-        LOGGER.debug('Looking for resources on: `{}`.'.format(path))
+        LOGGER.debug("Looking for resources on: `{}`.".format(path))
         if resp.status_code != HTTPOk.code:
             raise resp.raise_for_status()
         ems_resources = None
         try:
-            ems_resources = resp.json()['resources']['api']['ems']['resources']
+            ems_resources = resp.json()['resources']['api'][self.magpie_service]['resources']
             for res_id in ems_resources:
                 if ems_resources[res_id]['resource_name'] == 'processes':
                     ems_processes = ems_resources[res_id]['children']
@@ -132,7 +137,7 @@ def _get_process_resource_id(self, process_id, ems_processes_resources, request)
                         return ems_processes_resources[process_res_id]['resource_id']
                     raise resp.raise_for_status()
         except KeyError:
-            LOGGER.debug("Content of ems processes resources: `{!r}`.".format(ems_processes_resources))
+            LOGGER.debug("Content of processes resources: `{!r}`.".format(ems_processes_resources))
             raise ProcessNotFound("Could not find process `{}` resource for visibility retrieval.".format(process_id))
         except Exception as ex:
             LOGGER.debug("Exception during process resource retrieval: [{}]".format(repr(ex)))
@@ -140,9 +145,34 @@ def _get_process_resource_id(self, process_id, ems_processes_resources, request)
         LOGGER.debug("Could not find resource: `{}`.".format(process_id))
         return None
 
+    def _set_resource_permissions(self, resource_id, group_name, permission_names, request):
+        """
+        Sets group permissions on a resource.
+
+        :param resource_id: (int) magpie id of the resource to apply permissions on.
+        :param group_name: (str) name of the group for which to apply permissions, if any.
+        :param permission_names: (None, str, iterator) group permissions to apply to the resource, if any.
+        :param request: calling request for headers and credentials
+        """
+        if isinstance(group_name, six.string_types):
+            if permission_names is None:
+                permission_names = []
+            if isinstance(permission_names, six.string_types):
+                permission_names = [permission_names]
+
+            for perm in permission_names:
+                data = {u'permission_name': perm}
+                path = '{host}/groups/{grp}/resources/{id}/permissions' \
+                    .format(host=self.magpie_url, grp=group_name, id=resource_id)
+                resp = requests.post(path, data=data, cookies=request.cookies,
+                                     headers=self.json_headers, verify=self.twitcher_ssl_verify)
+                # permission is set if created or already exists
+                if resp.status_code not in (HTTPCreated.code, HTTPConflict.code):
+                    raise resp.raise_for_status()
+
     def _create_resource(self, resource_name, resource_parent_id, group_name, permission_names, request):
         """
-        Creates a resource under another parent resource, and sets basic user permissions on it.
+        Creates a resource under another parent resource, and sets basic group permissions on it.
 
         :param resource_name: (str) name of the resource to create.
         :param resource_parent_id: (int) id of the parent resource under which to create `resource_name`.
@@ -159,25 +189,10 @@ def _create_resource(self, resource_name, resource_parent_id, group_name, permis
             if resp.status_code != HTTPCreated.code:
                 raise resp.raise_for_status()
             res_id = resp.json()['resource']['resource_id']
-
-            if isinstance(group_name, six.string_types):
-                if permission_names is None:
-                    permission_names = []
-                if isinstance(permission_names, six.string_types):
-                    permission_names = [permission_names]
-
-                for perm in permission_names:
-                    data = {u'permission_name': perm}
-                    path = '{host}/groups/{grp}/resources/{id}/permissions' \
-                           .format(host=self.magpie_url, grp=group_name, id=res_id)
-                    resp = requests.post(path, data=data, cookies=request.cookies,
-                                         headers=self.json_headers, verify=self.twitcher_ssl_verify)
-                    if resp.status_code not in (HTTPCreated.code, HTTPConflict.code):
-                        raise resp.raise_for_status()
-
+            self._set_resource_permissions(res_id, group_name, permission_names, request)
             return res_id
         except KeyError:
-            raise ProcessRegistrationError('Failed adding process resource route `{}`.'.format(resource_name))
+            raise ProcessRegistrationError("Failed adding process resource route `{}`.".format(resource_name))
         except Exception as ex:
             LOGGER.debug("Exception during process resource creation: [{}]".format(repr(ex)))
             raise
@@ -186,22 +201,28 @@ def save_process(self, process, overwrite=True, request=None):
         """
         Save a new process.
 
-        If twitcher is not in EMS mode, simply delegate execution to default twitcher process store.
-        If twitcher is in EMS mode, user requesting creation must have sufficient permissions in magpie to do so.
+        If twitcher is not in EMS mode, delegate execution to default twitcher process store.
+        If twitcher is in EMS mode:
+            - user requesting creation must have sufficient administrator permissions in magpie to do so.
+            - assign any pre-requirement routes permissions to allow admins to edit '/ems/processes/...'
+
+            Requirements:
+                - service 'ems' of type 'api' must exist
+                - group 'administrators' must have ['read', 'write'] permissions on 'ems' service
         """
         if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
             try:
                 # get resource id of ems service
-                path = '{host}/services/ems'.format(host=self.magpie_url)
+                path = '{host}/services/{svc}'.format(host=self.magpie_url, svc=self.magpie_service)
                 resp = requests.get(path, cookies=request.cookies,
                                     headers=self.json_headers, verify=self.twitcher_ssl_verify)
                 if resp.status_code != HTTPOk.code:
                     raise resp.raise_for_status()
-                ems_res_id = resp.json()['ems']['resource_id']
+                ems_res_id = resp.json()[self.magpie_service]['resource_id']
             except KeyError:
-                raise ProcessRegistrationError('Failed retrieving EMS service resource.')
+                raise ProcessRegistrationError("Failed retrieving service resource.")
             except Exception as ex:
-                LOGGER.debug("Exception during `ems` resource retrieval: [{}]".format(repr(ex)))
+                LOGGER.debug("Exception during `{0}` resource retrieval: [{1}]".format(self.magpie_service, repr(ex)))
                 raise
 
             try:
@@ -220,19 +241,19 @@ def save_process(self, process, overwrite=True, request=None):
                 if not processes_res_id:
                     # all members of 'users' group can query '/ems/processes' (read exact route match),
                     # but visibility of each process will be filtered by specific '/ems/processes/{id}' permissions
-                    processes_res_id = self._create_resource(u'processes', ems_res_id, u'users', u'read-match', request)
+                    # members of 'administrators' automatically inherit read/write permissions from 'ems' service
+                    processes_res_id = self._create_resource(u'processes', ems_res_id,
+                                                             self.magpie_users, u'read-match', request)
             except KeyError:
-                raise ProcessRegistrationError('Failed retrieving EMS processes resource.')
+                raise ProcessRegistrationError("Failed retrieving processes resource.")
             except Exception as ex:
                 LOGGER.debug("Exception during `processes` resource retrieval: [{}]".format(repr(ex)))
                 raise
 
-            # create resource id of route '/ems/processes/{id}' and set minimal permissions
-            # use read permission so that users can execute any sub-route GET request on it
-            process_res_id = self._create_resource(process.id, processes_res_id, u'users', u'read', request)
-            # create resource id of route '/ems/processes/{id}/jobs' and set minimal permissions
-            # use write-match permission so that users can ONLY execute a job (cannot DELETE process, job, etc.)
-            self._create_resource(u'jobs', process_res_id, u'users', u'write-match', request)
+            # create resources of route '/ems/processes/{id}' and '/ems/processes/{id}/jobs'
+            # do not apply any permissions at first, so that the process is 'private' by default
+            process_res_id = self._create_resource(process.id, processes_res_id, None, None, request)
+            self._create_resource(u'jobs', process_res_id, None, None, request)
 
         return processstore_defaultfactory(request.registry).save_process(process, overwrite, request)
 
@@ -240,14 +261,16 @@ def delete_process(self, process_id, request=None):
         """
         Delete a process.
 
-        If twitcher is not in EMS mode, simply delegate execution to default twitcher process store.
-        If twitcher is in EMS mode, user requesting deletion must have sufficient permissions in magpie to do so.
+        Delegate execution to default twitcher process store.
+        If twitcher is in EMS mode:
+            - user requesting deletion must have administrator permissions in magpie (delete route blocked otherwise).
+            - also delete magpie resources tree corresponding to the process
         """
         if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
             resources = self._get_process_resources(request)
             resource_id = self._get_process_resource_id(process_id, resources, request)
             if not resource_id:
-                raise ProcessNotFound('Could not find process `{}` resource for deletion.'.format(process_id))
+                raise ProcessNotFound("Could not find process `{}` for resource for deletion.".format(process_id))
 
             path = '{host}/resources/{id}'.format(host=self.magpie_url, id=resource_id)
             resp = requests.delete(path, cookies=request.cookies,
@@ -259,31 +282,44 @@ def delete_process(self, process_id, request=None):
 
     def list_processes(self, request=None):
         """
-        List processes.
+        List publicly visible processes.
 
-        If twitcher is not in EMS mode, simply delegate execution to default twitcher process store.
-        If twitcher is in EMS mode, filter by corresponding resources with read permissions.
+        Delegate execution to default twitcher process store.
+        If twitcher is not in EMS mode, filter by only visible processes.
+        If twitcher is in EMS mode, filter according to magpie user group memberships:
+            - administrators: return everything
+            - any other group: return only visible processes
         """
-        process_list = processstore_defaultfactory(request.registry).list_processes(request)
-        LOGGER.debug('Found processes: {!r}.'.format(process_list))
-        if self.twitcher_config != TWITCHER_CONFIGURATION_EMS:
-            return process_list
-
-        ems_processes = self._get_process_resources(request)
-        ems_processes_visible = list()
-        for process in process_list:
-            # if id is returned, filtered group resource permission was set, therefore visibility is permitted
-            if self._get_process_resource_id(process.id, ems_processes, request):
-                ems_processes_visible.append(process)
-        return ems_processes_visible
+        visibility_filter = VISIBILITY_PUBLIC
+        if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
+            path = '{host}/users/{usr}/groups'.format(host=self.magpie_url, usr=self.magpie_current)
+            resp = requests.get(path, cookies=request.cookies,
+                                headers=self.json_headers, verify=self.twitcher_ssl_verify)
+            if resp.status_code != HTTPOk.code:
+                raise resp.raise_for_status()
+            try:
+                groups_memberships = resp.json()['group_names']
+                if self.magpie_admin in groups_memberships:
+                    visibility_filter = visibility_values
+            except KeyError:
+                raise ProcessNotFound("Failed retrieving processes read permissions for listing.")
+            except Exception as ex:
+                LOGGER.debug("Exception during processes listing: [{}]".format(repr(ex)))
+                raise
+
+        store = processstore_defaultfactory(request.registry)
+        process_list = store.list_processes(visibility=visibility_filter, request=request)
+        LOGGER.debug("Found visible processes: {!s}.".format(process_list))
+        return process_list
 
     def fetch_by_id(self, process_id, request=None):
         """
         Get a process if visible for user.
 
         Delegate operation to default twitcher process store.
-        If using twitcher proxy, magpie user/group permissions on corresponding resource (/ems/processes/{process_id})
-        will automatically handle Ok/Unauthorized responses using the API route's read access.
+        If twitcher is in EMS mode:
+            using twitcher proxy, magpie user/group permissions on corresponding resource (/ems/processes/{process_id})
+            will automatically handle Ok/Unauthorized responses using the API route's read access.
         """
         return processstore_defaultfactory(request.registry).fetch_by_id(process_id, request)
 
@@ -291,50 +327,31 @@ def get_visibility(self, process_id, request=None):
         """
         Get visibility of a process.
 
-        If twitcher is not in EMS mode, delegate operation to default twitcher process store.
-        If twitcher is in EMS mode, process visibility is checked using corresponding magpie resource permissions.
+        Delegate operation to default twitcher process store.
+        If twitcher is in EMS mode:
+            using twitcher proxy, only administrators get read permissions on '/ems/processes/{process_id}/visibility'
+            any other level user will get unauthorized on this route
         """
-        if self.twitcher_config != TWITCHER_CONFIGURATION_EMS:
-            return processstore_defaultfactory(request.registry).get_visibility(process_id, request)
-
-        # process visibility depends on permissions granted on corresponding magpie resource for group 'users'
-        # administrator MUST have read permissions on corresponding resource to retrieve its permissions
-        # non-admin users should not be able to even reach this code, since proxy will block them before
-        ems_processes = self._get_process_resources(request)
-        process_res_id = self._get_process_resource_id(process_id, ems_processes, request)
-        try:
-            path = '{host}/groups/users/resources/{id}/permissions'.format(host=self.magpie_url, id=process_res_id)
-            resp = requests.get(path, cookies=request.cookies,
-                                headers=self.json_headers, verify=self.twitcher_ssl_verify)
-            if resp.status_code != HTTPOk.code:
-                raise resp.raise_for_status()
-            process_res_perms = resp.json()['permission_names']
-            has_permissions = 'read' in process_res_perms or 'read-match' in process_res_perms
-            return VISIBILITY_PUBLIC if has_permissions else VISIBILITY_PRIVATE
-        except KeyError:
-            LOGGER.debug("Content of EMS processes resources: `{!r}`.".format(ems_processes))
-            LOGGER.debug("Value of process resource: `{!r}`.".format(process_res_id))
-            raise HTTPInternalServerError("Could not retrieve process `{}` visibility value.".format(process_id))
-        except Exception as ex:
-            LOGGER.debug("Exception during EMS process visibility permission retrieval: [{}]".format(repr(ex)))
-            raise
+        return processstore_defaultfactory(request.registry).get_visibility(process_id, request)
 
     def set_visibility(self, process_id, visibility, request=None):
         """
         Set visibility of a process.
 
         Delegate change of process visibility to default twitcher process store.
-        If twitcher is in EMS mode, also modify magpie permissions of corresponding process access point.
+        If twitcher is in EMS mode:
+            using twitcher proxy, only administrators get write permissions on '/ems/processes/{process_id}/visibility'
+            modify magpie permissions of corresponding process access points according to desired visibility.
         """
         if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
             ems_processes = self._get_process_resources(request)
             process_res_id = self._get_process_resource_id(process_id, ems_processes, request)
             if not process_res_id:
-                raise ProcessNotFound('Could not find process `{}` resource to change visibility.'.format(process_id))
+                raise ProcessNotFound("Could not find process `{}` resource to change visibility.".format(process_id))
 
             if visibility == VISIBILITY_PRIVATE:
-                path = '{host}/groups/users/resources/{id}/permissions/{perm}' \
-                       .format(host=self.magpie_url, id=process_res_id, perm='read')
+                path = '{host}/groups/{usr}/resources/{id}/permissions/{perm}' \
+                       .format(host=self.magpie_url, usr=self.magpie_users, id=process_res_id, perm=u'read')
                 reps = requests.delete(path, cookies=request.cookies,
                                        headers=self.json_headers, verify=self.twitcher_ssl_verify)
                 # permission is not set if deleted or non existing
@@ -342,13 +359,26 @@ def set_visibility(self, process_id, visibility, request=None):
                     raise reps.raise_for_status()
 
             elif visibility == VISIBILITY_PUBLIC:
-                path = '{host}/groups/users/resources/{id}/permissions'.format(host=self.magpie_url, id=process_res_id)
-                data = {u'permission_name': u'read'}
-                reps = requests.post(path, data=data, cookies=request.cookies,
-                                     headers=self.json_headers, verify=self.twitcher_ssl_verify)
-                # permission is set if created or already exists
-                if reps.status_code not in (HTTPCreated.code, HTTPConflict.code):
-                    raise reps.raise_for_status()
+                # read permission so that users can make any sub-route GET requests (ex: GET '/ems/processes/{id}/jobs')
+                self._set_resource_permissions(process_res_id, self.magpie_users, u'read', request)
+
+                # find resource corresponding to '/ems/processes/{id}/jobs'
+                path = '{host}/resources/{id}'.format(host=self.magpie_url, id=process_res_id)
+                resp = requests.get(path, cookies=request.cookies,
+                                    headers=self.json_headers, verify=self.twitcher_ssl_verify)
+                if resp.status_code != HTTPOk.code:
+                    raise resp.raise_for_status()
+                jobs_res_id = None
+                process_resources = resp.json()[str(process_res_id)]['children']
+                for res_id in process_resources:
+                    if process_resources[res_id]['resource_name'] == 'jobs':
+                        jobs_res_id = process_resources[res_id]['resource_id']
+                        break
+                if not jobs_res_id:
+                    raise ProcessNotFound("Could not find process `{}` jobs resource to set visibility."
+                                          .format(process_id))
+
+                # use write-match permission so that users can ONLY execute a job (cannot DELETE process, job, etc.)
+                self._set_resource_permissions(jobs_res_id, self.magpie_users, u'write-match', request)
 
-        # write visibility to store to remain consistent in processes structures even if using magpie permissions
         processstore_defaultfactory(request.registry).set_visibility(process_id, visibility, request)

From e4408d3e991dbe5edc3e59e96bd7328428e41232 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Fri, 5 Oct 2018 13:03:39 -0400
Subject: [PATCH 122/124] add visibility param

---
 magpie/adapter/magpieprocess.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index f9a6a9723..489470631 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -280,7 +280,7 @@ def delete_process(self, process_id, request=None):
 
         return processstore_defaultfactory(request.registry).delete_process(process_id, request)
 
-    def list_processes(self, request=None):
+    def list_processes(self, visibility=None, request=None):
         """
         List publicly visible processes.
 
@@ -290,7 +290,7 @@ def list_processes(self, request=None):
             - administrators: return everything
             - any other group: return only visible processes
         """
-        visibility_filter = VISIBILITY_PUBLIC
+        visibility_filter = visibility
         if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
             path = '{host}/users/{usr}/groups'.format(host=self.magpie_url, usr=self.magpie_current)
             resp = requests.get(path, cookies=request.cookies,
@@ -301,6 +301,8 @@ def list_processes(self, request=None):
                 groups_memberships = resp.json()['group_names']
                 if self.magpie_admin in groups_memberships:
                     visibility_filter = visibility_values
+                else:
+                    visibility_filter = VISIBILITY_PUBLIC
             except KeyError:
                 raise ProcessNotFound("Failed retrieving processes read permissions for listing.")
             except Exception as ex:
@@ -321,7 +323,7 @@ def fetch_by_id(self, process_id, request=None):
             using twitcher proxy, magpie user/group permissions on corresponding resource (/ems/processes/{process_id})
             will automatically handle Ok/Unauthorized responses using the API route's read access.
         """
-        return processstore_defaultfactory(request.registry).fetch_by_id(process_id, request)
+        return processstore_defaultfactory(request.registry).fetch_by_id(process_id, request=request)
 
     def get_visibility(self, process_id, request=None):
         """
@@ -332,7 +334,7 @@ def get_visibility(self, process_id, request=None):
             using twitcher proxy, only administrators get read permissions on '/ems/processes/{process_id}/visibility'
             any other level user will get unauthorized on this route
         """
-        return processstore_defaultfactory(request.registry).get_visibility(process_id, request)
+        return processstore_defaultfactory(request.registry).get_visibility(process_id, request=request)
 
     def set_visibility(self, process_id, visibility, request=None):
         """
@@ -381,4 +383,4 @@ def set_visibility(self, process_id, visibility, request=None):
                 # use write-match permission so that users can ONLY execute a job (cannot DELETE process, job, etc.)
                 self._set_resource_permissions(jobs_res_id, self.magpie_users, u'write-match', request)
 
-        processstore_defaultfactory(request.registry).set_visibility(process_id, visibility, request)
+        processstore_defaultfactory(request.registry).set_visibility(process_id, visibility=visibility, request=request)

From 9fd0e5d709ad142bf84f29d653b07949189b19e7 Mon Sep 17 00:00:00 2001
From: David Caron <dcaron05@gmail.com>
Date: Fri, 5 Oct 2018 14:30:21 -0400
Subject: [PATCH 123/124] in geoserver-api: keep the "workspaces" folder

instead of only its resources
---
 magpie/helpers/sync_services.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/magpie/helpers/sync_services.py b/magpie/helpers/sync_services.py
index 36e10e7a2..3c4bc18e6 100644
--- a/magpie/helpers/sync_services.py
+++ b/magpie/helpers/sync_services.py
@@ -66,8 +66,12 @@ def get_resources(self):
 
         workspaces = {w["name"]: {"children": {}, "resource_type": resource_type} for w in workspaces_list}
 
-        resources = {self.service_name: {"children": workspaces,
+        workspace_tree = {"workspaces": {"children": workspaces,
                                          "resource_type": resource_type}}
+
+        resources = {"geoserver-api": {"children": workspace_tree,
+                                       "resource_type": resource_type}}
+
         assert is_valid_resource_schema(resources), "Error in Interface implementation"
         return resources
 

From 401d73e6f9ea13b8b979e7cc71ba993aa08a2749 Mon Sep 17 00:00:00 2001
From: Francis Charette Migneault <francis.charette.migneault@gmail.com>
Date: Fri, 5 Oct 2018 14:41:41 -0400
Subject: [PATCH 124/124] use admin resources, visibility working

---
 magpie/adapter/magpieprocess.py | 233 +++++++++++++++-----------------
 1 file changed, 110 insertions(+), 123 deletions(-)

diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py
index 489470631..e937b241a 100644
--- a/magpie/adapter/magpieprocess.py
+++ b/magpie/adapter/magpieprocess.py
@@ -79,17 +79,40 @@ def _get_service_public_url(self, request):
                 raise
         return self.twitcher_service_url
 
-    def _get_process_resources(self, request):
+    def _find_child_resource_id(self, parent_resource_id, child_resource_name, request):
         """
-        Gets all 'process' magpie resources corresponding to results under '/ems/processes'.
-        Resources are filtered by user/groups permissions (only processes visible by this user).
+        Finds the resource id corresponding to a child resource by name of the specified parent resource.
+        :param parent_resource_id: id of the resource from which to search children resources.
+        :param child_resource_name: name of the sub resource to find.
+        :param request: calling request for headers and credentials.
+        :return:
+        """
+        path = '{host}/resources/{id}'.format(host=self.magpie_url, id=parent_resource_id)
+        resp = requests.get(path, cookies=request.cookies,
+                            headers=self.json_headers, verify=self.twitcher_ssl_verify)
+        if resp.status_code != HTTPOk.code:
+            raise resp.raise_for_status()
+        child_res_id = None
+        parent_resource_info = resp.json()[str(parent_resource_id)]
+        children_resources = parent_resource_info['children']
+        for res_id in children_resources:
+            if children_resources[res_id]['resource_name'] == child_resource_name:
+                child_res_id = children_resources[res_id]['resource_id']
+                return child_res_id
+        if not child_res_id:
+            raise HTTPNotFound("Could not find resource `{}` under resource `{}`."
+                               .format(child_resource_name, parent_resource_info['resource_name']))
+
+    def _get_service_processes_resource(self, request):
+        """
+        Finds the magpie resource 'processes' corresponding to '/ems/processes'.
+        User requesting resources must have administrator permissions in magpie.
 
-        :returns: list of magpie resources.
+        :returns: id of the 'processes' resource.
         """
-        path = '{host}/users/{usr}/resources?inherit=true'.format(host=self.magpie_url, usr=self.magpie_current)
+        path = '{host}/resources'.format(host=self.magpie_url)
         resp = requests.get(path, cookies=request.cookies,
                             headers=self.json_headers, verify=self.twitcher_ssl_verify)
-        LOGGER.debug("Looking for resources on: `{}`.".format(path))
         if resp.status_code != HTTPOk.code:
             raise resp.raise_for_status()
         ems_resources = None
@@ -97,82 +120,66 @@ def _get_process_resources(self, request):
             ems_resources = resp.json()['resources']['api'][self.magpie_service]['resources']
             for res_id in ems_resources:
                 if ems_resources[res_id]['resource_name'] == 'processes':
-                    ems_processes = ems_resources[res_id]['children']
-                    return ems_processes
+                    ems_processes_id = ems_resources[res_id]['resource_id']
+                    return ems_processes_id
         except KeyError:
-            LOGGER.debug("Content of ems service resources: `{!r}`.".format(ems_resources))
-            raise ProcessNotFound("Could not find resource `processes` endpoint for visibility retrieval.")
+            LOGGER.debug("Content of `{}` service resources: `{!r}`.".format(self.magpie_service, ems_resources))
+            raise ProcessNotFound("Could not find resource `processes` endpoint.")
         except Exception as ex:
-            LOGGER.debug("Exception during ems resources retrieval: [{}]".format(repr(ex)))
+            LOGGER.debug("Exception during `{}` resources retrieval: [{}]".format(self.magpie_service, repr(ex)))
             raise
         LOGGER.debug("Could not find resource: `processes`.")
-        return list()
+        return None
 
-    def _get_process_resource_id(self, process_id, ems_processes_resources, request):
+    def _create_resource_permissions(self, resource_id, group_name, permission_names, request):
         """
-        Requests for a 'process' resource corresponding to '/ems/processes/{id}' and returns its corresponding
-        magpie resource id if reading permission was granted to requesting user.
+        Creates group permission(s) on a resource.
 
-        :returns: id of the found 'process' resource, or None.
-        :raises:
-            HTTPException if not matching Ok or Unauthorized statuses.
-            ProcessNotFound if some response parsing error occurred.
+        :param resource_id: (int) magpie id of the resource to apply permissions on.
+        :param group_name: (str) name of the group for which to apply permissions, if any.
+        :param permission_names: (None, str, iterator) group permissions to apply to the resource, if any.
+        :param request: calling request for headers and credentials
         """
-        if not ems_processes_resources:
-            raise ProcessNotFound("Could not parse undefined processes resource endpoint for visibility retrieval.")
-        try:
-            for process_res_id in ems_processes_resources:
-                # find the requested process resource by matching ids
-                if ems_processes_resources[process_res_id]['resource_name'] == process_id:
-                    LOGGER.debug("Found process resource: `{}`.".format(process_id))
-
-                    # if read permission is granted on corresponding magpie resource route, twitcher
-                    # '/ems/process/{process_id}' will be accessible, otherwise unauthorized on private process
-                    path = '{host}/processes/{id}'.format(host=self._get_service_public_url(request), id=process_id)
-                    resp = requests.get(path, cookies=request.cookies,
-                                        headers=self.json_headers, verify=self.twitcher_ssl_verify)
-                    if resp.status_code == HTTPUnauthorized:
-                        return None
-                    elif resp.status_code == HTTPOk.code:
-                        return ems_processes_resources[process_res_id]['resource_id']
-                    raise resp.raise_for_status()
-        except KeyError:
-            LOGGER.debug("Content of processes resources: `{!r}`.".format(ems_processes_resources))
-            raise ProcessNotFound("Could not find process `{}` resource for visibility retrieval.".format(process_id))
-        except Exception as ex:
-            LOGGER.debug("Exception during process resource retrieval: [{}]".format(repr(ex)))
-            raise
-        LOGGER.debug("Could not find resource: `{}`.".format(process_id))
-        return None
+        if permission_names is None:
+            permission_names = []
+        if isinstance(permission_names, six.string_types):
+            permission_names = [permission_names]
+        for perm in permission_names:
+            data = {u'permission_name': perm}
+            path = '{host}/groups/{grp}/resources/{id}/permissions' \
+                .format(host=self.magpie_url, grp=group_name, id=resource_id)
+            resp = requests.post(path, data=data, cookies=request.cookies,
+                                 headers=self.json_headers, verify=self.twitcher_ssl_verify)
+            # permission is set if created or already exists
+            if resp.status_code not in (HTTPCreated.code, HTTPConflict.code):
+                raise resp.raise_for_status()
 
-    def _set_resource_permissions(self, resource_id, group_name, permission_names, request):
+    def _delete_resource_permissions(self, resource_id, group_name, permission_names, request):
         """
-        Sets group permissions on a resource.
+        Deletes group permission(s) on a resource.
 
-        :param resource_id: (int) magpie id of the resource to apply permissions on.
+        :param resource_id: (int) magpie id of the resource to remove permissions from.
         :param group_name: (str) name of the group for which to apply permissions, if any.
         :param permission_names: (None, str, iterator) group permissions to apply to the resource, if any.
         :param request: calling request for headers and credentials
         """
-        if isinstance(group_name, six.string_types):
-            if permission_names is None:
-                permission_names = []
-            if isinstance(permission_names, six.string_types):
-                permission_names = [permission_names]
-
-            for perm in permission_names:
-                data = {u'permission_name': perm}
-                path = '{host}/groups/{grp}/resources/{id}/permissions' \
-                    .format(host=self.magpie_url, grp=group_name, id=resource_id)
-                resp = requests.post(path, data=data, cookies=request.cookies,
-                                     headers=self.json_headers, verify=self.twitcher_ssl_verify)
-                # permission is set if created or already exists
-                if resp.status_code not in (HTTPCreated.code, HTTPConflict.code):
-                    raise resp.raise_for_status()
+        if permission_names is None:
+            permission_names = []
+        if isinstance(permission_names, six.string_types):
+            permission_names = [permission_names]
+        for perm in permission_names:
+            path = '{host}/groups/{grp}/resources/{id}/permissions/{perm}' \
+                   .format(host=self.magpie_url, grp=group_name, id=resource_id, perm=perm)
+            reps = requests.delete(path, cookies=request.cookies,
+                                   headers=self.json_headers, verify=self.twitcher_ssl_verify)
+            # permission is not set if deleted or non existing
+            if reps.status_code not in (HTTPOk.code, HTTPNotFound.code):
+                raise reps.raise_for_status()
 
     def _create_resource(self, resource_name, resource_parent_id, group_name, permission_names, request):
         """
         Creates a resource under another parent resource, and sets basic group permissions on it.
+        If the resource already exists for some reason, use it instead of the created one, and apply permissions.
 
         :param resource_name: (str) name of the resource to create.
         :param resource_parent_id: (int) id of the parent resource under which to create `resource_name`.
@@ -186,10 +193,14 @@ def _create_resource(self, resource_name, resource_parent_id, group_name, permis
             path = '{host}/resources'.format(host=self.magpie_url)
             resp = requests.post(path, data=data, cookies=request.cookies,
                                  headers=self.json_headers, verify=self.twitcher_ssl_verify)
-            if resp.status_code != HTTPCreated.code:
+            if resp.status_code == HTTPCreated.code:
+                res_id = resp.json()['resource']['resource_id']
+            elif resp.status_code == HTTPConflict.code:
+                res_id = self._find_child_resource_id(resource_parent_id, resource_name, request)
+            else:
                 raise resp.raise_for_status()
-            res_id = resp.json()['resource']['resource_id']
-            self._set_resource_permissions(res_id, group_name, permission_names, request)
+            if isinstance(group_name, six.string_types):
+                self._create_resource_permissions(res_id, group_name, permission_names, request)
             return res_id
         except KeyError:
             raise ProcessRegistrationError("Failed adding process resource route `{}`.".format(resource_name))
@@ -227,23 +238,12 @@ def save_process(self, process, overwrite=True, request=None):
 
             try:
                 # get resource id of route '/ems/processes', create it as necessary
-                path = '{host}/resources/{id}'.format(host=self.magpie_url, id=ems_res_id)
-                resp = requests.get(path, cookies=request.cookies,
-                                    headers=self.json_headers, verify=self.twitcher_ssl_verify)
-                if resp.status_code != HTTPOk.code:
-                    raise resp.raise_for_status()
-                processes_res_id = None
-                ems_resources = resp.json()[str(ems_res_id)]['children']
-                for child_resource in ems_resources:
-                    if ems_resources[child_resource]['resource_name'] == 'processes':
-                        processes_res_id = ems_resources[child_resource]['resource_id']
-                        break
-                if not processes_res_id:
-                    # all members of 'users' group can query '/ems/processes' (read exact route match),
-                    # but visibility of each process will be filtered by specific '/ems/processes/{id}' permissions
-                    # members of 'administrators' automatically inherit read/write permissions from 'ems' service
-                    processes_res_id = self._create_resource(u'processes', ems_res_id,
-                                                             self.magpie_users, u'read-match', request)
+                proc_res_id = self._find_child_resource_id(ems_res_id, 'processes', request)
+            except HTTPNotFound:
+                # all members of 'users' group can query '/ems/processes' (read exact route match),
+                # but visibility of each process will be filtered by specific '/ems/processes/{id}' permissions
+                # members of 'administrators' automatically inherit read/write permissions from 'ems' service
+                proc_res_id = self._create_resource(u'processes', ems_res_id, self.magpie_users, u'read-match', request)
             except KeyError:
                 raise ProcessRegistrationError("Failed retrieving processes resource.")
             except Exception as ex:
@@ -252,7 +252,7 @@ def save_process(self, process, overwrite=True, request=None):
 
             # create resources of route '/ems/processes/{id}' and '/ems/processes/{id}/jobs'
             # do not apply any permissions at first, so that the process is 'private' by default
-            process_res_id = self._create_resource(process.id, processes_res_id, None, None, request)
+            process_res_id = self._create_resource(process.id, proc_res_id, None, None, request)
             self._create_resource(u'jobs', process_res_id, None, None, request)
 
         return processstore_defaultfactory(request.registry).save_process(process, overwrite, request)
@@ -267,12 +267,11 @@ def delete_process(self, process_id, request=None):
             - also delete magpie resources tree corresponding to the process
         """
         if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
-            resources = self._get_process_resources(request)
-            resource_id = self._get_process_resource_id(process_id, resources, request)
-            if not resource_id:
-                raise ProcessNotFound("Could not find process `{}` for resource for deletion.".format(process_id))
+            ems_processes_id = self._get_service_processes_resource(request)
+            process_res_id = self._get_process_resource_id(ems_processes_id, process_id, request)
 
-            path = '{host}/resources/{id}'.format(host=self.magpie_url, id=resource_id)
+            # deleting the top-resource, magpie should automatically handle deletion of all sub-resources/permissions
+            path = '{host}/resources/{id}'.format(host=self.magpie_url, id=process_res_id)
             resp = requests.delete(path, cookies=request.cookies,
                                    headers=self.json_headers, verify=self.twitcher_ssl_verify)
             if resp.status_code != HTTPOk.code:
@@ -346,41 +345,29 @@ def set_visibility(self, process_id, visibility, request=None):
             modify magpie permissions of corresponding process access points according to desired visibility.
         """
         if self.twitcher_config == TWITCHER_CONFIGURATION_EMS:
-            ems_processes = self._get_process_resources(request)
-            process_res_id = self._get_process_resource_id(process_id, ems_processes, request)
-            if not process_res_id:
-                raise ProcessNotFound("Could not find process `{}` resource to change visibility.".format(process_id))
-
-            if visibility == VISIBILITY_PRIVATE:
-                path = '{host}/groups/{usr}/resources/{id}/permissions/{perm}' \
-                       .format(host=self.magpie_url, usr=self.magpie_users, id=process_res_id, perm=u'read')
-                reps = requests.delete(path, cookies=request.cookies,
-                                       headers=self.json_headers, verify=self.twitcher_ssl_verify)
-                # permission is not set if deleted or non existing
-                if reps.status_code not in (HTTPOk.code, HTTPNotFound.code):
-                    raise reps.raise_for_status()
-
-            elif visibility == VISIBILITY_PUBLIC:
-                # read permission so that users can make any sub-route GET requests (ex: GET '/ems/processes/{id}/jobs')
-                self._set_resource_permissions(process_res_id, self.magpie_users, u'read', request)
+            ems_processes_id = self._get_service_processes_resource(request)
+            process_res_id = self._find_child_resource_id(ems_processes_id, process_id, request)
 
+            try:
                 # find resource corresponding to '/ems/processes/{id}/jobs'
-                path = '{host}/resources/{id}'.format(host=self.magpie_url, id=process_res_id)
-                resp = requests.get(path, cookies=request.cookies,
-                                    headers=self.json_headers, verify=self.twitcher_ssl_verify)
-                if resp.status_code != HTTPOk.code:
-                    raise resp.raise_for_status()
-                jobs_res_id = None
-                process_resources = resp.json()[str(process_res_id)]['children']
-                for res_id in process_resources:
-                    if process_resources[res_id]['resource_name'] == 'jobs':
-                        jobs_res_id = process_resources[res_id]['resource_id']
-                        break
-                if not jobs_res_id:
-                    raise ProcessNotFound("Could not find process `{}` jobs resource to set visibility."
-                                          .format(process_id))
-
-                # use write-match permission so that users can ONLY execute a job (cannot DELETE process, job, etc.)
-                self._set_resource_permissions(jobs_res_id, self.magpie_users, u'write-match', request)
+                jobs_res_id = self._find_child_resource_id(process_res_id, 'jobs', request)
+
+                if visibility == VISIBILITY_PRIVATE:
+                    # remove write-match permissions of users on the process, cannot execute POST /jobs anymore
+                    self._delete_resource_permissions(jobs_res_id, self.magpie_users, u'write-match', request)
+                    # remove user read permissions on the process, cannot GET any info from it, not even see it in list
+                    self._delete_resource_permissions(process_res_id, self.magpie_users, u'read', request)
+
+                elif visibility == VISIBILITY_PUBLIC:
+                    # read permission so that users can make any sub-route GET requests (ex: '/ems/processes/{id}/jobs')
+                    self._create_resource_permissions(process_res_id, self.magpie_users, u'read', request)
+                    # use write-match permission so that users can ONLY execute a job (cannot DELETE process, job, etc.)
+                    self._create_resource_permissions(jobs_res_id, self.magpie_users, u'write-match', request)
+
+            except HTTPNotFound:
+                raise ProcessNotFound("Could not find process `{}` jobs resource to set visibility.".format(process_id))
+            except Exception as ex:
+                LOGGER.debug("Exception when trying to set process visibility: [{}]".format(repr(ex)))
+                raise
 
         processstore_defaultfactory(request.registry).set_visibility(process_id, visibility=visibility, request=request)