diff --git a/AUTHORS.rst b/AUTHORS.rst index bc6dfc048..07096adf9 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -4,9 +4,12 @@ Credits Development Lead ---------------- -* Francois-Xavier +* Francis Charette Migneault Contributors ------------ +* David Byrns +* David Caron * Francis Charette Migneault +* Francois-Xavier Derue diff --git a/Dockerfile b/Dockerfile index ec871796b..8c92648c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ MAINTAINER Francis Charette-Migneault RUN apt-get update && apt-get install -y \ build-essential \ supervisor \ + cron \ curl \ libssl-dev \ libffi-dev \ @@ -23,4 +24,11 @@ COPY ./ $MAGPIE_DIR RUN make install -f $MAGPIE_DIR/Makefile RUN make docs -f $MAGPIE_DIR/Makefile -CMD ["make", "start"] +# magpie cron service +ADD magpie-cron /etc/cron.d/magpie-cron +RUN chmod 0644 /etc/cron.d/magpie-cron +RUN touch ~/magpie_cron_status.log +# set /etc/environment so that cron runs using the environment variables set by docker +RUN env >> /etc/environment + +CMD make start diff --git a/HISTORY.rst b/HISTORY.rst index 0b964e4f9..36870a596 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,15 @@ History ======= +0.7.x +--------------------- + +`Magpie REST API latest documentation`_ + +* add service resource auto-sync feature +* return user/group services if any sub-resource has permissions +* [WIP] add inherited resource permission with querystring (deprecate `inherited_<>` routes) + 0.6.x --------------------- @@ -97,3 +106,4 @@ History .. _Magpie REST API 0.4.x documentation: magpie_api_0.4.x_ .. _Magpie REST API 0.5.x documentation: magpie_api_0.5.x_ .. _Magpie REST API 0.6.x documentation: magpie_api_0.6.x_ +.. _Magpie REST API latest documentation: _magpie_api_latest diff --git a/Makefile b/Makefile index 9cd755ae3..17bf99d38 100644 --- a/Makefile +++ b/Makefile @@ -143,7 +143,12 @@ install: sysinstall @echo "Installing Magpie..." python setup.py install +.PHONY: cron +cron: + @echo "Starting Cron service..." + cron + .PHONY: start -start: install +start: cron install @echo "Starting Magpie..." exec gunicorn -b 0.0.0.0:2001 --paste "$(CUR_DIR)/magpie/magpie.ini" --workers 10 --preload diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 000000000..d8b9e5175 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,2 @@ +magpie*.rst +modules.rst diff --git a/env/magpie.env.example b/env/magpie.env.example index 08ac5c3bf..ba6fbf117 100644 --- a/env/magpie.env.example +++ b/env/magpie.env.example @@ -6,6 +6,7 @@ MAGPIE_ADMIN_USER=admin MAGPIE_ADMIN_PASSWORD=qwerty MAGPIE_ANONYMOUS_USER=anonymous MAGPIE_USERS_GROUP=users +MAGPIE_CRON_LOG=~/magpie_cron.log PHOENIX_USER=phoenix PHOENIX_PASSWORD=qwerty PHOENIX_PORT=8443 diff --git a/magpie-cron b/magpie-cron new file mode 100644 index 000000000..fcb43530c --- /dev/null +++ b/magpie-cron @@ -0,0 +1 @@ +0 * * * * root /bin/bash -c "set -a ; . $MAGPIE_DIR/env/magpie.env ; . $MAGPIE_DIR/magpie/env/magpie.env ; . $MAGPIE_DIR/env/postgres.env ; . $MAGPIE_DIR/magpie/env/postgres.env ; set +a ; python -c 'from magpie.helpers.sync_resources import main; main()'" > ~/magpie_cron_status.log 2>&1 diff --git a/magpie/__meta__.py b/magpie/__meta__.py index b50c30ad7..786edc8af 100644 --- a/magpie/__meta__.py +++ b/magpie/__meta__.py @@ -2,7 +2,7 @@ General meta information on the magpie package. """ -__version__ = '0.6.5-dev' +__version__ = '0.7.0' __author__ = "Francois-Xavier Derue, Francis Charette-Migneault" __maintainer__ = "Francis Charette-Migneault" __email__ = 'francis.charette-migneault@crim.ca' diff --git a/magpie/adapter/__init__.py b/magpie/adapter/__init__.py index 505461691..4c6d730b6 100644 --- a/magpie/adapter/__init__.py +++ b/magpie/adapter/__init__.py @@ -2,13 +2,13 @@ from magpie.definitions.ziggurat_definitions import * from magpie.definitions.twitcher_definitions import * from magpie.adapter.magpieowssecurity import * -from magpie.adapter.magpieservice import * +from magpie.adapter.magpieservice import MagpieServiceStore from magpie.models import get_user from magpie.security import auth_config_from_settings from magpie.db import * from magpie import __meta__ import logging -logger = logging.getLogger(__name__) +logger = logging.getLogger("TWITCHER") class MagpieAdapter(AdapterInterface): @@ -19,9 +19,9 @@ def servicestore_factory(self, registry, headers=None): return MagpieServiceStore(registry=registry) def processstore_factory(self, registry): - # no reimplementation of processes on magpie side - # simply return the default twitcher process store - return DefaultAdapter().processstore_factory(registry) + # import here to avoid import errors on default twitcher not implementing processes + from magpie.adapter.magpieprocess import MagpieProcessStore + return MagpieProcessStore(registry=registry) def jobstore_factory(self, registry): # no reimplementation of jobs on magpie side diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py new file mode 100644 index 000000000..e937b241a --- /dev/null +++ b/magpie/adapter/magpieprocess.py @@ -0,0 +1,373 @@ +""" +Store adapters to read data from magpie. +""" + +from six.moves.urllib.parse import urlparse +import logging +import requests +import six +LOGGER = logging.getLogger("TWITCHER") + +from magpie.constants import get_constant +from magpie.definitions.twitcher_definitions import * +from magpie.definitions.pyramid_definitions import ( + ConfigurationError, + HTTPOk, + HTTPCreated, + HTTPNotFound, + HTTPConflict, + HTTPUnauthorized, + HTTPInternalServerError, + asbool +) + +# import 'process' elements separately than 'twitcher_definitions' because not defined in master +from twitcher.config import get_twitcher_configuration, TWITCHER_CONFIGURATION_EMS +from twitcher.exceptions import ProcessNotFound, ProcessRegistrationError +from twitcher.store import processstore_defaultfactory +from twitcher.store.base import ProcessStore +from twitcher.visibility import VISIBILITY_PUBLIC, VISIBILITY_PRIVATE, visibility_values + + +class MagpieProcessStore(ProcessStore): + """ + Registry for OWS processes. + Uses default process store for most operations. + Uses magpie to update process access and visibility. + """ + + def __init__(self, registry): + try: + # add 'http' scheme to url if omitted from config since further 'requests' calls fail without it + # mostly for testing when only 'localhost' is specified + # otherwise twitcher config should explicitly define it in MAGPIE_URL + url_parsed = urlparse(registry.settings.get('magpie.url').strip('/')) + if url_parsed.scheme in ['http', 'https']: + self.magpie_url = url_parsed.geturl() + else: + self.magpie_url = 'http://{}'.format(url_parsed.geturl()) + LOGGER.warn("Missing scheme from MagpieServiceStore url, new value: '{}'".format(self.magpie_url)) + self.magpie_users = get_constant('MAGPIE_USERS_GROUP') + self.magpie_admin = get_constant('MAGPIE_ADMIN_GROUP') + self.magpie_current = get_constant('MAGPIE_LOGGED_USER') + self.magpie_service = 'ems' + self.twitcher_config = get_twitcher_configuration(registry.settings) + self.twitcher_ssl_verify = asbool(registry.settings.get('twitcher.ows_proxy_ssl_verify', True)) + self.twitcher_service_url = None + self.json_headers = {'Accept': 'application/json'} + except AttributeError: + #If magpie.url does not exist, calling strip fct over None will raise this issue + raise ConfigurationError('magpie.url config cannot be found') + + def _get_service_public_url(self, request): + if not self.twitcher_service_url and self.twitcher_config == TWITCHER_CONFIGURATION_EMS: + # use generic 'current' user route to fetch service URL to ensure that even + # a user with minimal privileges will still return a match + path = '{host}/users/{usr}/services?inherit=true&cascade=true' \ + .format(host=self.magpie_url, usr=self.magpie_current) + resp = requests.get(path, cookies=request.cookies, + headers=self.json_headers, verify=self.twitcher_ssl_verify) + if resp.status_code != HTTPOk.code: + raise resp.raise_for_status() + try: + self.twitcher_service_url = resp.json()['services']['api'][self.magpie_service]['public_url'] + LOGGER.debug("Found service proxy url: {}".format(self.twitcher_service_url)) + except KeyError: + raise ProcessNotFound("Could not find service `{}` endpoint.".format(self.magpie_service)) + except Exception as ex: + LOGGER.debug("Exception during ems service url retrieval: [{}]".format(repr(ex))) + raise + return self.twitcher_service_url + + def _find_child_resource_id(self, parent_resource_id, child_resource_name, request): + """ + Finds the resource id corresponding to a child resource by name of the specified parent resource. + :param parent_resource_id: id of the resource from which to search children resources. + :param child_resource_name: name of the sub resource to find. + :param request: calling request for headers and credentials. + :return: + """ + path = '{host}/resources/{id}'.format(host=self.magpie_url, id=parent_resource_id) + resp = requests.get(path, cookies=request.cookies, + headers=self.json_headers, verify=self.twitcher_ssl_verify) + if resp.status_code != HTTPOk.code: + raise resp.raise_for_status() + child_res_id = None + parent_resource_info = resp.json()[str(parent_resource_id)] + children_resources = parent_resource_info['children'] + for res_id in children_resources: + if children_resources[res_id]['resource_name'] == child_resource_name: + child_res_id = children_resources[res_id]['resource_id'] + return child_res_id + if not child_res_id: + raise HTTPNotFound("Could not find resource `{}` under resource `{}`." + .format(child_resource_name, parent_resource_info['resource_name'])) + + def _get_service_processes_resource(self, request): + """ + Finds the magpie resource 'processes' corresponding to '/ems/processes'. + User requesting resources must have administrator permissions in magpie. + + :returns: id of the 'processes' resource. + """ + path = '{host}/resources'.format(host=self.magpie_url) + resp = requests.get(path, cookies=request.cookies, + headers=self.json_headers, verify=self.twitcher_ssl_verify) + if resp.status_code != HTTPOk.code: + raise resp.raise_for_status() + ems_resources = None + try: + ems_resources = resp.json()['resources']['api'][self.magpie_service]['resources'] + for res_id in ems_resources: + if ems_resources[res_id]['resource_name'] == 'processes': + ems_processes_id = ems_resources[res_id]['resource_id'] + return ems_processes_id + except KeyError: + LOGGER.debug("Content of `{}` service resources: `{!r}`.".format(self.magpie_service, ems_resources)) + raise ProcessNotFound("Could not find resource `processes` endpoint.") + except Exception as ex: + LOGGER.debug("Exception during `{}` resources retrieval: [{}]".format(self.magpie_service, repr(ex))) + raise + LOGGER.debug("Could not find resource: `processes`.") + return None + + def _create_resource_permissions(self, resource_id, group_name, permission_names, request): + """ + Creates group permission(s) on a resource. + + :param resource_id: (int) magpie id of the resource to apply permissions on. + :param group_name: (str) name of the group for which to apply permissions, if any. + :param permission_names: (None, str, iterator) group permissions to apply to the resource, if any. + :param request: calling request for headers and credentials + """ + if permission_names is None: + permission_names = [] + if isinstance(permission_names, six.string_types): + permission_names = [permission_names] + for perm in permission_names: + data = {u'permission_name': perm} + path = '{host}/groups/{grp}/resources/{id}/permissions' \ + .format(host=self.magpie_url, grp=group_name, id=resource_id) + resp = requests.post(path, data=data, cookies=request.cookies, + headers=self.json_headers, verify=self.twitcher_ssl_verify) + # permission is set if created or already exists + if resp.status_code not in (HTTPCreated.code, HTTPConflict.code): + raise resp.raise_for_status() + + def _delete_resource_permissions(self, resource_id, group_name, permission_names, request): + """ + Deletes group permission(s) on a resource. + + :param resource_id: (int) magpie id of the resource to remove permissions from. + :param group_name: (str) name of the group for which to apply permissions, if any. + :param permission_names: (None, str, iterator) group permissions to apply to the resource, if any. + :param request: calling request for headers and credentials + """ + if permission_names is None: + permission_names = [] + if isinstance(permission_names, six.string_types): + permission_names = [permission_names] + for perm in permission_names: + path = '{host}/groups/{grp}/resources/{id}/permissions/{perm}' \ + .format(host=self.magpie_url, grp=group_name, id=resource_id, perm=perm) + reps = requests.delete(path, cookies=request.cookies, + headers=self.json_headers, verify=self.twitcher_ssl_verify) + # permission is not set if deleted or non existing + if reps.status_code not in (HTTPOk.code, HTTPNotFound.code): + raise reps.raise_for_status() + + def _create_resource(self, resource_name, resource_parent_id, group_name, permission_names, request): + """ + Creates a resource under another parent resource, and sets basic group permissions on it. + If the resource already exists for some reason, use it instead of the created one, and apply permissions. + + :param resource_name: (str) name of the resource to create. + :param resource_parent_id: (int) id of the parent resource under which to create `resource_name`. + :param group_name: (str) name of the group for which to apply permissions to the created resource, if any. + :param permission_names: (None, str, iterator) group permissions to apply to the created resource, if any. + :param request: calling request for headers and credentials + :returns: id of the created resource + """ + try: + data = {u'parent_id': resource_parent_id, u'resource_name': resource_name, u'resource_type': u'route'} + path = '{host}/resources'.format(host=self.magpie_url) + resp = requests.post(path, data=data, cookies=request.cookies, + headers=self.json_headers, verify=self.twitcher_ssl_verify) + if resp.status_code == HTTPCreated.code: + res_id = resp.json()['resource']['resource_id'] + elif resp.status_code == HTTPConflict.code: + res_id = self._find_child_resource_id(resource_parent_id, resource_name, request) + else: + raise resp.raise_for_status() + if isinstance(group_name, six.string_types): + self._create_resource_permissions(res_id, group_name, permission_names, request) + return res_id + except KeyError: + raise ProcessRegistrationError("Failed adding process resource route `{}`.".format(resource_name)) + except Exception as ex: + LOGGER.debug("Exception during process resource creation: [{}]".format(repr(ex))) + raise + + def save_process(self, process, overwrite=True, request=None): + """ + Save a new process. + + If twitcher is not in EMS mode, delegate execution to default twitcher process store. + If twitcher is in EMS mode: + - user requesting creation must have sufficient administrator permissions in magpie to do so. + - assign any pre-requirement routes permissions to allow admins to edit '/ems/processes/...' + + Requirements: + - service 'ems' of type 'api' must exist + - group 'administrators' must have ['read', 'write'] permissions on 'ems' service + """ + if self.twitcher_config == TWITCHER_CONFIGURATION_EMS: + try: + # get resource id of ems service + path = '{host}/services/{svc}'.format(host=self.magpie_url, svc=self.magpie_service) + resp = requests.get(path, cookies=request.cookies, + headers=self.json_headers, verify=self.twitcher_ssl_verify) + if resp.status_code != HTTPOk.code: + raise resp.raise_for_status() + ems_res_id = resp.json()[self.magpie_service]['resource_id'] + except KeyError: + raise ProcessRegistrationError("Failed retrieving service resource.") + except Exception as ex: + LOGGER.debug("Exception during `{0}` resource retrieval: [{1}]".format(self.magpie_service, repr(ex))) + raise + + try: + # get resource id of route '/ems/processes', create it as necessary + proc_res_id = self._find_child_resource_id(ems_res_id, 'processes', request) + except HTTPNotFound: + # all members of 'users' group can query '/ems/processes' (read exact route match), + # but visibility of each process will be filtered by specific '/ems/processes/{id}' permissions + # members of 'administrators' automatically inherit read/write permissions from 'ems' service + proc_res_id = self._create_resource(u'processes', ems_res_id, self.magpie_users, u'read-match', request) + except KeyError: + raise ProcessRegistrationError("Failed retrieving processes resource.") + except Exception as ex: + LOGGER.debug("Exception during `processes` resource retrieval: [{}]".format(repr(ex))) + raise + + # create resources of route '/ems/processes/{id}' and '/ems/processes/{id}/jobs' + # do not apply any permissions at first, so that the process is 'private' by default + process_res_id = self._create_resource(process.id, proc_res_id, None, None, request) + self._create_resource(u'jobs', process_res_id, None, None, request) + + return processstore_defaultfactory(request.registry).save_process(process, overwrite, request) + + def delete_process(self, process_id, request=None): + """ + Delete a process. + + Delegate execution to default twitcher process store. + If twitcher is in EMS mode: + - user requesting deletion must have administrator permissions in magpie (delete route blocked otherwise). + - also delete magpie resources tree corresponding to the process + """ + if self.twitcher_config == TWITCHER_CONFIGURATION_EMS: + ems_processes_id = self._get_service_processes_resource(request) + process_res_id = self._get_process_resource_id(ems_processes_id, process_id, request) + + # deleting the top-resource, magpie should automatically handle deletion of all sub-resources/permissions + path = '{host}/resources/{id}'.format(host=self.magpie_url, id=process_res_id) + resp = requests.delete(path, cookies=request.cookies, + headers=self.json_headers, verify=self.twitcher_ssl_verify) + if resp.status_code != HTTPOk.code: + raise resp.raise_for_status() + + return processstore_defaultfactory(request.registry).delete_process(process_id, request) + + def list_processes(self, visibility=None, request=None): + """ + List publicly visible processes. + + Delegate execution to default twitcher process store. + If twitcher is not in EMS mode, filter by only visible processes. + If twitcher is in EMS mode, filter according to magpie user group memberships: + - administrators: return everything + - any other group: return only visible processes + """ + visibility_filter = visibility + if self.twitcher_config == TWITCHER_CONFIGURATION_EMS: + path = '{host}/users/{usr}/groups'.format(host=self.magpie_url, usr=self.magpie_current) + resp = requests.get(path, cookies=request.cookies, + headers=self.json_headers, verify=self.twitcher_ssl_verify) + if resp.status_code != HTTPOk.code: + raise resp.raise_for_status() + try: + groups_memberships = resp.json()['group_names'] + if self.magpie_admin in groups_memberships: + visibility_filter = visibility_values + else: + visibility_filter = VISIBILITY_PUBLIC + except KeyError: + raise ProcessNotFound("Failed retrieving processes read permissions for listing.") + except Exception as ex: + LOGGER.debug("Exception during processes listing: [{}]".format(repr(ex))) + raise + + store = processstore_defaultfactory(request.registry) + process_list = store.list_processes(visibility=visibility_filter, request=request) + LOGGER.debug("Found visible processes: {!s}.".format(process_list)) + return process_list + + def fetch_by_id(self, process_id, request=None): + """ + Get a process if visible for user. + + Delegate operation to default twitcher process store. + If twitcher is in EMS mode: + using twitcher proxy, magpie user/group permissions on corresponding resource (/ems/processes/{process_id}) + will automatically handle Ok/Unauthorized responses using the API route's read access. + """ + return processstore_defaultfactory(request.registry).fetch_by_id(process_id, request=request) + + def get_visibility(self, process_id, request=None): + """ + Get visibility of a process. + + Delegate operation to default twitcher process store. + If twitcher is in EMS mode: + using twitcher proxy, only administrators get read permissions on '/ems/processes/{process_id}/visibility' + any other level user will get unauthorized on this route + """ + return processstore_defaultfactory(request.registry).get_visibility(process_id, request=request) + + def set_visibility(self, process_id, visibility, request=None): + """ + Set visibility of a process. + + Delegate change of process visibility to default twitcher process store. + If twitcher is in EMS mode: + using twitcher proxy, only administrators get write permissions on '/ems/processes/{process_id}/visibility' + modify magpie permissions of corresponding process access points according to desired visibility. + """ + if self.twitcher_config == TWITCHER_CONFIGURATION_EMS: + ems_processes_id = self._get_service_processes_resource(request) + process_res_id = self._find_child_resource_id(ems_processes_id, process_id, request) + + try: + # find resource corresponding to '/ems/processes/{id}/jobs' + jobs_res_id = self._find_child_resource_id(process_res_id, 'jobs', request) + + if visibility == VISIBILITY_PRIVATE: + # remove write-match permissions of users on the process, cannot execute POST /jobs anymore + self._delete_resource_permissions(jobs_res_id, self.magpie_users, u'write-match', request) + # remove user read permissions on the process, cannot GET any info from it, not even see it in list + self._delete_resource_permissions(process_res_id, self.magpie_users, u'read', request) + + elif visibility == VISIBILITY_PUBLIC: + # read permission so that users can make any sub-route GET requests (ex: '/ems/processes/{id}/jobs') + self._create_resource_permissions(process_res_id, self.magpie_users, u'read', request) + # use write-match permission so that users can ONLY execute a job (cannot DELETE process, job, etc.) + self._create_resource_permissions(jobs_res_id, self.magpie_users, u'write-match', request) + + except HTTPNotFound: + raise ProcessNotFound("Could not find process `{}` jobs resource to set visibility.".format(process_id)) + except Exception as ex: + LOGGER.debug("Exception when trying to set process visibility: [{}]".format(repr(ex))) + raise + + processstore_defaultfactory(request.registry).set_visibility(process_id, visibility=visibility, request=request) diff --git a/magpie/adapter/magpieservice.py b/magpie/adapter/magpieservice.py index 8b61fcbb4..d9acbc87c 100644 --- a/magpie/adapter/magpieservice.py +++ b/magpie/adapter/magpieservice.py @@ -6,10 +6,10 @@ import logging import requests import json -LOGGER = logging.getLogger(__name__) +LOGGER = logging.getLogger("TWITCHER") from magpie.definitions.twitcher_definitions import * -from magpie.definitions.pyramid_definitions import ConfigurationError +from magpie.definitions.pyramid_definitions import ConfigurationError, HTTPOk class MagpieServiceStore(ServiceStore): @@ -48,9 +48,10 @@ def list_services(self, request=None): Lists all services registered in magpie. """ my_services = [] - response = requests.get('{url}/users/current/services'.format(url=self.magpie_url), + path = '/users/current/services?inherit=True&cascade=True' + response = requests.get('{url}{path}'.format(url=self.magpie_url, path=path), cookies=request.cookies) - if response.status_code != 200: + if response.status_code != HTTPOk.code: raise response.raise_for_status() services = json.loads(response.text) for service_type in services['services']: diff --git a/magpie/alembic/utils.py b/magpie/alembic/utils.py new file mode 100644 index 000000000..53e68b199 --- /dev/null +++ b/magpie/alembic/utils.py @@ -0,0 +1,9 @@ +from magpie.definitions.sqlalchemy_definitions import * + + +def has_column(context, table_name, column_name): + inspector = reflection.Inspector.from_engine(context.connection.engine) + for column in inspector.get_columns(table_name=table_name): + if column_name in column['name']: + return True + return False diff --git a/magpie/alembic/versions/73639c63c4fc_unified_api_service.py b/magpie/alembic/versions/73639c63c4fc_unified_api_service.py new file mode 100644 index 000000000..7096e2b52 --- /dev/null +++ b/magpie/alembic/versions/73639c63c4fc_unified_api_service.py @@ -0,0 +1,85 @@ +"""unified api service + +Revision ID: 73639c63c4fc +Revises: d01af1f2e445 +Create Date: 2018-09-27 16:12:02.282830 + +""" +import os, sys + +cur_file = os.path.abspath(__file__) +root_dir = os.path.dirname(cur_file) # version +root_dir = os.path.dirname(root_dir) # alembic +root_dir = os.path.dirname(root_dir) # magpie +root_dir = os.path.dirname(root_dir) # root +sys.path.insert(0, root_dir) + +from alembic import op +from alembic.context import get_context +from magpie.definitions.sqlalchemy_definitions import * +# from magpie.models import Service +from sqlalchemy.sql import table +from sqlalchemy import func + +Session = sessionmaker() + +# revision identifiers, used by Alembic. +revision = '73639c63c4fc' +down_revision = 'd01af1f2e445' +branch_labels = None +depends_on = None + + +def upgrade(): + context = get_context() + if isinstance(context.connection.engine.dialect, PGDialect): + # add 'sync_type' column if missing + op.add_column('services', sa.Column('sync_type', sa.UnicodeText(), nullable=True)) + + services = table('services', + sa.Column('url', sa.UnicodeText()), + sa.Column('type', sa.UnicodeText()), + sa.Column('sync_type', sa.UnicodeText()), + ) + + # transfer 'api' service types + op.execute(services. + update(). + where(services.c.type == op.inline_literal('project-api')). + values({'type': op.inline_literal('api'), + 'url': services.c.url + "/api", + 'sync_type': op.inline_literal('project-api') + }) + ) + op.execute(services. + update(). + where(services.c.type == op.inline_literal('geoserver-api')). + values({'type': op.inline_literal('api'), + 'sync_type': op.inline_literal('geoserver-api') + }) + ) + + +def downgrade(): + service = table('services', + sa.Column('url', sa.UnicodeText()), + sa.Column('type', sa.UnicodeText()), + sa.Column('sync_type', sa.UnicodeText()), + ) + + # transfer 'api' service types + op.execute(service. + update(). + where(service.c.sync_type == op.inline_literal('project-api')). + values({'type': op.inline_literal('project-api'), + 'url': func.replace(service.c.url, '/api', ''), + }) + ) + op.execute(service. + update(). + where(service.c.sync_type == op.inline_literal('geoserver-api')). + values({'type': op.inline_literal('geoserver-api'), + }) + ) + + op.drop_column('services', 'sync_type') diff --git a/magpie/alembic/versions/73b872478d87_add_resource_label.py b/magpie/alembic/versions/73b872478d87_add_resource_label.py new file mode 100644 index 000000000..b13ebd4ef --- /dev/null +++ b/magpie/alembic/versions/73b872478d87_add_resource_label.py @@ -0,0 +1,26 @@ +"""empty message + +Revision ID: 73b872478d87 +Revises: d01af1f2e445 +Create Date: 2018-09-24 11:29:38.108819 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '73b872478d87' +down_revision = '73639c63c4fc' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('resources', sa.Column('resource_display_name', sa.Unicode(100), nullable=True)) + op.add_column('remote_resources', sa.Column('resource_display_name', sa.Unicode(100), nullable=True)) + + +def downgrade(): + op.drop_column('resources', 'resource_display_name') + op.drop_column('remote_resources', 'resource_display_name') diff --git a/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py b/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py index 00b11a2ca..810707941 100644 --- a/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py +++ b/magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py @@ -39,20 +39,10 @@ def upgrade(): context.connection.engine.dialect.supports_sane_multi_rowcount = False if isinstance(context.connection.engine.dialect, PGDialect): - - # check if column exists, add it otherwise - inspector = reflection.Inspector.from_engine(context.connection.engine) - has_root_service_column = False - for column in inspector.get_columns(table_name='resources'): - if 'root_service_id' in column['name']: - has_root_service_column = True - break - - if not has_root_service_column: - op.add_column('resources', sa.Column('root_service_id', sa.Integer(), nullable=True)) + op.add_column('resources', sa.Column('root_service_id', sa.Integer(), nullable=True)) # add existing resource references to their root service, loop through reference tree chain - all_resources = session.query(models.Resource) + all_resources = session.query(models.Resource.parent_id) for resource in all_resources: service_resource = get_resource_root_service(resource, session) if service_resource.resource_id != resource.resource_id: @@ -61,4 +51,6 @@ def upgrade(): def downgrade(): - pass + context = get_context() + if isinstance(context.connection.engine.dialect, PGDialect): + op.drop_column('resources', 'root_service_id') diff --git a/magpie/alembic/versions/c352a98d570e_project_api_route_resource.py b/magpie/alembic/versions/c352a98d570e_project_api_route_resource.py index c4a2f048e..308018e25 100644 --- a/magpie/alembic/versions/c352a98d570e_project_api_route_resource.py +++ b/magpie/alembic/versions/c352a98d570e_project_api_route_resource.py @@ -32,12 +32,13 @@ def change_project_api_resource_type(new_type_name): if isinstance(context.connection.engine.dialect, PGDialect): # obtain service 'project-api' session = Session(bind=op.get_bind()) - project_api_svc = models.Service.by_service_name('project-api', db_session=session) + project_api_svc = session.query(models.Service.resource_id).filter_by(resource_name='project-api').first() # nothing to edit if it doesn't exist, otherwise change resource types to 'route' if project_api_svc: project_api_id = project_api_svc.resource_id - project_api_res = session.query(models.Resource).filter(models.Resource.root_service_id == project_api_id) + columns = models.Resource.resource_type, models.Resource.root_service_id + project_api_res = session.query(columns).filter(models.Resource.root_service_id == project_api_id) for res in project_api_res: res.resource_type = 'route' diff --git a/magpie/alembic/versions/d01af1f2e445_create_remote_sync_tables.py b/magpie/alembic/versions/d01af1f2e445_create_remote_sync_tables.py new file mode 100644 index 000000000..bd03a779b --- /dev/null +++ b/magpie/alembic/versions/d01af1f2e445_create_remote_sync_tables.py @@ -0,0 +1,64 @@ +"""create remote sync tables + +Revision ID: d01af1f2e445 +Revises: c352a98d570e +Create Date: 2018-09-11 10:56:23.779143 + +""" +import os, sys + +cur_file = os.path.abspath(__file__) +root_dir = os.path.dirname(cur_file) # version +root_dir = os.path.dirname(root_dir) # alembic +root_dir = os.path.dirname(root_dir) # magpie +root_dir = os.path.dirname(root_dir) # root +sys.path.insert(0, root_dir) + +from alembic import op +from magpie.definitions.sqlalchemy_definitions import * + +Session = sessionmaker() + +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'd01af1f2e445' +down_revision = 'c352a98d570e' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table('remote_resources', + sa.Column('resource_id', sa.Integer(), primary_key=True, + nullable=False, autoincrement=True), + sa.Column('service_id', + sa.Integer(), + sa.ForeignKey('services.resource_id', onupdate='CASCADE', ondelete='CASCADE'), + index=True, + nullable=False), + sa.Column('parent_id', + sa.Integer(), + sa.ForeignKey('remote_resources.resource_id', onupdate='CASCADE', ondelete='SET NULL'), + nullable=True), + sa.Column('ordering', sa.Integer(), default=0, nullable=False), + sa.Column('resource_name', sa.Unicode(100), nullable=False), + sa.Column('resource_type', sa.Unicode(30), nullable=False), + ) + op.create_table('remote_resources_sync_info', + sa.Column('id', sa.Integer(), primary_key=True, nullable=False, autoincrement=True), + sa.Column('service_id', + sa.Integer(), + sa.ForeignKey('services.resource_id', onupdate='CASCADE', ondelete='CASCADE'), + index=True, + nullable=False), + sa.Column('remote_resource_id', + sa.Integer(), + sa.ForeignKey('remote_resources.resource_id', onupdate='CASCADE', ondelete='CASCADE')), + sa.Column('last_sync', sa.DateTime(), nullable=True), + ) + + +def downgrade(): + op.drop_table('remote_resources_sync_info') + op.drop_table('remote_resources') diff --git a/magpie/api/api_requests.py b/magpie/api/api_requests.py index db93289ab..ae96735aa 100644 --- a/magpie/api/api_requests.py +++ b/magpie/api/api_requests.py @@ -152,3 +152,13 @@ def get_value_matchdict_checked(request, key): verify_param(val, notNone=True, notEmpty=True, httpError=HTTPUnprocessableEntity, paramName=key, msgOnFail=UnprocessableEntityResponseSchema.description) return val + + +def get_query_param(request, case_insensitive_key, default=None): + for p in request.params: + if p.lower() == case_insensitive_key: + value = request.params.get(p) + if isinstance(value, six.string_types): + return value.lower() + return value + return default diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py index ea1c1a5e2..68984bdb2 100644 --- a/magpie/api/api_rest_schemas.py +++ b/magpie/api/api_rest_schemas.py @@ -254,12 +254,21 @@ class HeaderRequestSchema(colander.MappingSchema): content_type.name = 'Content-Type' -class BaseBodySchema(colander.MappingSchema): +QueryInheritGroupsPermissions = colander.SchemaNode( + colander.Boolean(), default=False, missing=colander.drop, + description='User groups memberships inheritance to resolve service resource permissions.') +QueryCascadeResourcesPermissions = colander.SchemaNode( + colander.Boolean(), default=False, missing=colander.drop, + description='Display any service that has at least one sub-resource user permission, ' + 'or only services that have user permissions directly set on them.', ) + + +class BaseResponseBodySchema(colander.MappingSchema): __code = None __desc = None def __init__(self, code, description): - super(BaseBodySchema, self).__init__() + super(BaseResponseBodySchema, self).__init__() assert isinstance(code, int) assert isinstance(description, six.string_types) self.__code = code @@ -301,7 +310,7 @@ class ErrorVerifyParamBodySchema(colander.MappingSchema): missing=colander.drop) -class ErrorRequestInfoBodySchema(BaseBodySchema): +class ErrorRequestInfoBodySchema(BaseResponseBodySchema): def __init__(self, **kw): super(ErrorRequestInfoBodySchema, self).__init__(**kw) assert kw.get('code') >= 400 @@ -326,7 +335,7 @@ def __init__(self, **kw): super(InternalServerErrorResponseBodySchema, self).__init__(**kw) -class UnauthorizedResponseBodySchema(BaseBodySchema): +class UnauthorizedResponseBodySchema(BaseResponseBodySchema): def __init__(self, **kw): kw['code'] = HTTPUnauthorized.code super(UnauthorizedResponseBodySchema, self).__init__(**kw) @@ -356,7 +365,7 @@ class MethodNotAllowedResponseSchema(colander.MappingSchema): class UnprocessableEntityResponseSchema(colander.MappingSchema): description = "Invalid value specified." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPUnprocessableEntity.code, description=description) + body = BaseResponseBodySchema(code=HTTPUnprocessableEntity.code, description=description) class InternalServerErrorResponseSchema(colander.MappingSchema): @@ -366,42 +375,42 @@ class InternalServerErrorResponseSchema(colander.MappingSchema): class ProvidersListSchema(colander.SequenceSchema): - item = colander.SchemaNode( + provider_name = colander.SchemaNode( colander.String(), description="Available login providers.", - example=["ziggurat", "openid"], + example="openid", ) class ResourceTypesListSchema(colander.SequenceSchema): - item = colander.SchemaNode( + resource_type = colander.SchemaNode( colander.String(), description="Available resource type under root service.", - example=["file", "dictionary"], + example="file", ) class GroupNamesListSchema(colander.SequenceSchema): - item = colander.SchemaNode( + group_name = colander.SchemaNode( colander.String(), description="List of groups depending on context.", - example=["anonymous"] + example="administrators" ) class UserNamesListSchema(colander.SequenceSchema): - item = colander.SchemaNode( + user_name = colander.SchemaNode( colander.String(), description="Users registered in the db", - example=["anonymous", "admin", "toto"] + example="bob" ) class PermissionListSchema(colander.SequenceSchema): - item = colander.SchemaNode( + permission_name = colander.SchemaNode( colander.String(), description="Permissions applicable to the service/resource", - example=["read", "write"] + example="read" ) @@ -414,7 +423,9 @@ class UserBodySchema(colander.MappingSchema): colander.String(), description="Email of the user.", example="toto@mail.com") - group_names = GroupNamesListSchema() + group_names = GroupNamesListSchema( + example=['administrators', 'users'] + ) class GroupBodySchema(colander.MappingSchema): @@ -436,7 +447,10 @@ class GroupBodySchema(colander.MappingSchema): description="Number of users member of the group.", example=2, missing=colander.drop) - user_names = UserNamesListSchema(missing=colander.drop) + user_names = UserNamesListSchema( + example=['alice', 'bob'], + missing=colander.drop + ) class ServiceBodySchema(colander.MappingSchema): @@ -444,7 +458,9 @@ class ServiceBodySchema(colander.MappingSchema): colander.Integer(), description="Resource identification number", ) - permission_names = PermissionListSchema() + permission_names = PermissionListSchema( + example=['read', 'write'] + ) service_name = colander.SchemaNode( colander.String(), description="Name of the service", @@ -455,6 +471,11 @@ class ServiceBodySchema(colander.MappingSchema): description="Type of the service", example="thredds" ) + service_sync_type = colander.SchemaNode( + colander.String(), + description="Type of resource synchronization implementation.", + example="thredds" + ) public_url = colander.SchemaNode( colander.String(), description="Proxy URL available for public access with permissions", @@ -477,6 +498,11 @@ class ResourceBodySchema(colander.MappingSchema): description="Name of the resource", example="thredds" ) + resource_display_name = colander.SchemaNode( + colander.String(), + description="Display name of the resource", + example="Birdhouse Thredds Data Server" + ) resource_type = colander.SchemaNode( colander.String(), description="Type of the resource", @@ -542,29 +568,29 @@ class ResourcesSchemaNode(colander.MappingSchema): thredds = Resource_ServiceType_thredds_SchemaNode() -class Resources_ResponseBodySchema(BaseBodySchema): +class Resources_ResponseBodySchema(BaseResponseBodySchema): resources = ResourcesSchemaNode() class Resource_MatchDictCheck_ForbiddenResponseSchema(colander.MappingSchema): description = "Resource query by id refused by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class Resource_MatchDictCheck_NotFoundResponseSchema(colander.MappingSchema): description = "Resource ID not found in db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotFound.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) class Resource_MatchDictCheck_NotAcceptableResponseSchema(colander.MappingSchema): description = "Resource ID is an invalid literal for `int` type." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotAcceptable.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description) -class Resource_GET_ResponseBodySchema(BaseBodySchema): +class Resource_GET_ResponseBodySchema(BaseResponseBodySchema): resource_id = Resource_ChildResourceWithChildrenContainerBodySchema() resource_id.name = '{resource_id}' @@ -598,7 +624,7 @@ class Resource_PUT_RequestSchema(colander.MappingSchema): body = Resource_PUT_RequestBodySchema() -class Resource_PUT_ResponseBodySchema(BaseBodySchema): +class Resource_PUT_ResponseBodySchema(BaseResponseBodySchema): resource_id = colander.SchemaNode( colander.String(), description="Updated resource identification number." @@ -626,7 +652,7 @@ class Resource_PUT_OkResponseSchema(colander.MappingSchema): class Resource_PUT_ForbiddenResponseSchema(colander.MappingSchema): description = "Failed to update resource with new name." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class Resource_DELETE_RequestBodySchema(colander.MappingSchema): @@ -646,13 +672,13 @@ class Resource_DELETE_RequestSchema(colander.MappingSchema): class Resource_DELETE_OkResponseSchema(colander.MappingSchema): description = "Delete resource successful." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPOk.code, description=description) + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) class Resource_DELETE_ForbiddenResponseSchema(colander.MappingSchema): description = "Delete resource from db failed." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class Resources_GET_OkResponseSchema(colander.MappingSchema): @@ -666,6 +692,11 @@ class Resources_POST_BodySchema(colander.MappingSchema): colander.String(), description="Name of the resource to create" ) + resource_display_name = colander.SchemaNode( + colander.String(), + description="Display name of the resource to create, defaults to resource_name.", + missing=colander.drop + ) resource_type = colander.SchemaNode( colander.String(), description="Type of the resource to create" @@ -682,7 +713,7 @@ class Resources_POST_RequestBodySchema(colander.MappingSchema): body = Resources_POST_BodySchema() -class Resource_POST_ResponseBodySchema(BaseBodySchema): +class Resource_POST_ResponseBodySchema(BaseResponseBodySchema): resource_id = Resource_ChildResourceWithChildrenContainerBodySchema() resource_id.name = '{resource_id}' @@ -696,28 +727,28 @@ class Resources_POST_CreatedResponseSchema(colander.MappingSchema): class Resources_POST_BadRequestResponseSchema(colander.MappingSchema): description = "Invalid [`resource_name`|`resource_type`|`parent_id`] specified for child resource creation." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPBadRequest.code, description=description) + body = BaseResponseBodySchema(code=HTTPBadRequest.code, description=description) class Resources_POST_ForbiddenResponseSchema(colander.MappingSchema): description = "Failed to insert new resource in service tree using parent id." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class Resources_POST_NotFoundResponseSchema(colander.MappingSchema): description = "Could not find specified resource parent id." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotFound.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) class Resources_POST_ConflictResponseSchema(colander.MappingSchema): description = "Resource name already exists at requested tree level for creation." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPConflict.code, description=description) + body = BaseResponseBodySchema(code=HTTPConflict.code, description=description) -class ResourcePermissions_GET_ResponseBodySchema(BaseBodySchema): +class ResourcePermissions_GET_ResponseBodySchema(BaseResponseBodySchema): permission_names = PermissionListSchema() @@ -730,7 +761,7 @@ class ResourcePermissions_GET_OkResponseSchema(colander.MappingSchema): class ResourcePermissions_GET_NotAcceptableResponseSchema(colander.MappingSchema): description = "Invalid resource type to extract permissions." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotAcceptable.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description) class ServiceResourcesBodySchema(ServiceBodySchema): @@ -793,7 +824,7 @@ class ServicesSchemaNode(colander.MappingSchema): wps = ServiceType_wps_SchemaNode(missing=colander.drop) -class Service_FailureBodyResponseSchema(BaseBodySchema): +class Service_FailureBodyResponseSchema(BaseResponseBodySchema): service_name = colander.SchemaNode( colander.String(), description="Service name extracted from path" @@ -803,7 +834,7 @@ class Service_FailureBodyResponseSchema(BaseBodySchema): class Service_MatchDictCheck_ForbiddenResponseSchema(colander.MappingSchema): description = "Service query by name refused by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class Service_MatchDictCheck_NotFoundResponseSchema(colander.MappingSchema): @@ -812,7 +843,7 @@ class Service_MatchDictCheck_NotFoundResponseSchema(colander.MappingSchema): body = Service_FailureBodyResponseSchema(code=HTTPNotFound.code, description=description) -class Service_GET_ResponseBodySchema(BaseBodySchema): +class Service_GET_ResponseBodySchema(BaseResponseBodySchema): service_name = ServiceBodySchema() service_name.name = '{service_name}' @@ -823,7 +854,7 @@ class Service_GET_OkResponseSchema(colander.MappingSchema): body = Service_GET_ResponseBodySchema(code=HTTPOk.code, description=description) -class Services_GET_ResponseBodySchema(BaseBodySchema): +class Services_GET_ResponseBodySchema(BaseResponseBodySchema): services = ServicesSchemaNode() @@ -833,7 +864,7 @@ class Services_GET_OkResponseSchema(colander.MappingSchema): body = Services_GET_ResponseBodySchema(code=HTTPOk.code, description=description) -class Services_GET_NotAcceptableResponseBodySchema(BaseBodySchema): +class Services_GET_NotAcceptableResponseBodySchema(BaseResponseBodySchema): service_type = colander.SchemaNode( colander.String(), description="Name of the service type filter employed when applicable", @@ -857,6 +888,11 @@ class Services_POST_BodySchema(colander.MappingSchema): description="Type of the service to create", example="wps" ) + service_sync_type = colander.SchemaNode( + colander.String(), + description="Type of the service to create", + example="wps" + ) service_url = colander.SchemaNode( colander.String(), description="Private URL of the service to create", @@ -872,25 +908,25 @@ class Services_POST_RequestBodySchema(colander.MappingSchema): class Services_POST_CreatedResponseSchema(colander.MappingSchema): description = "Service registration to db successful." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPOk.code, description=description) + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) class Services_POST_BadRequestResponseSchema(colander.MappingSchema): description = "Invalid `service_type` value does not correspond to any of the existing service types." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPBadRequest.code, description=description) + body = BaseResponseBodySchema(code=HTTPBadRequest.code, description=description) class Services_POST_ForbiddenResponseSchema(colander.MappingSchema): description = "Service registration forbidden by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class Services_POST_ConflictResponseSchema(colander.MappingSchema): description = "Specified `service_name` value already exists." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPConflict.code, description=description) + body = BaseResponseBodySchema(code=HTTPConflict.code, description=description) class Service_PUT_ResponseBodySchema(colander.MappingSchema): @@ -921,7 +957,7 @@ class Service_PUT_RequestBodySchema(colander.MappingSchema): body = Service_PUT_ResponseBodySchema() -class Service_SuccessBodyResponseSchema(BaseBodySchema): +class Service_SuccessBodyResponseSchema(BaseResponseBodySchema): service = ServiceBodySchema() @@ -965,7 +1001,7 @@ class Service_DELETE_ForbiddenResponseSchema(colander.MappingSchema): body = Service_FailureBodyResponseSchema(code=HTTPForbidden.code, description=description) -class ServicePermissions_ResponseBodySchema(BaseBodySchema): +class ServicePermissions_ResponseBodySchema(BaseResponseBodySchema): permission_names = PermissionListSchema() @@ -975,7 +1011,7 @@ class ServicePermissions_GET_OkResponseSchema(colander.MappingSchema): body = ServicePermissions_ResponseBodySchema(code=HTTPOk.code, description=description) -class ServicePermissions_GET_NotAcceptableResponseBodySchema(BaseBodySchema): +class ServicePermissions_GET_NotAcceptableResponseBodySchema(BaseResponseBodySchema): service = ServiceBodySchema() @@ -1001,7 +1037,7 @@ class ServicePermissions_GET_NotAcceptableResponseSchema(colander.MappingSchema) ServiceResource_DELETE_OkResponseSchema = Resource_DELETE_OkResponseSchema -class ServiceResources_GET_ResponseBodySchema(BaseBodySchema): +class ServiceResources_GET_ResponseBodySchema(BaseResponseBodySchema): service_name = Resource_ServiceWithChildrenResourcesContainerBodySchema() service_name.name = '{service_name}' @@ -1012,7 +1048,7 @@ class ServiceResources_GET_OkResponseSchema(colander.MappingSchema): body = ServiceResources_GET_ResponseBodySchema(code=HTTPOk.code, description=description) -class ServiceResourceTypes_GET_ResponseBodySchema(BaseBodySchema): +class ServiceResourceTypes_GET_ResponseBodySchema(BaseResponseBodySchema): resource_types = ResourceTypesListSchema() @@ -1022,7 +1058,7 @@ class ServiceResourceTypes_GET_OkResponseSchema(colander.MappingSchema): body = ServiceResourceTypes_GET_ResponseBodySchema(code=HTTPOk.code, description=description) -class ServiceResourceTypes_GET_FailureBodyResponseSchema(BaseBodySchema): +class ServiceResourceTypes_GET_FailureBodyResponseSchema(BaseResponseBodySchema): service_type = colander.SchemaNode( colander.String(), description="Service type retrieved from route path." @@ -1041,7 +1077,7 @@ class ServiceResourceTypes_GET_NotFoundResponseSchema(colander.MappingSchema): body = ServiceResourceTypes_GET_FailureBodyResponseSchema(code=HTTPNotFound.code, description=description) -class Users_GET_ResponseBodySchema(BaseBodySchema): +class Users_GET_ResponseBodySchema(BaseResponseBodySchema): user_names = UserNamesListSchema() @@ -1054,10 +1090,10 @@ class Users_GET_OkResponseSchema(colander.MappingSchema): class Users_GET_ForbiddenResponseSchema(colander.MappingSchema): description = "Get users query refused by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) -class Users_CheckInfo_ResponseBodySchema(BaseBodySchema): +class Users_CheckInfo_ResponseBodySchema(BaseResponseBodySchema): param = ErrorVerifyParamBodySchema() @@ -1101,13 +1137,13 @@ class Users_CheckInfo_Login_ConflictResponseSchema(colander.MappingSchema): class User_Check_ForbiddenResponseSchema(colander.MappingSchema): description = "User check query was refused by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class User_Check_ConflictResponseSchema(colander.MappingSchema): description = "User name matches an already existing user name." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class User_POST_RequestBodySchema(colander.MappingSchema): @@ -1138,7 +1174,7 @@ class Users_POST_RequestSchema(colander.MappingSchema): body = User_POST_RequestBodySchema() -class Users_POST_ResponseBodySchema(BaseBodySchema): +class Users_POST_ResponseBodySchema(BaseResponseBodySchema): user = UserBodySchema() @@ -1151,13 +1187,13 @@ class Users_POST_CreatedResponseSchema(colander.MappingSchema): class Users_POST_ForbiddenResponseSchema(colander.MappingSchema): description = "Failed to add user to db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class UserNew_POST_ForbiddenResponseSchema(colander.MappingSchema): description = "New user query was refused by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class User_PUT_RequestBodySchema(colander.MappingSchema): @@ -1189,7 +1225,7 @@ class User_PUT_RequestSchema(colander.MappingSchema): class Users_PUT_OkResponseSchema(colander.MappingSchema): description = "Update user successful." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPOk.code, description=description) + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) # PUT method uses same sub-function as POST method (same responses) @@ -1199,10 +1235,10 @@ class Users_PUT_OkResponseSchema(colander.MappingSchema): class User_PUT_ConflictResponseSchema(colander.MappingSchema): description = "New name user already exists." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPConflict.code, description=description) + body = BaseResponseBodySchema(code=HTTPConflict.code, description=description) -class User_GET_ResponseBodySchema(BaseBodySchema): +class User_GET_ResponseBodySchema(BaseResponseBodySchema): user = UserBodySchema() @@ -1215,25 +1251,25 @@ class User_GET_OkResponseSchema(colander.MappingSchema): class User_CheckAnonymous_ForbiddenResponseSchema(colander.MappingSchema): description = "Anonymous user query refused by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class User_CheckAnonymous_NotFoundResponseSchema(colander.MappingSchema): description = "Anonymous user not found in db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotFound.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) class User_GET_ForbiddenResponseSchema(colander.MappingSchema): description = "User name query refused by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class User_GET_NotFoundResponseSchema(colander.MappingSchema): description = "User name not found in db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotFound.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) class User_DELETE_RequestSchema(colander.MappingSchema): @@ -1244,34 +1280,34 @@ class User_DELETE_RequestSchema(colander.MappingSchema): class User_DELETE_OkResponseSchema(colander.MappingSchema): description = "Delete user successful." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class User_DELETE_ForbiddenResponseSchema(colander.MappingSchema): description = "Delete user by name refused by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class UserGroup_GET_ForbiddenResponseSchema(colander.MappingSchema): description = "Group query was refused by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class UserGroup_GET_NotAcceptableResponseSchema(colander.MappingSchema): description = "Group for new user doesn't exist." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotAcceptable.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description) class UserGroup_Check_ForbiddenResponseSchema(colander.MappingSchema): description = "Failed to add user-group to db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) -class UserGroups_GET_ResponseBodySchema(BaseBodySchema): +class UserGroups_GET_ResponseBodySchema(BaseResponseBodySchema): group_names = GroupNamesListSchema() @@ -1299,7 +1335,7 @@ class UserGroups_POST_RequestSchema(colander.MappingSchema): body = UserGroups_POST_RequestBodySchema() -class UserGroups_POST_ResponseBodySchema(BaseBodySchema): +class UserGroups_POST_ResponseBodySchema(BaseResponseBodySchema): user_name = colander.SchemaNode( colander.String(), description="Name of the user in the user-group relationship", @@ -1321,19 +1357,19 @@ class UserGroups_POST_CreatedResponseSchema(colander.MappingSchema): class UserGroups_POST_GroupNotFoundResponseSchema(colander.MappingSchema): description = "Can't find the group to assign to." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotFound.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) class UserGroups_POST_ForbiddenResponseSchema(colander.MappingSchema): description = "Group query by name refused by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class UserGroups_POST_ConflictResponseSchema(colander.MappingSchema): description = "User already belongs to this group." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPConflict.code, description=description) + body = BaseResponseBodySchema(code=HTTPConflict.code, description=description) class UserGroup_DELETE_RequestSchema(colander.MappingSchema): @@ -1344,16 +1380,16 @@ class UserGroup_DELETE_RequestSchema(colander.MappingSchema): class UserGroup_DELETE_OkResponseSchema(colander.MappingSchema): description = "Delete user-group successful." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPOk.code, description=description) + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) class UserGroup_DELETE_NotFoundResponseSchema(colander.MappingSchema): description = "Invalid user-group combination for delete." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotFound.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) -class UserResources_GET_ResponseBodySchema(BaseBodySchema): +class UserResources_GET_ResponseBodySchema(BaseResponseBodySchema): resources = ResourcesSchemaNode() @@ -1363,7 +1399,7 @@ class UserResources_GET_OkResponseSchema(colander.MappingSchema): body = UserResources_GET_ResponseBodySchema(code=HTTPOk.code, description=description) -class UserResources_GET_NotFoundResponseBodySchema(BaseBodySchema): +class UserResources_GET_NotFoundResponseBodySchema(BaseResponseBodySchema): user_name = colander.SchemaNode(colander.String(), description="User name value read from path") resource_types = ResourceTypesListSchema(description="Resource types searched for") @@ -1374,7 +1410,7 @@ class UserResources_GET_NotFoundResponseSchema(colander.MappingSchema): body = UserResources_GET_NotFoundResponseBodySchema(code=HTTPNotFound.code, description=description) -class UserResourcePermissions_GET_ResponseBodySchema(BaseBodySchema): +class UserResourcePermissions_GET_ResponseBodySchema(BaseResponseBodySchema): permission_names = PermissionListSchema() @@ -1419,7 +1455,7 @@ class UserResourcePermissions_GET_NotAcceptableResourceTypeResponseSchema(coland class UserResourcePermissions_GET_NotFoundResponseSchema(colander.MappingSchema): description = "Specified user not found to obtain resource permissions." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotFound.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) class UserResourcePermissions_POST_RequestBodySchema(colander.MappingSchema): @@ -1439,7 +1475,7 @@ class UserResourcePermissions_POST_RequestSchema(colander.MappingSchema): body = UserResourcePermissions_POST_RequestBodySchema() -class UserResourcePermissions_POST_ResponseBodySchema(BaseBodySchema): +class UserResourcePermissions_POST_ResponseBodySchema(BaseResponseBodySchema): resource_id = colander.SchemaNode( colander.Integer(), description="resource_id of the created user-resource-permission reference.") @@ -1456,7 +1492,7 @@ class UserResourcePermissions_POST_ParamResponseBodySchema(colander.MappingSchem value = colander.SchemaNode(colander.String(), description="Specified parameter value.") -class UserResourcePermissions_POST_BadResponseBodySchema(BaseBodySchema): +class UserResourcePermissions_POST_BadResponseBodySchema(BaseResponseBodySchema): resource_type = colander.SchemaNode(colander.String(), description="Specified resource_type.") resource_name = colander.SchemaNode(colander.String(), description="Specified resource_name.") param = UserResourcePermissions_POST_ParamResponseBodySchema() @@ -1498,7 +1534,7 @@ class UserResourcePermission_DELETE_RequestSchema(colander.MappingSchema): class UserResourcePermissions_DELETE_OkResponseSchema(colander.MappingSchema): description = "Delete user resource permission successful." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPOk.code, description=description) + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) class UserResourcePermissions_DELETE_NotFoundResponseSchema(colander.MappingSchema): @@ -1507,7 +1543,7 @@ class UserResourcePermissions_DELETE_NotFoundResponseSchema(colander.MappingSche body = UserResourcePermissions_DELETE_BadResponseBodySchema(code=HTTPOk.code, description=description) -class UserServiceResources_GET_ResponseBodySchema(BaseBodySchema): +class UserServiceResources_GET_ResponseBodySchema(BaseResponseBodySchema): service = ServiceResourcesBodySchema() @@ -1517,6 +1553,15 @@ class UserServiceResources_GET_OkResponseSchema(colander.MappingSchema): body = UserServiceResources_GET_ResponseBodySchema(code=HTTPOk.code, description=description) +class UserServiceResources_GET_QuerySchema(colander.MappingSchema): + inherit = QueryInheritGroupsPermissions + + +class UserServiceResources_GET_RequestSchema(colander.MappingSchema): + header = HeaderRequestSchema() + querystring = UserServiceResources_GET_QuerySchema() + + class UserServicePermissions_POST_RequestBodySchema(colander.MappingSchema): permission_name = colander.SchemaNode(colander.String(), description="Name of the permission to create.") @@ -1531,7 +1576,20 @@ class UserServicePermission_DELETE_RequestSchema(colander.MappingSchema): body = colander.MappingSchema(default={}) -class UserServices_GET_ResponseBodySchema(BaseBodySchema): +class UserServices_GET_QuerySchema(colander.MappingSchema): + cascade = QueryCascadeResourcesPermissions + inherit = QueryInheritGroupsPermissions + list = colander.SchemaNode( + colander.Boolean(), default=False, missing=colander.drop, + description='Return services as a list of dicts. Default is a dict by service type, and by service name.') + + +class UserServices_GET_RequestSchema(colander.MappingSchema): + header = HeaderRequestSchema() + querystring = UserServices_GET_QuerySchema() + + +class UserServices_GET_ResponseBodySchema(BaseResponseBodySchema): services = ServicesSchemaNode() @@ -1541,7 +1599,16 @@ class UserServices_GET_OkResponseSchema(colander.MappingSchema): body = UserServices_GET_ResponseBodySchema -class UserServicePermissions_GET_ResponseBodySchema(BaseBodySchema): +class UserServicePermissions_GET_QuerySchema(colander.MappingSchema): + inherit = QueryInheritGroupsPermissions + + +class UserServicePermissions_GET_RequestSchema(colander.MappingSchema): + header = HeaderRequestSchema() + querystring = UserServicePermissions_GET_QuerySchema() + + +class UserServicePermissions_GET_ResponseBodySchema(BaseResponseBodySchema): permission_names = PermissionListSchema() @@ -1554,34 +1621,34 @@ class UserServicePermissions_GET_OkResponseSchema(colander.MappingSchema): class UserServicePermissions_GET_NotFoundResponseSchema(colander.MappingSchema): description = "Could not find permissions using specified `service_name` and `user_name`." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotFound.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) class Group_MatchDictCheck_ForbiddenResponseSchema(colander.MappingSchema): description = "Group query by name refused by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class Group_MatchDictCheck_NotFoundResponseSchema(colander.MappingSchema): description = "Group name not found in db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotFound.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) class Groups_CheckInfo_NotFoundResponseSchema(colander.MappingSchema): description = "User name not found in db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotFound.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) class Groups_CheckInfo_ForbiddenResponseSchema(colander.MappingSchema): description = "Failed to obtain groups of user." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) -class Groups_GET_ResponseBodySchema(BaseBodySchema): +class Groups_GET_ResponseBodySchema(BaseResponseBodySchema): group_names = GroupNamesListSchema() @@ -1594,14 +1661,14 @@ class Groups_GET_OkResponseSchema(colander.MappingSchema): class Groups_GET_ForbiddenResponseSchema(colander.MappingSchema): description = "Obtain group names refused by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) class Groups_POST_RequestSchema(colander.MappingSchema): group_name = colander.SchemaNode(colander.String(), description="Name of the group to create.") -class Groups_POST_ResponseBodySchema(BaseBodySchema): +class Groups_POST_ResponseBodySchema(BaseResponseBodySchema): group = GroupBodySchema() @@ -1626,10 +1693,10 @@ class Groups_POST_ForbiddenAddResponseSchema(colander.MappingSchema): class Groups_POST_ConflictResponseSchema(colander.MappingSchema): description = "Group name matches an already existing group name." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPConflict.code, description=description) + body = BaseResponseBodySchema(code=HTTPConflict.code, description=description) -class Group_GET_ResponseBodySchema(BaseBodySchema): +class Group_GET_ResponseBodySchema(BaseResponseBodySchema): group = GroupBodySchema() @@ -1642,7 +1709,7 @@ class Group_GET_OkResponseSchema(colander.MappingSchema): class Group_GET_NotFoundResponseSchema(colander.MappingSchema): description = "Group name was not found." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotFound.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) class Group_PUT_RequestSchema(colander.MappingSchema): @@ -1652,32 +1719,32 @@ class Group_PUT_RequestSchema(colander.MappingSchema): class Group_PUT_OkResponseSchema(colander.MappingSchema): description = "Update group successful." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPOk.code, description=description) + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) class Group_PUT_Name_NotAcceptableResponseSchema(colander.MappingSchema): description = "Invalid `group_name` value specified." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotAcceptable.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description) class Group_PUT_Size_NotAcceptableResponseSchema(colander.MappingSchema): description = "Invalid `group_name` length specified (>{length} characters)." \ .format(length=MAGPIE_USER_NAME_MAX_LENGTH) header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotAcceptable.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description) class Group_PUT_Same_NotAcceptableResponseSchema(colander.MappingSchema): description = "Invalid `group_name` must be different than current name." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPNotAcceptable.code, description=description) + body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description) class Group_PUT_ConflictResponseSchema(colander.MappingSchema): description = "Group name already exists." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPConflict.code, description=description) + body = BaseResponseBodySchema(code=HTTPConflict.code, description=description) class Group_DELETE_RequestSchema(colander.MappingSchema): @@ -1688,28 +1755,28 @@ class Group_DELETE_RequestSchema(colander.MappingSchema): class Group_DELETE_OkResponseSchema(colander.MappingSchema): description = "Delete group successful." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPOk.code, description=description) + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) class Group_DELETE_ForbiddenResponseSchema(colander.MappingSchema): description = "Delete group forbidden by db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPOk.code, description=description) + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) class GroupUsers_GET_OkResponseSchema(colander.MappingSchema): description = "Get group users successful." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPOk.code, description=description) + body = BaseResponseBodySchema(code=HTTPOk.code, description=description) class GroupUsers_GET_ForbiddenResponseSchema(colander.MappingSchema): description = "Failed to obtain group user names from db." header = HeaderResponseSchema() - body = BaseBodySchema(code=HTTPForbidden.code, description=description) + body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) -class GroupServices_GET_ResponseBodySchema(BaseBodySchema): +class GroupServices_GET_ResponseBodySchema(BaseResponseBodySchema): services = ServicesSchemaNode() @@ -1730,7 +1797,7 @@ class GroupServices_InternalServerErrorResponseSchema(colander.MappingSchema): code=HTTPInternalServerError.code, description=description) -class GroupServicePermissions_GET_ResponseBodySchema(BaseBodySchema): +class GroupServicePermissions_GET_ResponseBodySchema(BaseResponseBodySchema): permission_names = PermissionListSchema() @@ -1759,7 +1826,7 @@ class GroupServicePermissions_POST_RequestSchema(colander.MappingSchema): GroupResourcePermissions_POST_RequestSchema = GroupServicePermissions_POST_RequestSchema -class GroupResourcePermissions_POST_ResponseBodySchema(BaseBodySchema): +class GroupResourcePermissions_POST_ResponseBodySchema(BaseResponseBodySchema): permission_name = colander.SchemaNode(colander.String(), description="Name of the permission requested.") resource = ResourceBodySchema() group = GroupBodySchema() @@ -1825,7 +1892,7 @@ class GroupResourcePermissions_InternalServerErrorResponseSchema(colander.Mappin code=HTTPInternalServerError.code, description=description) -class GroupResources_GET_ResponseBodySchema(BaseBodySchema): +class GroupResources_GET_ResponseBodySchema(BaseResponseBodySchema): resources = ResourcesSchemaNode() @@ -1846,7 +1913,7 @@ class GroupResources_GET_InternalServerErrorResponseSchema(colander.MappingSchem code=HTTPInternalServerError.code, description=description) -class GroupResourcePermissions_GET_ResponseBodySchema(BaseBodySchema): +class GroupResourcePermissions_GET_ResponseBodySchema(BaseResponseBodySchema): permissions_names = PermissionListSchema() @@ -1856,7 +1923,7 @@ class GroupResourcePermissions_GET_OkResponseSchema(colander.MappingSchema): body = GroupResourcePermissions_GET_ResponseBodySchema(code=HTTPOk.code, description=description) -class GroupServiceResources_GET_ResponseBodySchema(BaseBodySchema): +class GroupServiceResources_GET_ResponseBodySchema(BaseResponseBodySchema): service = ServiceResourcesBodySchema() @@ -1870,7 +1937,7 @@ class GroupServicePermission_DELETE_RequestSchema(colander.MappingSchema): permission_name = colander.SchemaNode(colander.String(), description="Name of the permission to delete.") -class GroupServicePermission_DELETE_ResponseBodySchema(BaseBodySchema): +class GroupServicePermission_DELETE_ResponseBodySchema(BaseResponseBodySchema): permission_name = colander.SchemaNode(colander.String(), description="Name of the permission requested.") resource = ResourceBodySchema() group = GroupBodySchema() @@ -1900,7 +1967,7 @@ class GroupServicePermission_DELETE_NotFoundResponseSchema(colander.MappingSchem body = GroupServicePermission_DELETE_ResponseBodySchema(code=HTTPNotFound.code, description=description) -class Session_GET_ResponseBodySchema(BaseBodySchema): +class Session_GET_ResponseBodySchema(BaseResponseBodySchema): user = UserBodySchema(missing=colander.drop) authenticated = colander.SchemaNode( colander.Boolean(), @@ -1919,19 +1986,19 @@ class Session_GET_InternalServerErrorResponseSchema(colander.MappingSchema): body = InternalServerErrorResponseSchema() -class Providers_GET_ResponseBodySchema(BaseBodySchema): +class Providers_GET_ResponseBodySchema(BaseResponseBodySchema): provider_names = ProvidersListSchema() internal_providers = ProvidersListSchema() external_providers = ProvidersListSchema() -class Providers_GET_OkResponseSchema(BaseBodySchema): +class Providers_GET_OkResponseSchema(BaseResponseBodySchema): description = "Get providers successful." header = HeaderResponseSchema() body = Providers_GET_ResponseBodySchema(code=HTTPOk.code, description=description) -class Version_GET_ResponseBodySchema(BaseBodySchema): +class Version_GET_ResponseBodySchema(BaseResponseBodySchema): version = colander.SchemaNode( colander.String(), description="Magpie version string", diff --git a/magpie/api/management/resource/resource_formats.py b/magpie/api/management/resource/resource_formats.py index 1aec3aa36..013cabeae 100644 --- a/magpie/api/management/resource/resource_formats.py +++ b/magpie/api/management/resource/resource_formats.py @@ -6,21 +6,25 @@ def format_resource(resource, permissions=None, basic_info=False): def fmt_res(res, perms, info): - if info: - return { - u'resource_name': str(res.resource_name), - u'resource_type': str(res.resource_type), - u'resource_id': res.resource_id - } - return { - u'resource_name': str(res.resource_name), + resource_name = str(res.resource_name) + resource_display_name = resource_name + if res.resource_display_name: + resource_display_name = res.resource_display_name.encode('utf-8') + + result = { + u'resource_name': resource_name, + u'resource_display_name': resource_display_name, u'resource_type': str(res.resource_type), - u'resource_id': res.resource_id, - u'parent_id': res.parent_id, - u'root_service_id': res.root_service_id, - u'children': {}, - u'permission_names': list() if perms is None else sorted(perms) + u'resource_id': res.resource_id } + if not info: + result.update({ + u'parent_id': res.parent_id, + u'root_service_id': res.root_service_id, + u'children': {}, + u'permission_names': list() if perms is None else sorted(perms) + }) + return result return evaluate_call( lambda: fmt_res(resource, permissions, basic_info), diff --git a/magpie/api/management/resource/resource_utils.py b/magpie/api/management/resource/resource_utils.py index 70919a1b0..7486341a8 100644 --- a/magpie/api/management/resource/resource_utils.py +++ b/magpie/api/management/resource/resource_utils.py @@ -125,7 +125,7 @@ def get_resource_root_service(resource, db_session): return None -def create_resource(resource_name, resource_type, parent_id, db_session): +def create_resource(resource_name, resource_display_name, resource_type, parent_id, db_session): verify_param(resource_name, paramName=u'resource_name', notNone=True, notEmpty=True, httpError=HTTPBadRequest, msgOnFail="Invalid `resource_name` specified for child resource creation.") verify_param(resource_type, paramName=u'resource_type', notNone=True, notEmpty=True, httpError=HTTPBadRequest, @@ -142,6 +142,7 @@ def create_resource(resource_name, resource_type, parent_id, db_session): root_service = check_valid_service_resource(parent_resource, resource_type, db_session) new_resource = resource_factory(resource_type=resource_type, resource_name=resource_name, + resource_display_name=resource_display_name or resource_name, root_service_id=root_service.resource_id, parent_id=parent_resource.resource_id) diff --git a/magpie/api/management/resource/resource_views.py b/magpie/api/management/resource/resource_views.py index 4238c51aa..23f8dd3ac 100644 --- a/magpie/api/management/resource/resource_views.py +++ b/magpie/api/management/resource/resource_views.py @@ -42,9 +42,10 @@ def get_resource_view(request): def create_resource_view(request): """Register a new resource.""" resource_name = get_value_multiformat_post_checked(request, 'resource_name') + resource_display_name = get_multiformat_any(request, 'resource_display_name', default=resource_name) resource_type = get_value_multiformat_post_checked(request, 'resource_type') parent_id = get_value_multiformat_post_checked(request, 'parent_id') - return create_resource(resource_name, resource_type, parent_id, request.db) + return create_resource(resource_name, resource_display_name, resource_type, parent_id, request.db) @ResourceAPI.delete(schema=Resource_DELETE_RequestSchema(), tags=[ResourcesTag], diff --git a/magpie/api/management/service/service_formats.py b/magpie/api/management/service/service_formats.py index 2a0b3d350..6b0beee15 100644 --- a/magpie/api/management/service/service_formats.py +++ b/magpie/api/management/service/service_formats.py @@ -13,6 +13,7 @@ def fmt_svc(svc, perms): u'service_url': str(svc.url), u'service_name': str(svc.resource_name), u'service_type': str(svc.type), + u'service_sync_type': svc.sync_type, u'resource_id': svc.resource_id, u'permission_names': sorted(service_type_dict[svc.type].permission_names if perms is None else perms) } diff --git a/magpie/api/management/service/service_views.py b/magpie/api/management/service/service_views.py index ac6999447..d110f57aa 100644 --- a/magpie/api/management/service/service_views.py +++ b/magpie/api/management/service/service_views.py @@ -195,11 +195,13 @@ def create_service_direct_resource(request): """Register a new resource directly under a service.""" service = get_service_matchdict_checked(request) resource_name = get_multiformat_post(request, 'resource_name') + resource_display_name = get_multiformat_post(request, 'resource_display_name', default=resource_name) resource_type = get_multiformat_post(request, 'resource_type') parent_id = get_multiformat_post(request, 'parent_id') # no check because None/empty is allowed if not parent_id: parent_id = service.resource_id - return create_resource(resource_name, resource_type, parent_id=parent_id, db_session=request.db) + return create_resource(resource_name, resource_display_name, resource_type, parent_id=parent_id, + db_session=request.db) @ServiceResourceTypesAPI.get(tags=[ServicesTag], response_schemas=ServiceResource_GET_responses) diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py index 3250fc845..8caeebd72 100644 --- a/magpie/api/management/user/user_utils.py +++ b/magpie/api/management/user/user_utils.py @@ -1,5 +1,6 @@ from magpie.api.api_except import * from magpie.api.api_rest_schemas import * +from magpie.api.management.service.service_formats import format_service from magpie.api.management.resource.resource_utils import check_valid_service_resource_permission from magpie.api.management.user.user_formats import * from magpie.definitions.ziggurat_definitions import * @@ -71,35 +72,92 @@ def filter_user_permission(resource_permission_tuple_list, user): resource_permission_tuple_list) -def get_user_resource_permissions(user, resource, db_session, inherited_permissions=True): +def get_user_resource_permissions(user, resource, db_session, inherit_groups_permissions=True): if resource.owner_user_id == user.id: permission_names = models.resource_type_dict[resource.type].permission_names else: res_perm_tuple_list = resource.perms_for_user(user, db_session=db_session) - if not inherited_permissions: + if not inherit_groups_permissions: res_perm_tuple_list = filter_user_permission(res_perm_tuple_list, user) permission_names = [permission.perm_name for permission in res_perm_tuple_list] - return list(set(permission_names)) # remove any duplicates that could be incorporated by multiple groups - - -def get_user_service_permissions(user, service, db_session, inherited_permissions=True): + return sorted(set(permission_names)) # remove any duplicates that could be incorporated by multiple groups + + +def get_user_services(user, db_session, cascade_resources=False, + inherit_groups_permissions=False, format_as_list=False): + """ + Returns services by type with corresponding services by name containing sub-dict information. + + :param user: user for which to find services + :param db_session: database session connection + :param cascade_resources: + If `False`, return only services with *Direct* user permissions on their corresponding service-resource. + Otherwise, return every service that has at least one sub-resource with user permissions. + :param inherit_groups_permissions: + If `False`, return only user-specific service/sub-resources permissions. + Otherwise, resolve inherited permissions using all groups the user is member of. + :param format_as_list: + returns as list of service dict information (not grouped by type and by name) + :return: only services which the user as *Direct* or *Inherited* permissions, according to `inherit_from_resources` + :rtype: + dict of services by type with corresponding services by name containing sub-dict information, + unless `format_as_dict` is `True` + """ + resource_type = None if cascade_resources else ['service'] + res_perm_dict = get_user_resources_permissions_dict(user, resource_types=resource_type, db_session=db_session, + inherit_groups_permissions=inherit_groups_permissions) + + services = {} + for resource_id, perms in res_perm_dict.items(): + svc = models.Service.by_resource_id(resource_id=resource_id, db_session=db_session) + if svc.resource_type != 'service' and cascade_resources: + svc = models.Service.by_resource_id(resource_id=svc.root_service_id, db_session=db_session) + perms = service_type_dict[svc.type].permission_names + if svc.type not in services: + services[svc.type] = {} + if svc.resource_name not in services[svc.type]: + services[svc.type][svc.resource_name] = format_service(svc, perms) + + if not format_as_list: + return services + + services_list = list() + for svc_type in services: + for svc_name in services[svc_type]: + services_list.append(services[svc_type][svc_name]) + return services_list + + +def get_user_service_permissions(user, service, db_session, inherit_groups_permissions=True): if service.owner_user_id == user.id: permission_names = service_type_dict[service.type].permission_names else: svc_perm_tuple_list = service.perms_for_user(user, db_session=db_session) - if not inherited_permissions: + if not inherit_groups_permissions: svc_perm_tuple_list = filter_user_permission(svc_perm_tuple_list, user) permission_names = [permission.perm_name for permission in svc_perm_tuple_list] - return list(set(permission_names)) # remove any duplicates that could be incorporated by multiple groups + return sorted(set(permission_names)) # remove any duplicates that could be incorporated by multiple groups def get_user_resources_permissions_dict(user, db_session, resource_types=None, - resource_ids=None, inherited_permissions=True): + resource_ids=None, inherit_groups_permissions=True): + """ + Creates a dictionary of resources by id with corresponding permissions of the user. + + :param user: user for which to find services + :param db_session: database session connection + :param resource_types: (list) filter the search query with specified resource types + :param resource_ids: (list) filter the search query with specified resource ids + :param inherit_groups_permissions: + If `False`, return only user-specific resource permissions. + Otherwise, resolve inherited permissions using all groups the user is member of. + :return: only services which the user as *Direct* or *Inherited* permissions, according to `inherit_from_resources` + """ verify_param(user, notNone=True, httpError=HTTPNotFound, msgOnFail=UserResourcePermissions_GET_NotFoundResponseSchema.description) res_perm_tuple_list = user.resources_with_possible_perms(resource_ids=resource_ids, resource_types=resource_types, db_session=db_session) - if not inherited_permissions: + if not inherit_groups_permissions: res_perm_tuple_list = filter_user_permission(res_perm_tuple_list, user) resources_permissions_dict = {} for res_perm in res_perm_tuple_list: @@ -110,17 +168,17 @@ def get_user_resources_permissions_dict(user, db_session, resource_types=None, # remove any duplicates that could be incorporated by multiple groups for res_id in resources_permissions_dict: - resources_permissions_dict[res_id] = list(set(resources_permissions_dict[res_id])) + resources_permissions_dict[res_id] = sorted(set(resources_permissions_dict[res_id])) return resources_permissions_dict -def get_user_service_resources_permissions_dict(user, service, db_session, inherited_permissions=True): +def get_user_service_resources_permissions_dict(user, service, db_session, inherit_groups_permissions=True): resources_under_service = models.resource_tree_service.from_parent_deeper(parent_id=service.resource_id, db_session=db_session) resource_ids = [resource.Resource.resource_id for resource in resources_under_service] return get_user_resources_permissions_dict(user, db_session, resource_types=None, resource_ids=resource_ids, - inherited_permissions=inherited_permissions) + inherit_groups_permissions=inherit_groups_permissions) def check_user_info(user_name, email, password, group_name): diff --git a/magpie/api/management/user/user_views.py b/magpie/api/management/user/user_views.py index a992d9f64..0d9346220 100644 --- a/magpie/api/management/user/user_views.py +++ b/magpie/api/management/user/user_views.py @@ -5,7 +5,11 @@ from magpie.api.management.user.user_formats import * from magpie.api.management.user.user_utils import * from magpie.api.management.group.group_utils import * +from magpie.api.management.service.service_utils import get_services_by_type from magpie.api.management.service.service_formats import format_service, format_service_resources +from magpie.common import str2bool +import logging +LOGGER = logging.getLogger(__name__) @UsersAPI.get(tags=[UsersTag], response_schemas=Users_GET_responses) @@ -148,11 +152,11 @@ def build_json_user_resource_tree(usr): json_res = {} for svc in models.Service.all(db_session=db): svc_perms = get_user_service_permissions(user=usr, service=svc, db_session=db, - inherited_permissions=inherit_perms) + inherit_groups_permissions=inherit_perms) if svc.type not in json_res: json_res[svc.type] = {} res_perms_dict = get_user_service_resources_permissions_dict(user=usr, service=svc, db_session=db, - inherited_permissions=inherit_perms) + inherit_groups_permissions=inherit_perms) json_res[svc.type][svc.resource_name] = format_service_resources( svc, db_session=db, @@ -176,7 +180,8 @@ def build_json_user_resource_tree(usr): @view_config(route_name=UserResourcesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_user_resources_view(request): """List all resources a user has direct permission on (not including his groups permissions).""" - return get_user_resources_runner(request, inherited_group_resources_permissions=False) + inherit_groups_perms = str2bool(get_query_param(request, 'inherit')) + return get_user_resources_runner(request, inherited_group_resources_permissions=inherit_groups_perms) @UserInheritedResourcesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, @@ -186,18 +191,11 @@ def get_user_resources_view(request): @view_config(route_name=UserInheritedResourcesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_user_inherited_resources_view(request): """List all resources a user has permission on with his inherited user and groups permissions.""" + LOGGER.warn("Route deprecated: [{0}], Instead Use: [{1}]" + .format(UserInheritedResourcesAPI.path, UserResourcesAPI.path + "?inherit=true")) return get_user_resources_runner(request, inherited_group_resources_permissions=True) -def get_user_resource_permissions_runner(request, inherited_permissions=True): - user = get_user_matchdict_checked_or_logged(request) - resource = get_resource_matchdict_checked(request, 'resource_id') - perm_names = get_user_resource_permissions(resource=resource, user=user, db_session=request.db, - inherited_permissions=inherited_permissions) - return valid_http(httpSuccess=HTTPOk, detail=UserResourcePermissions_GET_OkResponseSchema.description, - content={u'permission_names': sorted(perm_names)}) - - @UserResourcePermissionsAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, response_schemas=UserResourcePermissions_GET_responses) @LoggedUserResourcePermissionsAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI, @@ -205,7 +203,13 @@ def get_user_resource_permissions_runner(request, inherited_permissions=True): @view_config(route_name=UserResourcePermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_user_resource_permissions_view(request): """List all direct permissions a user has on a specific resource (not including his groups permissions).""" - return get_user_resource_permissions_runner(request, inherited_permissions=False) + user = get_user_matchdict_checked_or_logged(request) + resource = get_resource_matchdict_checked(request, 'resource_id') + inherit_groups_perms = str2bool(get_query_param(request, 'inherit')) + perm_names = get_user_resource_permissions(resource=resource, user=user, db_session=request.db, + inherit_groups_permissions=inherit_groups_perms) + return valid_http(httpSuccess=HTTPOk, detail=UserResourcePermissions_GET_OkResponseSchema.description, + content={u'permission_names': sorted(perm_names)}) @UserResourceInheritedPermissionsAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, @@ -214,9 +218,17 @@ def get_user_resource_permissions_view(request): response_schemas=LoggedUserResourcePermissions_GET_responses) @view_config(route_name=UserResourceInheritedPermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) -def get_user_resource_inherited_permissions_view(request): +def get_user_resource_inherit_groups_permissions_view(request): """List all permissions a user has on a specific resource with his inherited user and groups permissions.""" - return get_user_resource_permissions_runner(request, inherited_permissions=True) + LOGGER.warn("Route deprecated: [{0}], Instead Use: [{1}]" + .format(UserResourceInheritedPermissionsAPI.path, UserResourcePermissionsAPI.path + "?inherit=true")) + + user = get_user_matchdict_checked_or_logged(request) + resource = get_resource_matchdict_checked(request, 'resource_id') + perm_names = get_user_resource_permissions(resource=resource, user=user, db_session=request.db, + inherit_groups_permissions=True) + return valid_http(httpSuccess=HTTPOk, detail=UserResourcePermissions_GET_OkResponseSchema.description, + content={u'permission_names': sorted(perm_names)}) @UserResourcePermissionsAPI.post(schema=UserResourcePermissions_POST_RequestSchema(), tags=[UsersTag], @@ -245,29 +257,24 @@ def delete_user_resource_permission_view(request): return delete_user_resource_permission(perm_name, resource, user.id, request.db) -def get_user_services_runner(request, inherited_group_services_permissions): - user = get_user_matchdict_checked_or_logged(request) - res_perm_dict = get_user_resources_permissions_dict(user, resource_types=['service'], db_session=request.db, - inherited_permissions=inherited_group_services_permissions) - - svc_json = {} - for resource_id, perms in res_perm_dict.items(): - svc = models.Service.by_resource_id(resource_id=resource_id, db_session=request.db) - if svc.type not in svc_json: - svc_json[svc.type] = {} - svc_json[svc.type][svc.resource_name] = format_service(svc, perms) - - return valid_http(httpSuccess=HTTPOk, detail=UserServices_GET_OkResponseSchema.description, - content={u'services': svc_json}) - - -@UserServicesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, response_schemas=UserServices_GET_responses) +@UserServicesAPI.get(tags=[UsersTag], schema=UserServices_GET_RequestSchema, + api_security=SecurityEveryoneAPI, response_schemas=UserServices_GET_responses) @LoggedUserServicesAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI, response_schemas=LoggedUserServices_GET_responses) @view_config(route_name=UserServicesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_user_services_view(request): - """List all services a user has direct permission on (not including his groups permissions).""" - return get_user_services_runner(request, inherited_group_services_permissions=False) + """List all services a user has permission on.""" + user = get_user_matchdict_checked_or_logged(request) + cascade_resources = str2bool(get_query_param(request, 'cascade')) + inherit_groups_perms = str2bool(get_query_param(request, 'inherit')) + format_as_list = str2bool(get_query_param(request, 'list')) + + svc_json = get_user_services(user, db_session=request.db, + cascade_resources=cascade_resources, + inherit_groups_permissions=inherit_groups_perms, + format_as_list=format_as_list) + return valid_http(httpSuccess=HTTPOk, detail=UserServices_GET_OkResponseSchema.description, + content={u'services': svc_json}) @UserInheritedServicesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, @@ -277,14 +284,29 @@ def get_user_services_view(request): @view_config(route_name=UserInheritedServicesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_user_inherited_services_view(request): """List all services a user has permission on with his inherited user and groups permissions.""" - return get_user_services_runner(request, inherited_group_services_permissions=True) + LOGGER.warn("Route deprecated: [{0}], Instead Use: [{1}]" + .format(LoggedUserInheritedServicesAPI.path, LoggedUserServicesAPI.path + "?inherit=true")) + user = get_user_matchdict_checked_or_logged(request) + svc_json = get_user_services(user, db_session=request.db, cascade_resources=False, inherit_groups_permissions=True) + return valid_http(httpSuccess=HTTPOk, detail=UserServices_GET_OkResponseSchema.description, + content={u'services': svc_json}) -def get_user_service_permissions_runner(request, inherited_permissions): +@UserServiceInheritedPermissionsAPI.get(schema=UserServicePermissions_GET_RequestSchema, + tags=[UsersTag], api_security=SecurityEveryoneAPI, + response_schemas=UserServicePermissions_GET_responses) +@LoggedUserServiceInheritedPermissionsAPI.get(schema=UserServicePermissions_GET_RequestSchema, + tags=[LoggedUserTag], api_security=SecurityEveryoneAPI, + response_schemas=LoggedUserServicePermissions_GET_responses) +@view_config(route_name=UserServiceInheritedPermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) +def get_user_service_inherited_permissions_view(request): + """List all permissions a user has on a service using all his inherited user and groups permissions.""" + LOGGER.warn("Route deprecated: [{0}], Instead Use: [{1}]" + .format(UserServiceInheritedPermissionsAPI.path, UserServicePermissionsAPI.path + "?inherit=true")) user = get_user_matchdict_checked_or_logged(request) service = get_service_matchdict_checked(request) perms = evaluate_call(lambda: get_user_service_permissions(service=service, user=user, db_session=request.db, - inherited_permissions=inherited_permissions), + inherit_groups_permissions=True), fallback=lambda: request.db.rollback(), httpError=HTTPNotFound, msgOnFail=UserServicePermissions_GET_NotFoundResponseSchema.description, content={u'service_name': str(service.resource_name), u'user_name': str(user.user_name)}) @@ -292,30 +314,30 @@ def get_user_service_permissions_runner(request, inherited_permissions): content={u'permission_names': sorted(perms)}) -@UserServicePermissionsAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, +@UserServicePermissionsAPI.get(schema=UserServicePermissions_GET_RequestSchema, + tags=[UsersTag], api_security=SecurityEveryoneAPI, response_schemas=UserServicePermissions_GET_responses) -@LoggedUserServicePermissionsAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI, +@LoggedUserServicePermissionsAPI.get(schema=UserServicePermissions_GET_RequestSchema, + tags=[LoggedUserTag], api_security=SecurityEveryoneAPI, response_schemas=LoggedUserServicePermissions_GET_responses) @view_config(route_name=UserServicePermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_user_service_permissions_view(request): - """List all direct permissions a user has on a service (not including his groups permissions).""" - return get_user_service_permissions_runner(request, inherited_permissions=False) - - -@UserServiceInheritedPermissionsAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, - response_schemas=UserServicePermissions_GET_responses) -@LoggedUserServiceInheritedPermissionsAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI, - response_schemas=LoggedUserServicePermissions_GET_responses) -@view_config(route_name=UserServiceInheritedPermissionsAPI.name, request_method='GET', - permission=NO_PERMISSION_REQUIRED) -def get_user_service_inherited_permissions_view(request): - """List all permissions a user has on a service using all his inherited user and groups permissions.""" - return get_user_service_permissions_runner(request, inherited_permissions=True) + """List all permissions a user has on a service.""" + user = get_user_matchdict_checked_or_logged(request) + service = get_service_matchdict_checked(request) + inherit_groups_perms = str2bool(get_query_param(request, 'inherit')) + perms = evaluate_call(lambda: get_user_service_permissions(service=service, user=user, db_session=request.db, + inherit_groups_permissions=inherit_groups_perms), + fallback=lambda: request.db.rollback(), httpError=HTTPNotFound, + msgOnFail=UserServicePermissions_GET_NotFoundResponseSchema.description, + content={u'service_name': str(service.resource_name), u'user_name': str(user.user_name)}) + return valid_http(httpSuccess=HTTPOk, detail=UserServicePermissions_GET_OkResponseSchema.description, + content={u'permission_names': sorted(perms)}) -@UserServicePermissionsAPI.post(schema=UserServicePermissions_POST_RequestBodySchema, tags=[UsersTag], +@UserServicePermissionsAPI.post(schema=UserServicePermissions_POST_RequestSchema, tags=[UsersTag], response_schemas=UserServicePermissions_POST_responses) -@LoggedUserServicePermissionsAPI.post(schema=UserServicePermissions_POST_RequestBodySchema, tags=[LoggedUserTag], +@LoggedUserServicePermissionsAPI.post(schema=UserServicePermissions_POST_RequestSchema, tags=[LoggedUserTag], response_schemas=LoggedUserServicePermissions_POST_responses) @view_config(route_name=UserServicePermissionsAPI.name, request_method='POST') def create_user_service_permission(request): @@ -332,27 +354,28 @@ def create_user_service_permission(request): response_schemas=LoggedUserServicePermission_DELETE_responses) @view_config(route_name=UserServicePermissionAPI.name, request_method='DELETE') def delete_user_service_permission(request): - """Delete a permission on a service for a user (not including his groups permissions).""" + """Delete a direct permission on a service for a user (not including his groups permissions).""" user = get_user_matchdict_checked_or_logged(request) service = get_service_matchdict_checked(request) perm_name = get_permission_multiformat_post_checked(request, service) return delete_user_resource_permission(perm_name, service, user.id, request.db) -def get_user_service_resource_permissions_runner(request, inherited_permissions): +def get_user_service_resource_permissions_runner(request, inherit_groups_permissions): """ Resource permissions a user as on a specific service :param request: - :param inherited_permissions: only direct permissions if False, else resolve permissions with user and his groups. + :param inherit_groups_permissions: + only direct permissions if False, otherwise resolve permissions with user and his groups. :return: """ user = get_user_matchdict_checked_or_logged(request) service = get_service_matchdict_checked(request) - service_perms = get_user_service_permissions(user, service, db_session=request.db, - inherited_permissions=inherited_permissions) - resources_perms_dict = get_user_service_resources_permissions_dict(user, service, db_session=request.db, - inherited_permissions=inherited_permissions) + service_perms = get_user_service_permissions( + user, service, db_session=request.db, inherit_groups_permissions=inherit_groups_permissions) + resources_perms_dict = get_user_service_resources_permissions_dict( + user, service, db_session=request.db, inherit_groups_permissions=inherit_groups_permissions) user_svc_res_json = format_service_resources( service=service, db_session=request.db, @@ -364,14 +387,17 @@ def get_user_service_resource_permissions_runner(request, inherited_permissions) content={u'service': user_svc_res_json}) -@UserServiceResourcesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, +@UserServiceResourcesAPI.get(schema=UserServiceResources_GET_RequestSchema, + tags=[UsersTag], api_security=SecurityEveryoneAPI, response_schemas=UserServiceResources_GET_responses) -@LoggedUserServiceResourcesAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI, +@LoggedUserServiceResourcesAPI.get(schema=UserServiceResources_GET_RequestSchema, + tags=[LoggedUserTag], api_security=SecurityEveryoneAPI, response_schemas=LoggedUserServiceResources_GET_responses) @view_config(route_name=UserServiceResourcesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_user_service_resources_view(request): - """List all resources under a service a user has direct permission on (not including his groups permissions).""" - return get_user_service_resource_permissions_runner(request, inherited_permissions=False) + """List all resources under a service a user has permission on.""" + inherit_groups_perms = str2bool(get_query_param(request, 'inherit')) + return get_user_service_resource_permissions_runner(request, inherit_groups_permissions=inherit_groups_perms) @UserServiceInheritedResourcesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, @@ -382,4 +408,6 @@ def get_user_service_resources_view(request): def get_user_service_inherited_resources_view(request): """List all resources under a service a user has permission on using all his inherited user and groups permissions.""" - return get_user_service_resource_permissions_runner(request, inherited_permissions=True) + LOGGER.warn("Route deprecated: [{0}], Instead Use: [{1}]" + .format(UserServiceInheritedResourcesAPI.path, UserServiceResourcesAPI.path + "?inherit=true")) + return get_user_service_resource_permissions_runner(request, inherit_groups_permissions=True) diff --git a/magpie/constants.py b/magpie/constants.py index dc15a59ba..f495210bc 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -53,6 +53,7 @@ MAGPIE_ANONYMOUS_EMAIL = '{}@mail.com'.format(MAGPIE_ANONYMOUS_USER) MAGPIE_ANONYMOUS_GROUP = MAGPIE_ANONYMOUS_USER MAGPIE_USERS_GROUP = os.getenv('MAGPIE_USERS_GROUP', 'users') +MAGPIE_CRON_LOG = os.getenv('MAGPIE_CRON_LOG', '~/magpie-cron.log') PHOENIX_USER = os.getenv('PHOENIX_USER', 'phoenix') PHOENIX_PASSWORD = os.getenv('PHOENIX_PASSWORD', 'qwerty') PHOENIX_PORT = int(os.getenv('PHOENIX_PORT', 8443)) diff --git a/magpie/definitions/pyramid_definitions.py b/magpie/definitions/pyramid_definitions.py index 3871c1085..1ce3cffc2 100644 --- a/magpie/definitions/pyramid_definitions.py +++ b/magpie/definitions/pyramid_definitions.py @@ -16,6 +16,7 @@ HTTPUnprocessableEntity, HTTPInternalServerError, ) +from pyramid.settings import asbool from pyramid.interfaces import IAuthenticationPolicy, IAuthorizationPolicy from pyramid.response import Response, FileResponse from pyramid.view import ( diff --git a/magpie/definitions/twitcher_definitions.py b/magpie/definitions/twitcher_definitions.py index d532808cc..3f1bf0024 100644 --- a/magpie/definitions/twitcher_definitions.py +++ b/magpie/definitions/twitcher_definitions.py @@ -3,7 +3,7 @@ from twitcher.owsproxy import owsproxy from twitcher.owssecurity import OWSSecurityInterface from twitcher.owsexceptions import OWSAccessForbidden -from twitcher.utils import parse_service_name +from twitcher.utils import parse_service_name, get_twitcher_url from twitcher.esgf import fetch_certificate, ESGF_CREDENTIALS from twitcher.datatype import Service from twitcher.store.base import ServiceStore diff --git a/magpie/definitions/ziggurat_definitions.py b/magpie/definitions/ziggurat_definitions.py index a6e6b0dd4..5936178b4 100644 --- a/magpie/definitions/ziggurat_definitions.py +++ b/magpie/definitions/ziggurat_definitions.py @@ -1,7 +1,7 @@ # import required definitions from ziggurat_foundations import ziggurat_model_init from ziggurat_foundations.models import groupfinder -from ziggurat_foundations.models.base import get_db_session +from ziggurat_foundations.models.base import get_db_session, BaseModel from ziggurat_foundations.models.external_identity import ExternalIdentityMixin from ziggurat_foundations.models.group import GroupMixin from ziggurat_foundations.models.group_permission import GroupPermissionMixin diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py new file mode 100644 index 000000000..108a30dce --- /dev/null +++ b/magpie/helpers/sync_resources.py @@ -0,0 +1,376 @@ +""" +Sychronize local and remote resources. + +To implement a new service, see the _SyncServiceInterface class. +""" + +import copy +import datetime +from collections import OrderedDict, defaultdict +import logging +import os + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +from magpie import db, models, constants +from magpie.helpers import sync_services + +LOGGER = logging.getLogger(__name__) + +CRON_SERVICE = False + +OUT_OF_SYNC = datetime.timedelta(hours=3) + +SYNC_SERVICES_TYPES = defaultdict(lambda: sync_services._SyncServiceDefault) +# noinspection PyTypeChecker +SYNC_SERVICES_TYPES.update({ + "thredds": sync_services._SyncServiceThreads, + "geoserver-api": sync_services._SyncServiceGeoserver, + "project-api": sync_services._SyncServiceProjectAPI, +}) + +# try to instantiate classes right away +for sync_service_class in SYNC_SERVICES_TYPES.values(): + name, url = "", "" + sync_service_class(name, url) + + +def merge_local_and_remote_resources(resources_local, service_sync_type, service_id, session): + """Main function to sync resources with remote server""" + if not get_last_sync(service_id, session): + return resources_local + remote_resources = _query_remote_resources_in_database(service_id, session=session) + max_depth = _get_max_depth(service_sync_type) + merged_resources = _merge_resources(resources_local, remote_resources, max_depth) + _sort_resources(merged_resources) + return merged_resources + + +def _merge_resources(resources_local, resources_remote, max_depth=None): + """ + Merge resources_local and resources_remote, adding the following keys to the output: + + - remote_id: id of the RemoteResource + - matches_remote: True or False depending if the resource is present on the remote server + + returns a dictionary of the form validated by 'sync_services.is_valid_resource_schema' + + """ + if not resources_remote: + return resources_local + + assert sync_services.is_valid_resource_schema(resources_local) + assert sync_services.is_valid_resource_schema(resources_remote) + + if not resources_local: + raise ValueError("The resources must contain at least the service name.") + + # don't overwrite the input arguments + merged_resources = copy.deepcopy(resources_local) + + def recurse(_resources_local, _resources_remote, depth=0): + # loop local resources, looking for matches in remote resources + for resource_name_local, values in _resources_local.items(): + matches_remote = resource_name_local in _resources_remote + remote_id = _resources_remote.get(resource_name_local, {}).get('remote_id', '') + deeper_than_fetched = depth >= max_depth if max_depth is not None else False + + values['remote_id'] = remote_id + values['matches_remote'] = matches_remote or deeper_than_fetched + + resource_remote_children = _resources_remote[resource_name_local]['children'] if matches_remote else {} + recurse(values['children'], resource_remote_children, depth + 1) + + # loop remote resources, looking for matches in local resources + for resource_name_remote, values in _resources_remote.items(): + if resource_name_remote not in _resources_local: + new_resource = {'permission_names': [], + 'children': {}, + 'id': None, + 'remote_id': values['remote_id'], + 'resource_display_name': values.get('resource_display_name', resource_name_remote), + 'matches_remote': True} + _resources_local[resource_name_remote] = new_resource + recurse(new_resource['children'], values['children'], depth + 1) + + recurse(merged_resources, resources_remote) + + assert sync_services.is_valid_resource_schema(merged_resources) + + return merged_resources + + +def _sort_resources(resources): + """ + Sorts a resource dictionary of the type validated by 'sync_services.is_valid_resource_schema' + by using an OrderedDict + :return: None + """ + for resource_name, values in resources.items(): + values['children'] = OrderedDict(sorted(values['children'].items())) + return _sort_resources(values['children']) + + +def _ensure_sync_info_exists(service_resource_id, session): + """ + Make sure the RemoteResourcesSyncInfo entry exists in the database. + :param service_resource_id: + :param session: + """ + service_sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_resource_id, session) + if not service_sync_info: + sync_info = models.RemoteResourcesSyncInfo(service_id=service_resource_id) + session.add(sync_info) + session.flush() + _create_main_resource(service_resource_id, session) + + +def _get_remote_resources(service): + """ + Request remote resources, depending on service type. + :param service: (models.Service) + :return: + """ + service_url = service.url + if service_url.endswith("/"): # remove trailing slash + service_url = service_url[:-1] + + sync_service_class = SYNC_SERVICES_TYPES.get(service.sync_type.lower(), sync_services._SyncServiceDefault) + sync_service = sync_service_class(service.resource_name, service_url) + return sync_service.get_resources() + + +def _delete_records(service_id, session): + """ + Delete all RemoteResource based on a Service.resource_id + :param service_id: + :param session: + """ + session.query(models.RemoteResource).filter_by(service_id=service_id).delete() + session.flush() + + +def _create_main_resource(service_id, session): + """ + Creates a main resource for a service, whether one currently exists or not. + + Each RemoteResourcesSyncInfo has a main RemoteResource of the same name as the service. + This is similar to the Service and Resource relationship. + :param service_id: + :param session: + """ + sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session) + main_resource = models.RemoteResource(service_id=service_id, + resource_name=unicode(sync_info.service.resource_name), + resource_type=u"directory") + session.add(main_resource) + session.flush() + sync_info.remote_resource_id = main_resource.resource_id + session.flush() + + +def _update_db(remote_resources, service_id, session): + """ + Writes remote resources to database. + :param remote_resources: + :param service_id: + :param session: + """ + sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session) + + def add_children(resources, parent_id, position=0): + for resource_name, values in resources.items(): + resource_display_name = unicode(values.get('resource_display_name', resource_name)) + new_resource = models.RemoteResource(service_id=sync_info.service_id, + resource_name=unicode(resource_name), + resource_display_name=resource_display_name, + resource_type=values['resource_type'], + parent_id=parent_id, + ordering=position) + session.add(new_resource) + session.flush() + position += 1 + add_children(values['children'], new_resource.resource_id) + + first_item = list(remote_resources)[0] + add_children(remote_resources[first_item]['children'], sync_info.remote_resource_id) + + sync_info.last_sync = datetime.datetime.now() + + session.flush() + + +def _get_resource_children(resource, db_session): + """ + Mostly copied from ziggurat_foundations to use RemoteResource instead of Resource + :param resource: + :param db_session: + :return: + """ + query = models.remote_resource_tree_service.from_parent_deeper(resource.resource_id, db_session=db_session) + + def build_subtree_strut(result): + """ + Returns a dictionary in form of + {node:Resource, children:{node_id: RemoteResource}} + """ + items = list(result) + root_elem = {'node': None, 'children': OrderedDict()} + if len(items) == 0: + return root_elem + for i, node in enumerate(items): + new_elem = {'node': node.RemoteResource, 'children': OrderedDict()} + path = list(map(int, node.path.split('/'))) + parent_node = root_elem + normalized_path = path[:-1] + if normalized_path: + for path_part in normalized_path: + parent_node = parent_node['children'][path_part] + parent_node['children'][new_elem['node'].resource_id] = new_elem + return root_elem + + return build_subtree_strut(query)['children'] + + +def _format_resource_tree(children): + fmt_res_tree = {} + for child_id, child_dict in children.items(): + resource = child_dict[u'node'] + new_children = child_dict[u'children'] + resource_display_name = resource.resource_display_name or resource.resource_name + resource_dict = {'children': _format_resource_tree(new_children), + 'remote_id': resource.resource_id, + 'resource_display_name': resource_display_name} + fmt_res_tree[resource.resource_name] = resource_dict + return fmt_res_tree + + +def _get_max_depth(service_type): + name, url = "", "" + return SYNC_SERVICES_TYPES[service_type](name, url).max_depth + + +def _query_remote_resources_in_database(service_id, session): + """ + Reads remote resources from the RemoteResources table. No external request is made. + :param service_type: + :param session: + :return: a dictionary of the form defined in 'sync_services.is_valid_resource_schema' + """ + service = session.query(models.Service).filter_by(resource_id=service_id).first() + _ensure_sync_info_exists(service_id, session) + + sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session) + main_resource = session.query(models.RemoteResource).filter_by( + resource_id=sync_info.remote_resource_id).first() + tree = _get_resource_children(main_resource, session) + + remote_resources = _format_resource_tree(tree) + return {service.resource_name: {'children': remote_resources, 'remote_id': main_resource.resource_id}} + + +def get_last_sync(service_id, session): + last_sync = None + _ensure_sync_info_exists(service_id, session) + sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session) + if sync_info: + last_sync = sync_info.last_sync + return last_sync + + +def fetch_all_services_by_type(service_type, session): + """ + Get remote resources for all services of a certain type. + :param service_type: + :param session: + """ + for service in session.query(models.Service).filter_by(type=service_type): + # noinspection PyBroadException + try: + fetch_single_service(service, session) + except: + if CRON_SERVICE: + LOGGER.exception("There was an error when fetching data from the url: %s" % service.url) + pass + else: + raise + + +def fetch_single_service(service, session): + """ + Get remote resources for a single service. + :param service: (models.Service) or service_id + :param session: + """ + if isinstance(service, int): + service = session.query(models.Service).filter_by(resource_id=service).first() + LOGGER.info("Requesting remote resources") + remote_resources = _get_remote_resources(service) + service_id = service.resource_id + LOGGER.info("Deleting RemoteResource records for service: %s" % service.resource_name) + _delete_records(service_id, session) + _ensure_sync_info_exists(service.resource_id, session) + LOGGER.info("Writing RemoteResource records to database") + _update_db(remote_resources, service_id, session) + + +def fetch(): + """ + Main function to get all remote resources for each service and write to database. + """ + LOGGER.info("Getting database url") + url = db.get_db_url() + + LOGGER.debug("Database url: %s" % url) + engine = create_engine(url) + + session = Session(bind=engine) + + for service_type in SYNC_SERVICES_TYPES: + LOGGER.info("Fetching data for service type: %s" % service_type) + fetch_all_services_by_type(service_type, session) + + session.commit() + session.close() + + +def setup_cron_logger(): + log_path = constants.get_constant("MAGPIE_CRON_LOG") + log_path = os.path.expandvars(log_path) + log_path = os.path.expanduser(log_path) + file_handler = logging.FileHandler(log_path) + file_handler.setLevel(logging.INFO) + formatter = logging.Formatter("%(asctime)s %(levelname)8s %(message)s") + file_handler.setFormatter(formatter) + LOGGER.addHandler(file_handler) + LOGGER.setLevel(logging.INFO) + + +def main(): + """ + Main entry point for cron service. + """ + global CRON_SERVICE + CRON_SERVICE = True + + setup_cron_logger() + + LOGGER.info("Magpie cron started.") + + try: + db_ready = db.is_database_ready() + if not db_ready: + LOGGER.info("Database isn't ready") + return + LOGGER.info("Starting to fetch data for all service types") + fetch() + except Exception: + LOGGER.exception("An error occured") + raise + + LOGGER.info("Success, exiting.") + + +if __name__ == '__main__': + fetch() diff --git a/magpie/helpers/sync_services.py b/magpie/helpers/sync_services.py new file mode 100644 index 000000000..3c4bc18e6 --- /dev/null +++ b/magpie/helpers/sync_services.py @@ -0,0 +1,142 @@ +import abc +from collections import OrderedDict + +import requests +import threddsclient + + +def is_valid_resource_schema(resources): + """ + Returns True if the structure of the input dictionary is a tree of the form: + + {'resource_name_1': {'children': {'resource_name_3': {'children': {}}, + 'resource_name_4': {'children': {}} + }, + }, + 'resource_name_2': {'children': {}} + } + :return: bool + """ + for resource_name, values in resources.items(): + if 'children' not in values: + return False + if not isinstance(values['children'], (OrderedDict, dict)): + return False + return is_valid_resource_schema(values['children']) + return True + + +class _SyncServiceInterface: + __metaclass__ = abc.ABCMeta + + def __init__(self, service_name, url): + self.service_name = service_name + self.url = url + + @abc.abstractproperty + def max_depth(self): + """ + The max depth at which remote resources are fetched + :return: (int) + """ + + @abc.abstractmethod + def get_resources(self): + """ + This is the function actually fetching the data from the remote service. + Implement this for every specific service. + + :return: The returned dictionary must be validated by 'is_valid_resource_schema' + """ + pass + + +class _SyncServiceGeoserver(_SyncServiceInterface): + @property + def max_depth(self): + return None + + def get_resources(self): + # Only workspaces are fetched for now + resource_type = "route" + workspaces_url = "{}/{}".format(self.url, "workspaces") + resp = requests.get(workspaces_url, headers={"Accept": "application/json"}) + resp.raise_for_status() + workspaces_list = resp.json().get("workspaces", {}).get("workspace", {}) + + workspaces = {w["name"]: {"children": {}, "resource_type": resource_type} for w in workspaces_list} + + workspace_tree = {"workspaces": {"children": workspaces, + "resource_type": resource_type}} + + resources = {"geoserver-api": {"children": workspace_tree, + "resource_type": resource_type}} + + assert is_valid_resource_schema(resources), "Error in Interface implementation" + return resources + + +class _SyncServiceProjectAPI(_SyncServiceInterface): + @property + def max_depth(self): + return None + + def get_resources(self): + # Only workspaces are fetched for now + resource_type = "route" + projects_url = "/".join([self.url, "Projects"]) + resp = requests.get(projects_url) + resp.raise_for_status() + + projects = {p["id"]: {"children": {}, + "resource_type": resource_type, + "resource_display_name": p["name"]} + for p in resp.json()} + + resources = {self.service_name: {"children": projects, "resource_type": resource_type}} + assert is_valid_resource_schema(resources), "Error in Interface implementation" + return resources + + +class _SyncServiceThreads(_SyncServiceInterface): + @property + def max_depth(self): + return 3 + + @staticmethod + def _resource_id(resource): + id_ = resource.name + if len(resource.datasets) > 0: + id_ = resource.datasets[0].ID.split("/")[-1] + return id_ + + def get_resources(self): + def thredds_get_resources(url, depth): + cat = threddsclient.read_url(url) + name = self._resource_id(cat) + if depth == self.max_depth: + name = self.service_name + resource_type = 'directory' + if cat.datasets and cat.datasets[0].content_type != "application/directory": + resource_type = 'file' + + tree_item = {name: {'children': {}, 'resource_type': resource_type}} + + if depth > 0: + for reference in cat.flat_references(): + tree_item[name]['children'].update(thredds_get_resources(reference.url, depth - 1)) + + return tree_item + + resources = thredds_get_resources(self.url, self.max_depth) + assert is_valid_resource_schema(resources), 'Error in Interface implementation' + return resources + + +class _SyncServiceDefault(_SyncServiceInterface): + @property + def max_depth(self): + return None + + def get_resources(self): + return {} diff --git a/magpie/magpiectl.py b/magpie/magpiectl.py index b58e3b15c..0657afb1c 100644 --- a/magpie/magpiectl.py +++ b/magpie/magpiectl.py @@ -5,12 +5,12 @@ Magpie is a service for AuthN and AuthZ based on Ziggurat-Foundations """ -# -- Standard library --403------------------------------------------------------ -import logging.config +# -- Standard library -------------------------------------------------------- import argparse import time import warnings import logging +import logging.config LOGGER = logging.getLogger(__name__) # -- Definitions diff --git a/magpie/models.py b/magpie/models.py index 822858056..3e9347439 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -3,7 +3,6 @@ from magpie.definitions.sqlalchemy_definitions import * from magpie.api.api_except import * - Base = declarative_base() @@ -41,6 +40,8 @@ class Resource(ResourceMixin, Base): permission_names = [] child_resource_allowed = True + resource_display_name = sa.Column(sa.Unicode(100), nullable=True) + # reference to top-most service under which the resource is nested # if the resource is the service, id is None (NULL) @declared_attr @@ -107,8 +108,22 @@ class Service(Resource): __mapper_args__ = {u'polymorphic_identity': u'service', u'inherit_condition': resource_id == Resource.resource_id} # ... your own properties.... - url = sa.Column(sa.UnicodeText(), unique=True) # http://localhost:8083 - type = sa.Column(sa.UnicodeText()) # wps, wms, thredds, ... + + @declared_attr + def url(self): + # http://localhost:8083 + return sa.Column(sa.UnicodeText(), unique=True) + + @declared_attr + def type(self): + # wps, wms, thredds, ... + return sa.Column(sa.UnicodeText()) + + @declared_attr + def sync_type(self): + # project-api, geoserver-api, ... + return sa.Column(sa.UnicodeText(), nullable=True) + resource_type_name = u'service' @staticmethod @@ -166,15 +181,90 @@ class Workspace(Resource): class Route(Resource): __mapper_args__ = {u'polymorphic_identity': u'route'} permission_names = [u'read', - u'write'] + u'write', + u'read-match', + u'write-match'] resource_type_name = u'route' +class RemoteResource(BaseModel, Base): + __tablename__ = "remote_resources" + + __possible_permissions__ = () + _ziggurat_services = [ResourceTreeService] + + resource_id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True) + service_id = sa.Column(sa.Integer(), + sa.ForeignKey('services.resource_id', + onupdate='CASCADE', + ondelete='CASCADE'), + index=True, + nullable=False) + parent_id = sa.Column(sa.Integer(), + sa.ForeignKey('remote_resources.resource_id', + onupdate='CASCADE', + ondelete='SET NULL'), + nullable=True) + ordering = sa.Column(sa.Integer(), default=0, nullable=False) + resource_name = sa.Column(sa.Unicode(100), nullable=False) + resource_display_name = sa.Column(sa.Unicode(100), nullable=True) + resource_type = sa.Column(sa.Unicode(30), nullable=False) + + def __repr__(self): + info = self.resource_type, self.resource_name, self.resource_id, self.ordering, self.parent_id + return '' % info + + +class RemoteResourcesSyncInfo(Base): + __tablename__ = "remote_resources_sync_info" + + id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True) + service_id = sa.Column(sa.Integer(), + sa.ForeignKey('services.resource_id', + onupdate='CASCADE', + ondelete='CASCADE'), + index=True, + nullable=False) + service = relationship("Service", foreign_keys=[service_id]) + remote_resource_id = sa.Column(sa.Integer(), + sa.ForeignKey('remote_resources.resource_id', onupdate='CASCADE', + ondelete='CASCADE')) + last_sync = sa.Column(sa.DateTime(), nullable=True) + + @staticmethod + def by_service_id(service_id, session): + condition = RemoteResourcesSyncInfo.service_id == service_id + service_info = session.query(RemoteResourcesSyncInfo).filter(condition).first() + return service_info + + def __repr__(self): + last_modified = self.last_sync.strftime("%Y-%m-%dT%H:%M:%S") if self.last_sync else None + info = self.service_id, last_modified, self.id + return '' % info + + +class RemoteResourceTreeService(ResourceTreeService): + def __init__(self, service_cls): + self.model = RemoteResource + super(RemoteResourceTreeService, self).__init__(service_cls) + + +class RemoteResourceTreeServicePostgreSQL(ResourceTreeServicePostgreSQL): + """ + This is necessary, because ResourceTreeServicePostgreSQL.model is the Resource class. + If we want to change it for a RemoteResource, we need this class. + + The ResourceTreeService.__init__ call sets the model. + """ + pass + + ziggurat_model_init(User, Group, UserGroup, GroupPermission, UserPermission, UserResourcePermission, GroupResourcePermission, Resource, ExternalIdentity, passwordmanager=None) resource_tree_service = ResourceTreeService(ResourceTreeServicePostgreSQL) +remote_resource_tree_service = RemoteResourceTreeService(RemoteResourceTreeServicePostgreSQL) resource_type_dict = { Service.resource_type_name: Service, @@ -204,5 +294,6 @@ def get_all_resource_permission_names(): def find_children_by_name(name, parent_id, db_session): tree_struct = resource_tree_service.from_parent_deeper(parent_id=parent_id, limit_depth=1, db_session=db_session) tree_level_entries = [node for node in tree_struct] - tree_level_filtered = [node.Resource for node in tree_level_entries if node.Resource.resource_name.lower() == name.lower()] + tree_level_filtered = [node.Resource for node in tree_level_entries if + node.Resource.resource_name.lower() == name.lower()] return tree_level_filtered.pop() if len(tree_level_filtered) else None diff --git a/magpie/register.py b/magpie/register.py index d789c7f93..ea869cec0 100644 --- a/magpie/register.py +++ b/magpie/register.py @@ -163,18 +163,19 @@ def get_magpie_url(): def get_twitcher_protected_service_url(magpie_service_name, hostname=None): - hostname = hostname or get_constant('HOSTNAME') twitcher_proxy_url = get_constant('TWITCHER_PROTECTED_URL', raise_not_set=False) - if twitcher_proxy_url: - return twitcher_proxy_url - twitcher_proxy = get_constant('TWITCHER_PROTECTED_PATH', raise_not_set=False) - if not twitcher_proxy.endswith('/'): - twitcher_proxy = twitcher_proxy + '/' - if not twitcher_proxy.startswith('/'): - twitcher_proxy = '/' + twitcher_proxy - if not twitcher_proxy.startswith('/twitcher'): - twitcher_proxy = '/twitcher' + twitcher_proxy - return "https://{0}{1}{2}".format(hostname, twitcher_proxy, magpie_service_name) + if not twitcher_proxy_url: + twitcher_proxy = get_constant('TWITCHER_PROTECTED_PATH', raise_not_set=False) + if not twitcher_proxy.endswith('/'): + twitcher_proxy = twitcher_proxy + '/' + if not twitcher_proxy.startswith('/'): + twitcher_proxy = '/' + twitcher_proxy + if not twitcher_proxy.startswith('/twitcher'): + twitcher_proxy = '/twitcher' + twitcher_proxy + hostname = hostname or get_constant('HOSTNAME') + twitcher_proxy_url = "https://{0}{1}".format(hostname, twitcher_proxy) + twitcher_proxy_url.rstrip('/') + return "{0}/{1}".format(twitcher_proxy_url, magpie_service_name) def register_services(register_service_url, services_dict, cookies, @@ -352,14 +353,15 @@ def magpie_register_services(services_dict, push_to_phoenix, user, password, pro def magpie_register_services_with_db_session(services_dict, db_session, push_to_phoenix=False, force_update=False, update_getcapabilities_permissions=False): - existing_services = models.Service.all(db_session=db_session) - existing_services_names = [svc.resource_name for svc in existing_services] + existing_services_names = [n[0] for n in db_session.query(models.Service.resource_name)] magpie_anonymous_user = get_constant('MAGPIE_ANONYMOUS_USER') anonymous_user = models.User.by_user_name(magpie_anonymous_user, db_session=db_session) - for svc_name in services_dict: - svc_new_url = os.path.expandvars(services_dict[svc_name]['url']) - svc_type = services_dict[svc_name]['type'] + for svc_name, svc_values in services_dict.items(): + svc_new_url = os.path.expandvars(svc_values['url']) + svc_type = svc_values['type'] + + svc_sync_type = svc_values.get('sync_type') if force_update and svc_name in existing_services_names: svc = models.Service.by_service_name(svc_name, db_session=db_session) if svc.url == svc_new_url: @@ -368,17 +370,22 @@ def magpie_register_services_with_db_session(services_dict, db_session, push_to_ print_log("Service URL update [{url_old}] => [{url_new}] ({svc})" .format(url_old=svc.url, url_new=svc_new_url, svc=svc_name)) svc.url = svc_new_url + svc.sync_type = svc_sync_type elif not force_update and svc_name in existing_services_names: print_log("Skipping service [{svc}] (conflict)" .format(svc=svc_name)) else: print_log("Adding service [{svc}]".format(svc=svc_name)) - svc = models.Service(resource_name=svc_name, resource_type=u'service', url=svc_new_url, type=svc_type) + svc = models.Service(resource_name=svc_name, + resource_type=u'service', + url=svc_new_url, + type=svc_type, + sync_type=svc_sync_type) db_session.add(svc) if update_getcapabilities_permissions and anonymous_user is None: print_log("Cannot update 'getcapabilities' permission of non existing anonymous user", level=logging.WARN) elif update_getcapabilities_permissions and 'getcapabilities' in service_type_dict[svc_type].permission_names: - svc = models.Service.by_service_name(svc_name, db_session=db_session) + svc = db_session.query(models.Service.resource_id).filter_by(resource_name=svc_name).first() svc_perm_getcapabilities = models.UserResourcePermissionService.by_resource_user_and_perm( user_id=anonymous_user.id, perm_name='getcapabilities', resource_id=svc.resource_id, db_session=db_session diff --git a/magpie/services.py b/magpie/services.py index db32e7dcf..78008910c 100644 --- a/magpie/services.py +++ b/magpie/services.py @@ -258,12 +258,12 @@ def __init__(self, service, request): @property def __acl__(self): - raise NotImplementedError + return self.route_acl() - @property def route_acl(self, sub_api_route=None): self.expand_acl(self.service, self.request.user) + match_index = 0 route_parts = self.request.path.split('/') route_api_base = self.service.resource_name if sub_api_route is None else sub_api_route @@ -273,38 +273,33 @@ def route_acl(self, sub_api_route=None): if len(route_parts) - 1 > api_idx: route_parts = route_parts[api_idx + 1::] route_child = self.service + + # process read/write inheritance permission access while route_child and route_parts: part_name = route_parts.pop(0) route_res_id = route_child.resource_id route_child = models.find_children_by_name(part_name, parent_id=route_res_id, db_session=self.request.db) + match_index = len(self.acl) self.expand_acl(route_child, self.request.user) + + # process read/write-match specific permission access + # (convert exact route '-match' to read/write only if matching last item's permissions) + for i in range(match_index, len(self.acl)): + if self.acl[i][2] == 'read-match': + self.acl[i] = (self.acl[i][0], self.acl[i][1], 'read') + if self.acl[i][2] == 'write-match': + self.acl[i] = (self.acl[i][0], self.acl[i][1], 'write') + return self.acl def permission_requested(self): - if self.request.method == 'GET': + # only read/write are used for 'real' access control, '-match' permissions must be updated accordingly + if self.request.method.upper() in ['GET', 'HEAD']: return u'read' return u'write' -class ServiceGeoserverAPI(ServiceAPI): - def __init__(self, service, request): - super(ServiceGeoserverAPI, self).__init__(service, request) - - @property - def __acl__(self): - return ServiceAPI.route_acl.fget(self) - - -class ServiceProjectAPI(ServiceAPI): - def __init__(self, service, request): - super(ServiceProjectAPI, self).__init__(service, request) - - @property - def __acl__(self): - return ServiceAPI.route_acl.fget(self, sub_api_route='api') - - class ServiceWFS(ServiceI): permission_names = [ @@ -411,10 +406,9 @@ def permission_requested(self): service_type_dict = { u'access': ServiceAccess, - u'geoserver-api': ServiceGeoserverAPI, + u'api': ServiceAPI, u'geoserverwms': ServiceGeoserver, u'ncwms': ServiceNCWMS2, - u'project-api': ServiceProjectAPI, u'thredds': ServiceTHREDDS, u'wfs': ServiceWFS, u'wps': ServiceWPS, diff --git a/magpie/ui/home/static/style.css b/magpie/ui/home/static/style.css index d0f2b8868..1c936469b 100644 --- a/magpie/ui/home/static/style.css +++ b/magpie/ui/home/static/style.css @@ -403,6 +403,20 @@ div.perm_title { margin: 0.1em 0 0 -4em; } +.tree_item_message { + text-align: left; + color: gray; + display: inline-flex; + float: right; + margin: 0.25em 0 0 0; +} + +.tree_item_message>img { + width: 1.5em; + height: 1.5em; + margin: -0.15em 0 0 0; +} + div.perm_checkbox { width:3em; float:right; diff --git a/magpie/ui/home/static/warning_exclamation_orange.png b/magpie/ui/home/static/warning_exclamation_orange.png new file mode 100644 index 000000000..4fe4ea20f Binary files /dev/null and b/magpie/ui/home/static/warning_exclamation_orange.png differ diff --git a/magpie/ui/management/templates/edit_group.mako b/magpie/ui/management/templates/edit_group.mako index d8b8e7169..e0c215ffa 100644 --- a/magpie/ui/management/templates/edit_group.mako +++ b/magpie/ui/management/templates/edit_group.mako @@ -2,19 +2,27 @@ <%namespace name="tree" file="ui.management:templates/tree_scripts.mako"/> <%def name="render_item(key, value, level)"> + %for perm in permissions: - % if perm in value['permission_names']: -
- -
- % else: -
- -
- % endif +
+ % if perm in value['permission_names']: + + % else: + + % endif +
%endfor + % if not value.get('matches_remote', True): +
+ +
+

+ +

+ % endif % if level == 0:
@@ -99,11 +107,34 @@
+ %if error_message: +
${error_message}
+ %endif +
+

+ Last synchronization with remote services: + %if sync_implemented: + ${last_sync} + + %else: + Not implemented for this service type. + %endif +

+ %if ids_to_clean and not out_of_sync: +

+ Note: + Some resources are absent from the remote server + + +

+ %endif +
+
-
Resources
- %for perm in permissions: -
${perm}
- %endfor +
Resources
+ %for perm in permissions: +
${perm}
+ %endfor
${tree.render_tree(render_item, resources)} diff --git a/magpie/ui/management/templates/edit_user.mako b/magpie/ui/management/templates/edit_user.mako index 82f7896aa..c19430fb1 100644 --- a/magpie/ui/management/templates/edit_user.mako +++ b/magpie/ui/management/templates/edit_user.mako @@ -3,12 +3,13 @@ <%namespace name="tree" file="ui.management:templates/tree_scripts.mako"/> <%def name="render_item(key, value, level)"> + %for perm in permissions: % if perm in value['permission_names']:
@@ -16,14 +17,23 @@ % else:
% endif %endfor + % if not value.get('matches_remote', True): +
+ +
+

+ +

+ % endif % if level == 0:
@@ -129,14 +139,14 @@

Permissions

- - + %else: > - + %endif View inherited group permissions
@@ -153,6 +163,28 @@
+ %if error_message: +
${error_message}
+ %endif +
+

+ Last synchronization with remote services: + %if sync_implemented: + ${last_sync} + + %else: + Not implemented for this service type. + %endif +

+ %if ids_to_clean and not out_of_sync: +

+ Note: + Some resources are absent from the remote server + + +

+ %endif +
Resources
%for perm in permissions: diff --git a/magpie/ui/management/templates/tree_scripts.mako b/magpie/ui/management/templates/tree_scripts.mako index bca77e8f7..22e13ee78 100644 --- a/magpie/ui/management/templates/tree_scripts.mako +++ b/magpie/ui/management/templates/tree_scripts.mako @@ -42,14 +42,16 @@ li.Collapsed {
    %for key in tree:
    -
    + % if tree[key]['children']:
  • % else:
  • % endif -
    ${key}
    - +
    ${tree[key].get('resource_display_name', key)}
    + + + ${item_renderer(key, tree[key], level)}
    • diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py index 7ce623237..296b194cb 100644 --- a/magpie/ui/management/views.py +++ b/magpie/ui/management/views.py @@ -1,9 +1,16 @@ +import datetime +from collections import OrderedDict + +import humanize + from magpie.definitions.pyramid_definitions import * from magpie.constants import get_constant from magpie.common import str2bool +from magpie.helpers.sync_resources import OUT_OF_SYNC from magpie.services import service_type_dict -from magpie.models import resource_type_dict +from magpie.models import resource_type_dict, remote_resource_tree_service from magpie.ui.management import check_response +from magpie.helpers import sync_resources from magpie.ui.home import add_template_data from magpie import register, __meta__ from distutils.version import LooseVersion @@ -227,19 +234,32 @@ def add_user(self): def edit_user(self): user_name = self.request.matchdict['user_name'] cur_svc_type = self.request.matchdict['cur_svc_type'] - inherited_permissions = self.request.matchdict.get('inherited_permissions', False) + inherit_groups_permissions = self.request.matchdict.get('inherit_groups_permissions', False) user_url = '{url}/users/{usr}'.format(url=self.magpie_url, usr=user_name) own_groups = self.get_user_groups(user_name) all_groups = self.get_all_groups(first_default_group=get_constant('MAGPIE_USERS_GROUP')) + error_message = "" + + # Todo: + # Until the api is modified to make it possible to request from the RemoteResource table, + # we have to access the database directly here + session = self.request.db + + try: + # The service type is 'default'. This function replaces cur_svc_type with the first service type. + svc_types, cur_svc_type, services = self.get_services(cur_svc_type) + except Exception as e: + raise HTTPBadRequest(detail=repr(e)) + user_resp = requests.get(user_url, cookies=self.request.cookies) check_response(user_resp) user_info = user_resp.json()['user'] user_info[u'edit_mode'] = u'no_edit' user_info[u'own_groups'] = own_groups user_info[u'groups'] = all_groups - user_info[u'inherited_permissions'] = inherited_permissions + user_info[u'inherit_groups_permissions'] = inherit_groups_permissions if self.request.method == 'POST': res_id = self.request.POST.get(u'resource_id') @@ -247,16 +267,23 @@ def edit_user(self): is_save_user_info = False requires_update_name = False - if u'inherited_permissions' in self.request.POST: - inherited_permissions = str2bool(self.request.POST[u'inherited_permissions']) - user_info[u'inherited_permissions'] = inherited_permissions + if u'inherit_groups_permissions' in self.request.POST: + inherit_groups_permissions = str2bool(self.request.POST[u'inherit_groups_permissions']) + user_info[u'inherit_groups_permissions'] = inherit_groups_permissions if u'delete' in self.request.POST: check_response(requests.delete(user_url, cookies=self.request.cookies)) return HTTPFound(self.request.route_url('view_users')) elif u'goto_service' in self.request.POST: return self.goto_service(res_id) - elif u'resource_id' in self.request.POST: + elif u'clean_resource' in self.request.POST: + # 'clean_resource' must be above 'edit_permissions' because they're in the same form. + self.delete_resource(res_id) + elif u'edit_permissions' in self.request.POST: + if not res_id or res_id == 'None': + remote_id = int(self.request.POST.get('remote_id')) + services_names = [s['service_name'] for s in services.values()] + res_id = self.add_remote_resource(cur_svc_type, services_names, user_name, remote_id, is_user=True) self.edit_user_or_group_resource_permissions(user_name, res_id, is_user=True) elif u'edit_group_membership' in self.request.POST: is_edit_group_membership = True @@ -276,6 +303,19 @@ def edit_user(self): elif u'save_email' in self.request.POST: user_info[u'email'] = self.request.POST.get(u'new_user_email') is_save_user_info = True + elif u'force_sync' in self.request.POST: + errors = [] + for service_info in services.values(): + try: + sync_resources.fetch_single_service(service_info['resource_id'], session) + except Exception: + errors.append(service_info['service_name']) + if errors: + error_message += self.make_sync_error_message(errors) + elif u'clean_all' in self.request.POST: + ids_to_clean = self.request.POST.get('ids_to_clean').split(";") + for id_ in ids_to_clean: + self.delete_resource(id_) if is_save_user_info: check_response(requests.put(user_url, data=user_info, cookies=self.request.cookies)) @@ -305,16 +345,30 @@ def edit_user(self): # display resources permissions per service type tab try: - svc_types, cur_svc_type, services = self.get_services(cur_svc_type) res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict( - user_name, services, cur_svc_type, is_user=True, is_inherited_permissions=inherited_permissions) - user_info[u'cur_svc_type'] = cur_svc_type - user_info[u'svc_types'] = svc_types - user_info[u'resources'] = res_perms - user_info[u'permissions'] = res_perm_names + user_name, services, cur_svc_type, is_user=True, is_inherit_groups_permissions=inherit_groups_permissions) except Exception as e: raise HTTPBadRequest(detail=repr(e)) + sync_types = [s["service_sync_type"] for s in services.values()] + sync_implemented = any(s in sync_resources.SYNC_SERVICES_TYPES for s in sync_types) + + info = self.get_remote_resources_info(res_perms, services, session) + res_perms, ids_to_clean, last_sync_humanized, out_of_sync = info + + if out_of_sync: + error_message = self.make_sync_error_message(out_of_sync) + + user_info[u'error_message'] = error_message + user_info[u'ids_to_clean'] = ";".join(ids_to_clean) + user_info[u'last_sync'] = last_sync_humanized + user_info[u'sync_implemented'] = sync_implemented + user_info[u'out_of_sync'] = out_of_sync + user_info[u'cur_svc_type'] = cur_svc_type + user_info[u'svc_types'] = svc_types + user_info[u'resources'] = res_perms + user_info[u'permissions'] = res_perm_names + return add_template_data(self.request, data=user_info) @view_config(route_name='view_groups', renderer='templates/view_groups.mako') @@ -361,7 +415,11 @@ def resource_tree_parser(self, raw_resources_tree, permission): for r_id, resource in raw_resources_tree.items(): perm_names = self.default_get(permission, r_id, []) children = self.resource_tree_parser(resource['children'], permission) - resources_tree[resource['resource_name']] = dict(id=r_id, permission_names=perm_names, children=children) + children = OrderedDict(sorted(children.items())) + resources_tree[resource['resource_name']] = dict(id=r_id, + permission_names=perm_names, + resource_display_name=resource['resource_display_name'], + children=children) return resources_tree def perm_tree_parser(self, raw_perm_tree): @@ -395,7 +453,7 @@ def edit_group_users(self, group_name): def edit_user_or_group_resource_permissions(self, user_or_group_name, resource_id, is_user=False): usr_grp_type = 'users' if is_user else 'groups' res_perms_url = '{url}/{grp_type}/{grp}/resources/{res_id}/permissions' \ - .format(url=self.magpie_url, grp_type=usr_grp_type, grp=user_or_group_name, res_id=resource_id) + .format(url=self.magpie_url, grp_type=usr_grp_type, grp=user_or_group_name, res_id=resource_id) try: res_perms_resp = requests.get(res_perms_url, cookies=self.request.cookies) res_perms = res_perms_resp.json()['permission_names'] @@ -403,6 +461,7 @@ def edit_user_or_group_resource_permissions(self, user_or_group_name, resource_i raise HTTPBadRequest(detail=repr(e)) selected_perms = self.request.POST.getall('permission') + removed_perms = list(set(res_perms) - set(selected_perms)) new_perms = list(set(selected_perms) - set(res_perms)) @@ -414,9 +473,9 @@ def edit_user_or_group_resource_permissions(self, user_or_group_name, resource_i check_response(requests.post(res_perms_url, data=data, cookies=self.request.cookies)) def get_user_or_group_resources_permissions_dict(self, user_or_group_name, services, service_type, - is_user=False, is_inherited_permissions=False): + is_user=False, is_inherit_groups_permissions=False): user_or_group_type = 'users' if is_user else 'groups' - inherit_type = 'inherited_' if is_inherited_permissions and is_user else '' + inherit_type = 'inherited_' if is_inherit_groups_permissions and is_user else '' group_perms_url = '{url}/{usr_grp_type}/{usr_grp}/{inherit}resources' \ .format(url=self.magpie_url, usr_grp_type=user_or_group_type, @@ -427,17 +486,20 @@ def get_user_or_group_resources_permissions_dict(self, user_or_group_name, servi svc_perm_url = '{url}/services/types/{svc_type}'.format(url=self.magpie_url, svc_type=service_type) resp_svc_type = check_response(requests.get(svc_perm_url, cookies=self.request.cookies)) resp_available_svc_types = resp_svc_type.json()['services'][service_type] + + # remove possible duplicate permissions from different services resources_permission_names = set() for svc in resp_available_svc_types: resources_permission_names.update(set(resp_available_svc_types[svc]['permission_names'])) - resources_permission_names = list(resources_permission_names) + # inverse sort so that displayed permissions are sorted, since added from right to left in tree view + resources_permission_names = sorted(resources_permission_names, reverse=True) - resources = {} - for service in services: + resources = OrderedDict() + for service in sorted(services): if not service: continue - permission = {} + permission = OrderedDict() try: raw_perms = resp_group_perms_json['resources'][service_type][service] permission[raw_perms['resource_id']] = raw_perms['permission_names'] @@ -449,9 +511,10 @@ def get_user_or_group_resources_permissions_dict(self, user_or_group_name, servi .format(url=self.magpie_url, svc=service), cookies=self.request.cookies)) raw_resources = resp_resources.json()[service] - resources[service] = dict(id=raw_resources['resource_id'], - permission_names=self.default_get(permission, raw_resources['resource_id'], []), - children=self.resource_tree_parser(raw_resources['resources'], permission)) + resources[service] = OrderedDict( + id=raw_resources['resource_id'], + permission_names=self.default_get(permission, raw_resources['resource_id'], []), + children=self.resource_tree_parser(raw_resources['resources'], permission)) return resources_permission_names, resources @view_config(route_name='edit_group', renderer='templates/edit_group.mako') @@ -460,6 +523,19 @@ def edit_group(self): cur_svc_type = self.request.matchdict['cur_svc_type'] group_info = {u'edit_mode': u'no_edit', u'group_name': group_name, u'cur_svc_type': cur_svc_type} + error_message = "" + + # Todo: + # Until the api is modified to make it possible to request from the RemoteResource table, + # we have to access the database directly here + session = self.request.db + + try: + # The service type is 'default'. This function replaces cur_svc_type with the first service type. + svc_types, cur_svc_type, services = self.get_services(cur_svc_type) + except Exception as e: + raise HTTPBadRequest(detail=repr(e)) + # move to service or edit requested group/permission changes if self.request.method == 'POST': res_id = self.request.POST.get('resource_id') @@ -477,19 +553,55 @@ def edit_group(self): return HTTPFound(self.request.route_url('edit_group', **group_info)) elif u'goto_service' in self.request.POST: return self.goto_service(res_id) - elif u'resource_id' in self.request.POST: + elif u'clean_resource' in self.request.POST: + # 'clean_resource' must be above 'edit_permissions' because they're in the same form. + self.delete_resource(res_id) + elif u'edit_permissions' in self.request.POST: + if not res_id or res_id == 'None': + remote_id = int(self.request.POST.get('remote_id')) + services_names = [s['service_name'] for s in services.values()] + res_id = self.add_remote_resource(cur_svc_type, services_names, group_name, remote_id, is_user=False) self.edit_user_or_group_resource_permissions(group_name, res_id, is_user=False) - else: + elif u'member' in self.request.POST: self.edit_group_users(group_name) + elif u'force_sync' in self.request.POST: + errors = [] + for service_info in services.values(): + try: + sync_resources.fetch_single_service(service_info['resource_id'], session) + except Exception: + errors.append(service_info['service_name']) + if errors: + error_message += self.make_sync_error_message(errors) + + elif u'clean_all' in self.request.POST: + ids_to_clean = self.request.POST.get('ids_to_clean').split(";") + for id_ in ids_to_clean: + self.delete_resource(id_) + else: + return HTTPBadRequest(detail="Invalid POST request.") # display resources permissions per service type tab try: - svc_types, cur_svc_type, services = self.get_services(cur_svc_type) res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(group_name, services, cur_svc_type, is_user=False) except Exception as e: raise HTTPBadRequest(detail=repr(e)) + sync_types = [s["service_sync_type"] for s in services.values()] + sync_implemented = any(s in sync_resources.SYNC_SERVICES_TYPES for s in sync_types) + + info = self.get_remote_resources_info(res_perms, services, session) + res_perms, ids_to_clean, last_sync_humanized, out_of_sync = info + + if out_of_sync: + error_message = self.make_sync_error_message(out_of_sync) + + group_info[u'error_message'] = error_message + group_info[u'ids_to_clean'] = ";".join(ids_to_clean) + group_info[u'last_sync'] = last_sync_humanized + group_info[u'sync_implemented'] = sync_implemented + group_info[u'out_of_sync'] = out_of_sync group_info[u'group_name'] = group_name group_info[u'cur_svc_type'] = cur_svc_type group_info[u'users'] = self.get_user_names() @@ -500,6 +612,100 @@ def edit_group(self): group_info[u'permissions'] = res_perm_names return add_template_data(self.request, data=group_info) + def make_sync_error_message(self, service_names): + this = "this service" if len(service_names) == 1 else "these services" + error_message = ("There seems to be an issue synchronizing resources from " + "{}: {}".format(this, ", ".join(service_names))) + return error_message + + def get_remote_resources_info(self, res_perms, services, session): + last_sync_humanized = "Never" + ids_to_clean, out_of_sync = [], [] + now = datetime.datetime.now() + + service_ids = [s["resource_id"] for s in services.values()] + last_sync_datetimes = self.get_last_sync_datetimes(service_ids, session) + + if any(last_sync_datetimes): + last_sync_datetime = min(filter(bool, last_sync_datetimes)) + last_sync_humanized = humanize.naturaltime(now - last_sync_datetime) + res_perms = self.merge_remote_resources(res_perms, services, session) + + for last_sync, service_name in zip(last_sync_datetimes, services): + if last_sync: + ids_to_clean += self.get_ids_to_clean(res_perms[service_name]["children"]) + if now - last_sync > OUT_OF_SYNC: + out_of_sync.append(service_name) + return res_perms, ids_to_clean, last_sync_humanized, out_of_sync + + def merge_remote_resources(self, res_perms, services, session): + merged_resources = {} + for service_name, service_values in services.items(): + service_id = service_values["resource_id"] + merge = sync_resources.merge_local_and_remote_resources + resources_for_service = merge(res_perms, service_values["service_sync_type"], service_id, session) + merged_resources[service_name] = resources_for_service[service_name] + return merged_resources + + def get_last_sync_datetimes(self, service_ids, session): + return [sync_resources.get_last_sync(s, session) for s in service_ids] + + def delete_resource(self, res_id): + url = '{url}/resources/{resource_id}'.format(url=self.magpie_url, resource_id=res_id) + try: + check_response(requests.delete(url, cookies=self.request.cookies)) + except HTTPNotFound: + # Some resource ids are already deleted because they were a child + # of another just deleted parent resource. + # We just skip them. + pass + + def get_ids_to_clean(self, resources): + ids = [] + for resource_name, values in resources.items(): + if "matches_remote" in values and not values["matches_remote"]: + ids.append(values['id']) + ids += self.get_ids_to_clean(values['children']) + return ids + + def add_remote_resource(self, service_type, services_names, user_or_group, remote_id, is_user=False): + try: + res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(user_or_group, + services=services_names, + service_type=service_type, + is_user=is_user) + except Exception as e: + raise HTTPBadRequest(detail=repr(e)) + + # Todo: + # Until the api is modified to make it possible to request from the RemoteResource table, + # we have to access the database directly here + session = self.request.db + + # get the parent resources for this remote_id + parents = remote_resource_tree_service.path_upper(remote_id, db_session=session) + parents = list(reversed(list(parents))) + + parent_id = None + current_resources = res_perms + for remote_resource in parents: + name = remote_resource.resource_name + if name in current_resources: + parent_id = current_resources[name]['id'] + current_resources = current_resources[name]['children'] + else: + data = { + 'resource_name': name, + 'resource_display_name': remote_resource.resource_display_name, + 'resource_type': remote_resource.resource_type, + 'parent_id': parent_id, + } + resources_url = '{url}/resources'.format(url=self.magpie_url) + response = check_response(requests.post(resources_url, data=data, cookies=self.request.cookies)) + parent_id = response.json()['resource']['resource_id'] + + return parent_id + @view_config(route_name='view_services', renderer='templates/view_services.mako') def view_services(self): if 'delete' in self.request.POST: @@ -544,7 +750,7 @@ def add_service(self): u'service_url': service_url, u'service_type': service_type, u'service_push': service_push} - check_response(requests.post(self.magpie_url+'/services', data=data, cookies=self.request.cookies)) + check_response(requests.post(self.magpie_url + '/services', data=data, cookies=self.request.cookies)) return HTTPFound(self.request.route_url('view_services', cur_svc_type=service_type)) services_keys_sorted = sorted(service_type_dict) diff --git a/providers.cfg b/providers.cfg index 523f95813..95c96dfa0 100644 --- a/providers.cfg +++ b/providers.cfg @@ -5,6 +5,7 @@ providers: public: true c4i: false type: wps + sync_type: wps malleefowl: url: http://${HOSTNAME}:8091/wps @@ -12,6 +13,7 @@ providers: public: true c4i: false type: wps + sync_type: wps lb_flyingpigeon: url: http://${HOSTNAME}:58093/wps @@ -19,6 +21,7 @@ providers: public: true c4i: false type: wps + sync_type: wps thredds: url: http://${HOSTNAME}:8083/thredds @@ -26,6 +29,7 @@ providers: public: true c4i: false type: thredds + sync_type: thredds ncWMS2: url: http://${HOSTNAME}:8080/ncWMS2 @@ -33,6 +37,7 @@ providers: public: true c4i: false type: ncwms + sync_type: ncwms geoserverwms: url: http://${HOSTNAME}:8087/geoserver @@ -40,6 +45,7 @@ providers: public: true c4i: false type: geoserverwms + sync_type: geoserverwms geoserver: url: http://${HOSTNAME}:8087/geoserver @@ -47,6 +53,7 @@ providers: public: true c4i: false type: wfs + sync_type: wfs geoserver-web: url: http://${HOSTNAME}:8087/geoserver/web/ @@ -54,17 +61,20 @@ providers: public: true c4i: false type: access + sync_type: access geoserver-api: - url: http://${HOSTNAME}:8087/geoserver/rest/ + url: http://${HOSTNAME}:8087/geoserver/rest title: geoserver-api public: true c4i: false - type: geoserver-api + type: api + sync_type: geoserver-api project-api: - url: http://${HOSTNAME}:3005 + url: http://${HOSTNAME}:3005/api title: project-api public: true c4i: false - type: project-api + type: api + sync_type: project-api diff --git a/requirements.txt b/requirements.txt index 789b2fc22..ec558560f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ lxml>=3.7 bcrypt==3.1.3 futures==3.1.1 zope.sqlalchemy -gunicorn +gunicorn==19.8.1 alembic==0.9.6 paste python-dotenv @@ -29,4 +29,6 @@ python-dotenv cornice_swagger>=0.7.0 cornice colander +threddsclient +humanize requests_file diff --git a/setup.py b/setup.py index e48d9810d..cf6b04f12 100644 --- a/setup.py +++ b/setup.py @@ -97,7 +97,7 @@ # -- script entry points ----------------------------------------------- entry_points="""\ [paste.app_factory] - main = magpiectl:main + main = magpie.magpiectl:main [console_scripts] """, ) diff --git a/tests/interfaces.py b/tests/interfaces.py index 7feb5bad0..ce728334e 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -248,6 +248,7 @@ def test_GetUserInheritedResources(self): utils.check_val_is_in('resource_id', svc_dict) utils.check_val_is_in('service_name', svc_dict) utils.check_val_is_in('service_type', svc_dict) + utils.check_val_is_in('service_sync_type', svc_dict) utils.check_val_is_in('service_url', svc_dict) utils.check_val_is_in('public_url', svc_dict) utils.check_val_is_in('permission_names', svc_dict) @@ -256,6 +257,7 @@ def test_GetUserInheritedResources(self): utils.check_val_type(svc_dict['service_name'], six.string_types) utils.check_val_type(svc_dict['service_url'], six.string_types) utils.check_val_type(svc_dict['service_type'], six.string_types) + utils.check_val_type(svc_dict['service_sync_type'], six.string_types) utils.check_val_type(svc_dict['public_url'], six.string_types) utils.check_val_type(svc_dict['permission_names'], list) utils.check_val_type(svc_dict['resources'], dict) @@ -381,6 +383,7 @@ def test_GetServiceResources(self): utils.check_val_is_in('resource_id', svc_dict) utils.check_val_is_in('service_name', svc_dict) utils.check_val_is_in('service_type', svc_dict) + utils.check_val_is_in('service_sync_type', svc_dict) utils.check_val_is_in('service_url', svc_dict) utils.check_val_is_in('public_url', svc_dict) utils.check_val_is_in('permission_names', svc_dict) @@ -389,6 +392,7 @@ def test_GetServiceResources(self): utils.check_val_type(svc_dict['service_name'], six.string_types) utils.check_val_type(svc_dict['service_url'], six.string_types) utils.check_val_type(svc_dict['service_type'], six.string_types) + utils.check_val_type(svc_dict['service_sync_type'], six.string_types) utils.check_val_type(svc_dict['public_url'], six.string_types) utils.check_val_type(svc_dict['permission_names'], list) utils.check_resource_children(svc_dict['resources'], svc_dict['resource_id'], svc_dict['resource_id']) @@ -551,13 +555,33 @@ def test_PostResources_DirectServiceResource(self): data = { "resource_name": self.test_resource_name, + "resource_display_name": self.test_resource_name, "resource_type": self.test_resource_type, "parent_id": service_resource_id } resp = utils.test_request(self.url, 'POST', '/resources', headers=self.json_headers, cookies=self.cookies, data=data) json_body = utils.check_response_basic_info(resp, 201) - utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type, self.version) + utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type, + self.test_resource_name, self.version) + + @pytest.mark.resources + @unittest.skipUnless(runner.MAGPIE_TEST_RESOURCES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('resources')) + def test_PostResources_DirectServiceResourceOptional(self): + service_info = utils.TestSetup.get_ExistingTestServiceInfo(self) + service_resource_id = service_info['resource_id'] + + data = { + "resource_name": self.test_resource_name, + # resource_display_name should default to self.test_resource_name, + "resource_type": self.test_resource_type, + "parent_id": service_resource_id + } + resp = utils.test_request(self.url, 'POST', '/resources', + headers=self.json_headers, cookies=self.cookies, data=data) + json_body = utils.check_response_basic_info(resp, 201) + utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type, + self.test_resource_name, self.version) @pytest.mark.resources @unittest.skipUnless(runner.MAGPIE_TEST_RESOURCES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('resources')) @@ -570,13 +594,15 @@ def test_PostResources_ChildrenResource(self): data = { "resource_name": self.test_resource_name, + "resource_display_name": self.test_resource_name, "resource_type": self.test_resource_type, "parent_id": direct_resource_id } resp = utils.test_request(self.url, 'POST', '/resources', headers=self.json_headers, cookies=self.cookies, data=data) json_body = utils.check_response_basic_info(resp, 201) - utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type, self.version) + utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type, + self.test_resource_name, self.version) @pytest.mark.resources @unittest.skipUnless(runner.MAGPIE_TEST_RESOURCES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('resources')) diff --git a/tests/utils.py b/tests/utils.py index a9d509451..5e82b337a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -288,12 +288,13 @@ def check_error_param_structure(json_body, paramValue=Null, paramName=Null, para check_val_equal(json_body['paramCompare'], paramCompare) -def check_post_resource_structure(json_body, resource_name, resource_type, version=None): +def check_post_resource_structure(json_body, resource_name, resource_type, resource_display_name, version=None): """ Validates POST /resource response information based on different Magpie version formats. :param json_body: json body of the response to validate. :param resource_name: name of the resource to validate. :param resource_type: type of the resource to validate. + :param resource_display_name: display name of the resource to validate. :param version: version of application/remote server to use for format validation, use local Magpie version if None. :raise failing condition """ @@ -302,9 +303,11 @@ def check_post_resource_structure(json_body, resource_name, resource_type, versi check_val_is_in('resource', json_body) check_val_type(json_body['resource'], dict) check_val_is_in('resource_name', json_body['resource']) + check_val_is_in('resource_display_name', json_body['resource']) check_val_is_in('resource_type', json_body['resource']) check_val_is_in('resource_id', json_body['resource']) check_val_equal(json_body['resource']['resource_name'], resource_name) + check_val_equal(json_body['resource']['resource_display_name'], resource_display_name) check_val_equal(json_body['resource']['resource_type'], resource_type) check_val_type(json_body['resource']['resource_id'], int) else: @@ -340,6 +343,8 @@ def check_resource_children(resource_dict, parent_resource_id, root_service_id): check_val_equal(resource_info['parent_id'], parent_resource_id) check_val_is_in('resource_name', resource_info) check_val_type(resource_info['resource_name'], six.string_types) + check_val_is_in('resource_display_name', resource_info) + check_val_type(resource_info['resource_display_name'], six.string_types) check_val_is_in('permission_names', resource_info) check_val_type(resource_info['permission_names'], list) check_val_is_in('children', resource_info)