diff --git a/docs/sync.py b/docs/sync.py index c5ff7c2dc..4f7810f71 100644 --- a/docs/sync.py +++ b/docs/sync.py @@ -5,7 +5,7 @@ """ from subprocess import check_call -from urllib.parse import urljoin +from six.moves.urllib.parse import urljoin from shlex import split import argparse diff --git a/magpie/__init__.py b/magpie/__init__.py index 476479186..22a28d9bb 100644 --- a/magpie/__init__.py +++ b/magpie/__init__.py @@ -16,6 +16,8 @@ USER_GROUP = os.getenv('USER_GROUP', 'users') ANONYMOUS_USER = os.getenv('ANONYMOUS_USER', 'anonymous') +ANONYMOUS_GROUP = ANONYMOUS_USER +ANONYMOUS_PASSWORD = ANONYMOUS_USER ADMIN_PERM = 'admin' #ADMIN_PERM = NO_PERMISSION_REQUIRED diff --git a/magpie/api/esgf/esgfopenid.py b/magpie/api/esgf/esgfopenid.py index 65b24de23..511078b18 100644 --- a/magpie/api/esgf/esgfopenid.py +++ b/magpie/api/esgf/esgfopenid.py @@ -7,8 +7,8 @@ This providers are dependent on the |pyopenid|_ package. """ -import urllib2 import ssl +from six.moves.urllib.request import urlopen from authomatic.providers.openid import OpenID from openid.fetchers import setDefaultFetcher, Urllib2Fetcher @@ -21,10 +21,8 @@ class MyFetcher(Urllib2Fetcher): @staticmethod - def _urlopen(req): - return urllib2.urlopen(req, context=ssl._create_unverified_context()) - - urlopen = _urlopen + def urlopen(req): + return urlopen(req, context=ssl._create_unverified_context()) class ESGFOpenID(OpenID): diff --git a/magpie/db.py b/magpie/db.py index 4094b5e38..5b3db701f 100644 --- a/magpie/db.py +++ b/magpie/db.py @@ -1,6 +1,7 @@ from definitions.alembic_definitions import * from definitions.sqlalchemy_definitions import * -import ConfigParser +# noinspection PyCompatibility +import configparser import transaction import models import inspect @@ -9,7 +10,6 @@ import logging logger = logging.getLogger(__name__) - # import or define all models here to ensure they are attached to the # Base.metadata prior to any initialization routines from models import * @@ -78,7 +78,7 @@ def get_db_session_from_config_ini(config_ini_path, ini_main_section_name='app:m def get_settings_from_config_ini(config_ini_path, ini_main_section_name='app:magpie_app'): - parser = ConfigParser.ConfigParser() + parser = configparser.ConfigParser() parser.read([config_ini_path]) settings = dict(parser.items(ini_main_section_name)) return settings diff --git a/magpie/helpers/register_default_users.py b/magpie/helpers/register_default_users.py index 6e70006bf..1c4b2b639 100644 --- a/magpie/helpers/register_default_users.py +++ b/magpie/helpers/register_default_users.py @@ -47,9 +47,9 @@ def register_user_with_group(user_name, group_name, email, password, db_session) def init_anonymous(db_session): register_user_with_group(user_name=ANONYMOUS_USER, - group_name=ANONYMOUS_USER, + group_name=ANONYMOUS_GROUP, email=ANONYMOUS_USER + '@mail.com', - password=ANONYMOUS_USER, + password=ANONYMOUS_PASSWORD, db_session=db_session) diff --git a/magpie/register.py b/magpie/register.py index fdf5bce90..d204b0526 100644 --- a/magpie/register.py +++ b/magpie/register.py @@ -183,9 +183,9 @@ def get_magpie_url(): return 'http://{0}{1}'.format(hostname, magpie_port) -def get_twitcher_protected_service_url(magpie_service_name): +def get_twitcher_protected_service_url(magpie_service_name, hostname=None): try: - hostname = os.getenv('HOSTNAME') + hostname = hostname or os.getenv('HOSTNAME') twitcher_proxy = os.getenv('TWITCHER_PROTECTED_PATH') if hostname is None: raise ValueError("Environment variable was None", 'HOSTNAME') diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py index 004cede94..9adc8789b 100644 --- a/tests/test_magpie_api.py +++ b/tests/test_magpie_api.py @@ -11,10 +11,12 @@ import unittest import pytest import pyramid.testing -import six import yaml +import six +from six.moves.urllib.parse import urlparse from magpie import * from magpie.services import service_type_dict +from magpie.register import get_twitcher_protected_service_url from magpie import magpie from test_utils import * @@ -232,6 +234,14 @@ def test_GetUsers(self): check_val_is_in('anonymous', json_body['user_names']) # anonymous always in users check_val_is_in(self.usr, json_body['user_names']) # current test user in users + @pytest.mark.users + def test_ValidateDefaultUsers(self): + resp = test_request(self.url, 'GET', '/users', headers=self.json_headers, cookies=self.cookies) + json_body = check_response_basic_info(resp, 200) + users = json_body['user_names'] + check_val_is_in(ANONYMOUS_USER, users) + check_val_is_in(ADMIN_USER, users) + @classmethod def check_GetUserResourcesPermissions(cls, user_name): route = '/users/{usr}/resources/{res_id}/permissions'.format(res_id=cls.test_service_resource_id, usr=user_name) @@ -283,6 +293,15 @@ def test_GetUserInheritedResources(self): check_val_type(svc_dict['permission_names'], list) check_val_type(svc_dict['resources'], dict) + @pytest.mark.users + def test_ValidateDefaultGroups(self): + resp = test_request(self.url, 'GET', '/groups', headers=self.json_headers, cookies=self.cookies) + json_body = check_response_basic_info(resp, 200) + groups = json_body['group_names'] + check_val_is_in(ANONYMOUS_GROUP, groups) + check_val_is_in(USER_GROUP, groups) + check_val_is_in(ADMIN_GROUP, groups) + @pytest.mark.groups def test_GetGroupUsers(self): route = '/groups/{grp}/users'.format(grp=ADMIN_GROUP) @@ -437,6 +456,45 @@ def test_PostServiceResources_DirectResource_Conflict(self): isParamValueLiteralUnicode=True, paramCompareExists=True, paramValue=self.test_resource_name, paramName=u'resource_name') + @pytest.mark.services + def test_ValidateDefaultServiceProviders(self): + resp = test_request(self.url, 'GET', '/services', headers=self.json_headers, cookies=self.cookies) + json_body = check_response_basic_info(resp, 200) + + # prepare a flat list of registered services + services_list = list() + for svc_type in json_body['services']: + services_of_type = json_body['services'][svc_type] + services_list.extend(services_of_type.values()) + + # ensure that registered services information are all matching the providers in config file + # ignore registered services not from providers as their are not explicitly required from the config + for svc in services_list: + svc_name = svc['service_name'] + if svc_name in self.test_services_info: + check_val_equal(svc['service_type'], self.test_services_info[svc_name]['type']) + hostname = urlparse(self.url).hostname + check_val_equal(svc['public_url'], get_twitcher_protected_service_url(svc_name, hostname=hostname)) + svc_url = self.test_services_info[svc_name]['url'].replace('${HOSTNAME}', hostname) + check_val_equal(svc['service_url'], svc_url) + + # ensure that no providers are missing from registered services + registered_svc_names = [svc['service_name'] for svc in services_list] + for svc_name in self.test_services_info: + check_val_is_in(svc_name, registered_svc_names) + + # ensure that 'getcapabilities' permission is given to anonymous for applicable services + services_list_getcap = [svc for svc in services_list if 'getcapabilities' in svc['permission_names']] + route = '/users/{usr}/services'.format(usr=ANONYMOUS_USER) + resp = test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) + json_body = check_response_basic_info(resp, 200) + services = json_body['services'] + for svc in services_list_getcap: + svc_name = svc['service_name'] + svc_type = svc['service_type'] + check_val_is_in(svc_name, services[svc_type]) + check_val_is_in('getcapabilities', services[svc_type][svc_name]['permission_names']) + @pytest.mark.resources def test_PostResources_DirectServiceResource(self): service_info = self.setup_GetExistingTestServiceInfo() diff --git a/travis_pypi_setup.py b/travis_pypi_setup.py index d2cc617b7..092de8c60 100644 --- a/travis_pypi_setup.py +++ b/travis_pypi_setup.py @@ -13,12 +13,7 @@ from cryptography.hazmat.primitives.serialization import load_pem_public_key from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 - - -try: - from urllib import urlopen -except: - from urllib.request import urlopen +from six.moves.urllib.request import urlopen GITHUB_REPO = 'fderue/magpie'