valid providers+getcapabilities (issue #74, PR #72) + adjust backport…

… imports

docs/sync.py

@@ -5,7 +5,7 @@
 """
 
 from subprocess import check_call
-from urllib.parse import urljoin
+from six.moves.urllib.parse import urljoin
 from shlex import split
 import argparse
 

magpie/__init__.py

@@ -16,6 +16,8 @@
 USER_GROUP = os.getenv('USER_GROUP', 'users')
 
 ANONYMOUS_USER = os.getenv('ANONYMOUS_USER', 'anonymous')
+ANONYMOUS_GROUP = ANONYMOUS_USER
+ANONYMOUS_PASSWORD = ANONYMOUS_USER
 
 ADMIN_PERM = 'admin'
 #ADMIN_PERM = NO_PERMISSION_REQUIRED

magpie/api/esgf/esgfopenid.py

@@ -7,8 +7,8 @@
     This providers are dependent on the |pyopenid|_ package.
 """
 
-import urllib2
 import ssl
+from six.moves.urllib.request import urlopen
 from authomatic.providers.openid import OpenID
 from openid.fetchers import setDefaultFetcher, Urllib2Fetcher
 
@@ -21,10 +21,8 @@
 
 class MyFetcher(Urllib2Fetcher):
     @staticmethod
-    def _urlopen(req):
-        return urllib2.urlopen(req, context=ssl._create_unverified_context())
-
-    urlopen = _urlopen
+    def urlopen(req):
+        return urlopen(req, context=ssl._create_unverified_context())
 
 
 class ESGFOpenID(OpenID):

magpie/db.py

@@ -1,6 +1,7 @@
 from definitions.alembic_definitions import *
 from definitions.sqlalchemy_definitions import *
-import ConfigParser
+# noinspection PyCompatibility
+import configparser
 import transaction
 import models
 import inspect
@@ -9,7 +10,6 @@
 import logging
 logger = logging.getLogger(__name__)
 
-
 # import or define all models here to ensure they are attached to the
 # Base.metadata prior to any initialization routines
 from models import *
@@ -78,7 +78,7 @@ def get_db_session_from_config_ini(config_ini_path, ini_main_section_name='app:m
 
 
 def get_settings_from_config_ini(config_ini_path, ini_main_section_name='app:magpie_app'):
-    parser = ConfigParser.ConfigParser()
+    parser = configparser.ConfigParser()
     parser.read([config_ini_path])
     settings = dict(parser.items(ini_main_section_name))
     return settings

magpie/helpers/register_default_users.py

@@ -47,9 +47,9 @@ def register_user_with_group(user_name, group_name, email, password, db_session)
 
 def init_anonymous(db_session):
     register_user_with_group(user_name=ANONYMOUS_USER,
-                             group_name=ANONYMOUS_USER,
+                             group_name=ANONYMOUS_GROUP,
                              email=ANONYMOUS_USER + '@mail.com',
-                             password=ANONYMOUS_USER,
+                             password=ANONYMOUS_PASSWORD,
                              db_session=db_session)
 
 

magpie/register.py

@@ -183,9 +183,9 @@ def get_magpie_url():
     return 'http://{0}{1}'.format(hostname, magpie_port)
 
 
-def get_twitcher_protected_service_url(magpie_service_name):
+def get_twitcher_protected_service_url(magpie_service_name, hostname=None):
     try:
-        hostname = os.getenv('HOSTNAME')
+        hostname = hostname or os.getenv('HOSTNAME')
         twitcher_proxy = os.getenv('TWITCHER_PROTECTED_PATH')
         if hostname is None:
             raise ValueError("Environment variable was None", 'HOSTNAME')

tests/test_magpie_api.py

@@ -11,10 +11,12 @@
 import unittest
 import pytest
 import pyramid.testing
-import six
 import yaml
+import six
+from six.moves.urllib.parse import urlparse
 from magpie import *
 from magpie.services import service_type_dict
+from magpie.register import get_twitcher_protected_service_url
 from magpie import magpie
 from test_utils import *
 
@@ -232,6 +234,14 @@ def test_GetUsers(self):
         check_val_is_in('anonymous', json_body['user_names'])       # anonymous always in users
         check_val_is_in(self.usr, json_body['user_names'])          # current test user in users
 
+    @pytest.mark.users
+    def test_ValidateDefaultUsers(self):
+        resp = test_request(self.url, 'GET', '/users', headers=self.json_headers, cookies=self.cookies)
+        json_body = check_response_basic_info(resp, 200)
+        users = json_body['user_names']
+        check_val_is_in(ANONYMOUS_USER, users)
+        check_val_is_in(ADMIN_USER, users)
+
     @classmethod
     def check_GetUserResourcesPermissions(cls, user_name):
         route = '/users/{usr}/resources/{res_id}/permissions'.format(res_id=cls.test_service_resource_id, usr=user_name)
@@ -283,6 +293,15 @@ def test_GetUserInheritedResources(self):
                 check_val_type(svc_dict['permission_names'], list)
                 check_val_type(svc_dict['resources'], dict)
 
+    @pytest.mark.users
+    def test_ValidateDefaultGroups(self):
+        resp = test_request(self.url, 'GET', '/groups', headers=self.json_headers, cookies=self.cookies)
+        json_body = check_response_basic_info(resp, 200)
+        groups = json_body['group_names']
+        check_val_is_in(ANONYMOUS_GROUP, groups)
+        check_val_is_in(USER_GROUP, groups)
+        check_val_is_in(ADMIN_GROUP, groups)
+
     @pytest.mark.groups
     def test_GetGroupUsers(self):
         route = '/groups/{grp}/users'.format(grp=ADMIN_GROUP)
@@ -437,6 +456,45 @@ def test_PostServiceResources_DirectResource_Conflict(self):
                                     isParamValueLiteralUnicode=True, paramCompareExists=True,
                                     paramValue=self.test_resource_name, paramName=u'resource_name')
 
+    @pytest.mark.services
+    def test_ValidateDefaultServiceProviders(self):
+        resp = test_request(self.url, 'GET', '/services', headers=self.json_headers, cookies=self.cookies)
+        json_body = check_response_basic_info(resp, 200)
+
+        # prepare a flat list of registered services
+        services_list = list()
+        for svc_type in json_body['services']:
+            services_of_type = json_body['services'][svc_type]
+            services_list.extend(services_of_type.values())
+
+        # ensure that registered services information are all matching the providers in config file
+        # ignore registered services not from providers as their are not explicitly required from the config
+        for svc in services_list:
+            svc_name = svc['service_name']
+            if svc_name in self.test_services_info:
+                check_val_equal(svc['service_type'], self.test_services_info[svc_name]['type'])
+                hostname = urlparse(self.url).hostname
+                check_val_equal(svc['public_url'], get_twitcher_protected_service_url(svc_name, hostname=hostname))
+                svc_url = self.test_services_info[svc_name]['url'].replace('${HOSTNAME}', hostname)
+                check_val_equal(svc['service_url'], svc_url)
+
+        # ensure that no providers are missing from registered services
+        registered_svc_names = [svc['service_name'] for svc in services_list]
+        for svc_name in self.test_services_info:
+            check_val_is_in(svc_name, registered_svc_names)
+
+        # ensure that 'getcapabilities' permission is given to anonymous for applicable services
+        services_list_getcap = [svc for svc in services_list if 'getcapabilities' in svc['permission_names']]
+        route = '/users/{usr}/services'.format(usr=ANONYMOUS_USER)
+        resp = test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies)
+        json_body = check_response_basic_info(resp, 200)
+        services = json_body['services']
+        for svc in services_list_getcap:
+            svc_name = svc['service_name']
+            svc_type = svc['service_type']
+            check_val_is_in(svc_name, services[svc_type])
+            check_val_is_in('getcapabilities', services[svc_type][svc_name]['permission_names'])
+
     @pytest.mark.resources
     def test_PostResources_DirectServiceResource(self):
         service_info = self.setup_GetExistingTestServiceInfo()

travis_pypi_setup.py

@@ -13,12 +13,7 @@
 from cryptography.hazmat.primitives.serialization import load_pem_public_key
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15
-
-
-try:
-    from urllib import urlopen
-except:
-    from urllib.request import urlopen
+from six.moves.urllib.request import urlopen
 
 
 GITHUB_REPO = 'fderue/magpie'