diff --git a/nodeshot/interop/sync/settings.py b/nodeshot/interop/sync/settings.py index c7c5a2db..c60401cb 100755 --- a/nodeshot/interop/sync/settings.py +++ b/nodeshot/interop/sync/settings.py @@ -5,20 +5,7 @@ ('nodeshot.interop.sync.synchronizers.Nodeshot', 'Nodeshot (RESTful translator)'), ('nodeshot.interop.sync.synchronizers.GeoJson', 'GeoJSON (periodic sync)'), ('nodeshot.interop.sync.synchronizers.GeoRss', 'GeoRSS (periodic sync)'), - ('nodeshot.interop.sync.synchronizers.OpenWisp', 'OpenWisp (periodic sync)'), - ('nodeshot.interop.sync.synchronizers.OpenWispCitySdkTourism', 'OpenWISP CitySDK Tourism (periodic sync)'), - ('nodeshot.interop.sync.synchronizers.ProvinciaWifi', 'Provincia WiFi (periodic sync)'), - ('nodeshot.interop.sync.synchronizers.ProvinciaWifiCitySdkTourism', 'Provincia WiFi CitySDK Tourism (periodic sync)'), - ('nodeshot.interop.sync.synchronizers.ProvinciaWifiCitySdkMobility', 'Provincia WiFi CitySDK Mobility (periodic sync)'), - ('nodeshot.interop.sync.synchronizers.CitySdkMobility', 'CitySDK Mobility (event driven)'), - ('nodeshot.interop.sync.synchronizers.CitySdkTourism', 'CitySDK Tourism (event driven)'), - ('nodeshot.interop.sync.synchronizers.GeoJsonCitySdkMobility', 'GeoJSON CitySDK Mobility (periodic sync)'), - ('nodeshot.interop.sync.synchronizers.GeoJsonCitySdkTourism', 'GeoJSON CitySDK Tourism (periodic sync)'), - ('nodeshot.interop.sync.synchronizers.OpenLabor', 'OpenLabor (RESTful translator + event driven)'), - ('nodeshot.interop.sync.synchronizers.ProvinceRomeTraffic', 'Province of Rome Traffic') + ('nodeshot.interop.sync.synchronizers.OpenWisp', 'OpenWisp (periodic sync)') ] SYNCHRONIZERS = DEFAULT_SYNCHRONIZERS + getattr(settings, 'NODESHOT_SYNCHRONIZERS', []) - -CITYSDK_TOURISM_TEST_CONFIG = getattr(settings, 'NODESHOT_CITYSDK_TOURISM_TEST_CONFIG', False) -CITYSDK_MOBILITY_TEST_CONFIG = getattr(settings, 'NODESHOT_CITYSDK_MOBILITY_TEST_CONFIG', False) diff --git a/nodeshot/interop/sync/synchronizers/__init__.py b/nodeshot/interop/sync/synchronizers/__init__.py index 3c1c87be..ab0e4cd5 100755 --- a/nodeshot/interop/sync/synchronizers/__init__.py +++ b/nodeshot/interop/sync/synchronizers/__init__.py @@ -11,29 +11,3 @@ 'GeoRss', 'OpenWisp' ] - -# --- citysdk (will be moved in a separate package soon) --- # - -from .citysdk_mobility import CitySdkMobility -from .citysdk_tourism import CitySdkTourism -from .geojson_citysdk_mobility import GeoJsonCitySdkMobility -from .geojson_citysdk_tourism import GeoJsonCitySdkTourism -from .openwisp_citysdk_tourism import OpenWispCitySdkTourism -from .provinciawifi import ProvinciaWifi -from .provinciawifi_citysdk_mobility import ProvinciaWifiCitySdkMobility -from .provinciawifi_citysdk_tourism import ProvinciaWifiCitySdkTourism -from .provincerometraffic import ProvinceRomeTraffic -from .openlabor import OpenLabor - -__all__ = __all__ + [ - 'CitySdkMobility', - 'CitySdkTourism', - 'GeoJsonCitySdkMobility', - 'GeoJsonCitySdkTourism', - 'OpenWispCitySdkTourism', - 'ProvinciaWifi', - 'ProvinciaWifiCitySdkMobility', - 'ProvinciaWifiCitySdkTourism', - 'ProvinceRomeTraffic', - 'OpenLabor', -] diff --git a/nodeshot/interop/sync/synchronizers/citysdk_mobility.py b/nodeshot/interop/sync/synchronizers/citysdk_mobility.py deleted file mode 100755 index c47e0918..00000000 --- a/nodeshot/interop/sync/synchronizers/citysdk_mobility.py +++ /dev/null @@ -1,266 +0,0 @@ -from __future__ import absolute_import - -import requests -import simplejson as json - -from django.core.exceptions import ImproperlyConfigured -from django.utils.translation import ugettext_lazy as _ - -from nodeshot.interop.sync.synchronizers.base import BaseSynchronizer, GenericGisSynchronizer -from nodeshot.interop.sync.models import NodeExternal - -from celery.utils.log import get_logger -logger = get_logger(__name__) - - -class CitySdkMobilityMixin(object): - """ - CitySdkMobility synchronizer mixin - Provides methods to perform following operations: - * perform authentication into citysdk mobility API - * add new records - * change existing records - * delete existing records - """ - SCHEMA = [ - { - 'name': 'citysdk_url', - 'class': 'URLField', - 'kwargs': { - 'help_text': _('CitySDK Mobility API URL') - } - }, - { - 'name': 'citysdk_username', - 'class': 'CharField', - 'kwargs': { - 'max_length': 128, - 'help_text': _('Username of user who has write permission') - } - }, - { - 'name': 'citysdk_password', - 'class': 'CharField', - 'kwargs': { - 'max_length': 128, - 'help_text': _('Password of user who has write permission'), - } - } - ] - - def __init__(self, *args, **kwargs): - super(CitySdkMobilityMixin, self).__init__(*args, **kwargs) - self._init_config() - - def _init_config(self): - """ Init required attributes if necessary (for internal use only) """ - try: - citysdk_url = self.config['citysdk_url'] - except KeyError as e: - raise ImproperlyConfigured(e) - # add trailing slash if missing - if citysdk_url.endswith('/'): - self.citysdk_url = citysdk_url - else: - self.citysdk_url = '%s/' % citysdk_url - - def clean(self): - """ - Custom Validation, is executed by ExternalLayer.clean(); - These validation methods will be called before saving an object into the DB - * verify authentication works - """ - session = self.get_session() - self.release_session(session) - - def get_session(self): - """ authenticate into the CitySDK Mobility API and return session token """ - self.verbose('Authenticating to CitySDK') - logger.info('== Authenticating to CitySDK ==') - - authentication_url = '%sget_session?e=%s&p=%s' % ( - self.citysdk_url, - self.config['citysdk_username'], - self.config['citysdk_password'] - ) - - try: - response = requests.get( - authentication_url, - verify=self.verify_ssl - ) - except Exception as e: - message = 'API Authentication Error: "%s"' % e - logger.error(message) - raise ImproperlyConfigured(message) - - if response.status_code != 200: - try: - message = 'API Authentication Error: "%s"' % json.loads(response.content)['message'] - except Exception: - message = 'API Authentication Error: "%s"' % response.content - logger.error(message) - raise ImproperlyConfigured(message) - - # store session token - # will be valid for 1 minute after each request - session = json.loads(response.content)['results'][0] - - return session - - def release_session(self, session): - release_url = '%srelease_session' % self.citysdk_url - response = requests.get( - release_url, - verify=self.verify_ssl, - headers={ 'Content-type': 'application/json', 'X-Auth': session } - ) - - if response.status_code == 200: - return True - else: - return False - - def convert_format(self, node, create_type="create"): - """ Prepares the JSON that will be sent to the CitySDK API """ - data = node.data or {} - - if node.status: data['status'] = node.status.slug - if node.description: data['description'] = node.description - if node.address: data['address'] = node.address - if node.elev: data['elevation'] = node.elev - if node.user: data['owner'] = node.user.get_full_name() - - result = { - "create": { - "params": { - "create_type": create_type, - "srid": 4326 - } - }, - "nodes": [ - { - "name": node.name, - "geom" : json.loads(node.geometry.json), - "data" : data - } - ] - } - - if create_type == "create": - result['nodes'][0]['id'] = node.slug - elif create_type == "update": - result['nodes'][0]['cdk_id'] = node.external.external_id - - return result - - def add(self, node, authenticate=True): - """ Add a new record into CitySDK db """ - session = self.get_session() - - citysdk_record = self.convert_format(node) - citysdk_api_url = '%snodes/%s' % (self.citysdk_url, self.config['citysdk_layer']) - - # citysdk sync - response = requests.put( - citysdk_api_url, - data=json.dumps(citysdk_record), - verify=self.verify_ssl, - headers={ 'Content-type': 'application/json', 'X-Auth': session } - ) - - self.release_session(session) - - if response.status_code != 200: - message = 'ERROR while creating "%s". Response: %s' % (node.name, response.content) - logger.error(message) - return False - else: - ext = NodeExternal() - ext.node = node - ext.external_id = '{layer}.{slug}'.format( - layer=self.config['citysdk_layer'], - slug=node.slug.replace('-', '.') - ) - ext.save() - - try: - data = json.loads(response.content) - except json.JSONDecodeError as e: - logger.error('== ERROR: JSONDecodeError %s ==' % e) - return False - - message = 'New record "%s" saved in CitySDK through the HTTP API"' % node.name - self.verbose(message) - logger.info(message) - - return True - - def change(self, node, authenticate=True): - """ Add a new record into CitySDK db """ - session = self.get_session() - - citysdk_record = self.convert_format(node, create_type='update') - citysdk_api_url = '%snodes/%s' % (self.citysdk_url, self.config['citysdk_layer']) - - # citysdk sync - response = requests.put( - citysdk_api_url, - data=json.dumps(citysdk_record), - verify=self.verify_ssl, - headers={ 'Content-type': 'application/json', 'X-Auth': session } - ) - - self.release_session(session) - - if response.status_code != 200: - message = 'ERROR while updating record "%s" through CitySDK API\n%s' % (node.name, response.content) - logger.error(message) - return False - - try: - json.loads(response.content) - except json.JSONDecodeError as e: - logger.error(e) - return False - - message = 'Updated record "%s" through the CitySDK HTTP API' % node.name - self.verbose(message) - logger.info(message) - - return True - - def delete(self, external_id, authenticate=True): - """ Delete record from CitySDK db """ - session = self.get_session() - - citysdk_api_url = '%s%s/%s' % ( - self.citysdk_url, - external_id, - self.config['citysdk_layer'] - ) - - response = requests.delete( - citysdk_api_url, - params={ 'delete_node': True }, - verify=self.verify_ssl, - headers={ 'Content-type': 'application/json', 'X-Auth': session } - ) - - self.release_session(session) - - if response.status_code != 200: - message = 'Failed to delete a record through the CitySDK HTTP API' - self.verbose(message) - logger.info(message) - return False - - message = 'Deleted a record through the CitySDK HTTP API' - self.verbose(message) - logger.info(message) - - return True - - -class CitySdkMobility(CitySdkMobilityMixin, BaseSynchronizer): - SCHEMA = CitySdkMobilityMixin.SCHEMA + [GenericGisSynchronizer.SCHEMA[1]] diff --git a/nodeshot/interop/sync/synchronizers/citysdk_tourism.py b/nodeshot/interop/sync/synchronizers/citysdk_tourism.py deleted file mode 100755 index 40df884a..00000000 --- a/nodeshot/interop/sync/synchronizers/citysdk_tourism.py +++ /dev/null @@ -1,400 +0,0 @@ -from __future__ import absolute_import - -import requests -import simplejson as json - -from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist -from django.utils.translation import ugettext_lazy as _ - -from nodeshot.interop.sync.synchronizers.base import BaseSynchronizer, GenericGisSynchronizer -from nodeshot.interop.sync.models import NodeExternal - -from celery.utils.log import get_logger -logger = get_logger(__name__) - - -class CitySdkTourismMixin(object): - """ - CitySdkTourismMixin synchronizer mixin - Provides methods to perform following operations: - * perform authentication into citysdk API - * create or find a category - * add new records - * change existing records - * delete existing records - """ - SCHEMA = [ - { - 'name': 'citysdk_url', - 'class': 'URLField', - 'kwargs': { - 'help_text': _('CitySDK Tourism API URL') - } - }, - { - 'name': 'citysdk_category', - 'class': 'CharField', - 'kwargs': { - 'max_length': 128, - 'help_text': _('CitySDK Tourism API category name') - } - }, - { - 'name': 'citysdk_category_id', - 'class': 'CharField', - 'kwargs': { - 'max_length': 128, - 'blank': True, - 'help_text': _('CitySDK Tourism API category ID, if left blank a new category will be created') - } - }, - { - 'name': 'citysdk_username', - 'class': 'CharField', - 'kwargs': { - 'max_length': 128, - 'help_text': _('Username of user who has write permission') - } - }, - { - 'name': 'citysdk_password', - 'class': 'CharField', - 'kwargs': { - 'max_length': 128, - 'help_text': _('Password of user who has write permission'), - } - }, - { - 'name': 'citysdk_lang', - 'class': 'CharField', - 'kwargs': { - 'max_length': 8, - 'help_text': _('language code'), - } - }, - { - 'name': 'citysdk_term', - 'class': 'CharField', - 'kwargs': { - 'max_length': 16, - 'help_text': _('term (eg: center)'), - } - }, - { - 'name': 'verify_ssl', - 'class': 'BooleanField', - 'kwargs': { 'default': True } - } - ] - - _persisted_cookies = None - - def __init__(self, *args, **kwargs): - super(CitySdkTourismMixin, self).__init__(*args, **kwargs) - try: - self._init_config() - except KeyError as e: - raise ImproperlyConfigured(e) - - def _init_config(self): - """ Init required attributes if necessary (for internal use only) """ - if getattr(self, 'citysdk_categories_url', None) is None: - self.citysdk_resource_url = '%s%ss/' % (self.config['citysdk_url'], self.config['citysdk_type']) - self.citysdk_categories_url = '%scategories?list=%s&limit=0&format=json' % (self.config['citysdk_url'], self.config['citysdk_type']) - self.citysdk_category_id = self.config.get('citysdk_category_id') - - def clean(self): - """ - Custom Validation, is executed by ExternalLayer.clean(); - These validation methods will be called before saving an object into the DB - * verify authentication works - """ - self.authenticate() - - def after_external_layer_saved(self, layer_config=None): - """ - Method that will be called after the external layer has been saved - """ - self.find_citysdk_category(layer_config) - - def before_start(self, *args, **kwargs): - """ before the import starts do authentication (1 time only) """ - # first time - self.authenticate(force_http_request=True) - # store cookies in a string - self._persisted_cookies = self.cookies - self.layer.external.config = self.config - self.layer.external.save(after_save=False) - - def after_complete(self, *args, **kwargs): - """ clear self._persisted_cookies """ - self._persisted_cookies = None - - def authenticate(self, force_http_request=False): - """ authenticate into the CitySDK API if necessary """ - # if session cookie is persisted in memory no need to reauthenticate - # if force_http_request is True do HTTP request anyway - if force_http_request is False and self._persisted_cookies is not None: - self.cookies = self._persisted_cookies - return True - - self.verbose('Authenticating to CitySDK') - logger.info('== Authenticating to CitySDK ==') - - citysdk_auth_url = '%sauth?format=json' % self.config['citysdk_url'] - - response = requests.post(citysdk_auth_url, { - 'username': self.config['citysdk_username'], - 'password': self.config['citysdk_password'], - }) - - if response.status_code != 200: - message = 'API Authentication Error: "%s"' % json.loads(response.content)['ResponseStatus']['Message'] - logger.error(message) - raise ImproperlyConfigured(message) - - self.cookies = response.cookies.get_dict() - - return True - - def find_citysdk_category(self, layer_config=None): - """ - Automatically finds the citysdk category ID - * ensure the ID specified in config is correct otherwise auto-correct - * create category if it does not exist - * if category exist find the ID - * store category ID in config - """ - logger.info('== Going to find CitySDK category ID ==') - - self._init_config() - - if layer_config: - self.config = layer_config - - citysdk_category_id = self.config.get('citysdk_category_id', False) - self.authenticate() - response = requests.get(self.citysdk_categories_url, cookies=self.cookies) - - # do we already have the category id in the db config? - # And is the category present in the API response? - if citysdk_category_id is not False and citysdk_category_id in response.content: - message = 'category with ID "%s" already present in config' % citysdk_category_id - self.verbose(message) - logger.info(message) - - # exit here - return False - # if not go and find it! - else: - # category does not exist, create it - if self.config['citysdk_category'] not in response.content: - - category = { - "list": self.config['citysdk_type'], # poi, event, route - "category": { - "label": [ - { - "lang": self.config['citysdk_lang'], - "term": "primary", - "value": self.config['citysdk_category'] - } - ], - "lang": self.config['citysdk_lang'], - "term": "category", - "value": self.config['citysdk_category'] - } - } - - self.verbose('Creating new category in CitySDK DB') - logger.info('== Creating new category in CitySDK DB ==') - # put to create - response = requests.put(self.citysdk_categories_url, data=json.dumps(category), - headers={'content-type': 'application/json'}, - cookies=self.cookies) - - # raise exception if something has gone wrong - if response.status_code is not 200: - message = 'ERROR: %s' % response.content - self.verbose(message) - logger.error(message) - raise ImproperlyConfigured(response.content) - - # get ID - citysdk_category_id = json.loads(response.content) - - message = 'category with ID "%s" has been created' % citysdk_category_id - self.verbose(message) - logger.info(message) - # category already exists, find ID - else: - categories = json.loads(response.content)['categories'] - - for category in categories: - if category['value'] == self.config['citysdk_category']: - citysdk_category_id = category['id'] - - # raise exception if not found - should not happen but who knows - if citysdk_category_id is None: - message = 'Category was thought to be there but could not be found!' - logger.info(message) - raise ImproperlyConfigured(message) - - # now store ID in the database both in case category has been created or not - self.config['citysdk_category_id'] = citysdk_category_id - self.layer.external.config = self.config - self.layer.external.save() - # verbose output - message = 'category with ID "%s" has been stored in config' % citysdk_category_id - self.verbose(message) - logger.info(message) - - def convert_format(self, node): - """ Prepares the JSON that will be sent to the CitySDK API """ - - # determine description or fill some hopefully useful value - if not node.description.strip(): - description = '%s in %s' % (node.name, node.address) - else: - description = node.description - - return { - self.config['citysdk_type'] :{ - "location":{ - "point":[ - { - "Point":{ - "posList":"%s %s" % (float(node.point.coords[1]), float(node.point.coords[0])), - "srsName":"http://www.opengis.net/def/crs/EPSG/0/4326" - }, - "term": self.config['citysdk_term'] - } - ], - "address": { - "value":"""BEGIN:VCARD -N:;%s;;;; -ADR;INTL;PARCEL;WORK:;;%s;%s;%s;;%s -END:VCARD""" % ( - node.name, - node.data['address'], - node.data['city'], - node.data['province'], - node.data['country'], - ), - "type": "text/vcard" - }, - }, - "label":[ - { - "term": "primary", - "value": node.name - }, - ], - "description":[ - { - "value": description, - "lang": self.config['citysdk_lang'] - }, - ], - "category":[ - { - "id": self.citysdk_category_id - } - ], - "base": self.citysdk_resource_url, - "lang": self.config['citysdk_lang'], - "created": unicode(node.added), - "author":{ - "term": "primary", - "value": self.layer.organization - }, - "license":{ - "term": "primary", - "value": "open-data" - } - } - } - - def add(self, node, authenticate=True): - """ Add a new record into CitySDK db """ - if authenticate: - self.authenticate() - - citysdk_record = self.convert_format(node) - - # citysdk sync - response = requests.put(self.citysdk_resource_url, data=json.dumps(citysdk_record), - headers={ 'content-type': 'application/json' }, cookies=self.cookies) - - if response.status_code != 200: - message = 'ERROR while creating "%s". Response: %s' % (node.name, response.content) - logger.error(message) - return False - - try: - data = json.loads(response.content) - except json.JSONDecodeError as e: - logger.error('== ERROR: JSONDecodeError %s ==' % e) - return False - - external = NodeExternal.objects.create(node=node, external_id=data['id']) - message = 'New record "%s" saved in CitySDK through the HTTP API"' % node.name - self.verbose(message) - logger.info(message) - - return True - - def change(self, node, authenticate=True): - """ Edit existing record in CitySDK db """ - if authenticate: - self.authenticate() - - citysdk_record = self.convert_format(node) - - # citysdk sync - try: - citysdk_record['poi']['id'] = node.external.external_id - response = requests.post( - self.citysdk_resource_url, - data=json.dumps(citysdk_record), - headers={ 'content-type': 'application/json' }, - cookies=self.cookies) - - if response.status_code == 200: - message = 'Updated record "%s" through the CitySDK HTTP API' % node.name - self.verbose(message) - logger.info(message) - else: - message = 'ERROR while updating record "%s" through CitySDK API\n%s' % (node.name, response.content) - logger.error(message) - raise ImproperlyConfigured(message) - - return True - - # in case external_id is not in the local DB we need to create instead - except ObjectDoesNotExist: - return self.add(node, authenticate=False) - - def delete(self, external_id, authenticate=True): - """ Delete record from CitySDK db """ - if authenticate: - self.authenticate() - - response = requests.delete(self.citysdk_resource_url, data='{"id":"%s"}' % external_id, - headers={ 'content-type': 'application/json' }, cookies=self.cookies) - - if response.status_code != 200: - message = 'Failed to delete a record through the CitySDK HTTP API' - self.verbose(message) - logger.info(message) - return False - - message = 'Deleted a record through the CitySDK HTTP API' - self.verbose(message) - logger.info(message) - - return True - - -class CitySdkTourism(CitySdkTourismMixin, BaseSynchronizer): - SCHEMA = CitySdkTourismMixin.SCHEMA + [GenericGisSynchronizer.SCHEMA[1]] diff --git a/nodeshot/interop/sync/synchronizers/geojson_citysdk_mobility.py b/nodeshot/interop/sync/synchronizers/geojson_citysdk_mobility.py deleted file mode 100755 index 70ed819c..00000000 --- a/nodeshot/interop/sync/synchronizers/geojson_citysdk_mobility.py +++ /dev/null @@ -1,9 +0,0 @@ -from __future__ import absolute_import - -from nodeshot.interop.sync.synchronizers import GeoJson -from .citysdk_mobility import CitySdkMobilityMixin - - -class GeoJsonCitySdkMobility(CitySdkMobilityMixin, GeoJson): - """ Import GeoJson and sync CitySDK """ - SCHEMA = CitySdkMobilityMixin.SCHEMA + GeoJson.SCHEMA diff --git a/nodeshot/interop/sync/synchronizers/geojson_citysdk_tourism.py b/nodeshot/interop/sync/synchronizers/geojson_citysdk_tourism.py deleted file mode 100755 index be0bac2a..00000000 --- a/nodeshot/interop/sync/synchronizers/geojson_citysdk_tourism.py +++ /dev/null @@ -1,70 +0,0 @@ -from __future__ import absolute_import - -from nodeshot.interop.sync.synchronizers import GeoJson -from .citysdk_tourism import CitySdkTourismMixin - - -class GeoJsonCitySdkTourism(CitySdkTourismMixin, GeoJson): - """ Import GeoJson and sync CitySDK tourism API """ - SCHEMA = CitySdkTourismMixin.SCHEMA + GeoJson.SCHEMA - - def convert_format(self, node): - # determine description or fill some hopefully useful value - if node.description.strip() == '': - description = node.name - else: - description = node.description - - return { - self.config['citysdk_type'] :{ - "location":{ - "point":[ - { - "Point":{ - "posList":"%s %s" % (float(node.point.coords[1]), float(node.point.coords[0])), - "srsName":"http://www.opengis.net/def/crs/EPSG/0/4326" - }, - "term": self.config['citysdk_term'] - } - ], - "address": { - "value":"""BEGIN:VCARD -N:;%s;;;; -ADR;INTL;PARCEL;WORK:;;%s; -END:VCARD""" % ( - node.name, - description - ), - "type": "text/vcard" - }, - }, - "label":[ - { - "term": "primary", - "value": node.name - }, - ], - "description":[ - { - "value": description, - "lang": self.config['citysdk_lang'] - }, - ], - "category":[ - { - "id": self.citysdk_category_id - } - ], - "base": self.citysdk_resource_url, - "lang": self.config['citysdk_lang'], - "created": unicode(node.added), - "author":{ - "term": "primary", - "value": self.layer.organization - }, - "license":{ - "term": "primary", - "value": "open-data" - } - } - } diff --git a/nodeshot/interop/sync/synchronizers/openlabor.py b/nodeshot/interop/sync/synchronizers/openlabor.py deleted file mode 100755 index 080993be..00000000 --- a/nodeshot/interop/sync/synchronizers/openlabor.py +++ /dev/null @@ -1,281 +0,0 @@ -from __future__ import absolute_import - -import requests -import simplejson as json -from datetime import datetime - -from django.utils.translation import ugettext_lazy as _ -from django.contrib.gis.geos import Point -from django.core.cache import cache -from django.core.exceptions import ImproperlyConfigured - -from rest_framework import serializers -from rest_framework_gis import serializers as geoserializers - -from nodeshot.core.base.utils import now_after -from nodeshot.core.nodes.models import Node, Status -from nodeshot.core.nodes.serializers import NodeListSerializer -from nodeshot.interop.sync.models import NodeExternal -from nodeshot.interop.sync.synchronizers.base import BaseSynchronizer, GenericGisSynchronizer - -from celery.utils.log import get_logger -logger = get_logger(__name__) - - -class OpenLaborSerializer(NodeListSerializer): - """ node list """ - layer_name = serializers.Field('layer_name') - details = serializers.SerializerMethodField('get_details') - - def get_details(self, obj): - """ return job detail """ - return obj.data.get('more_info', None) - - -class OpenLaborGeoSerializer(geoserializers.GeoFeatureModelSerializer, OpenLaborSerializer): - pass - - -class OpenLabor(BaseSynchronizer): - """ - OpenLabor RESTful translator - """ - SCHEMA = [ - { - 'name': 'open311_url', - 'class': 'URLField' - }, - { - 'name': 'service_code_get', - 'class': 'CharField', - 'kwargs': { 'max_length': 16 } - }, - { - 'name': 'service_code_post', - 'class': 'CharField', - 'kwargs': { 'max_length': 16 } - }, - { - 'name': 'default_status', - 'class': 'CharField', - 'kwargs': { 'max_length': 255, 'blank': True } - }, - GenericGisSynchronizer.SCHEMA[1] # verify_ssl - ] - - def __init__(self, *args, **kwargs): - super(OpenLabor, self).__init__(*args, **kwargs) - self._init_config() - - def _init_config(self): - """ init config attributes """ - try: - url = self.config['open311_url'] - service_code = self.config['service_code_get'] - except KeyError as e: - raise ImproperlyConfigured(e) - - # add trailing slash if missing - if self.config['open311_url'].endswith('/'): - self.open311_url = url - else: - self.open311_url = '%s/' % url - - # url from where to fetch nodes - self.get_url = '%srequests.json?service_code=%s' % ( - self.open311_url, - service_code - ) - - # url for POST - self.post_url = '%srequests.json' % self.open311_url - # api_key - self.api_key = self.config.get('api_key', '') - - # default status - try: - self.default_status = Status.objects.get(slug=self.config.get('default_status', '')) - except Status.DoesNotExist: - self.default_status = None - - def to_nodeshot(self, node): - """ - takes an openlabor structure as input - and returns a dictionary representing a nodeshot node model instance - """ - latitude = node.get('latitude', False) - longitude = node.get('longitude', False) - - address = node.get('address', None) - city = node.get('city', None) - cap = node.get('CAP', None) - - full_address = address - # if city info available insert it in full_address - if city: - full_address = '%s, %s' % (full_address, city) - # same for cap - if cap: - full_address = '%s - %s' % (full_address, cap) - - additional_data = { - "professional_profile": node.get('professionalProfile', None), - "qualification_required": node.get('qualificationRequired', None), - "phone": node.get('phone', None), - "fax": node.get('fax', None), - "email": node.get('email', None), - "address": address, - "city": city, - "cap": cap, - "more_info": node.get('linkMoreInfo', None) - } - - added = node.get('dateInsert', None) - # convert timestamp to date - if added: - added = datetime.utcfromtimestamp(int(added)) - - return { - "name": node.get('position', ''), - "slug": node.get('idJobOriginal', None), - "layer_id": self.layer.id, - "user": None, - "status": self.default_status, - "geometry": Point(float(longitude), float(latitude)), - "elev": None, - "address": full_address, - "description": node.get('notes', ''), - "data": additional_data, - "updated": added, - "added": added, # no updated info, set insertion date as last update - } - - def to_external(self, node): - """ - takes a nodeshot Node instance as input - and returns a dictionary representing JSON structure of OpenLabor - """ - # calculated automatically 1 month after now - job_expiration = int(now_after(days=30).strftime('%s')) - - if node.user is not None: - user_email = node.user.email - user_first_name = node.user.first_name - user_last_name = node.user.last_name - else: - user_email = None - user_first_name = None - user_last_name = None - - return { - "service_code": self.config['service_code_post'], - "latitude": node.point[1], - "longitude": node.point[0], - "j_latitude": node.point[1], - "j_longitude": node.point[0], - "address": node.address, - "j_address": node.address, - "address_string": node.address, - "email": user_email, - "first_name": user_first_name, - "last_name": user_last_name, - "description": node.description, - "api_key": self.config.get('api_key', ''), - "locale": "it_IT", - "position": node.name, - "professionalProfile": node.data.get('professional_profile'), - "qualificationRequired": node.data.get('qualification_required'), - "contractType": node.data.get('contract_type', ''), - "workersRequired": node.data.get('workers_required', 1), - "jobExpiration": job_expiration, - "notes": node.description, - "zipCode": node.data.get('zip_code', None), - "cityCompany": node.data.get('city', None), - "sourceJob": user_email, - "sourceJobName": user_first_name, - "sourceJobSurname": user_last_name - } - - def get_nodes(self, class_name, params): - """ get nodes """ - # determine if response is going to be JSON or GeoJSON - if 'geojson' in class_name.lower(): - response_format = 'geojson' - SerializerClass = OpenLaborGeoSerializer - else: - response_format = 'json' - SerializerClass = OpenLaborSerializer - - layer_name = self.layer.name - cache_key = 'layer_%s_nodes.%s' % (self.layer.id, response_format) - serialized_nodes = cache.get(cache_key, False) - - if serialized_nodes is False: - try: - response = requests.get( - self.get_url, - verify=self.verify_ssl - ) - except requests.exceptions.ConnectionError as e: - return { - 'error': _('external layer not reachable'), - 'exception': list(e.message) - } - - try: - response.data = json.loads(response.content) - except json.scanner.JSONDecodeError as e: - return { - 'error': _('external layer is experiencing some issues because it returned invalid data'), - 'exception': list(e) - } - - nodes = [] - - # loop over all the entries and convert to nodeshot format - for job in response.data: - # skip records which do not have geographic information - if not job.get('latitude', False) or not job.get('longitude', False): - continue - # convert response in nodeshot format - node_dictionary = self.to_nodeshot(job) - # create Node model instance (needed for rest_framework serializer) - node = Node(**node_dictionary) - node.layer_name = layer_name # hack to avoid too many queries to get layer name each time - nodes.append(node) - - # serialize with rest framework to achieve consistency - serialized_nodes = SerializerClass(nodes, many=True).data - cache.set(cache_key, serialized_nodes, 86400) # cache for 1 day - - return serialized_nodes - - def add(self, node): - """ Add a new record into OpenLabor db """ - openlabor_record = self.to_external(node) - - # openlabor sync - response=requests.post(self.post_url,openlabor_record) - - if response.status_code != 200: - message = 'ERROR while creating "%s". Response: %s' % (node.name, response.content) - logger.error('== %s ==' % message) - return False - - try: - data = response.json() - except json.JSONDecodeError as e: - logger.error('== ERROR: JSONDecodeError %s ==' % e) - return False - - NodeExternal.objects.create(node=node, external_id=int(data['AddedJobId'])) - message = 'New record "%s" saved in CitySDK through the HTTP API"' % node.name - self.verbose(message) - logger.info('== %s ==' % message) - - # clear cache - cache_key1 = 'layer_%s_nodes.json' % self.layer.id - cache_key2 = 'layer_%s_nodes.geojson' % self.layer.id - cache.delete_many([cache_key1, cache_key2]) - - return True diff --git a/nodeshot/interop/sync/synchronizers/openwisp_citysdk_tourism.py b/nodeshot/interop/sync/synchronizers/openwisp_citysdk_tourism.py deleted file mode 100755 index f890157f..00000000 --- a/nodeshot/interop/sync/synchronizers/openwisp_citysdk_tourism.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import absolute_import - -from nodeshot.interop.sync.synchronizers import OpenWisp -from .citysdk_tourism import CitySdkTourismMixin - - -class OpenWispCitySdkTourism(CitySdkTourismMixin, OpenWisp): - """ - OpenWispCitySdkTourism synchronizer class - Imports data from OpenWISP GeoRSS and then exports the data to the CitySDK database - """ - SCHEMA = CitySdkTourismMixin.SCHEMA + OpenWisp.SCHEMA - - def convert_format(self, node): - # determine description or fill some hopefully useful value - if node.description.strip() == '': - description = '%s in %s' % (node.name, node.address) - else: - description = node.description - - return { - self.config['citysdk_type'] :{ - "location":{ - "point":[ - { - "Point":{ - "posList":"%s %s" % (float(node.point.coords[1]), float(node.point.coords[0])), - "srsName":"http://www.opengis.net/def/crs/EPSG/0/4326" - }, - "term": self.config['citysdk_term'] - } - ], - "address": { - "value":"""BEGIN:VCARD -N:;%s;;;; -ADR;INTL;PARCEL;WORK:;;%s; -END:VCARD""" % ( - node.name, - node.address - ), - "type": "text/vcard" - }, - }, - "label":[ - { - "term": "primary", - "value": node.name - }, - ], - "description":[ - { - "value": description, - "lang": self.config['citysdk_lang'] - }, - ], - "category":[ - { - "id": self.citysdk_category_id - } - ], - "base": self.citysdk_resource_url, - "lang": self.config['citysdk_lang'], - "created": unicode(node.added), - "author":{ - "term": "primary", - "value": self.layer.organization - }, - "license":{ - "term": "primary", - "value": "open-data" - } - } - } diff --git a/nodeshot/interop/sync/synchronizers/provincerometraffic.py b/nodeshot/interop/sync/synchronizers/provincerometraffic.py deleted file mode 100755 index 3eb1ae3e..00000000 --- a/nodeshot/interop/sync/synchronizers/provincerometraffic.py +++ /dev/null @@ -1,231 +0,0 @@ -from __future__ import absolute_import - -import requests -import simplejson as json -from datetime import date, datetime, timedelta - -from django.template.defaultfilters import slugify -from django.contrib.gis.geos import GEOSGeometry -from django.core.exceptions import ValidationError - -from nodeshot.core.nodes.models import Node, Status -from nodeshot.interop.sync.synchronizers.base import BaseSynchronizer - - -class ProvinceRomeTraffic(BaseSynchronizer): - """ Province of Rome Traffic synchronizer class """ - SCHEMA = [ - { - 'name': 'streets_url', - 'class': 'URLField' - }, - { - 'name': 'measurements_url', - 'class': 'URLField' - }, - { - 'name': 'check_streets_every_n_days', - 'class': 'IntegerField', - 'kwargs': { 'default': 2 } - } - ] - - def retrieve_data(self): - """ retrieve data """ - # shortcuts for readability - streets_url = self.config.get('streets_url') - check_streets_every_n_days = int(self.config.get('check_streets_every_n_days', 2)) - last_time_streets_checked = self.config.get('last_time_streets_checked', None) - measurements_url = self.config.get('measurements_url') - - # do HTTP request and store content - self.measurements = requests.get(measurements_url, verify=self.verify_ssl).content - - try: - last_time_streets_checked = datetime.strptime(last_time_streets_checked, '%Y-%m-%d').date() - except TypeError: - pass - - # if last time checked more than days specified - if last_time_streets_checked is None or last_time_streets_checked < date.today() - timedelta(days=check_streets_every_n_days): - # get huge streets file - self.streets = requests.get(streets_url, verify=self.verify_ssl).content - else: - self.streets = False - - def parse(self): - """ parse data """ - self.measurements = json.loads(self.measurements)["features"] - if self.streets: - self.streets = json.loads(self.streets)["features"] - - def save(self): - """ synchronize DB """ - self.process_streets() - self.process_measurements() - - def process_measurements(self): - items = self.measurements - if len(items) < 1: - self.message += """ - No measurements found. - """ - else: - saved_measurements = 0 - for item in items: - try: - node = Node.objects.get(pk=int(item['id'])) - except Node.DoesNotExist: - print "Could not retrieve node #%s" % item['id'] - continue - try: - node.data['last_measurement'] = item['properties']['TIMESTAMP'] - node.data['velocity'] = item['properties']['VELOCITY'] - node.save() - self.verbose('Updated measurement for node %s' % node.id) - saved_measurements += 1 - except KeyError: - pass - self.message += """ - Updated measurements of %d street segments out of %d - """ % (saved_measurements, len(items)) - - def process_streets(self): - if not self.streets: - self.message = """ - Street data not processed. - """ - return False - # retrieve all items - items = self.streets - - # init empty lists - added_nodes = [] - changed_nodes = [] - unmodified_nodes = [] - - # retrieve a list of local nodes in DB - local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) - # init empty list of slug of external nodes that will be needed to perform delete operations - external_nodes_slug = [] - deleted_nodes_count = 0 - - try: - self.status = Status.objects.get(slug=self.config.get('status', None)) - except Status.DoesNotExist: - self.status = None - - # loop over every parsed item - for item in items: - # retrieve info in auxiliary variables - # readability counts! - pk = item['id'] - name = item['properties'].get('LOCATION', '')[0:70] - address = name - slug = slugify(name) - - number = 1 - original_name = name - needed_different_name = False - - while True: - # items might have the same name... so we add a number.. - # check in DB too - if slug in external_nodes_slug or Node.objects.filter(slug__exact=slug).exclude(pk=pk).count() > 0: - needed_different_name = True - number = number + 1 - name = "%s - %d" % (original_name, number) - slug = slug = slugify(name) - else: - if needed_different_name: - self.verbose('needed a different name for %s, trying "%s"' % (original_name, name)) - break - - # geometry object - geometry = GEOSGeometry(json.dumps(item["geometry"])) - - # default values - added = False - changed = False - - try: - # edit existing node - node = Node.objects.get(pk=pk) - except Node.DoesNotExist: - # add a new node - node = Node() - node.id = pk - node.layer = self.layer - node.status = self.status - node.data = {} - added = True - - if node.name != name: - node.name = name - changed = True - - if node.slug != slug: - node.slug = slug - changed = True - - if added is True or node.geometry.equals(geometry) is False: - node.geometry = geometry - changed = True - - if node.address != address: - node.address = address - changed = True - - # perform save or update only if necessary - if added or changed: - try: - node.full_clean() - node.save() - except ValidationError as e: - raise Exception("%s errors: %s" % (name, e.messages)) - - if added: - added_nodes.append(node) - self.verbose('new node saved with name "%s"' % node.name) - elif changed: - changed_nodes.append(node) - self.verbose('node "%s" updated' % node.name) - else: - unmodified_nodes.append(node) - self.verbose('node "%s" unmodified' % node.name) - - # fill node list container - external_nodes_slug.append(node.slug) - - # delete old nodes - for local_node in local_nodes_slug: - # if local node not found in external nodes - if local_node not in external_nodes_slug: - node_name = node.name - # retrieve from DB and delete - node = Node.objects.get(slug=local_node) - node.delete() - # then increment count that will be included in message - deleted_nodes_count = deleted_nodes_count + 1 - self.verbose('node "%s" deleted' % node_name) - - self.config['last_time_streets_checked'] = str(date.today()) - self.layer.external.config = self.config - self.layer.external.save() - - # message that will be returned - self.message = """ - %s streets added - %s streets changed - %s streets deleted - %s streets unmodified - %s total external records processed - %s total local records for this layer - """ % ( - len(added_nodes), - len(changed_nodes), - deleted_nodes_count, - len(unmodified_nodes), - len(items), - Node.objects.filter(layer=self.layer).count() - ) diff --git a/nodeshot/interop/sync/synchronizers/provinciawifi.py b/nodeshot/interop/sync/synchronizers/provinciawifi.py deleted file mode 100755 index 53019b28..00000000 --- a/nodeshot/interop/sync/synchronizers/provinciawifi.py +++ /dev/null @@ -1,163 +0,0 @@ -from __future__ import absolute_import - -from django.template.defaultfilters import slugify -from django.contrib.gis.geos import Point -from django.core.exceptions import ValidationError -from django.conf import settings - -from nodeshot.core.nodes.models import Node, Status -from nodeshot.interop.sync.synchronizers.base import XmlSynchronizer, GenericGisSynchronizer - - -class ProvinciaWifi(XmlSynchronizer): - """ ProvinciaWifi synchronizer class """ - SCHEMA = GenericGisSynchronizer.SCHEMA[0:2] - - def save(self): - """ synchronize DB """ - # retrieve all items - items = self.parsed_data.getElementsByTagName('AccessPoint') - - # init empty lists - added_nodes = [] - changed_nodes = [] - unmodified_nodes = [] - - # retrieve a list of local nodes in DB - local_nodes_slug = Node.objects.filter(layer=self.layer).values_list('slug', flat=True) - # init empty list of slug of external nodes that will be needed to perform delete operations - external_nodes_slug = [] - deleted_nodes_count = 0 - - try: - self.status = Status.objects.get(slug=self.config.get('status', None)) - except Status.DoesNotExist: - self.status = None - - # loop over every parsed item - for item in items: - # retrieve info in auxiliary variables - # readability counts! - name = self.get_text(item, 'Denominazione')[0:70] - slug = slugify(name) - - number = 1 - original_name = name - needed_different_name = False - - while True: - # items might have the same name... so we add a number.. - if slug in external_nodes_slug: - needed_different_name = True - number = number + 1 - name = "%s - %d" % (original_name, number) - slug = slug = slugify(name) - else: - if needed_different_name: - self.verbose('needed a different name for %s, trying "%s"' % (original_name, name)) - break - - lat = self.get_text(item, 'Latitudine') - lng = self.get_text(item, 'longitudine') - description = 'Indirizzo: %s, %s; Tipologia: %s' % ( - self.get_text(item, 'Indirizzo'), - self.get_text(item, 'Comune'), - self.get_text(item, 'Tipologia') - ) - address = '%s, %s' % ( - self.get_text(item, 'Indirizzo'), - self.get_text(item, 'Comune') - ) - - # point object - point = Point(float(lng), float(lat)) - - # default values - added = False - changed = False - - try: - # edit existing node - node = Node.objects.get(slug=slug) - except Node.DoesNotExist: - # add a new node - node = Node() - node.layer = self.layer - node.status = self.status - added = True - - if node.name != name: - node.name = name - changed = True - - if node.slug != slug: - node.slug = slug - changed = True - - if added is True or node.geometry.equals(point) is False: - node.geometry = point - changed = True - - if node.description != description: - node.description = description - changed = True - - if node.address != address: - node.address = address # complete address - node.data = { - 'address': self.get_text(item, 'Indirizzo'), - 'city': self.get_text(item, 'Comune'), - 'province': 'Roma', - 'country': 'Italia' - } - changed = True - - # perform save or update only if necessary - if added or changed: - try: - node.full_clean() - node.save() - except ValidationError as e: - raise Exception("%s errors: %s" % (name, e.messages)) - - if added: - added_nodes.append(node) - self.verbose('new node saved with name "%s"' % node.name) - elif changed: - changed_nodes.append(node) - self.verbose('node "%s" updated' % node.name) - else: - unmodified_nodes.append(node) - self.verbose('node "%s" unmodified' % node.name) - - # fill node list container - external_nodes_slug.append(node.slug) - - # delete old nodes - for local_node in local_nodes_slug: - # if local node not found in external nodes - if not local_node in external_nodes_slug: - node_name = node.name - # retrieve from DB and delete - node = Node.objects.get(slug=local_node) - node.delete() - # then increment count that will be included in message - deleted_nodes_count = deleted_nodes_count + 1 - self.verbose('node "%s" deleted' % node_name) - - # message that will be returned - self.message = """ - %s nodes added - %s nodes changed - %s nodes deleted - %s nodes unmodified - %s total external records processed - %s total local nodes for this layer - """ % ( - len(added_nodes), - len(changed_nodes), - deleted_nodes_count, - len(unmodified_nodes), - len(items), - Node.objects.filter(layer=self.layer).count() - ) diff --git a/nodeshot/interop/sync/synchronizers/provinciawifi_citysdk_mobility.py b/nodeshot/interop/sync/synchronizers/provinciawifi_citysdk_mobility.py deleted file mode 100755 index 09ccb637..00000000 --- a/nodeshot/interop/sync/synchronizers/provinciawifi_citysdk_mobility.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import absolute_import - -from .provinciawifi import ProvinciaWifi -from .citysdk_mobility import CitySdkMobilityMixin - - -class ProvinciaWifiCitySdkMobility(CitySdkMobilityMixin, ProvinciaWifi): - """ - ProvinciaWifiCitySdkMobility synchronizer - Imports data from Provincia WIFI in the local DB and it also sends it - to CitySDK mobility API - """ - SCHEMA = CitySdkMobilityMixin.SCHEMA + ProvinciaWifi.SCHEMA diff --git a/nodeshot/interop/sync/synchronizers/provinciawifi_citysdk_tourism.py b/nodeshot/interop/sync/synchronizers/provinciawifi_citysdk_tourism.py deleted file mode 100755 index 6adde675..00000000 --- a/nodeshot/interop/sync/synchronizers/provinciawifi_citysdk_tourism.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import absolute_import - -from .provinciawifi import ProvinciaWifi -from .citysdk_tourism import CitySdkTourismMixin - - -class ProvinciaWifiCitySdkTourism(CitySdkTourismMixin, ProvinciaWifi): - """ - ProvinciaWifiCitySdkTourism synchronizer - Imports data from Provincia WIFI in the local DB and it also sends it - to CitySDK tourism API - """ - SCHEMA = CitySdkTourismMixin.SCHEMA + ProvinciaWifi.SCHEMA diff --git a/nodeshot/interop/sync/tests.py b/nodeshot/interop/sync/tests.py index d96e10cd..3e4b6512 100755 --- a/nodeshot/interop/sync/tests.py +++ b/nodeshot/interop/sync/tests.py @@ -1,7 +1,6 @@ import sys import simplejson as json import requests -from time import sleep from cStringIO import StringIO from datetime import date, timedelta @@ -17,7 +16,7 @@ from nodeshot.core.base.tests import user_fixtures from .models import LayerExternal, NodeExternal -from .settings import settings, SYNCHRONIZERS, CITYSDK_TOURISM_TEST_CONFIG, CITYSDK_MOBILITY_TEST_CONFIG +from .settings import settings, SYNCHRONIZERS from .tasks import synchronize_external_layers @@ -309,191 +308,6 @@ def test_openwisp(self): self.assertEqual(node.updated.strftime('%Y-%m-%d'), '2013-07-10') self.assertEqual(node.added.strftime('%Y-%m-%d'), '2013-06-14') - def test_provinciawifi(self): - """ test ProvinciaWifi synchronizer """ - layer = Layer.objects.external()[0] - layer.minimum_distance = 0 - layer.area = None - layer.new_nodes_allowed = False - layer.save() - layer = Layer.objects.get(pk=layer.pk) - - external = LayerExternal(layer=layer) - external.synchronizer_path = 'nodeshot.interop.sync.synchronizers.ProvinciaWifi' - external._reload_schema() - external.url = '%s/provincia-wifi.xml' % TEST_FILES_PATH - external.full_clean() - external.save() - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('5 nodes added', output) - self.assertIn('0 nodes changed', output) - self.assertIn('5 total external', output) - self.assertIn('5 total local', output) - - # start checking DB too - nodes = layer.node_set.all() - - # ensure all nodes have been imported - self.assertEqual(nodes.count(), 5) - - # check one particular node has the data we expect it to have - node = Node.objects.get(slug='viale-di-valle-aurelia-73') - self.assertEqual(node.name, 'viale di valle aurelia, 73') - self.assertEqual(node.address, 'viale di valle aurelia, 73, Roma') - point = Point(12.4373, 41.9025) - self.assertTrue(node.geometry.equals(point)) - - # ensure itmes with the same name on the XML get a different name in the DB - node = Node.objects.get(slug='largo-agostino-gemelli-8') - node = Node.objects.get(slug='largo-agostino-gemelli-8-2') - node = Node.objects.get(slug='largo-agostino-gemelli-8-3') - node = Node.objects.get(slug='largo-agostino-gemelli-8-4') - - ### --- with the following step we expect some nodes to be deleted and some to be added --- ### - - external.url = '%s/provincia-wifi2.xml' % TEST_FILES_PATH - external.full_clean() - external.save() - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('1 nodes added', output) - self.assertIn('2 nodes unmodified', output) - self.assertIn('3 nodes deleted', output) - self.assertIn('0 nodes changed', output) - self.assertIn('3 total external', output) - self.assertIn('3 total local', output) - - # ensure all nodes have been imported - self.assertEqual(nodes.count(), 3) - - # check one particular node has the data we expect it to have - node = Node.objects.get(slug='via-g-pullino-97') - self.assertEqual(node.name, 'Via G. Pullino 97') - self.assertEqual(node.address, 'Via G. Pullino 97, Roma') - self.assertEqual(node.description, 'Indirizzo: Via G. Pullino 97, Roma; Tipologia: Privati federati') - point = Point(12.484, 41.8641) - self.assertTrue(node.geometry.equals(point)) - - def test_province_rome_traffic(self): - """ test ProvinceRomeTraffic converter """ - layer = Layer.objects.external()[0] - layer.minimum_distance = 0 - layer.area = None - layer.new_nodes_allowed = False - layer.save() - layer = Layer.objects.get(pk=layer.pk) - - streets_url = '%s/citysdk-wp4-streets.json' % TEST_FILES_PATH - measurements_url = '%s/citysdk-wp4-measurements.json' % TEST_FILES_PATH - - external = LayerExternal(layer=layer) - external.synchronizer_path = 'nodeshot.interop.sync.synchronizers.ProvinceRomeTraffic' - external._reload_schema() - external.config = { - "streets_url": streets_url, - "measurements_url": measurements_url, - "check_streets_every_n_days": 2 - } - external.full_clean() - external.save() - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('20 streets added', output) - self.assertIn('0 streets changed', output) - self.assertIn('20 total external', output) - self.assertIn('20 total local', output) - self.assertIn('Updated measurements of 4 street segments', output) - - # start checking DB too - nodes = layer.node_set.all() - - # ensure all nodes have been imported - self.assertEqual(nodes.count(), 20) - - # check one particular node has the data we expect it to have - node = Node.objects.get(slug='via-di-santa-prisca') - self.assertEqual(node.name, 'VIA DI SANTA PRISCA') - self.assertEqual(node.address, 'VIA DI SANTA PRISCA') - geometry = GEOSGeometry('SRID=4326;LINESTRING (12.4837894439700001 41.8823699951170028, 12.4839096069340005 41.8820686340329971, 12.4839801788330007 41.8818206787110014)') - self.assertTrue(node.geometry.equals(geometry)) - - # check measurements - node = Node.objects.get(slug='via-casilina') - self.assertEqual(node.name, 'VIA CASILINA') - self.assertEqual(node.data['last_measurement'], '09-09-2013 22:31:00') - self.assertEqual(node.data['velocity'], '44') - - # ensure last_time_streets_checked is today - layer = Layer.objects.get(pk=layer.id) - self.assertEqual(layer.external.config['last_time_streets_checked'], str(date.today())) - - ### --- not much should happen --- ### - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('Street data not processed', output) - - # set last_time_streets_checked to 6 days ago - layer.external.config['last_time_streets_checked'] = str(date.today() - timedelta(days=6)) - external.full_clean() - layer.external.save() - - ### --- with the following step we expect some nodes to be deleted and some to be added --- ### - - streets_url = '%s/citysdk-wp4-streets2.json' % TEST_FILES_PATH - measurements_url = '%s/citysdk-wp4-measurements2.json' % TEST_FILES_PATH - - external.config['streets_url'] = streets_url - external.config['measurements_url'] = measurements_url - external.full_clean() - external.save() - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('5 streets added', output) - self.assertIn('16 streets unmodified', output) - self.assertIn('4 streets deleted', output) - self.assertIn('0 streets changed', output) - self.assertIn('21 total external', output) - self.assertIn('21 total local', output) - self.assertIn('No measurements found', output) - - # ensure all nodes have been imported - self.assertEqual(nodes.count(), 21) - - # ensure last_time_streets_checked is today - layer = Layer.objects.get(pk=layer.id) - self.assertEqual(layer.external.config['last_time_streets_checked'], str(date.today())) - def test_geojson_sync(self): """ test GeoJSON sync """ layer = Layer.objects.external()[0] @@ -784,85 +598,6 @@ def test_georss_w3c(self): self.assertIn('2 total external', output) self.assertIn('2 total local', output) - def test_openlabor_get_nodes(self): - layer = Layer.objects.external()[0] - layer.minimum_distance = 0 - layer.area = None - layer.new_nodes_allowed = True - layer.save() - layer = Layer.objects.get(pk=layer.pk) - - external = LayerExternal(layer=layer) - external.synchronizer_path = 'nodeshot.interop.sync.synchronizers.OpenLabor' - external._reload_schema() - external.config = { - "open311_url": '%s/' % TEST_FILES_PATH, - "service_code_get": "001", - "service_code_post": "002", - "default_status": "active", - "api_key": "DEVO1395445966" - } - external.full_clean() - external.save() - - url = reverse('api_layer_nodes_list', args=[layer.slug]) - response = self.client.get(url) - nodes = response.data['nodes'] - self.assertEqual(response.status_code, 200) - self.assertEqual(len(nodes), 2) - self.assertEqual(nodes[0]['name'], 'SARTO CONFEZIONISTA') - self.assertEqual(nodes[0]['address'], 'Via Lussemburgo snc, Anzio - 00042') - - # test geojson - url = reverse('api_layer_nodes_geojson', args=[layer.slug]) - response = self.client.get(url) - nodes = response.data['features'] - self.assertEqual(response.status_code, 200) - self.assertEqual(len(nodes), 2) - self.assertEqual(nodes[0]['properties']['name'], 'SARTO CONFEZIONISTA') - self.assertEqual(nodes[0]['properties']['address'], 'Via Lussemburgo snc, Anzio - 00042') - - def test_openlabor_add_node(self): - layer = Layer.objects.external()[0] - layer.minimum_distance = 0 - layer.area = None - layer.new_nodes_allowed = True - layer.save() - layer = Layer.objects.get(pk=layer.pk) - - url = 'http://devopenlabor.lynxlab.com/api/v1' - - external = LayerExternal(layer=layer) - external.synchronizer_path = 'nodeshot.interop.sync.synchronizers.OpenLabor' - external._reload_schema() - external.config = { - "open311_url": url, - "service_code_get": "001", - "service_code_post": "002", - "default_status": "active", - "api_key": "DEVO1395445966" - } - external.full_clean() - external.save() - - node = Node() - node.name = 'offerta di lavoro di test' - node.description = 'altra offerta di lavoro inserita automaticamente tramite unit test' - node.geometry = 'POINT (12.5823391919000012 41.8721429276999820)' - node.layer = layer - node.user_id = 1 - node.address = 'via del test' - node.data = { - "professional_profile": "professional_profile test", - "qualification_required": "qualification_required test", - "contract_type": "contract_type test", - "zip_code": "zip code test", - "city": "city test" - } - - node.save() - self.assertIsNotNone(node.external.external_id) - def test_nodeshot_sync(self): layer = Layer.objects.external()[0] layer.minimum_distance = 0 @@ -969,444 +704,3 @@ def test_nodeshot_sync_exceptions(self): self.assertEqual(response.status_code, 200) self.assertIn('error', response.data) self.assertIn('exception', response.data) - - if CITYSDK_TOURISM_TEST_CONFIG: - def test_openwisp_citysdk_tourism(self): - layer = Layer.objects.external()[0] - layer.minimum_distance = 0 - layer.area = None - layer.new_nodes_allowed = False - layer.save() - layer = Layer.objects.get(pk=layer.pk) - - xml_url = '%s/openwisp-georss.xml' % TEST_FILES_PATH - - external = LayerExternal(layer=layer) - external.synchronizer_path = 'nodeshot.interop.sync.synchronizers.OpenWispCitySdkTourism' - external._reload_schema() - external.config = CITYSDK_TOURISM_TEST_CONFIG.copy() - external.config.update({ - "status": "active", - "url": xml_url, - "verify_ssl": False - }) - external.full_clean() - external.save() - - querystring_params = { - 'category': CITYSDK_TOURISM_TEST_CONFIG['citysdk_category'], - 'limit': '-1' - } - - data = json.loads(requests.get(CITYSDK_TOURISM_TEST_CONFIG['search_url'], params=querystring_params).content) - self.assertEqual(len(data['poi']), 0) - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('42 nodes added', output) - self.assertIn('0 nodes changed', output) - self.assertIn('42 total external', output) - self.assertIn('42 total local', output) - - sleep(1) # wait 1 second - - data = json.loads(requests.get(CITYSDK_TOURISM_TEST_CONFIG['search_url'], params=querystring_params).content) - self.assertEqual(len(data['poi']), 42) - - ### --- with the following step we expect some nodes to be deleted --- ### - - xml_url = '%s/openwisp-georss2.xml' % TEST_FILES_PATH - external.config['url'] = xml_url - external.save() - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('4 nodes unmodified', output) - self.assertIn('38 nodes deleted', output) - self.assertIn('0 nodes changed', output) - self.assertIn('4 total external', output) - self.assertIn('4 total local', output) - - data = json.loads(requests.get(CITYSDK_TOURISM_TEST_CONFIG['search_url'], params=querystring_params).content) - self.assertEqual(len(data['poi']), 4) - - ### --- delete everything --- ### - - for node in layer.node_set.all(): - node.delete() - - sleep(1) # wait 1 second - - data = json.loads(requests.get(CITYSDK_TOURISM_TEST_CONFIG['search_url'], params=querystring_params).content) - self.assertEqual(len(data['poi']), 0) - - def test_geojson_citysdk_tourism(self): - layer = Layer.objects.external()[0] - layer.minimum_distance = 0 - layer.area = None - layer.new_nodes_allowed = False - layer.save() - layer = Layer.objects.get(pk=layer.pk) - - url = '%s/geojson1.json' % TEST_FILES_PATH - - external = LayerExternal(layer=layer) - external.synchronizer_path = 'nodeshot.interop.sync.synchronizers.GeoJsonCitySdkTourism' - external._reload_schema() - external.config = CITYSDK_TOURISM_TEST_CONFIG.copy() - external.config.update({ - "status": "active", - "url": url, - "verify_ssl": False - }) - external.full_clean() - external.save() - - querystring_params = { - 'category': CITYSDK_TOURISM_TEST_CONFIG['citysdk_category'], - 'limit': '-1' - } - - data = json.loads(requests.get(CITYSDK_TOURISM_TEST_CONFIG['search_url'], params=querystring_params).content) - self.assertEqual(len(data['poi']), 0) - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('2 nodes added', output) - self.assertIn('0 nodes changed', output) - self.assertIn('2 total external', output) - self.assertIn('2 total local', output) - self.assertEqual(layer.node_set.count(), 2) - - sleep(1) # wait 1 second - - data = json.loads(requests.get(CITYSDK_TOURISM_TEST_CONFIG['search_url'], params=querystring_params).content) - self.assertEqual(len(data['poi']), 2) - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('2 nodes unmodified', output) - self.assertIn('0 nodes deleted', output) - self.assertIn('0 nodes changed', output) - self.assertIn('2 total external', output) - self.assertIn('2 total local', output) - self.assertEqual(layer.node_set.count(), 2) - - data = json.loads(requests.get(CITYSDK_TOURISM_TEST_CONFIG['search_url'], params=querystring_params).content) - self.assertEqual(len(data['poi']), 2) - - ### --- repeat with slightly different input --- ### - - url = '%s/geojson4.json' % TEST_FILES_PATH - external.config['url'] = url - external.save() - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('1 nodes unmodified', output) - self.assertIn('1 nodes deleted', output) - self.assertIn('1 total external', output) - self.assertIn('1 total local', output) - self.assertEqual(layer.node_set.count(), 1) - - data = json.loads(requests.get(CITYSDK_TOURISM_TEST_CONFIG['search_url'], params=querystring_params).content) - self.assertEqual(len(data['poi']), 1) - - ### --- delete everything --- ### - - for node in layer.node_set.all(): - node.delete() - - sleep(1) # wait 1 second - - data = json.loads(requests.get(CITYSDK_TOURISM_TEST_CONFIG['search_url'], params=querystring_params).content) - self.assertEqual(len(data['poi']), 0) - - def test_provinciawifi_citysdk_tourism(self): - layer = Layer.objects.external()[0] - layer.minimum_distance = 0 - layer.area = None - layer.new_nodes_allowed = False - layer.save() - layer = Layer.objects.get(pk=layer.pk) - - xml_url = '%s/provincia-wifi.xml' % TEST_FILES_PATH - - external = LayerExternal(layer=layer) - external.synchronizer_path = 'nodeshot.interop.sync.synchronizers.ProvinciaWifiCitySdkTourism' - external._reload_schema() - external.config = CITYSDK_TOURISM_TEST_CONFIG.copy() - external.config.update({ - "status": "active", - "url": xml_url, - "verify_ssl": False - }) - external.full_clean() - external.save() - - querystring_params = { - 'category': CITYSDK_TOURISM_TEST_CONFIG['citysdk_category'], - 'limit': '-1' - } - - data = json.loads(requests.get(CITYSDK_TOURISM_TEST_CONFIG['search_url'], params=querystring_params).content) - self.assertEqual(len(data['poi']), 0) - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('5 nodes added', output) - self.assertIn('0 nodes changed', output) - self.assertIn('5 total external', output) - self.assertIn('5 total local', output) - self.assertEqual(layer.node_set.count(), 5) - - sleep(1) - - data = json.loads(requests.get(CITYSDK_TOURISM_TEST_CONFIG['search_url'], params=querystring_params).content) - self.assertEqual(len(data['poi']), 5) - - ### --- with the following step we expect some nodes to be deleted and some to be added --- ### - - external.config['url'] = '%s/provincia-wifi2.xml' % TEST_FILES_PATH - external.save() - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('1 nodes added', output) - self.assertIn('2 nodes unmodified', output) - self.assertIn('3 nodes deleted', output) - self.assertIn('0 nodes changed', output) - self.assertIn('3 total external', output) - self.assertIn('3 total local', output) - self.assertEqual(layer.node_set.count(), 3) - - data = json.loads(requests.get(CITYSDK_TOURISM_TEST_CONFIG['search_url'], params=querystring_params).content) - self.assertEqual(len(data['poi']), 3) - - sleep(1) - - ### --- delete everything --- ### - - for node in layer.node_set.all(): - node.delete() - - sleep(1) # wait 1 second - - data = json.loads(requests.get(CITYSDK_TOURISM_TEST_CONFIG['search_url'], params=querystring_params).content) - self.assertEqual(len(data['poi']), 0) - - if CITYSDK_MOBILITY_TEST_CONFIG: - def test_geojson_citysdk_mobility(self): - layer = Layer.objects.external()[0] - layer.minimum_distance = 0 - layer.area = None - layer.new_nodes_allowed = False - layer.save() - layer = Layer.objects.get(pk=layer.pk) - - url = '%s/geojson1.json' % TEST_FILES_PATH - - external = LayerExternal(layer=layer) - external.synchronizer_path = 'nodeshot.interop.sync.synchronizers.GeoJsonCitySdkMobility' - external._reload_schema() - external.config = CITYSDK_MOBILITY_TEST_CONFIG.copy() - external.config.update({ - "url": url, - "verify_ssl": False, - }) - external.full_clean() - external.save() - - querystring_params = { - 'layer': CITYSDK_MOBILITY_TEST_CONFIG['citysdk_layer'], - 'per_page': '1000' - } - citysdk_nodes_url = '%s/nodes' % CITYSDK_MOBILITY_TEST_CONFIG['citysdk_url'] - data = json.loads(requests.get(citysdk_nodes_url, params=querystring_params, verify=False).content) - self.assertEqual(len(data['results']), 0) - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('2 nodes added', output) - self.assertIn('0 nodes changed', output) - self.assertIn('2 total external', output) - self.assertIn('2 total local', output) - self.assertEqual(layer.node_set.count(), 2) - self.assertNotEqual(layer.node_set.first().external.external_id, '') - - sleep(1) # wait 1 second - - data = json.loads(requests.get(citysdk_nodes_url, params=querystring_params, verify=False).content) - self.assertEqual(len(data['results']), 2) - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('2 nodes unmodified', output) - self.assertIn('0 nodes deleted', output) - self.assertIn('0 nodes changed', output) - self.assertIn('2 total external', output) - self.assertIn('2 total local', output) - self.assertEqual(layer.node_set.count(), 2) - - data = json.loads(requests.get(citysdk_nodes_url, params=querystring_params, verify=False).content) - self.assertEqual(len(data['results']), 2) - - ### --- repeat with slightly different input --- ### - - url = '%s/geojson4.json' % TEST_FILES_PATH - external.config['url'] = url - external.save() - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('1 nodes unmodified', output) - self.assertIn('1 nodes deleted', output) - self.assertIn('1 total external', output) - self.assertIn('1 total local', output) - self.assertEqual(layer.node_set.count(), 1) - - data = json.loads(requests.get(citysdk_nodes_url, params=querystring_params, verify=False).content) - self.assertEqual(len(data['results']), 1) - - ### --- delete everything --- ### - - for node in layer.node_set.all(): - node.delete() - - sleep(1) # wait 1 second - - data = json.loads(requests.get(citysdk_nodes_url, params=querystring_params, verify=False).content) - self.assertEqual(len(data['results']), 0) - - def test_provinciawifi_citysdk_mobility(self): - layer = Layer.objects.external()[0] - layer.minimum_distance = 0 - layer.area = None - layer.new_nodes_allowed = False - layer.save() - layer = Layer.objects.get(pk=layer.pk) - - xml_url = '%s/provincia-wifi.xml' % TEST_FILES_PATH - - external = LayerExternal(layer=layer) - external.synchronizer_path = 'nodeshot.interop.sync.synchronizers.ProvinciaWifiCitySdkMobility' - external._reload_schema() - external.config = CITYSDK_MOBILITY_TEST_CONFIG.copy() - external.config.update({ - "status": "active", - "url": xml_url, - "verify_ssl": False - }) - external.full_clean() - external.save() - - querystring_params = { - 'layer': CITYSDK_MOBILITY_TEST_CONFIG['citysdk_layer'], - 'per_page': '1000' - } - citysdk_nodes_url = '%s/nodes' % CITYSDK_MOBILITY_TEST_CONFIG['citysdk_url'] - data = json.loads(requests.get(citysdk_nodes_url, params=querystring_params, verify=False).content) - self.assertEqual(len(data['results']), 0) - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('5 nodes added', output) - self.assertIn('0 nodes changed', output) - self.assertIn('5 total external', output) - self.assertIn('5 total local', output) - self.assertEqual(layer.node_set.count(), 5) - - sleep(1) - - data = json.loads(requests.get(citysdk_nodes_url, params=querystring_params, verify=False).content) - self.assertEqual(len(data['results']), 5) - - ### --- with the following step we expect some nodes to be deleted and some to be added --- ### - - external.config['url'] = '%s/provincia-wifi2.xml' % TEST_FILES_PATH - external.save() - - output = capture_output( - management.call_command, - ['sync', 'vienna'], - kwargs={ 'verbosity': 0 } - ) - - # ensure following text is in output - self.assertIn('1 nodes added', output) - self.assertIn('2 nodes unmodified', output) - self.assertIn('3 nodes deleted', output) - self.assertIn('0 nodes changed', output) - self.assertIn('3 total external', output) - self.assertIn('3 total local', output) - self.assertEqual(layer.node_set.count(), 3) - - data = json.loads(requests.get(citysdk_nodes_url, params=querystring_params, verify=False).content) - self.assertEqual(len(data['results']), 3) - - sleep(1) - - ### --- delete everything --- ### - - for node in layer.node_set.all(): - node.delete() - - sleep(1) # wait 1 second - - data = json.loads(requests.get(citysdk_nodes_url, params=querystring_params, verify=False).content) - self.assertEqual(len(data['results']), 0)