diff --git a/ansible/roles/ckan/vars/main.yml b/ansible/roles/ckan/vars/main.yml index 74f656dcaa..34d136e62f 100644 --- a/ansible/roles/ckan/vars/main.yml +++ b/ansible/roles/ckan/vars/main.yml @@ -7,7 +7,7 @@ celery_user: "{{ www_user }}" ckan_plugins_default: stats scheming_datasets fluent # order matters, when templates call super() -ckan_plugins: harvest ckan_harvester hri_harvester dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface spatial_metadata spatial_query csw_harvester drupal7 datarequests report qa archiver ytp_organizations ytp_comments ytp_request hierarchy_display ytp_theme ytp_drupal ytp_tasks ytp_dataset ytp_user ytp_service datastore showcase datapusher recline_grid_view recline_graph_view recline_map_view text_view image_view pdf_view geo_view geojson_view +ckan_plugins: harvest ckan_harvester hri_harvester dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface spatial_metadata spatial_query csw_harvester drupal7 datarequests report qa archiver ytp_organizations ytp_comments ytp_request ytp_report hierarchy_display ytp_theme ytp_drupal ytp_tasks ytp_dataset ytp_spatial ytp_user ytp_service datastore showcase datapusher recline_grid_view recline_graph_view recline_map_view text_view image_view pdf_view geo_view geojson_view ckan_aws_plugins: cloudstorage diff --git a/modules/ckanext-ytp-main/ckanext/ytp/dataset/plugin.py b/modules/ckanext-ytp-main/ckanext/ytp/dataset/plugin.py index 82b4be7b23..928a4e7e1e 100644 --- a/modules/ckanext-ytp-main/ckanext/ytp/dataset/plugin.py +++ b/modules/ckanext-ytp-main/ckanext/ytp/dataset/plugin.py @@ -201,7 +201,6 @@ class YTPDatasetForm(plugins.SingletonPlugin, toolkit.DefaultDatasetForm): plugins.implements(plugins.IPackageController, inherit=True) plugins.implements(plugins.IActions) plugins.implements(plugins.IConfigurable) - plugins.implements(ISpatialHarvester, inherit=True) plugins.implements(plugins.IAuthFunctions) plugins.implements(plugins.IValidators) @@ -576,6 +575,29 @@ def before_index(self, pkg_dict): def get_actions(self): return {'package_show': action_package_show} + + + # IValidators + def get_validators(self): + return { + 'lower_if_exists': validators.lower_if_exists, + 'upper_if_exists': validators.upper_if_exists, + 'tag_string_or_tags_required': validators.tag_string_or_tags_required, + 'create_tags': validators.create_tags, + 'create_fluent_tags': validators.create_fluent_tags, + 'set_private_if_not_admin': validators.set_private_if_not_admin, + 'list_to_string': validators.list_to_string, + 'convert_to_list': validators.convert_to_list, + 'tag_list_output': validators.tag_list_output, + 'repeating_text': validators.repeating_text, + 'repeating_text_output': validators.repeating_text_output, + 'only_default_lang_required': validators.only_default_lang_required + } + + +class YTPSpatialHarvester(plugins.SingletonPlugin): + plugins.implements(ISpatialHarvester, inherit=True) + # ISpatialHarvester def get_package_dict(self, context, data_dict): @@ -690,20 +712,3 @@ def get_package_dict(self, context, data_dict): return package_dict - - # IValidators - def get_validators(self): - return { - 'lower_if_exists': validators.lower_if_exists, - 'upper_if_exists': validators.upper_if_exists, - 'tag_string_or_tags_required': validators.tag_string_or_tags_required, - 'create_tags': validators.create_tags, - 'create_fluent_tags': validators.create_fluent_tags, - 'set_private_if_not_admin': validators.set_private_if_not_admin, - 'list_to_string': validators.list_to_string, - 'convert_to_list': validators.convert_to_list, - 'tag_list_output': validators.tag_list_output, - 'repeating_text': validators.repeating_text, - 'repeating_text_output': validators.repeating_text_output, - 'only_default_lang_required': validators.only_default_lang_required - } diff --git a/modules/ckanext-ytp-main/ckanext/ytp/dataset/schemas/dataset.json b/modules/ckanext-ytp-main/ckanext/ytp/dataset/schemas/dataset.json index 04b26b8fe0..7cb8038c5a 100644 --- a/modules/ckanext-ytp-main/ckanext/ytp/dataset/schemas/dataset.json +++ b/modules/ckanext-ytp-main/ckanext/ytp/dataset/schemas/dataset.json @@ -50,7 +50,7 @@ }, { "field_name": "geographical_coverage", - "label": "Geographical Coverage", + "label": "Geographical coverage", "form_placeholder":"eg. tampere", "preset": "vocabulary_with_autocomplete", "validators": "ignore_missing convert_to_list create_tags(geographical_coverage)", @@ -102,7 +102,7 @@ }, { "field_name": "maintainer_email", - "label": "Maintainer Email", + "label": "Maintainer email", "form_placeholder": "joe@example.com", "display_property": "dc:contributor", "display_snippet": "email.html", @@ -110,7 +110,7 @@ }, { "field_name": "maintainer_website", - "label": "Maintainer Website", + "label": "Maintainer website", "form_placeholder": "http://www.example.com", "display_property": "dc:contributor", "display_snippet": "link.html" @@ -123,7 +123,7 @@ }, { "field_name": "author_email", - "label": "Author Email", + "label": "Author email", "form_placeholder": "joe@example.com", "display_property": "dc:creator", "display_snippet": "email.html", @@ -144,7 +144,7 @@ }, { "field_name": "update_frequency", - "label": "Update Frequency", + "label": "Update frequency", "form_placeholder":"eg. every second week", "form_languages": ["fi", "en", "sv"], "preset": "fluent_vocabulary_with_autocomplete", @@ -169,29 +169,29 @@ "preset": "fluent_core_markdown_translated", "form_placeholder": "eg. A detailed description", "form_languages": ["fi", "en", "sv"], - "label": "Copyright Notice", + "label": "Copyright notice", "display_snippet": null, "description": "An universal, compact and easy to understand description of the added dataset. Use as confining terms as possible to assist the user to understand what types of data, meters and dimensions the dataset contains." }, { "field_name": "collection_type", - "label": "Collection Type", + "label": "Collection type", "preset": "select", "choices": [ { "value": "Open Data", - "label": "Open Data" + "label": "Open data" }, { "value": "Interoperability Tools", - "label": "Interoperability Tools" + "label": "Interoperability tools" } ], "required": true }, { "field_name": "valid_from", - "label": "Valid From", + "label": "Valid from", "preset": "date" }, { @@ -256,7 +256,7 @@ }, { "field_name": "time_series_precision", - "label": "Time Series Precision", + "label": "Time series precision", "form_placeholder":"eg. 2 weeks", "preset": "fluent_vocabulary_with_autocomplete", "validators": "fluent_tags create_fluent_tags(time_series_precision)", @@ -269,22 +269,22 @@ }, { "field_name": "temporal_granularity", - "label": "Temporal Granularity", + "label": "Temporal granularity", "preset": "fluent_text" }, { "field_name": "update_frequency", - "label": "Update Frequency", + "label": "Update frequency", "preset": "fluent_text" }, { "field_name": "temporal_coverage_to", - "label": "Temporal Coverage To", + "label": "Temporal coverage to", "preset": "date" }, { "field_name": "temporal_coverage_from", - "label": "Temporal Coverage from", + "label": "Temporal coverage from", "preset": "date" } ] diff --git a/modules/ckanext-ytp-main/ckanext/ytp/report/reports.py b/modules/ckanext-ytp-main/ckanext/ytp/report/reports.py index e7111d9b25..9b381f5005 100644 --- a/modules/ckanext-ytp-main/ckanext/ytp/report/reports.py +++ b/modules/ckanext-ytp-main/ckanext/ytp/report/reports.py @@ -3,6 +3,10 @@ import itertools from datetime import timedelta, datetime +import logging + +log = logging.getLogger(__name__) + def test_report(): return { 'table' : [ @@ -36,24 +40,45 @@ def administrative_branch_summary_report(): ] context = {} + + # Optimization opportunity: Could fetch all orgs here and manually create the hierarchy orgs = get_action('organization_list')(context, {'organizations': org_names, 'all_fields': True}) - orgs_by_name = {org['name']: org for org in orgs} org_trees = [get_action('group_tree_section')(context, {'id': org['id'], 'type': 'organization'}) for org in orgs] - org_ids_by_tree = {r['name']: [x['id'] for x in flatten(r, lambda x: x['children'])] - for r in org_trees} - datasets_by_tree = {k: list(package_generator('owner_org:(%s)' % ' OR '.join(v), 1000, context)) - for k, v in org_ids_by_tree.iteritems()} + + def children(dataset): + return dataset['children'] + + org_levels = { + org['name']: level + for t in org_trees + for org, level in hierarchy_levels(t, children)} + + flat_orgs = (org for t in org_trees for org in flatten(t, children)) + root_tree_ids_pairs = ( + (r, [x['id'] for x in flatten(r, children)]) + for r in flat_orgs) + + # Optimization opportunity: Prefetch datasets for all related orgs in one go + root_datasets_pairs = ( + (k, list(package_generator('owner_org:(%s)' % ' OR '.join(v), 1000, context))) + for k, v in root_tree_ids_pairs) + return { 'table' : [{ - 'organization': orgs_by_name[org_name], + 'organization': org, + 'level': org_levels[org['name']], 'dataset_count': len(datasets), + 'dataset_count_1yr': glen(d for d in datasets if age(d) >= timedelta(1 * 365)), + 'dataset_count_2yr': glen(d for d in datasets if age(d) >= timedelta(2 * 365)), + 'dataset_count_3yr': glen(d for d in datasets if age(d) >= timedelta(3 * 365)), 'new_datasets_month': glen(d for d in datasets if age(d) <= timedelta(30)), - 'new_datasets_year': glen(d for d in datasets if age(d) <= timedelta(365)), - 'resource_formats': resource_formats(datasets) + 'new_datasets_6_months': glen(d for d in datasets if age(d) <= timedelta(6 * 30)), + 'resource_formats': resource_formats(datasets), + 'openness_score_avg': openness_score_avg(context, datasets) } - for org_name, datasets in datasets_by_tree.iteritems() + for org, datasets in root_datasets_pairs ] } @@ -98,5 +123,27 @@ def flatten(x, children): for cx in flatten(child, children): yield cx +def hierarchy_levels(x, children, level=0): + ''' + Provide hierarchy levels for nodes in a hierarchy + ''' + yield(x, level) + for child in children(x): + for cx, cl in hierarchy_levels(child, children, level + 1): + yield (cx, cl) + def resource_formats(datasets): return ', '.join({r['format'] for d in datasets for r in d['resources'] if r['format']}) + +def openness_score_avg(context, datasets): + openness_score = get_action('qa_package_openness_show') + scores = (openness_score(context, {"id": d['id']}) for d in datasets) + total, count = reduce(tuple_sum, ( + (s['openness_score'], 1) + for s in scores + if s.get('openness_score') is not None), + (0, 0)) + return total / count if count > 0 else None + +def tuple_sum(*xs): + return tuple(sum(x) for x in zip(*xs)) diff --git a/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html b/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html index a3b9813f35..dbb2801a59 100644 --- a/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html +++ b/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html @@ -1,20 +1,32 @@ - +
- - - - - + + + + + + + + + + + + + {% for row in table %} - + + + + - + + {% endfor %} diff --git a/modules/ckanext-ytp-main/setup.py b/modules/ckanext-ytp-main/setup.py index a29eb8d7c1..82650df262 100644 --- a/modules/ckanext-ytp-main/setup.py +++ b/modules/ckanext-ytp-main/setup.py @@ -34,6 +34,7 @@ hri_harvester=ckanext.ytp.organizations.harvesters.hriharvester:HRIHarvester ytp_theme=ckanext.ytp.theme.plugin:YtpThemePlugin ytp_dataset=ckanext.ytp.dataset.plugin:YTPDatasetForm + ytp_spatial=ckanext.ytp.dataset.plugin:YTPSpatialHarvester ytp_service=ckanext.ytp.service.plugin:YTPServiceForm ytp_report=ckanext.ytp.report.plugin:YtpReportPlugin diff --git a/modules/ytp-assets-common/src/less/ckan/ckan.less b/modules/ytp-assets-common/src/less/ckan/ckan.less index c42e6a6525..557dbccd01 100644 --- a/modules/ytp-assets-common/src/less/ckan/ckan.less +++ b/modules/ytp-assets-common/src/less/ckan/ckan.less @@ -36,10 +36,6 @@ } } -input[type="date"] { - height: 24px; -} - [role=main], .main { background: none; padding-top: initial; @@ -1101,3 +1097,12 @@ input[type=checkbox], input[type=radio] { .ckanext-datapreview { clear: both; } + +.control-label.group-label { + display: block; + float: none; +} + +.slug-preview { + padding-top: 0; +}
Administrative branchDataset countNew datasets last monthNew datasets last yearData formatsAdministrative branchDataset countNew datasetsAvg. openness scoreData formats
Current1 year ago2 years ago3 years agoLast monthLast 6 months
{{ row.organization.name }}{{ row.organization.title }} {{ row.dataset_count }}{{ row.dataset_count_1yr }}{{ row.dataset_count_2yr }}{{ row.dataset_count_3yr }} {{ row.new_datasets_month }}{{ row.new_datasets_year }}{{ row.new_datasets_6_months }}{% if row.openness_score_avg == None %}-{% else %}{{ row.openness_score_avg }}/5{% endif %} {{ row.resource_formats }}