From 0ce82e62723f6f4a82f092f1d64a64c04658ae1f Mon Sep 17 00:00:00 2001 From: Teemu Erkkola Date: Tue, 14 Nov 2017 10:28:40 +0200 Subject: [PATCH 1/7] AD-6: Administrative branch summary report, more columns and visible hierarchy --- ansible/roles/ckan/vars/main.yml | 2 +- .../ckanext/ytp/report/reports.py | 47 ++++++++++++++++--- .../administrative_branch_summary_report.html | 8 +++- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/ansible/roles/ckan/vars/main.yml b/ansible/roles/ckan/vars/main.yml index 74f656dcaa..209ebb63e4 100644 --- a/ansible/roles/ckan/vars/main.yml +++ b/ansible/roles/ckan/vars/main.yml @@ -7,7 +7,7 @@ celery_user: "{{ www_user }}" ckan_plugins_default: stats scheming_datasets fluent # order matters, when templates call super() -ckan_plugins: harvest ckan_harvester hri_harvester dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface spatial_metadata spatial_query csw_harvester drupal7 datarequests report qa archiver ytp_organizations ytp_comments ytp_request hierarchy_display ytp_theme ytp_drupal ytp_tasks ytp_dataset ytp_user ytp_service datastore showcase datapusher recline_grid_view recline_graph_view recline_map_view text_view image_view pdf_view geo_view geojson_view +ckan_plugins: harvest ckan_harvester hri_harvester dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface spatial_metadata spatial_query csw_harvester drupal7 datarequests report qa archiver ytp_organizations ytp_comments ytp_request ytp_report hierarchy_display ytp_theme ytp_drupal ytp_tasks ytp_dataset ytp_user ytp_service datastore showcase datapusher recline_grid_view recline_graph_view recline_map_view text_view image_view pdf_view geo_view geojson_view ckan_aws_plugins: cloudstorage diff --git a/modules/ckanext-ytp-main/ckanext/ytp/report/reports.py b/modules/ckanext-ytp-main/ckanext/ytp/report/reports.py index e7111d9b25..b2a4640e05 100644 --- a/modules/ckanext-ytp-main/ckanext/ytp/report/reports.py +++ b/modules/ckanext-ytp-main/ckanext/ytp/report/reports.py @@ -3,6 +3,10 @@ import itertools from datetime import timedelta, datetime +import logging + +log = logging.getLogger(__name__) + def test_report(): return { 'table' : [ @@ -36,24 +40,44 @@ def administrative_branch_summary_report(): ] context = {} + + # Optimization opportunity: Could fetch all orgs here and manually create the hierarchy orgs = get_action('organization_list')(context, {'organizations': org_names, 'all_fields': True}) - orgs_by_name = {org['name']: org for org in orgs} org_trees = [get_action('group_tree_section')(context, {'id': org['id'], 'type': 'organization'}) for org in orgs] - org_ids_by_tree = {r['name']: [x['id'] for x in flatten(r, lambda x: x['children'])] - for r in org_trees} - datasets_by_tree = {k: list(package_generator('owner_org:(%s)' % ' OR '.join(v), 1000, context)) - for k, v in org_ids_by_tree.iteritems()} + + def children(dataset): + return dataset['children'] + + org_levels = { + org['name']: level + for t in org_trees + for org, level in hierarchy_levels(t, children)} + + flat_orgs = (org for t in org_trees for org in flatten(t, children)) + root_tree_ids_pairs = ( + (r, [x['id'] for x in flatten(r, children)]) + for r in flat_orgs) + + # Optimization opportunity: Prefetch datasets for all related orgs in one go + root_datasets_pairs = ( + (k, list(package_generator('owner_org:(%s)' % ' OR '.join(v), 1000, context))) + for k, v in root_tree_ids_pairs) + return { 'table' : [{ - 'organization': orgs_by_name[org_name], + 'organization': org, + 'level': org_levels[org['name']], 'dataset_count': len(datasets), + 'dataset_count_1yr': glen(d for d in datasets if age(d) >= timedelta(1 * 365)), + 'dataset_count_2yr': glen(d for d in datasets if age(d) >= timedelta(2 * 365)), + 'dataset_count_3yr': glen(d for d in datasets if age(d) >= timedelta(3 * 365)), 'new_datasets_month': glen(d for d in datasets if age(d) <= timedelta(30)), 'new_datasets_year': glen(d for d in datasets if age(d) <= timedelta(365)), 'resource_formats': resource_formats(datasets) } - for org_name, datasets in datasets_by_tree.iteritems() + for org, datasets in root_datasets_pairs ] } @@ -98,5 +122,14 @@ def flatten(x, children): for cx in flatten(child, children): yield cx +def hierarchy_levels(x, children, level=0): + ''' + Provide hierarchy levels for nodes in a hierarchy + ''' + yield(x, level) + for child in children(x): + for cx, cl in hierarchy_levels(child, children, level + 1): + yield (cx, cl) + def resource_formats(datasets): return ', '.join({r['format'] for d in datasets for r in d['resources'] if r['format']}) diff --git a/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html b/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html index a3b9813f35..f8e610bb0d 100644 --- a/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html +++ b/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html @@ -3,6 +3,9 @@ Administrative branch Dataset count + Dataset count (1 year ago) + Dataset count (2 years ago) + Dataset count (3 years ago) New datasets last month New datasets last year Data formats @@ -11,8 +14,11 @@ {% for row in table %} - {{ row.organization.name }} + {{ row.organization.name }} {{ row.dataset_count }} + {{ row.dataset_count_1yr }} + {{ row.dataset_count_2yr }} + {{ row.dataset_count_3yr }} {{ row.new_datasets_month }} {{ row.new_datasets_year }} {{ row.resource_formats }} From 6e0d121a39b0bb927600b92699481bae75edac3d Mon Sep 17 00:00:00 2001 From: Teemu Erkkola Date: Tue, 14 Nov 2017 14:12:34 +0200 Subject: [PATCH 2/7] AD-6: ABS report tuning --- .../administrative_branch_summary_report.html | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html b/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html index f8e610bb0d..03d75a3011 100644 --- a/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html +++ b/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html @@ -1,20 +1,24 @@ - - - - - - - - + + + + + + + + + + + + {% for row in table %} - + From e46fe0a35ef34f3c009fdd0f5d43875defaa1401 Mon Sep 17 00:00:00 2001 From: Teemu Erkkola Date: Thu, 16 Nov 2017 08:28:28 +0200 Subject: [PATCH 3/7] AD-6: ABS report openness score --- .../ckanext/ytp/report/reports.py | 18 ++++++++++++++++-- .../administrative_branch_summary_report.html | 8 +++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/modules/ckanext-ytp-main/ckanext/ytp/report/reports.py b/modules/ckanext-ytp-main/ckanext/ytp/report/reports.py index b2a4640e05..9b381f5005 100644 --- a/modules/ckanext-ytp-main/ckanext/ytp/report/reports.py +++ b/modules/ckanext-ytp-main/ckanext/ytp/report/reports.py @@ -74,8 +74,9 @@ def children(dataset): 'dataset_count_2yr': glen(d for d in datasets if age(d) >= timedelta(2 * 365)), 'dataset_count_3yr': glen(d for d in datasets if age(d) >= timedelta(3 * 365)), 'new_datasets_month': glen(d for d in datasets if age(d) <= timedelta(30)), - 'new_datasets_year': glen(d for d in datasets if age(d) <= timedelta(365)), - 'resource_formats': resource_formats(datasets) + 'new_datasets_6_months': glen(d for d in datasets if age(d) <= timedelta(6 * 30)), + 'resource_formats': resource_formats(datasets), + 'openness_score_avg': openness_score_avg(context, datasets) } for org, datasets in root_datasets_pairs ] @@ -133,3 +134,16 @@ def hierarchy_levels(x, children, level=0): def resource_formats(datasets): return ', '.join({r['format'] for d in datasets for r in d['resources'] if r['format']}) + +def openness_score_avg(context, datasets): + openness_score = get_action('qa_package_openness_show') + scores = (openness_score(context, {"id": d['id']}) for d in datasets) + total, count = reduce(tuple_sum, ( + (s['openness_score'], 1) + for s in scores + if s.get('openness_score') is not None), + (0, 0)) + return total / count if count > 0 else None + +def tuple_sum(*xs): + return tuple(sum(x) for x in zip(*xs)) diff --git a/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html b/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html index 03d75a3011..0151e00930 100644 --- a/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html +++ b/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html @@ -1,9 +1,10 @@ -
Administrative branchDataset countDataset count (1 year ago)Dataset count (2 years ago)Dataset count (3 years ago)New datasets last monthNew datasets last yearData formatsAdministrative branchDataset countNew datasetsData formats
Current1 year ago2 years ago3 years agoLast monthLast year
{{ row.organization.name }}{{ row.organization.title }} {{ row.dataset_count }} {{ row.dataset_count_1yr }} {{ row.dataset_count_2yr }}
+
+ @@ -12,7 +13,7 @@ - + @@ -24,7 +25,8 @@ - + + {% endfor %} From 30a230f6a3980c231d4c9abe38b852e2a144aea1 Mon Sep 17 00:00:00 2001 From: Teemu Erkkola Date: Thu, 16 Nov 2017 09:13:21 +0200 Subject: [PATCH 4/7] AD-6: Added handling for missing openness score --- .../templates/report/administrative_branch_summary_report.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html b/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html index 0151e00930..dbb2801a59 100644 --- a/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html +++ b/modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html @@ -26,7 +26,7 @@ - + {% endfor %} From 3103905519f0155c90b1df432bf505ba89a3e19d Mon Sep 17 00:00:00 2001 From: Teemu Erkkola Date: Thu, 16 Nov 2017 14:06:53 +0200 Subject: [PATCH 5/7] Move spatial harvester customizations to separate plugin --- ansible/roles/ckan/vars/main.yml | 2 +- .../ckanext/ytp/dataset/plugin.py | 41 +++++++++++-------- modules/ckanext-ytp-main/setup.py | 1 + 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/ansible/roles/ckan/vars/main.yml b/ansible/roles/ckan/vars/main.yml index 209ebb63e4..34d136e62f 100644 --- a/ansible/roles/ckan/vars/main.yml +++ b/ansible/roles/ckan/vars/main.yml @@ -7,7 +7,7 @@ celery_user: "{{ www_user }}" ckan_plugins_default: stats scheming_datasets fluent # order matters, when templates call super() -ckan_plugins: harvest ckan_harvester hri_harvester dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface spatial_metadata spatial_query csw_harvester drupal7 datarequests report qa archiver ytp_organizations ytp_comments ytp_request ytp_report hierarchy_display ytp_theme ytp_drupal ytp_tasks ytp_dataset ytp_user ytp_service datastore showcase datapusher recline_grid_view recline_graph_view recline_map_view text_view image_view pdf_view geo_view geojson_view +ckan_plugins: harvest ckan_harvester hri_harvester dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface spatial_metadata spatial_query csw_harvester drupal7 datarequests report qa archiver ytp_organizations ytp_comments ytp_request ytp_report hierarchy_display ytp_theme ytp_drupal ytp_tasks ytp_dataset ytp_spatial ytp_user ytp_service datastore showcase datapusher recline_grid_view recline_graph_view recline_map_view text_view image_view pdf_view geo_view geojson_view ckan_aws_plugins: cloudstorage diff --git a/modules/ckanext-ytp-main/ckanext/ytp/dataset/plugin.py b/modules/ckanext-ytp-main/ckanext/ytp/dataset/plugin.py index 82b4be7b23..928a4e7e1e 100644 --- a/modules/ckanext-ytp-main/ckanext/ytp/dataset/plugin.py +++ b/modules/ckanext-ytp-main/ckanext/ytp/dataset/plugin.py @@ -201,7 +201,6 @@ class YTPDatasetForm(plugins.SingletonPlugin, toolkit.DefaultDatasetForm): plugins.implements(plugins.IPackageController, inherit=True) plugins.implements(plugins.IActions) plugins.implements(plugins.IConfigurable) - plugins.implements(ISpatialHarvester, inherit=True) plugins.implements(plugins.IAuthFunctions) plugins.implements(plugins.IValidators) @@ -576,6 +575,29 @@ def before_index(self, pkg_dict): def get_actions(self): return {'package_show': action_package_show} + + + # IValidators + def get_validators(self): + return { + 'lower_if_exists': validators.lower_if_exists, + 'upper_if_exists': validators.upper_if_exists, + 'tag_string_or_tags_required': validators.tag_string_or_tags_required, + 'create_tags': validators.create_tags, + 'create_fluent_tags': validators.create_fluent_tags, + 'set_private_if_not_admin': validators.set_private_if_not_admin, + 'list_to_string': validators.list_to_string, + 'convert_to_list': validators.convert_to_list, + 'tag_list_output': validators.tag_list_output, + 'repeating_text': validators.repeating_text, + 'repeating_text_output': validators.repeating_text_output, + 'only_default_lang_required': validators.only_default_lang_required + } + + +class YTPSpatialHarvester(plugins.SingletonPlugin): + plugins.implements(ISpatialHarvester, inherit=True) + # ISpatialHarvester def get_package_dict(self, context, data_dict): @@ -690,20 +712,3 @@ def get_package_dict(self, context, data_dict): return package_dict - - # IValidators - def get_validators(self): - return { - 'lower_if_exists': validators.lower_if_exists, - 'upper_if_exists': validators.upper_if_exists, - 'tag_string_or_tags_required': validators.tag_string_or_tags_required, - 'create_tags': validators.create_tags, - 'create_fluent_tags': validators.create_fluent_tags, - 'set_private_if_not_admin': validators.set_private_if_not_admin, - 'list_to_string': validators.list_to_string, - 'convert_to_list': validators.convert_to_list, - 'tag_list_output': validators.tag_list_output, - 'repeating_text': validators.repeating_text, - 'repeating_text_output': validators.repeating_text_output, - 'only_default_lang_required': validators.only_default_lang_required - } diff --git a/modules/ckanext-ytp-main/setup.py b/modules/ckanext-ytp-main/setup.py index a29eb8d7c1..82650df262 100644 --- a/modules/ckanext-ytp-main/setup.py +++ b/modules/ckanext-ytp-main/setup.py @@ -34,6 +34,7 @@ hri_harvester=ckanext.ytp.organizations.harvesters.hriharvester:HRIHarvester ytp_theme=ckanext.ytp.theme.plugin:YtpThemePlugin ytp_dataset=ckanext.ytp.dataset.plugin:YTPDatasetForm + ytp_spatial=ckanext.ytp.dataset.plugin:YTPSpatialHarvester ytp_service=ckanext.ytp.service.plugin:YTPServiceForm ytp_report=ckanext.ytp.report.plugin:YtpReportPlugin From fd7090037f9dd8d559d057df9d89bce120a47237 Mon Sep 17 00:00:00 2001 From: Teemu Erkkola Date: Mon, 20 Nov 2017 09:45:09 +0200 Subject: [PATCH 6/7] Fix dataset input form layout --- modules/ytp-assets-common/src/less/ckan/ckan.less | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/ytp-assets-common/src/less/ckan/ckan.less b/modules/ytp-assets-common/src/less/ckan/ckan.less index c42e6a6525..557dbccd01 100644 --- a/modules/ytp-assets-common/src/less/ckan/ckan.less +++ b/modules/ytp-assets-common/src/less/ckan/ckan.less @@ -36,10 +36,6 @@ } } -input[type="date"] { - height: 24px; -} - [role=main], .main { background: none; padding-top: initial; @@ -1101,3 +1097,12 @@ input[type=checkbox], input[type=radio] { .ckanext-datapreview { clear: both; } + +.control-label.group-label { + display: block; + float: none; +} + +.slug-preview { + padding-top: 0; +} From edcc1f196baaeca3c42e30d75a0444eb880b7c05 Mon Sep 17 00:00:00 2001 From: Teemu Erkkola Date: Mon, 20 Nov 2017 13:10:20 +0200 Subject: [PATCH 7/7] Harmonized data model label casing --- .../ckanext/ytp/dataset/schemas/dataset.json | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/modules/ckanext-ytp-main/ckanext/ytp/dataset/schemas/dataset.json b/modules/ckanext-ytp-main/ckanext/ytp/dataset/schemas/dataset.json index 04b26b8fe0..7cb8038c5a 100644 --- a/modules/ckanext-ytp-main/ckanext/ytp/dataset/schemas/dataset.json +++ b/modules/ckanext-ytp-main/ckanext/ytp/dataset/schemas/dataset.json @@ -50,7 +50,7 @@ }, { "field_name": "geographical_coverage", - "label": "Geographical Coverage", + "label": "Geographical coverage", "form_placeholder":"eg. tampere", "preset": "vocabulary_with_autocomplete", "validators": "ignore_missing convert_to_list create_tags(geographical_coverage)", @@ -102,7 +102,7 @@ }, { "field_name": "maintainer_email", - "label": "Maintainer Email", + "label": "Maintainer email", "form_placeholder": "joe@example.com", "display_property": "dc:contributor", "display_snippet": "email.html", @@ -110,7 +110,7 @@ }, { "field_name": "maintainer_website", - "label": "Maintainer Website", + "label": "Maintainer website", "form_placeholder": "http://www.example.com", "display_property": "dc:contributor", "display_snippet": "link.html" @@ -123,7 +123,7 @@ }, { "field_name": "author_email", - "label": "Author Email", + "label": "Author email", "form_placeholder": "joe@example.com", "display_property": "dc:creator", "display_snippet": "email.html", @@ -144,7 +144,7 @@ }, { "field_name": "update_frequency", - "label": "Update Frequency", + "label": "Update frequency", "form_placeholder":"eg. every second week", "form_languages": ["fi", "en", "sv"], "preset": "fluent_vocabulary_with_autocomplete", @@ -169,29 +169,29 @@ "preset": "fluent_core_markdown_translated", "form_placeholder": "eg. A detailed description", "form_languages": ["fi", "en", "sv"], - "label": "Copyright Notice", + "label": "Copyright notice", "display_snippet": null, "description": "An universal, compact and easy to understand description of the added dataset. Use as confining terms as possible to assist the user to understand what types of data, meters and dimensions the dataset contains." }, { "field_name": "collection_type", - "label": "Collection Type", + "label": "Collection type", "preset": "select", "choices": [ { "value": "Open Data", - "label": "Open Data" + "label": "Open data" }, { "value": "Interoperability Tools", - "label": "Interoperability Tools" + "label": "Interoperability tools" } ], "required": true }, { "field_name": "valid_from", - "label": "Valid From", + "label": "Valid from", "preset": "date" }, { @@ -256,7 +256,7 @@ }, { "field_name": "time_series_precision", - "label": "Time Series Precision", + "label": "Time series precision", "form_placeholder":"eg. 2 weeks", "preset": "fluent_vocabulary_with_autocomplete", "validators": "fluent_tags create_fluent_tags(time_series_precision)", @@ -269,22 +269,22 @@ }, { "field_name": "temporal_granularity", - "label": "Temporal Granularity", + "label": "Temporal granularity", "preset": "fluent_text" }, { "field_name": "update_frequency", - "label": "Update Frequency", + "label": "Update frequency", "preset": "fluent_text" }, { "field_name": "temporal_coverage_to", - "label": "Temporal Coverage To", + "label": "Temporal coverage to", "preset": "date" }, { "field_name": "temporal_coverage_from", - "label": "Temporal Coverage from", + "label": "Temporal coverage from", "preset": "date" } ]
Administrative branch Dataset count New datasetsAvg. openness score Data formats
2 years ago 3 years ago Last monthLast yearLast 6 months
{{ row.dataset_count_2yr }} {{ row.dataset_count_3yr }} {{ row.new_datasets_month }}{{ row.new_datasets_year }}{{ row.new_datasets_6_months }}{{ row.openness_score_avg }} {{ row.resource_formats }}
{{ row.dataset_count_3yr }} {{ row.new_datasets_month }} {{ row.new_datasets_6_months }}{{ row.openness_score_avg }}{% if row.openness_score_avg == None %}-{% else %}{{ row.openness_score_avg }}/5{% endif %} {{ row.resource_formats }}