Merge branch 'master' of https://github.com/yhteentoimivuuspalvelut/ytp

ansible/roles/ckan/vars/main.yml

@@ -7,7 +7,7 @@ celery_user: "{{ www_user }}"
 
 ckan_plugins_default: stats scheming_datasets fluent
 # order matters, when templates call super()
-ckan_plugins: harvest ckan_harvester hri_harvester dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface spatial_metadata spatial_query csw_harvester drupal7 datarequests report qa archiver ytp_organizations ytp_comments ytp_request hierarchy_display ytp_theme ytp_drupal ytp_tasks ytp_dataset ytp_user ytp_service datastore showcase datapusher recline_grid_view recline_graph_view recline_map_view text_view image_view pdf_view geo_view geojson_view
+ckan_plugins: harvest ckan_harvester hri_harvester dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface spatial_metadata spatial_query csw_harvester drupal7 datarequests report qa archiver ytp_organizations ytp_comments ytp_request ytp_report hierarchy_display ytp_theme ytp_drupal ytp_tasks ytp_dataset ytp_spatial ytp_user ytp_service datastore showcase datapusher recline_grid_view recline_graph_view recline_map_view text_view image_view pdf_view geo_view geojson_view
 
 
 ckan_aws_plugins: cloudstorage

modules/ckanext-ytp-main/ckanext/ytp/dataset/plugin.py

@@ -201,7 +201,6 @@ class YTPDatasetForm(plugins.SingletonPlugin, toolkit.DefaultDatasetForm):
     plugins.implements(plugins.IPackageController, inherit=True)
     plugins.implements(plugins.IActions)
     plugins.implements(plugins.IConfigurable)
-    plugins.implements(ISpatialHarvester, inherit=True)
     plugins.implements(plugins.IAuthFunctions)
     plugins.implements(plugins.IValidators)
 
@@ -576,6 +575,29 @@ def before_index(self, pkg_dict):
     def get_actions(self):
         return {'package_show': action_package_show}
 
+
+
+    # IValidators
+    def get_validators(self):
+        return {
+            'lower_if_exists': validators.lower_if_exists,
+            'upper_if_exists': validators.upper_if_exists,
+            'tag_string_or_tags_required': validators.tag_string_or_tags_required,
+            'create_tags': validators.create_tags,
+            'create_fluent_tags': validators.create_fluent_tags,
+            'set_private_if_not_admin': validators.set_private_if_not_admin,
+            'list_to_string': validators.list_to_string,
+            'convert_to_list': validators.convert_to_list,
+            'tag_list_output': validators.tag_list_output,
+            'repeating_text': validators.repeating_text,
+            'repeating_text_output': validators.repeating_text_output,
+            'only_default_lang_required': validators.only_default_lang_required
+        }
+
+
+class YTPSpatialHarvester(plugins.SingletonPlugin):
+    plugins.implements(ISpatialHarvester, inherit=True)
+
     # ISpatialHarvester
 
     def get_package_dict(self, context, data_dict):
@@ -690,20 +712,3 @@ def get_package_dict(self, context, data_dict):
 
         return package_dict
 
-
-    # IValidators
-    def get_validators(self):
-        return {
-            'lower_if_exists': validators.lower_if_exists,
-            'upper_if_exists': validators.upper_if_exists,
-            'tag_string_or_tags_required': validators.tag_string_or_tags_required,
-            'create_tags': validators.create_tags,
-            'create_fluent_tags': validators.create_fluent_tags,
-            'set_private_if_not_admin': validators.set_private_if_not_admin,
-            'list_to_string': validators.list_to_string,
-            'convert_to_list': validators.convert_to_list,
-            'tag_list_output': validators.tag_list_output,
-            'repeating_text': validators.repeating_text,
-            'repeating_text_output': validators.repeating_text_output,
-            'only_default_lang_required': validators.only_default_lang_required
-        }

modules/ckanext-ytp-main/ckanext/ytp/dataset/schemas/dataset.json

@@ -50,7 +50,7 @@
     },
     {
       "field_name": "geographical_coverage",
-      "label": "Geographical Coverage",
+      "label": "Geographical coverage",
       "form_placeholder":"eg. tampere",
       "preset": "vocabulary_with_autocomplete",
       "validators": "ignore_missing convert_to_list create_tags(geographical_coverage)",
@@ -102,15 +102,15 @@
     },
     {
       "field_name": "maintainer_email",
-      "label": "Maintainer Email",
+      "label": "Maintainer email",
       "form_placeholder": "joe@example.com",
       "display_property": "dc:contributor",
       "display_snippet": "email.html",
       "display_email_name_field": "maintainer"
     },
     {
       "field_name": "maintainer_website",
-      "label": "Maintainer Website",
+      "label": "Maintainer website",
       "form_placeholder": "http://www.example.com",
       "display_property": "dc:contributor",
       "display_snippet": "link.html"
@@ -123,7 +123,7 @@
     },
     {
       "field_name": "author_email",
-      "label": "Author Email",
+      "label": "Author email",
       "form_placeholder": "joe@example.com",
       "display_property": "dc:creator",
       "display_snippet": "email.html",
@@ -144,7 +144,7 @@
     },
     {
       "field_name": "update_frequency",
-      "label": "Update Frequency",
+      "label": "Update frequency",
       "form_placeholder":"eg. every second week",
       "form_languages": ["fi", "en", "sv"],
       "preset": "fluent_vocabulary_with_autocomplete",
@@ -169,29 +169,29 @@
       "preset": "fluent_core_markdown_translated",
       "form_placeholder": "eg. A detailed description",
       "form_languages": ["fi", "en", "sv"],
-      "label": "Copyright Notice",
+      "label": "Copyright notice",
       "display_snippet": null,
       "description": "An universal, compact and easy to understand description of the added dataset. Use as confining terms as possible to assist the user to understand what types of data, meters and dimensions the dataset contains."
     },
     {
       "field_name": "collection_type",
-      "label": "Collection Type",
+      "label": "Collection type",
       "preset": "select",
       "choices": [
         {
           "value": "Open Data",
-          "label": "Open Data"
+          "label": "Open data"
         },
         {
           "value": "Interoperability Tools",
-          "label": "Interoperability Tools"
+          "label": "Interoperability tools"
         }
       ],
       "required": true
     },
     {
       "field_name": "valid_from",
-      "label": "Valid From",
+      "label": "Valid from",
       "preset": "date"
     },
     {
@@ -256,7 +256,7 @@
     },
     {
       "field_name": "time_series_precision",
-      "label": "Time Series Precision",
+      "label": "Time series precision",
       "form_placeholder":"eg. 2 weeks",
       "preset": "fluent_vocabulary_with_autocomplete",
       "validators": "fluent_tags create_fluent_tags(time_series_precision)",
@@ -269,22 +269,22 @@
     },
     {
       "field_name": "temporal_granularity",
-      "label": "Temporal Granularity",
+      "label": "Temporal granularity",
       "preset": "fluent_text"
     },
     {
       "field_name": "update_frequency",
-      "label": "Update Frequency",
+      "label": "Update frequency",
       "preset": "fluent_text"
     },
     {
       "field_name": "temporal_coverage_to",
-      "label": "Temporal Coverage To",
+      "label": "Temporal coverage to",
       "preset": "date"
     },
     {
       "field_name": "temporal_coverage_from",
-      "label": "Temporal Coverage from",
+      "label": "Temporal coverage from",
       "preset": "date"
     }
   ]

modules/ckanext-ytp-main/ckanext/ytp/report/reports.py

@@ -3,6 +3,10 @@
 import itertools
 from datetime import timedelta, datetime
 
+import logging
+
+log = logging.getLogger(__name__)
+
 def test_report():
     return {
         'table' : [
@@ -36,24 +40,45 @@ def administrative_branch_summary_report():
             ]
 
     context = {}
+
+    # Optimization opportunity: Could fetch all orgs here and manually create the hierarchy
     orgs = get_action('organization_list')(context, {'organizations': org_names, 'all_fields': True})
-    orgs_by_name = {org['name']: org for org in orgs}
 
     org_trees = [get_action('group_tree_section')(context, {'id': org['id'], 'type': 'organization'})
                  for org in orgs]
-    org_ids_by_tree = {r['name']: [x['id'] for x in flatten(r, lambda x: x['children'])]
-        for r in org_trees}
-    datasets_by_tree = {k: list(package_generator('owner_org:(%s)' % ' OR '.join(v), 1000, context))
-            for k, v in org_ids_by_tree.iteritems()}
+
+    def children(dataset):
+        return dataset['children']
+
+    org_levels = {
+            org['name']: level
+            for t in org_trees
+            for org, level in hierarchy_levels(t, children)}
+
+    flat_orgs = (org for t in org_trees for org in flatten(t, children))
+    root_tree_ids_pairs = (
+            (r, [x['id'] for x in flatten(r, children)])
+            for r in flat_orgs)
+
+    # Optimization opportunity: Prefetch datasets for all related orgs in one go
+    root_datasets_pairs = (
+            (k, list(package_generator('owner_org:(%s)' % ' OR '.join(v), 1000, context)))
+            for k, v in root_tree_ids_pairs)
+
     return {
         'table' : [{
-            'organization': orgs_by_name[org_name],
+            'organization': org,
+            'level': org_levels[org['name']],
             'dataset_count': len(datasets),
+            'dataset_count_1yr': glen(d for d in datasets if age(d) >= timedelta(1 * 365)),
+            'dataset_count_2yr': glen(d for d in datasets if age(d) >= timedelta(2 * 365)),
+            'dataset_count_3yr': glen(d for d in datasets if age(d) >= timedelta(3 * 365)),
             'new_datasets_month': glen(d for d in datasets if age(d) <= timedelta(30)),
-            'new_datasets_year': glen(d for d in datasets if age(d) <= timedelta(365)),
-            'resource_formats': resource_formats(datasets)
+            'new_datasets_6_months': glen(d for d in datasets if age(d) <= timedelta(6 * 30)),
+            'resource_formats': resource_formats(datasets),
+            'openness_score_avg': openness_score_avg(context, datasets)
             }
-            for org_name, datasets in datasets_by_tree.iteritems()
+            for org, datasets in root_datasets_pairs
             ]
     }
 
@@ -98,5 +123,27 @@ def flatten(x, children):
         for cx in flatten(child, children):
             yield cx
 
+def hierarchy_levels(x, children, level=0):
+    '''
+    Provide hierarchy levels for nodes in a hierarchy
+    '''
+    yield(x, level)
+    for child in children(x):
+        for cx, cl in hierarchy_levels(child, children, level + 1):
+            yield (cx, cl)
+
 def resource_formats(datasets):
     return ', '.join({r['format'] for d in datasets for r in d['resources'] if r['format']})
+
+def openness_score_avg(context, datasets):
+    openness_score = get_action('qa_package_openness_show')
+    scores = (openness_score(context, {"id": d['id']}) for d in datasets)
+    total, count = reduce(tuple_sum, (
+        (s['openness_score'], 1)
+        for s in scores
+        if s.get('openness_score') is not None),
+        (0, 0))
+    return total / count if count > 0 else None
+
+def tuple_sum(*xs):
+    return tuple(sum(x) for x in zip(*xs))

modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html

@@ -1,20 +1,32 @@
-<table class="table table-bordered table-condensed tablesorter" id="report-table" style="width: 100%; table-layout:fixed; margin-top: 8px;">
+<table class="table table-condensed tablesorter" id="report-table">
     <thead>
       <tr>
-        <th>Administrative branch</th>
-        <th>Dataset count</th>
-        <th>New datasets last month</th>
-        <th>New datasets last year</th>
-        <th>Data formats</th>
+        <th rowspan="2">Administrative branch</th>
+        <th colspan="4">Dataset count</th>
+        <th colspan="2">New datasets</th>
+        <th rowspan="2">Avg. openness score</th>
+        <th rowspan="2">Data formats</th>
+      </tr>
+      <tr>
+        <th>Current</th>
+        <th>1 year ago</th>
+        <th>2 years ago</th>
+        <th>3 years ago</th>
+        <th>Last month</th>
+        <th>Last 6 months</th>
       </tr>
     </thead>
     <tbody>
       {% for row in table %}
         <tr>
-          <td>{{ row.organization.name }}</td>
+          <td style="padding-left: {{ row.level + 1 }}em">{{ row.organization.title }}</td>
           <td>{{ row.dataset_count }}</td>
+          <td>{{ row.dataset_count_1yr }}</td>
+          <td>{{ row.dataset_count_2yr }}</td>
+          <td>{{ row.dataset_count_3yr }}</td>
           <td>{{ row.new_datasets_month }}</td>
-          <td>{{ row.new_datasets_year }}</td>
+          <td>{{ row.new_datasets_6_months }}</td>
+          <td>{% if row.openness_score_avg == None %}-{% else %}{{ row.openness_score_avg }}/5{% endif %}</td>
           <td>{{ row.resource_formats }}</td>
         </tr>
       {% endfor %}

modules/ckanext-ytp-main/setup.py

@@ -34,6 +34,7 @@
         hri_harvester=ckanext.ytp.organizations.harvesters.hriharvester:HRIHarvester
         ytp_theme=ckanext.ytp.theme.plugin:YtpThemePlugin
         ytp_dataset=ckanext.ytp.dataset.plugin:YTPDatasetForm
+        ytp_spatial=ckanext.ytp.dataset.plugin:YTPSpatialHarvester
         ytp_service=ckanext.ytp.service.plugin:YTPServiceForm
         ytp_report=ckanext.ytp.report.plugin:YtpReportPlugin
 

modules/ytp-assets-common/src/less/ckan/ckan.less

@@ -36,10 +36,6 @@
   }
 }
 
-input[type="date"] {
-    height: 24px;
-}
-
 [role=main], .main {
   background: none;
   padding-top: initial;
@@ -1101,3 +1097,12 @@ input[type=checkbox], input[type=radio] {
 .ckanext-datapreview {
   clear: both;
 }
+
+.control-label.group-label {
+  display: block;
+  float: none;
+}
+
+.slug-preview {
+  padding-top: 0;
+}