AD-6: Administrative branch summary report prototype

ansible/roles/ckan/vars/main.yml

@@ -7,7 +7,7 @@ celery_user: "{{ www_user }}"
 
 ckan_plugins_default: stats
 # order matters, when templates call super()
-ckan_plugins: multilingual_dataset harvest ckan_harvester hri_harvester dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface spatial_metadata spatial_query csw_harvester drupal7 datarequests report qa archiver ytp_organizations ytp_comments ytp_request hierarchy_display ytp_theme ytp_drupal ytp_tasks ytp_dataset ytp_user ytp_service datastore showcase datapusher recline_grid_view recline_graph_view recline_map_view text_view image_view pdf_view geo_view geojson_view
+ckan_plugins: multilingual_dataset harvest ckan_harvester hri_harvester dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface spatial_metadata spatial_query csw_harvester drupal7 datarequests report qa archiver ytp_organizations ytp_comments ytp_request hierarchy_display ytp_theme ytp_drupal ytp_tasks ytp_dataset ytp_user ytp_service ytp_report datastore showcase datapusher recline_grid_view recline_graph_view recline_map_view text_view image_view pdf_view geo_view geojson_view
 
 ckan_aws_plugins: cloudstorage
 

modules/ckanext-ytp-main/ckanext/ytp/report/__init__.py

@@ -0,0 +1,8 @@
+# this is a namespace package
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)
+

modules/ckanext-ytp-main/ckanext/ytp/report/plugin.py

@@ -0,0 +1,16 @@
+import ckan.plugins as p
+from ckanext.report.interfaces import IReport
+
+class YtpReportPlugin(p.SingletonPlugin):
+    p.implements(IReport)
+    p.implements(p.IConfigurer, inherit=True)
+
+    # IReport
+
+    def register_reports(self):
+        import reports
+        return [reports.test_report_info, reports.administrative_branch_summary_report_info]
+
+    def update_config(self, config):
+        from ckan.plugins import toolkit
+        toolkit.add_template_directory(config, 'templates')

modules/ckanext-ytp-main/ckanext/ytp/report/reports.py

@@ -0,0 +1,102 @@
+from ckan.common import OrderedDict
+from ckan.logic import get_action, NotFound, NotAuthorized
+import itertools
+from datetime import timedelta, datetime
+
+def test_report():
+    return {
+        'table' : [
+            {str(d): (n*8 + d)**2 for d in range(8)}
+            for n in range(8)
+            ]
+    }
+
+test_report_info = {
+    'name': 'test-report',
+    'title': 'Test Report',
+    'description': 'Most Testy Reportie',
+    'option_defaults': None,
+    'option_combinations': None,
+    'generate': test_report,
+    'template': 'report/test_report.html',
+}
+
+def administrative_branch_summary_report():
+    org_names = [
+            'liikenne-ja-viestintaministerio',
+            'maa-ja-metsatalousministerio',
+            'oikeusministerio',
+            'opetus-ja-kulttuuriministerio',
+            'puolustusministerio',
+            'sosiaali-ja-terveysministerio',
+            'tyo-ja-elinkeinoministerio',
+            'valtioneuvoston-kanslia',
+            'valtiovarainministerio',
+            'ymparistoministerio',
+            ]
+
+    context = {}
+    orgs = get_action('organization_list')(context, {'organizations': org_names, 'all_fields': True})
+    orgs_by_name = {org['name']: org for org in orgs}
+
+    org_trees = [get_action('group_tree_section')(context, {'id': org['id'], 'type': 'organization'})
+                 for org in orgs]
+    org_ids_by_tree = {r['name']: [x['id'] for x in flatten(r, lambda x: x['children'])]
+        for r in org_trees}
+    datasets_by_tree = {k: list(package_generator('owner_org:(%s)' % ' OR '.join(v), 1000, context))
+            for k, v in org_ids_by_tree.iteritems()}
+    return {
+        'table' : [{
+            'organization': orgs_by_name[org_name],
+            'dataset_count': len(datasets),
+            'new_datasets_month': glen(d for d in datasets if age(d) <= timedelta(30)),
+            'new_datasets_year': glen(d for d in datasets if age(d) <= timedelta(365)),
+            'resource_formats': resource_formats(datasets)
+            }
+            for org_name, datasets in datasets_by_tree.iteritems()
+            ]
+    }
+
+administrative_branch_summary_report_info = {
+    'name': 'administrative-branch-summary-report',
+    'title': 'Administrative Branch Summary',
+    'description': 'Dataset statistics by administrative branch summary',
+    'option_defaults': None,
+    'option_combinations': None,
+    'generate': administrative_branch_summary_report,
+    'template': 'report/administrative_branch_summary_report.html',
+}
+
+def package_generator(query, page_size, context):
+    package_search = get_action('package_search')
+
+    for index in itertools.count(start=0, step=page_size):
+        data_dict = {'include_private': True, 'rows': page_size, 'q': query, 'start': index}
+        packages = package_search(context, data_dict).get('results', [])
+        for package in packages:
+            yield package
+        else:
+            return
+
+def age(dataset):
+    return datetime.now() - datetime.strptime(dataset['metadata_created'], '%Y-%m-%dT%H:%M:%S.%f')
+
+
+def glen(generator):
+    '''
+    Returns the number of items in a generator without interemediate lists
+    '''
+    return sum(1 for x in generator)
+
+
+def flatten(x, children):
+    '''
+    Flatten a hierarchy into an element iterator
+    '''
+    yield x
+    for child in children(x):
+        for cx in flatten(child, children):
+            yield cx
+
+def resource_formats(datasets):
+    return ', '.join(r['format'] for d in datasets for r in d['resources'] if r['format'])

modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/administrative_branch_summary_report.html

@@ -0,0 +1,23 @@
+<table class="table table-bordered table-condensed tablesorter" id="report-table" style="width: 100%; table-layout:fixed; margin-top: 8px;">
+    <thead>
+      <tr>
+        <th>Administrative branch</th>
+        <th>Dataset count</th>
+        <th>New datasets last month</th>
+        <th>New datasets last year</th>
+        <th>Data formats</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for row in table %}
+        <tr>
+          <td>{{ row.organization.name }}</td>
+          <td>{{ row.dataset_count }}</td>
+          <td>{{ row.new_datasets_month }}</td>
+          <td>{{ row.new_datasets_year }}</td>
+          <td>{{ row.resource_formats }}</td>
+        </tr>
+      {% endfor %}
+    </tbody>
+</table>
+

modules/ckanext-ytp-main/ckanext/ytp/report/templates/report/test_report.html

@@ -0,0 +1,34 @@
+{#
+Report (snippet)
+
+table - main data, as a list of rows, each row is a dict
+data - other data values, as a dict
+#}
+<table class="table table-bordered table-condensed tablesorter" id="report-table" style="width: 100%; table-layout:fixed; margin-top: 8px;">
+    <thead>
+      <tr>
+        <th>1</th>
+        <th>2</th>
+        <th>3</th>
+        <th>4</th>
+        <th>5</th>
+        <th>6</th>
+        <th>7</th>
+        <th>8</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for row in table %}
+        <tr>
+          <td>{{ row['0'] }}</td>
+          <td>{{ row['1'] }}</td>
+          <td>{{ row['2'] }}</td>
+          <td>{{ row['3'] }}</td>
+          <td>{{ row['4'] }}</td>
+          <td>{{ row['5'] }}</td>
+          <td>{{ row['6'] }}</td>
+          <td>{{ row['7'] }}</td>
+        </tr>
+      {% endfor %}
+    </tbody>
+</table>

modules/ckanext-ytp-main/setup.py

@@ -35,6 +35,7 @@
         ytp_theme=ckanext.ytp.theme.plugin:YtpThemePlugin
         ytp_dataset=ckanext.ytp.dataset.plugin:YTPDatasetForm
         ytp_service=ckanext.ytp.service.plugin:YTPServiceForm
+        ytp_report=ckanext.ytp.report.plugin:YtpReportPlugin
 
         [ckan.celery_task]
         tasks = ckanext.ytp.common.celery_import:task_imports