Skip to content

Commit

Permalink
more dataset model implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
@Zharktas
Zharktas committed Nov 1, 2017
1 parent 0d5717c commit 5cfe706
Show file tree
Hide file tree
Showing 4 changed files with 159 additions and 29 deletions.
1 change: 1 addition & 0 deletions ansible/roles/ckan/templates/ckan.ini.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ ckan.plugins = {{ ckan_plugins_default }} {{ ckan_plugins }} {% if google_analyt
scheming.dataset_schemas = ckanext.ytp.dataset.schemas:dataset.json
scheming.presets = ckanext.ytp.dataset:presets.json
ckanext.scheming:presets.json
ckanext.fluent:presets.json

ckan.site_title = Avoindata.fi
ckan.site_logo = {{www_root}}/resources/images/avoindata.fi.png
Expand Down
106 changes: 80 additions & 26 deletions modules/ckanext-ytp-main/ckanext/ytp/dataset/commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@

import ast

from ckan.lib.munge import munge_title_to_name

class YtpFacetTranslations(CkanCommand):
""" Command to add task to schedule table """
max_args = None
Expand Down Expand Up @@ -110,31 +112,83 @@ def migrate(ctx, config):
for dataset in datasets:
data_dict = {'id': dataset}
old_package_dict = get_action('package_show')(context, data_dict)

extras = {x['key']: x['value'] for x in old_package_dict.get('extras', {})}

original_language = default_lang
if extras.get('original_language'):
original_language = extras.get('original_language')

langs = []
if extras.get('translations'):
langs = ast.literal_eval(extras.get('translations'))

new_package_dict = {
'id': old_package_dict['id'],
'title_translated': {
original_language: old_package_dict['title']
},
'notes_translated':{
original_language: old_package_dict['notes']
},
'collection_type': extras.get('collection_type', 'Open Data')
}

for lang in langs:
new_package_dict['title_translated'][u'' + lang] = extras.get('title_' + lang)
new_package_dict['notes_translated'][u''+ lang] = extras.get('notes_' + lang)
print(new_package_dict)
if old_package_dict.get('title_translated'):
extras = {x['key']: x['value'] for x in old_package_dict.get('extras', {})}

original_language = default_lang
if extras.get('original_language'):
original_language = extras.get('original_language')

langs = []
if extras.get('translations'):
langs = ast.literal_eval(extras.get('translations'))

new_package_dict = {
'id': old_package_dict['id'],
'name': munge_title_to_name(old_package_dict['title']),
'title_translated': {
original_language: old_package_dict['title']
},
'notes_translated':{
original_language: old_package_dict['notes']
},
'collection_type': extras.get('collection_type', 'Open Data'),
'keywords': {'fi': []},
'content_type': {'fi': []},
'copyright_notice_translated':{
original_language: extras.get('copyright_notice', '')
},
'external_urls': extras.get('extra_information', []),
'owner': old_package_dict.get('owner', ''),
'owner_org': old_package_dict['owner_org'],
'valid_from': old_package_dict.get('valid_from'),
'valid_till': old_package_dict.get('valid_till'),
'license_id': old_package_dict['license_id'],
'maintainer_email': old_package_dict.get('maintainer_email', ''),
'resources': []
}

for lang in langs:
new_package_dict['title_translated'][u'' + lang] = extras.get('title_' + lang)
new_package_dict['notes_translated'][u''+ lang] = extras.get('notes_' + lang)
new_package_dict['copyright_notice_translated'][u''+ lang] = extras.get('copyright_notice_' + lang)


new_package_dict['keywords']['fi'] = [ tag['name'] for tag in old_package_dict['tags'] if tag['vocabulary_id'] is None ]
new_package_dict['content_type']['fi'] = [ tag['name'] for tag in old_package_dict['tags'] if tag['vocabulary_id'] is not None ]

for resource in old_package_dict.get('resources', []):
new_resource = {
'name_translated': {
original_language: resource['name']
},
'description_translated': {
original_language: resource.get('description')
},
'temporal_granularity': {
original_language: resource.get('temporal_granularity')
},
'temporal_coverage_to': resource.get('temporal_coverage_to'),
'temporal_coverage_from': resource.get('temporal_coverage_from'),
'update_frequency': {
original_language: resource.get('update_frequency')
},
'format': resource.get('format'),
'created': resource.get('created'),
'url': resource.get('url'),
'modified': resource.get('modified')

}

for lang in langs:
new_resource['name_translated'][u'' + lang] = resource.get('name_' + lang)
new_resource['description_translated'][u''+ lang] = resource.get('description_' + lang)
new_resource['temporal_granularity'][u''+ lang] = resource.get('temporal_granularity_' + lang)
new_resource['update_frequency'][u''+ lang] = resource.get('update_frequency_' + lang)

new_package_dict['resources'].append(new_resource)
print(new_package_dict)
created_dataset = get_action('package_create')({}, new_package_dict)
print(new_package_dict)

print(datasets)
77 changes: 75 additions & 2 deletions modules/ckanext-ytp-main/ckanext/ytp/dataset/schemas/dataset.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,18 +35,32 @@
"only_default_lang_required": true,
"description": "Keywords or tags through which users are able to find this dataset easily through the search page or other datasets which have the same tag."
},
{
"field_name": "content_type",
"label": "Content type",
"form_placeholder":"eg. every second week",
"form_languages": ["fi", "en", "sv"],
"preset": "fluent_vocabulary_with_autocomplete",
"validators": "only_default_lang_required fluent_tags create_fluent_tags(content_type)",
"form_attrs": {
"data-module": "autocomplete",
"data-module-tags": "",
"data-module-source": "/api/2/util/tag/autocomplete?incomplete=?&vocabulary_id=content_type"
},
"only_default_lang_required": true,
"description": "Keywords or tags through which users are able to find this dataset easily through the search page or other datasets which have the same tag."
},
{
"field_name": "geographical_coverage",
"label": "Geographical Coverage",
"form_placeholder":"eg. tampere",
"preset": "vocabulary_with_autocomplete",
"validators": "not_empty convert_to_list create_tags(geographical_coverage)",
"validators": "ignore_missing convert_to_list create_tags(geographical_coverage)",
"form_attrs": {
"data-module": "autocomplete",
"data-module-tags": "",
"data-module-source": "/api/2/util/tag/autocomplete?incomplete=?&vocabulary_id=geographical_coverage"
},
"required": true,
"description": "Select the municipalities from which the dataset contains data."
},
{
Expand Down Expand Up @@ -104,6 +118,25 @@
"display_property": "dc:contributor",
"display_snippet": "link.html"
},
{
"field_name": "author",
"label": "Author",
"form_placeholder": "Joe Bloggs",
"display_property": "dc:creator"
},
{
"field_name": "author_email",
"label": "Author Email",
"form_placeholder": "joe@example.com",
"display_property": "dc:creator",
"display_snippet": "email.html",
"display_email_name_field": "author"
},
{
"field_name": "owner",
"label": "Owner",
"form_placeholder": "Joe Bloggs"
},
{
"field_name": "private",
"label": "Visibility",
Expand Down Expand Up @@ -134,6 +167,16 @@
"display_snippet": "license.html",
"required": true
},
{
"field_name": "copyright_notice_translated",
"preset": "fluent_core_markdown_translated",
"form_placeholder": "eg. A detailed description",
"form_languages": ["fi", "en", "sv"],
"label": "Copyright Notice",
"display_snippet": null,
"only_default_lang_required": true,
"description": "An universal, compact and easy to understand description of the added dataset. Use as confining terms as possible to assist the user to understand what types of data, meters and dimensions the dataset contains."
},
{
"field_name": "collection_type",
"label": "Collection Type",
Expand All @@ -149,6 +192,16 @@
}
],
"required": true
},
{
"field_name": "valid_from",
"label": "Valid From",
"preset": "date"
},
{
"field_name": "valid_till",
"label": "Valid till",
"preset": "date"
}
],
"resource_fields": [
Expand Down Expand Up @@ -218,6 +271,26 @@
"data-module-source": "/api/2/util/tag/autocomplete?incomplete=?&vocabulary_id=time_series_precision"
},
"description": "A string which describes the precision of the entered time series."
},
{
"field_name": "temporal_granularity",
"label": "Temporal Granularity",
"preset": "fluent_text"
},
{
"field_name": "update_frequency",
"label": "Update Frequency",
"preset": "fluent_text"
},
{
"field_name": "temporal_coverage_to",
"label": "Temporal Coverage To",
"preset": "date"
},
{
"field_name": "temporal_coverage_from",
"label": "Temporal Coverage from",
"preset": "date"
}
]
}
4 changes: 3 additions & 1 deletion modules/ckanext-ytp-main/ckanext/ytp/dataset/templates/package/snippets/dataset_info.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@ <h4 class="dataset-title sidebar-title">{{_('Tags')}}</h4>
<h4 class="dataset-title sidebar-title">{{_('Content Type')}}</h4>

{% if pkg.get( 'content_type') %}
{% set con_list = pkg.get( 'content_type').split(',') %}

{% set lang = h.get_lang_prefix() %}
{% set con_list = pkg.get( 'content_type' ).get(lang) %}
{%- for content in con_list -%}
<a href="/data/{{ h.lang() }}/dataset?content_type={{content | lower }}">{{_(content) | lower}}</a>
{%- if not loop.last %} <br /> {% endif -%}
Expand Down

0 comments on commit 5cfe706

Please sign in to comment.