fix: Fixes JSON-LD translation and related issues (#846)

* Add default value converter and remove dataset property from DatasetFile

* Removes type from Person context, fixes missing context in jsonld translation of project, fixes url encoding of dataset names

Author: Panaetius

renku/core/commands/checks/migration.py

@@ -240,7 +240,8 @@ def fix_dataset_files_urls(client):
     """Ensure dataset files have correct url format."""
     for dataset in client.datasets.values():
         for file_ in dataset.files:
-            file_.url = url_to_string(file_.url)
+            if file_.url:
+                file_.url = url_to_string(file_.url)
 
         dataset.to_yaml()
 

renku/core/commands/providers/dataverse.py

@@ -220,7 +220,6 @@ def as_dataset(self, client):
                 filename=file_.name,
                 filesize=file_.content_size,
                 filetype=file_.file_format,
-                dataset=dataset.name,
                 path='',
             )
             serialized_files.append(dataset_file)

renku/core/commands/providers/zenodo.py

@@ -245,7 +245,6 @@ def as_dataset(self, client):
                 filename=file_.filename,
                 filesize=file_.filesize,
                 filetype=file_.type,
-                dataset=dataset.name,
                 path='',
             )
             serialized_files.append(dataset_file)

renku/core/management/datasets.py

@@ -304,7 +304,6 @@ def _add_from_local(self, dataset, path, link, destination):
                     'path': path_in_repo,
                     'url': path_in_repo,
                     'creator': dataset.creator,
-                    'dataset': dataset.name,
                     'parent': self
                 }]
 
@@ -326,7 +325,6 @@ def _add_from_local(self, dataset, path, link, destination):
             'path': destination.relative_to(self.path),
             'url': 'file://' + os.path.relpath(str(src), str(self.path)),
             'creator': dataset.creator,
-            'dataset': dataset.name,
             'parent': self
         }]
 
@@ -352,7 +350,6 @@ def _add_from_url(self, dataset, url, destination):
             'path': destination.relative_to(self.path),
             'url': remove_credentials(url),
             'creator': dataset.creator,
-            'dataset': dataset.name,
             'parent': self
         }]
 
@@ -361,6 +358,7 @@ def _add_from_git(self, dataset, url, sources, destination, ref):
         from renku import LocalClient
 
         u = parse.urlparse(url)
+
         sources = self._resolve_paths(u.path, sources)
 
         # Get all files from repo that match sources
@@ -429,7 +427,6 @@ def _add_from_git(self, dataset, url, sources, destination, ref):
                     'path': path_in_dst_repo,
                     'url': remove_credentials(url),
                     'creator': creators,
-                    'dataset': dataset.name,
                     'parent': self,
                     'based_on': based_on
                 })

renku/core/models/datasets.py

@@ -172,8 +172,6 @@ class DatasetFile(Entity, CreatorMixin):
 
     checksum = attr.ib(default=None, kw_only=True)
 
-    dataset = jsonld.ib(context='schema:isPartOf', default=None, kw_only=True)
-
     filename = attr.ib(kw_only=True, converter=convert_filename_path)
 
     name = jsonld.ib(context='schema:name', kw_only=True, default=None)

renku/core/models/jsonld.py

@@ -22,6 +22,7 @@
 import weakref
 from copy import deepcopy
 from datetime import datetime, timezone
+from functools import partial
 from importlib import import_module
 from pathlib import Path
 
@@ -265,12 +266,23 @@ def _propagate_reference_contexts(
     return current_context, scoped_properties
 
 
+def _default_converter(cls, value):
+    """A default converter method that tries to deserialize objects."""
+    if isinstance(value, dict):
+        return cls.from_jsonld(value)
+
+    return value
+
+
 def attrib(context=None, type=None, **kwargs):
     """Create a new attribute with context."""
     kwargs.setdefault('metadata', {})
     kwargs['metadata'][KEY] = context
     if type:
         kwargs['metadata'][KEY_CLS] = type
+
+        if 'converter' not in kwargs and hasattr(type, 'from_jsonld'):
+            kwargs['converter'] = partial(_default_converter, type)
     return attr.ib(**kwargs)
 
 
@@ -523,6 +535,15 @@ def from_jsonld(
 
         for k, v in compacted.items():
             if k in fields:
+                no_value_context = isinstance(v, dict) and '@context' not in v
+                has_nested_context = (
+                    k in compacted['@context'] and
+                    '@context' in compacted['@context'][k]
+                )
+                if no_value_context and has_nested_context:
+                    # Propagate down context
+                    v['@context'] = compacted['@context'][k]['@context']
+
                 data_[k.lstrip('_')] = v
 
         if __reference__:

renku/core/models/migrations/dataset.py

@@ -30,7 +30,12 @@ def migrate_dataset_schema(data):
     )
 
     data['creator'] = data.pop('authors', {})
-    for file_name, file_ in data.get('files', {}).items():
+
+    files = data.get('files', [])
+
+    if isinstance(files, dict):
+        files = files.values()
+    for file_ in files:
         file_['creator'] = file_.pop('authors', {})
 
     return data
@@ -52,13 +57,13 @@ def migrate_absolute_paths(data):
     files = data.get('files', [])
 
     if isinstance(files, dict):
-        files = files.values()
+        files = list(files.values())
 
     for file_ in files:
         path = Path(file_.get('path'), '.')
         if path.is_absolute():
             file_['path'] = path.relative_to((os.getcwd()))
-
+    data['files'] = files
     return data
 
 

renku/core/models/projects.py

@@ -73,7 +73,6 @@ class Project(object):
         kw_only=True,
         context={
             '@id': 'schema:creator',
-            '@type': 'schema:Person',
         },
         type=Person
     )
@@ -120,6 +119,8 @@ def project_id(self):
             owner = remote.get('owner') or owner
             name = remote.get('name') or name
         host = os.environ.get('RENKU_DOMAIN') or host
+        if name:
+            name = urllib.parse.quote(name, safe='')
         project_url = urllib.parse.urljoin(
             'https://{host}'.format(host=host),
             pathlib.posixpath.join(PROJECT_URL_PATH, owner, name or 'NULL')