4 error at creating a dataset from url

[JR-1991]

Overview

Following up from issue #4 this PR fixes illegal attribute names such as e-mail or not/good and instead returns e_mail and not_good respectively. These changes though won't affect the resulting Dataverse JSON.

Fixes issues

fixes #4

[atrisovic]

It doesn't work :-S

[atrisovic]

This is the error

/Users/ant746/PycharmProjects/easyDataverse/easyDataverse/core/dataset.py:469: UserWarning: No 'API_TOKEN' found in the environment. Please be aware, that you might not have the rights to download this dataset.
  warnings.warn(
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[3], line 1
----> 1 dataset = Dataset.from_url("https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/SYNPBS&version=4.0")

File ~/opt/miniconda3/envs/easyDVcondaPR/lib/python3.10/site-packages/pydantic/decorator.py:40, in pydantic.decorator.validate_arguments.validate.wrapper_function()

File ~/opt/miniconda3/envs/easyDVcondaPR/lib/python3.10/site-packages/pydantic/decorator.py:134, in pydantic.decorator.ValidatedFunction.call()

File ~/opt/miniconda3/envs/easyDVcondaPR/lib/python3.10/site-packages/pydantic/decorator.py:201, in pydantic.decorator.ValidatedFunction.execute()

File ~/PycharmProjects/easyDataverse/easyDataverse/core/dataset.py:359, in Dataset.from_url(cls, url, filedir, download_files, api_token, lib_name)
    353     raise ValueError(
    354         f"Given URL '{url}' is not a valid Dataverse URL since no 'persistenID' is given"
    355     )
    357 dataverse_url = f"https://{parsed_url.hostname}/"
--> 359 return cls.from_dataverse_doi(
    360     doi=doi[0],
    361     filedir=filedir,
    362     lib_name=lib_name,
    363     dataverse_url=dataverse_url,
    364     api_token=api_token,
    365     download_files=download_files,
    366 )

File ~/opt/miniconda3/envs/easyDVcondaPR/lib/python3.10/site-packages/pydantic/decorator.py:40, in pydantic.decorator.validate_arguments.validate.wrapper_function()

File ~/opt/miniconda3/envs/easyDVcondaPR/lib/python3.10/site-packages/pydantic/decorator.py:134, in pydantic.decorator.ValidatedFunction.call()

File ~/opt/miniconda3/envs/easyDVcondaPR/lib/python3.10/site-packages/pydantic/decorator.py:201, in pydantic.decorator.ValidatedFunction.execute()

File ~/PycharmProjects/easyDataverse/easyDataverse/core/dataset.py:416, in Dataset.from_dataverse_doi(cls, doi, filedir, filenames, download_files, lib_name, dataverse_url, api_token)
    412     if not dataverse_url:
    413         raise ValueError(
    414             "Dataverse URL has not been specified in argument 'dataverse_url'. Please specify it to download datasets from your desired installation."
    415         )
--> 416     return cls._fetch_without_lib(
    417         dataset=cls(),
    418         doi=doi,
    419         filedir=filedir,
    420         dataverse_url=dataverse_url,
    421         api_token=api_token,
    422         filenames=filenames,
    423     )
    425 elif not lib_name and "EASYDATAVERSE_LIB_NAME" in os.environ:
    426     dataverse_url, api_token = cls._fetch_env_vars(api_token)

File ~/PycharmProjects/easyDataverse/easyDataverse/core/dataset.py:473, in Dataset._fetch_without_lib(**kwargs)
    468     except KeyError:
    469         warnings.warn(
    470             "No 'API_TOKEN' found in the environment. Please be aware, that you might not have the rights to download this dataset."
    471         )
--> 473 return download_from_dataverse_without_lib(**kwargs)

File ~/PycharmProjects/easyDataverse/easyDataverse/tools/downloader/downloader.py:99, in download_from_dataverse_without_lib(dataset, doi, filedir, dataverse_url, api_token, filenames)
     96 dataset.p_id = doi
     98 # Step 2: Extract all metadatablocks from the given dataset
---> 99 blocks = [
    100     create_block_definitions(block_name, block, dataverse_url)
    101     for block_name, block in metadatablocks.items()
    102 ]
    104 # Step 3: Populate data and assign to dataset
    105 for block in blocks:

File ~/PycharmProjects/easyDataverse/easyDataverse/tools/downloader/downloader.py:100, in <listcomp>(.0)
     96 dataset.p_id = doi
     98 # Step 2: Extract all metadatablocks from the given dataset
     99 blocks = [
--> 100     create_block_definitions(block_name, block, dataverse_url)
    101     for block_name, block in metadatablocks.items()
    102 ]
    104 # Step 3: Populate data and assign to dataset
    105 for block in blocks:

File ~/PycharmProjects/easyDataverse/easyDataverse/tools/downloader/nolibutils.py:95, in create_block_definitions(block_name, block, dataverse_url)
     93 # Turn raw definitions into classes
     94 for field in block["fields"]:
---> 95     _process_field(field, lookup, cls_def, add_funs)
     97 # Now, create the class
     98 block_cls = create_model(
     99     block_name.capitalize(), __base__=(DataverseBase,), **cls_def
    100 )()

File ~/PycharmProjects/easyDataverse/easyDataverse/tools/downloader/nolibutils.py:138, in _process_field(field, lookup, cls_def, add_funs)
    131 PROCESS_MAPPING = {
    132     "primitive": _process_primitive,
    133     "controlledVocabulary": _process_primitive,
    134     "compound": _process_compound,
    135 }
    137 fun = PROCESS_MAPPING[field["typeClass"]]
--> 138 fun(field, lookup, cls_def, add_funs)

File ~/PycharmProjects/easyDataverse/easyDataverse/tools/downloader/nolibutils.py:162, in _process_compound(field, lookup, cls_def, add_funs)
    159 field_name = _camel_to_snake(field_name)
    161 # Generate add method
--> 162 add_funs[f"add_{field_name}"] = _generate_add_method(cls, field_name)
    164 cls_def[field_name] = (dtype, Field(**field_meta))

File ~/PycharmProjects/easyDataverse/easyDataverse/tools/downloader/nolibutils.py:252, in _generate_add_method(target_cls, field)
    249 add_fun = copy.deepcopy(_generic_add_function)
    250 add_fun.__name__ = f"add_{field}"
--> 252 return forge.sign(
    253     forge.self,
    254     *[
    255         forge.kwarg(name, type=dtype, default=forge.empty)
    256         for name, dtype in target_cls.__annotations__.items()
    257     ],
    258     forge.kwarg("_target_cls", default=target_cls, bound=True),
    259     forge.kwarg("_field", default=field, bound=True),
    260 )(add_fun)

File ~/opt/miniconda3/envs/easyDVcondaPR/lib/python3.10/site-packages/forge/_revision.py:331, in Revision.__call__(self, callable)
    328         return callable(*mapped.args, **mapped.kwargs)
    330 next_.validate()
--> 331 inner.__mapper__ = Mapper(next_, callable)  # type: ignore
    332 inner.__signature__ = inner.__mapper__.public_signature  # type: ignore
    333 return inner

File ~/opt/miniconda3/envs/easyDVcondaPR/lib/python3.10/site-packages/forge/_revision.py:62, in Mapper.__init__(self, fsignature, callable)
     54 def __init__(
     55         self,
     56         fsignature: FSignature,
   (...)
     59     # pylint: disable=W0622, redefined-builtin
     60     # pylint: disable=W0621, redefined-outer-name
     61     private_signature = inspect.signature(callable)
---> 62     public_signature = fsignature.native
     63     parameter_map = self.map_parameters(fsignature, private_signature)
     64     context_param = get_context_parameter(fsignature)

File ~/opt/miniconda3/envs/easyDVcondaPR/lib/python3.10/site-packages/forge/_signature.py:1231, in FSignature.native(self)
   1224 @property
   1225 def native(self) -> inspect.Signature:
   1226     """
   1227     Provides a representation of this :class:`~forge.FSignature` as an
   1228     instance of :class:`inspect.Signature`
   1229     """
   1230     return inspect.Signature(
-> 1231         [param.native for param in self if not param.bound],
   1232         return_annotation=self.return_annotation,
   1233     )

File ~/opt/miniconda3/envs/easyDVcondaPR/lib/python3.10/site-packages/forge/_signature.py:1231, in <listcomp>(.0)
   1224 @property
   1225 def native(self) -> inspect.Signature:
   1226     """
   1227     Provides a representation of this :class:`~forge.FSignature` as an
   1228     instance of :class:`inspect.Signature`
   1229     """
   1230     return inspect.Signature(
-> 1231         [param.native for param in self if not param.bound],
   1232         return_annotation=self.return_annotation,
   1233     )

File ~/opt/miniconda3/envs/easyDVcondaPR/lib/python3.10/site-packages/forge/_signature.py:355, in FParameter.native(self)
    353 if not self.name:
    354     raise TypeError('Cannot generate an unnamed parameter')
--> 355 return inspect.Parameter(
    356     name=self.name,
    357     kind=self.kind,
    358     default=empty.ccoerce_native(self.default),
    359     annotation=empty.ccoerce_native(self.type),
    360 )

File ~/opt/miniconda3/envs/easyDVcondaPR/lib/python3.10/inspect.py:2673, in Parameter.__init__(self, name, kind, default, annotation)
   2670     name = 'implicit{}'.format(name[1:])
   2672 if not name.isidentifier():
-> 2673     raise ValueError('{!r} is not a valid parameter name'.format(name))
   2675 self._name = name

ValueError: 'e-mail' is not a valid parameter name

[JR-1991]

I have introduced a typo here, my fault. Just fixed it and tested your code in a Colab Notebook - Working here, but would be awesome if you could check on your machine too.

If its working, I am going to update the package at PyPi 👍

[atrisovic]

It works now!