forked from dmlc/gluon-nlp
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Improve word embeddings evaluation docs, tests and code style * Decrease default batch-size for word embedding evaluation to avoid OOM * Add dataset and keyword registry * Enable dataset registry for word embedding evaluation datasets * Update word embedding evaluation script based on dataset registry * Scope code in script behind __main__ Fix redefined-outer-name lint error * Enable Dataset registry for all Datasets * Workaround Py3 inspect.getargspec deprecation * Add logging option to word embedding evaluation script * Add __all__
- Loading branch information
Showing
13 changed files
with
535 additions
and
219 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,146 @@ | ||
| # coding: utf-8 | ||
|
|
||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
| """A registry for datasets | ||
| The registry makes it simple to construct a dataset given its name. | ||
| """ | ||
| __all__ = ['register', 'create', 'list_datasets'] | ||
|
|
||
| import inspect | ||
|
|
||
| from mxnet import registry | ||
| from mxnet.gluon.data import Dataset | ||
|
|
||
| _REGSITRY_NAME_KWARGS = {} | ||
|
|
||
|
|
||
| def register(class_=None, **kwargs): | ||
| """Registers a dataset with segment specific hyperparameters. | ||
| When passing keyword arguments to `register`, they are checked to be valid | ||
| keyword arguments for the registered Dataset class constructor and are | ||
| saved in the registry. Registered keyword arguments can be retrieved with | ||
| the `list_datasets` function. | ||
| All arguments that result in creation of separate datasets should be | ||
| registered. Examples are datasets divided in different segments or | ||
| categories, or datasets containing multiple languages. | ||
| Once registered, an instance can be created by calling | ||
| :func:`~gluonnlp.data.create` with the class name. | ||
| Parameters | ||
| ---------- | ||
| **kwargs : list or tuple of allowed argument values | ||
| For each keyword argument, it's value must be a list or tuple of the | ||
| allowed argument values. | ||
| Examples | ||
| -------- | ||
| >>> @gluonnlp.data.register(segment=['train', 'test', 'dev']) | ||
| ... class MyDataset(Dataset): | ||
| ... def __init__(self, segment='train'): | ||
| ... pass | ||
| >>> my_dataset = gluonnlp.embedding.create('MyDataset') | ||
| >>> print(type(my_dataset)) | ||
| <class '__main__.MyDataset'> | ||
| """ | ||
|
|
||
| def _real_register(class_): | ||
| # Assert that the passed kwargs are meaningful | ||
| for kwarg_name, values in kwargs.items(): | ||
| try: | ||
| real_args = inspect.getfullargspec(class_).args | ||
| except AttributeError: | ||
| # pylint: disable=deprecated-method | ||
| real_args = inspect.getargspec(class_.__init__).args | ||
|
|
||
| if not kwarg_name in real_args: | ||
| raise RuntimeError( | ||
| ('{} is not a valid argument for {}. ' | ||
| 'Only valid arguments can be registered.').format( | ||
| kwarg_name, class_.__name__)) | ||
|
|
||
| if not isinstance(values, (list, tuple)): | ||
| raise RuntimeError(('{} should be a list of ' | ||
| 'valid arguments for {}. ').format( | ||
| values, kwarg_name)) | ||
|
|
||
| # Save the kwargs associated with this class_ | ||
| _REGSITRY_NAME_KWARGS[class_] = kwargs | ||
|
|
||
| register_ = registry.get_register_func(Dataset, 'dataset') | ||
| return register_(class_) | ||
|
|
||
| if class_ is not None: | ||
| # Decorator was called without arguments | ||
| return _real_register(class_) | ||
|
|
||
| return _real_register | ||
|
|
||
|
|
||
| def create(name, **kwargs): | ||
| """Creates an instance of a registered dataset. | ||
| Parameters | ||
| ---------- | ||
| name : str | ||
| The dataset name (case-insensitive). | ||
| Returns | ||
| ------- | ||
| An instance of :class:`mxnet.gluon.data.Dataset` constructed with the | ||
| keyword arguments passed to the create function. | ||
| """ | ||
| create_ = registry.get_create_func(Dataset, 'dataset') | ||
| return create_(name, **kwargs) | ||
|
|
||
|
|
||
| def list_datasets(name=None): | ||
| """Get valid datasets and registered parameters. | ||
| Parameters | ||
| ---------- | ||
| name : str or None, default None | ||
| Return names and registered parameters of registered datasets. If name | ||
| is specified, only registered parameters of the respective dataset are | ||
| returned. | ||
| Returns | ||
| ------- | ||
| dict: | ||
| A dict of all the valid keyword parameters names for the specified | ||
| dataset. If name is set to None, returns a dict mapping each valid name | ||
| to its respective keyword parameter dict. The valid names can be | ||
| plugged in `gluonnlp.model.word_evaluation_model.create(name)`. | ||
| """ | ||
| reg = registry.get_registry(Dataset) | ||
|
|
||
| if name is not None: | ||
| class_ = reg[name.lower()] | ||
| return _REGSITRY_NAME_KWARGS[class_] | ||
| else: | ||
| return { | ||
| dataset_name: _REGSITRY_NAME_KWARGS[class_] | ||
| for dataset_name, class_ in registry.get_registry(Dataset).items() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.