Skip to content

Commit

Permalink
Fix read from code when config does not exists
Browse files Browse the repository at this point in the history
PiperOrigin-RevId: 347997824
  • Loading branch information
Conchylicultor authored and copybara-github committed Dec 17, 2020
1 parent 5180e49 commit b572fe9
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 1 deletion.
11 changes: 10 additions & 1 deletion tensorflow_datasets/core/load.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,10 @@ def builder(
raise not_found_error


def _try_load_from_files_first(cls, **builder_kwargs) -> bool:
def _try_load_from_files_first(
cls: Optional[Type[dataset_builder.DatasetBuilder]],
**builder_kwargs: Any,
) -> bool:
"""Returns True if files should be used rather than code."""
if set(builder_kwargs) - {'version', 'config', 'data_dir'}:
return False # Has extra kwargs, require original code.
Expand All @@ -181,6 +184,12 @@ def _try_load_from_files_first(cls, **builder_kwargs) -> bool:
return True # Code does not exists
elif 'version' in builder_kwargs:
return True # Version explicitly given (unlock backward compatibility)
elif (
'config' in builder_kwargs
and isinstance(builder_kwargs['config'], str)
and builder_kwargs['config'] not in cls.builder_configs
):
return True # Requested config isn't found in the code
else:
return False # Code exists and no version given, use code.

Expand Down
20 changes: 20 additions & 0 deletions tensorflow_datasets/core/read_only_builder_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,26 @@ def test_builder_files_exists(code_builder: dataset_builder.DatasetBuilder):
assert not isinstance(builder, read_only_builder.ReadOnlyBuilder)


def test_builder_config(code_builder: dataset_builder.DatasetBuilder):
"""Tests that code found but config not loads from files."""
if not code_builder.BUILDER_CONFIGS:
return

# Remove the registered configs
with mock.patch.object(type(code_builder), 'BUILDER_CONFIGS', []), \
mock.patch.object(type(code_builder), 'builder_configs', {}):
# Config isn't present in the code anymore
with pytest.raises(ValueError, match='BuilderConfig .* not found'):
load.builder(
f'{code_builder.name}/dummy_config', data_dir='/tmp/path/not-exists'
)

# But previously generated configs still be loaded from disk
builder = load.builder(f'{code_builder.name}/dummy_config')
assert not isinstance(builder, type(code_builder))
assert isinstance(builder, read_only_builder.ReadOnlyBuilder)


def test_builder_code_not_found(code_builder: dataset_builder.DatasetBuilder):
"""If the code isn't found, use files instead."""

Expand Down
2 changes: 2 additions & 0 deletions tensorflow_datasets/core/registered_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ def as_dataset(self, **kwargs):
self.as_dataset_kwargs = kwargs
return self

builder_configs = {}


class UnregisteredBuilder(EmptyDatasetBuilder):

Expand Down

0 comments on commit b572fe9

Please sign in to comment.