From f77ef54b8a1329d2b9a7de79c193a8095078d94d Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 27 Oct 2017 16:35:25 -0700 Subject: [PATCH 1/7] Add the ResourceReader ABC --- importlib_resources/abc.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 importlib_resources/abc.py diff --git a/importlib_resources/abc.py b/importlib_resources/abc.py new file mode 100644 index 00000000..8975cbf0 --- /dev/null +++ b/importlib_resources/abc.py @@ -0,0 +1,25 @@ +import abc +from typing.io import BinaryIO + + +class ResourceReader(abc.ABC): + + """Abstract base class for loaders to provide resource reading support.""" + + @abc.abstractmethod + def open_resource(self, path: str) -> BinaryIO: + """Return an opened, file-like object for binary reading of the resource. + + The 'path' argument is expected to represent only a file name. + If the resource cannot be found, FileNotFoundError is raised. + """ + raise FileNotFoundError + + def resource_path(self, path: str) -> str: + """Return the file system path to the specified resource. + + The 'path' argument is expected to represent only a file name. + If the resource does not exist on the file system, raise + FileNotFoundError. + """ + raise FileNotFoundError From e3d95724f671367934ace97f3384d61a5946e720 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 27 Oct 2017 16:35:40 -0700 Subject: [PATCH 2/7] Use the ResourceReader API --- importlib_resources/__init__.py | 83 +++++++++++++++++-------------- importlib_resources/tests/util.py | 45 +++++++++++++++++ 2 files changed, 90 insertions(+), 38 deletions(-) diff --git a/importlib_resources/__init__.py b/importlib_resources/__init__.py index 36cf5e1c..5f560291 100644 --- a/importlib_resources/__init__.py +++ b/importlib_resources/__init__.py @@ -47,27 +47,31 @@ def open(package: Package, file_name: FileName) -> BinaryIO: """Return a file-like object opened for binary-reading of the resource.""" file_name = _normalize_path(file_name) package = _get_package(package) - # Using pathlib doesn't work well here due to the lack of 'strict' argument - # for pathlib.Path.resolve() prior to Python 3.6. - absolute_package_path = os.path.abspath(package.__spec__.origin) - package_path = os.path.dirname(absolute_package_path) - full_path = os.path.join(package_path, file_name) try: - return builtins.open(full_path, 'rb') - except IOError: - # Just assume the loader is a resource loader; all the relevant - # importlib.machinery loaders are and an AttributeError for get_data() - # will make it clear what is needed from the loader. - loader = typing.cast(importlib.abc.ResourceLoader, - package.__spec__.loader) + return package.__spec__.loader.open_resource(file_name) + except AttributeError: + # Using pathlib doesn't work well here due to the lack of 'strict' + # argument for pathlib.Path.resolve() prior to Python 3.6. + absolute_package_path = os.path.abspath(package.__spec__.origin) + package_path = os.path.dirname(absolute_package_path) + full_path = os.path.join(package_path, file_name) try: - data = loader.get_data(full_path) + return builtins.open(full_path, 'rb') except IOError: - package_name = package.__spec__.name - message = '{!r} resource not found in {!r}'.format(file_name, package_name) - raise FileNotFoundError(message) - else: - return io.BytesIO(data) + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = typing.cast(importlib.abc.ResourceLoader, + package.__spec__.loader) + try: + data = loader.get_data(full_path) + except IOError: + package_name = package.__spec__.name + message = '{!r} resource not found in {!r}'.format(file_name, + package_name) + raise FileNotFoundError(message) + else: + return io.BytesIO(data) def read(package: Package, file_name: FileName, encoding: str = 'utf-8', @@ -98,25 +102,28 @@ def path(package: Package, file_name: FileName) -> Iterator[pathlib.Path]: raised if the file was deleted prior to the context manager exiting). """ - normalized_path = _normalize_path(file_name) + file_name = _normalize_path(file_name) package = _get_package(package) - package_directory = pathlib.Path(package.__spec__.origin).parent - file_path = package_directory / normalized_path - if file_path.exists(): - yield file_path - else: - with open(package, normalized_path) as file: - data = file.read() - # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' - # blocks due to the need to close the temporary file to work on - # Windows properly. - fd, raw_path = tempfile.mkstemp() - try: - os.write(fd, data) - os.close(fd) - yield pathlib.Path(raw_path) - finally: + try: + yield pathlib.Path(package.__spec__.loader.resource_path(file_name)) + except (AttributeError, FileNotFoundError): + package_directory = pathlib.Path(package.__spec__.origin).parent + file_path = package_directory / file_name + if file_path.exists(): + yield file_path + else: + with open(package, file_name) as file: + data = file.read() + # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' + # blocks due to the need to close the temporary file to work on + # Windows properly. + fd, raw_path = tempfile.mkstemp() try: - os.remove(raw_path) - except FileNotFoundError: - pass + os.write(fd, data) + os.close(fd) + yield pathlib.Path(raw_path) + finally: + try: + os.remove(raw_path) + except FileNotFoundError: + pass diff --git a/importlib_resources/tests/util.py b/importlib_resources/tests/util.py index 0340b45b..7bcff57c 100644 --- a/importlib_resources/tests/util.py +++ b/importlib_resources/tests/util.py @@ -1,12 +1,38 @@ import abc import importlib +import importlib.machinery +import importlib.util +import io import pathlib import sys import unittest +from .. import abc as resources_abc from . import data +def create_package(*, file, path): + class Reader(resources_abc.ResourceReader): + def open_resource(self, path): + self._path = path + if isinstance(file, Exception): + raise file + else: + return file + + def resource_path(self, path_): + self._path = path_ + if isinstance(path, Exception): + raise path + else: + return path + + spec = importlib.machinery.ModuleSpec('testingpackage', Reader(), + origin='does-not-exist', + is_package=True) + return importlib.util.module_from_spec(spec) + + class CommonTests(abc.ABC): @abc.abstractmethod @@ -54,6 +80,25 @@ def test_non_package(self): with self.assertRaises(TypeError): self.execute(__spec__.name, 'utf-8.file') + def test_resource_opener(self): + data = io.BytesIO(b'Hello, world!') + package = create_package(file=data, path=FileNotFoundError()) + self.execute(package, 'utf-8.file') + self.assertEqual(package.__spec__.loader._path, 'utf-8.file') + + def test_resource_path(self): + data = io.BytesIO(b'Hello, world!') + path = __file__ + package = create_package(file=data, path=path) + self.execute(package, 'utf-8.file') + self.assertEqual(package.__spec__.loader._path, 'utf-8.file') + + def test_useless_loader(self): + package = create_package(file=FileNotFoundError(), + path=FileNotFoundError()) + with self.assertRaises(FileNotFoundError): + self.execute(package, 'utf-8.file') + class ZipSetup: From eaeae38204cc7885658d5136ea292846668bc408 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 31 Oct 2017 10:56:00 -0700 Subject: [PATCH 3/7] Guarantee data files do not have their line endings changed by git --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..e6805180 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.file binary +*.zip binary From fc2337d28125e67b780c1328b6da3be8898e1667 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 31 Oct 2017 10:56:16 -0700 Subject: [PATCH 4/7] Use LBYL --- importlib_resources/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/importlib_resources/__init__.py b/importlib_resources/__init__.py index 5f560291..0d596534 100644 --- a/importlib_resources/__init__.py +++ b/importlib_resources/__init__.py @@ -47,9 +47,9 @@ def open(package: Package, file_name: FileName) -> BinaryIO: """Return a file-like object opened for binary-reading of the resource.""" file_name = _normalize_path(file_name) package = _get_package(package) - try: + if hasattr(package.__spec__.loader, 'open_resource'): return package.__spec__.loader.open_resource(file_name) - except AttributeError: + else: # Using pathlib doesn't work well here due to the lack of 'strict' # argument for pathlib.Path.resolve() prior to Python 3.6. absolute_package_path = os.path.abspath(package.__spec__.origin) From e02ddeacb8f930849d053d50c666ceb20990a6ae Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 31 Oct 2017 11:23:33 -0700 Subject: [PATCH 5/7] Make mypy happier --- importlib_resources/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/importlib_resources/__init__.py b/importlib_resources/__init__.py index 0d596534..4569e438 100644 --- a/importlib_resources/__init__.py +++ b/importlib_resources/__init__.py @@ -12,6 +12,8 @@ from typing import Iterator, Union from typing.io import BinaryIO +from . import abc as resources_abc + Package = Union[types.ModuleType, str] if sys.version_info >= (3, 6): @@ -48,7 +50,9 @@ def open(package: Package, file_name: FileName) -> BinaryIO: file_name = _normalize_path(file_name) package = _get_package(package) if hasattr(package.__spec__.loader, 'open_resource'): - return package.__spec__.loader.open_resource(file_name) + reader = typing.cast(resources_abc.ResourceReader, + package.__spec__.loader) + return reader.open_resource(file_name) else: # Using pathlib doesn't work well here due to the lack of 'strict' # argument for pathlib.Path.resolve() prior to Python 3.6. From d7e6df30a31d14f80c7df0c8513a4a4c0b0ddaa4 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 31 Oct 2017 14:15:31 -0700 Subject: [PATCH 6/7] More LBYL --- importlib_resources/__init__.py | 51 +++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/importlib_resources/__init__.py b/importlib_resources/__init__.py index 4569e438..a0683e1f 100644 --- a/importlib_resources/__init__.py +++ b/importlib_resources/__init__.py @@ -108,26 +108,33 @@ def path(package: Package, file_name: FileName) -> Iterator[pathlib.Path]: """ file_name = _normalize_path(file_name) package = _get_package(package) - try: - yield pathlib.Path(package.__spec__.loader.resource_path(file_name)) - except (AttributeError, FileNotFoundError): - package_directory = pathlib.Path(package.__spec__.origin).parent - file_path = package_directory / file_name - if file_path.exists(): - yield file_path - else: - with open(package, file_name) as file: - data = file.read() - # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' - # blocks due to the need to close the temporary file to work on - # Windows properly. - fd, raw_path = tempfile.mkstemp() + if hasattr(package.__spec__.loader, 'resource_path'): + reader = typing.cast(resources_abc.ResourceReader, + package.__spec__.loader) + try: + yield pathlib.Path(reader.resource_path(file_name)) + return + except FileNotFoundError: + pass + # Fall-through for both the lack of resource_path() *and* if resource_path() + # raises FileNotFoundError. + package_directory = pathlib.Path(package.__spec__.origin).parent + file_path = package_directory / file_name + if file_path.exists(): + yield file_path + else: + with open(package, file_name) as file: + data = file.read() + # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' + # blocks due to the need to close the temporary file to work on + # Windows properly. + fd, raw_path = tempfile.mkstemp() + try: + os.write(fd, data) + os.close(fd) + yield pathlib.Path(raw_path) + finally: try: - os.write(fd, data) - os.close(fd) - yield pathlib.Path(raw_path) - finally: - try: - os.remove(raw_path) - except FileNotFoundError: - pass + os.remove(raw_path) + except FileNotFoundError: + pass From 71a5fe3bea239d38f985a9dea0de516a6f00ed8d Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 31 Oct 2017 14:19:58 -0700 Subject: [PATCH 7/7] Make Python 3.4 happy --- importlib_resources/tests/util.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/importlib_resources/tests/util.py b/importlib_resources/tests/util.py index 7bcff57c..6b00fcb7 100644 --- a/importlib_resources/tests/util.py +++ b/importlib_resources/tests/util.py @@ -1,10 +1,10 @@ import abc import importlib import importlib.machinery -import importlib.util import io import pathlib import sys +import types import unittest from .. import abc as resources_abc @@ -27,10 +27,15 @@ def resource_path(self, path_): else: return path - spec = importlib.machinery.ModuleSpec('testingpackage', Reader(), + name = 'testingpackage' + spec = importlib.machinery.ModuleSpec(name, Reader(), origin='does-not-exist', is_package=True) - return importlib.util.module_from_spec(spec) + # Unforunately importlib.util.module_from_spec() was not introduced until + # Python 3.5. + module = types.ModuleType(name) + module.__spec__ = spec + return module class CommonTests(abc.ABC):