diff --git a/importlib_resources/_compat.py b/importlib_resources/_compat.py index dbfc6796..ab03fb9c 100644 --- a/importlib_resources/_compat.py +++ b/importlib_resources/_compat.py @@ -93,6 +93,10 @@ def _zip_reader(spec): with suppress(AttributeError): return readers.ZipReader(spec.loader, spec.name) + def _namespace_reader(spec): + with suppress(AttributeError, ValueError): + return readers.NamespaceReader(spec.submodule_search_locations) + def _available_reader(spec): with suppress(AttributeError): return spec.loader.get_resource_reader(spec.name) @@ -106,6 +110,8 @@ def _native_reader(spec): _native_reader(self.spec) or # local ZipReader if a zip module _zip_reader(self.spec) or + # local NamespaceReader if a namespace module + _namespace_reader(self.spec) or # local FileReader readers.FileReader(self) ) diff --git a/importlib_resources/_py3.py b/importlib_resources/_py3.py index 7b535d0b..9c198e8e 100644 --- a/importlib_resources/_py3.py +++ b/importlib_resources/_py3.py @@ -31,27 +31,30 @@ def open_binary(package: Package, resource: Resource) -> BinaryIO: return reader.open_resource(resource) # Using pathlib doesn't work well here due to the lack of 'strict' # argument for pathlib.Path.resolve() prior to Python 3.6. - absolute_package_path = os.path.abspath( - package.__spec__.origin or 'non-existent file') - package_path = os.path.dirname(absolute_package_path) - full_path = os.path.join(package_path, resource) - try: - return open(full_path, mode='rb') - except OSError: - # Just assume the loader is a resource loader; all the relevant - # importlib.machinery loaders are and an AttributeError for - # get_data() will make it clear what is needed from the loader. - loader = cast(ResourceLoader, package.__spec__.loader) - data = None - if hasattr(package.__spec__.loader, 'get_data'): - with suppress(OSError): - data = loader.get_data(full_path) - if data is None: - package_name = package.__spec__.name - message = '{!r} resource not found in {!r}'.format( - resource, package_name) - raise FileNotFoundError(message) - return BytesIO(data) + spec = package.__spec__ + if package.__spec__.submodule_search_locations is not None: + paths = package.__spec__.submodule_search_locations + elif package.__spec__.origin is not None: + paths = [os.path.dirname(os.path.abspath(package.__spec__.origin))] + + for package_path in paths: + full_path = os.path.join(package_path, resource) + try: + return open(full_path, mode='rb') + except OSError: + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = cast(ResourceLoader, package.__spec__.loader) + data = None + if hasattr(package.__spec__.loader, 'get_data'): + with suppress(OSError): + data = loader.get_data(full_path) + if data is not None: + return BytesIO(data) + + raise FileNotFoundError('{!r} resource not found in {!r}'.format( + resource, package.__spec__.name)) def open_text(package: Package, @@ -144,12 +147,7 @@ def contents(package: Package) -> Iterable[str]: reader = _common.get_resource_reader(package) if reader is not None: return reader.contents() - # Is the package a namespace package? By definition, namespace packages - # cannot have resources. - namespace = ( - package.__spec__.origin is None or - package.__spec__.origin == 'namespace' - ) - if namespace or not package.__spec__.has_location: - return () - return list(item.name for item in _common.from_package(package).iterdir()) + transversable = _common.from_package(package) + if transversable.is_dir(): + return list(item.name for item in transversable.iterdir()) + return [] diff --git a/importlib_resources/readers.py b/importlib_resources/readers.py index 3b07c76c..25c5045c 100644 --- a/importlib_resources/readers.py +++ b/importlib_resources/readers.py @@ -1,6 +1,9 @@ +import itertools +import os.path + from . import abc -from ._compat import Path, ZipPath +from ._compat import suppress, Path, ZipPath class FileReader(abc.TraversableResources): @@ -39,3 +42,82 @@ def is_resource(self, path): def files(self): return ZipPath(self.archive, self.prefix) + + +class MultiplexedPath(abc.Traversable): + """ + Given a series of Traversable objects, implement a merged + version of the interface across all objects. Useful for + namespace packages which may be multihomed at a single + name. + """ + def __init__(self, *paths): + self._paths = list(map(Path, set(paths))) + if not self._paths: + message = 'MultiplexedPath must contain at least one path' + raise FileNotFoundError(message) + if any(not path.is_dir() for path in self._paths): + raise NotADirectoryError( + 'MultiplexedPath only supports directories') + + def iterdir(self): + visited = [] + for path in self._paths: + for file in path.iterdir(): + if file.name in visited: + continue + visited.append(file.name) + yield file + + def read_bytes(self): + return self.open(mode='rb').read() + + def read_text(self, *args, **kwargs): + return self.open(mode='r', *args, **kwargs).read() + + def is_dir(self): + return True + + def is_file(self): + return False + + def joinpath(self, child): + # first try to find child in current paths + for file in self.iterdir(): + if file.name == child: + return file + # if it does not exist, construct it with the first path + return Path(os.path.join(self._paths[0], child)) + + __truediv__ = joinpath + + def open(self, *args, **kwargs): + for path in self._paths[:-1]: + with suppress(Exception): + return path.open(*args, **kwargs) + return self._paths[-1].open(*args, **kwargs) + + def name(self): + return os.path.basename(self._paths[0]) + + def __repr__(self): + return 'MultiplexedPath({})'.format( + ', '.join("'{}'".format(path) for path in self._paths)) + + +class NamespaceReader(abc.TraversableResources): + def __init__(self, namespace_path): + if 'NamespacePath' not in str(namespace_path): + raise ValueError('Invalid path') + self.path = MultiplexedPath(*list(namespace_path)) + + def resource_path(self, resource): + """ + Return the file system path to prevent + `resources.path()` from creating a temporary + copy. + """ + return str(self.path.joinpath(resource)) + + def files(self): + return self.path diff --git a/importlib_resources/tests/data03/__init__.py b/importlib_resources/tests/data03/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/importlib_resources/tests/data03/namespace/portion1/__init__.py b/importlib_resources/tests/data03/namespace/portion1/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/importlib_resources/tests/data03/namespace/portion2/__init__.py b/importlib_resources/tests/data03/namespace/portion2/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/importlib_resources/tests/data03/namespace/resource1.txt b/importlib_resources/tests/data03/namespace/resource1.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/importlib_resources/tests/namespacedata01/binary.file b/importlib_resources/tests/namespacedata01/binary.file new file mode 100644 index 00000000..eaf36c1d Binary files /dev/null and b/importlib_resources/tests/namespacedata01/binary.file differ diff --git a/importlib_resources/tests/namespacedata01/utf-16.file b/importlib_resources/tests/namespacedata01/utf-16.file new file mode 100644 index 00000000..2cb77229 Binary files /dev/null and b/importlib_resources/tests/namespacedata01/utf-16.file differ diff --git a/importlib_resources/tests/namespacedata01/utf-8.file b/importlib_resources/tests/namespacedata01/utf-8.file new file mode 100644 index 00000000..1c0132ad --- /dev/null +++ b/importlib_resources/tests/namespacedata01/utf-8.file @@ -0,0 +1 @@ +Hello, UTF-8 world! diff --git a/importlib_resources/tests/test_open.py b/importlib_resources/tests/test_open.py index 8a3429f2..b9f4a267 100644 --- a/importlib_resources/tests/test_open.py +++ b/importlib_resources/tests/test_open.py @@ -1,3 +1,4 @@ +import sys import unittest import importlib_resources as resources @@ -65,6 +66,16 @@ def setUp(self): self.data = data01 +@unittest.skipUnless( + sys.version_info[0] >= 3, + 'namespace packages not available on Python 2' +) +class OpenDiskNamespaceTests(OpenTests, unittest.TestCase): + def setUp(self): + from . import namespacedata01 + self.data = namespacedata01 + + class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): pass diff --git a/importlib_resources/tests/test_resource.py b/importlib_resources/tests/test_resource.py index 6920ac16..7c1eb8fc 100644 --- a/importlib_resources/tests/test_resource.py +++ b/importlib_resources/tests/test_resource.py @@ -1,3 +1,4 @@ +import os.path import sys import unittest import importlib_resources as resources @@ -140,36 +141,6 @@ def test_unrelated_contents(self): {'__init__.py', 'resource2.txt'}) -@unittest.skipIf(sys.version_info < (3,), 'No namespace packages in Python 2') -class NamespaceTest(unittest.TestCase): - def test_namespaces_cannot_have_resources(self): - contents = resources.contents( - 'importlib_resources.tests.data03.namespace') - self.assertFalse(list(contents)) - # Even though there is a file in the namespace directory, it is not - # considered a resource, since namespace packages can't have them. - self.assertFalse(resources.is_resource( - 'importlib_resources.tests.data03.namespace', - 'resource1.txt')) - # We should get an exception if we try to read it or open it. - self.assertRaises( - FileNotFoundError, - resources.open_text, - 'importlib_resources.tests.data03.namespace', 'resource1.txt') - self.assertRaises( - FileNotFoundError, - resources.open_binary, - 'importlib_resources.tests.data03.namespace', 'resource1.txt') - self.assertRaises( - FileNotFoundError, - resources.read_text, - 'importlib_resources.tests.data03.namespace', 'resource1.txt') - self.assertRaises( - FileNotFoundError, - resources.read_binary, - 'importlib_resources.tests.data03.namespace', 'resource1.txt') - - class DeletingZipsTest(unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. @@ -245,5 +216,39 @@ def test_read_text_does_not_keep_open(self): del c +@unittest.skipUnless( + sys.version_info[0] >= 3, + 'namespace packages not available on Python 2' +) +class ResourceFromNamespaceTest01(unittest.TestCase): + @classmethod + def setUpClass(cls): + sys.path.append(os.path.abspath(os.path.join(__file__, '..'))) + + def test_is_submodule_resource(self): + self.assertTrue( + resources.is_resource(import_module('namespacedata01'), 'binary.file')) + + def test_read_submodule_resource_by_name(self): + self.assertTrue( + resources.is_resource('namespacedata01', 'binary.file')) + + def test_submodule_contents(self): + contents = set(resources.contents(import_module('namespacedata01'))) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + + def test_submodule_contents_by_name(self): + contents = set(resources.contents('namespacedata01')) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + + if __name__ == '__main__': unittest.main()