Merge pull request #196 from FFY00/namespace-resources

Add support for resources in namespace packages

importlib_resources/_compat.py

@@ -93,6 +93,10 @@ def _zip_reader(spec):
             with suppress(AttributeError):
                 return readers.ZipReader(spec.loader, spec.name)
 
+        def _namespace_reader(spec):
+            with suppress(AttributeError, ValueError):
+                return readers.NamespaceReader(spec.submodule_search_locations)
+
         def _available_reader(spec):
             with suppress(AttributeError):
                 return spec.loader.get_resource_reader(spec.name)
@@ -106,6 +110,8 @@ def _native_reader(spec):
             _native_reader(self.spec) or
             # local ZipReader if a zip module
             _zip_reader(self.spec) or
+            # local NamespaceReader if a namespace module
+            _namespace_reader(self.spec) or
             # local FileReader
             readers.FileReader(self)
             )

importlib_resources/_py3.py

@@ -31,27 +31,30 @@ def open_binary(package: Package, resource: Resource) -> BinaryIO:
         return reader.open_resource(resource)
     # Using pathlib doesn't work well here due to the lack of 'strict'
     # argument for pathlib.Path.resolve() prior to Python 3.6.
-    absolute_package_path = os.path.abspath(
-        package.__spec__.origin or 'non-existent file')
-    package_path = os.path.dirname(absolute_package_path)
-    full_path = os.path.join(package_path, resource)
-    try:
-        return open(full_path, mode='rb')
-    except OSError:
-        # Just assume the loader is a resource loader; all the relevant
-        # importlib.machinery loaders are and an AttributeError for
-        # get_data() will make it clear what is needed from the loader.
-        loader = cast(ResourceLoader, package.__spec__.loader)
-        data = None
-        if hasattr(package.__spec__.loader, 'get_data'):
-            with suppress(OSError):
-                data = loader.get_data(full_path)
-        if data is None:
-            package_name = package.__spec__.name
-            message = '{!r} resource not found in {!r}'.format(
-                resource, package_name)
-            raise FileNotFoundError(message)
-        return BytesIO(data)
+    spec = package.__spec__
+    if package.__spec__.submodule_search_locations is not None:
+        paths = package.__spec__.submodule_search_locations
+    elif package.__spec__.origin is not None:
+        paths = [os.path.dirname(os.path.abspath(package.__spec__.origin))]
+
+    for package_path in paths:
+        full_path = os.path.join(package_path, resource)
+        try:
+            return open(full_path, mode='rb')
+        except OSError:
+            # Just assume the loader is a resource loader; all the relevant
+            # importlib.machinery loaders are and an AttributeError for
+            # get_data() will make it clear what is needed from the loader.
+            loader = cast(ResourceLoader, package.__spec__.loader)
+            data = None
+            if hasattr(package.__spec__.loader, 'get_data'):
+                with suppress(OSError):
+                    data = loader.get_data(full_path)
+            if data is not None:
+                return BytesIO(data)
+
+    raise FileNotFoundError('{!r} resource not found in {!r}'.format(
+        resource, package.__spec__.name))
 
 
 def open_text(package: Package,
@@ -144,12 +147,7 @@ def contents(package: Package) -> Iterable[str]:
     reader = _common.get_resource_reader(package)
     if reader is not None:
         return reader.contents()
-    # Is the package a namespace package?  By definition, namespace packages
-    # cannot have resources.
-    namespace = (
-        package.__spec__.origin is None or
-        package.__spec__.origin == 'namespace'
-        )
-    if namespace or not package.__spec__.has_location:
-        return ()
-    return list(item.name for item in _common.from_package(package).iterdir())
+    transversable = _common.from_package(package)
+    if transversable.is_dir():
+        return list(item.name for item in transversable.iterdir())
+    return []

importlib_resources/readers.py

@@ -1,6 +1,9 @@
+import itertools
+import os.path
+
 from . import abc
 
-from ._compat import Path, ZipPath
+from ._compat import suppress, Path, ZipPath
 
 
 class FileReader(abc.TraversableResources):
@@ -39,3 +42,82 @@ def is_resource(self, path):
 
     def files(self):
         return ZipPath(self.archive, self.prefix)
+
+
+class MultiplexedPath(abc.Traversable):
+    """
+    Given a series of Traversable objects, implement a merged
+    version of the interface across all objects. Useful for
+    namespace packages which may be multihomed at a single
+    name.
+    """
+    def __init__(self, *paths):
+        self._paths = list(map(Path, set(paths)))
+        if not self._paths:
+            message = 'MultiplexedPath must contain at least one path'
+            raise FileNotFoundError(message)
+        if any(not path.is_dir() for path in self._paths):
+            raise NotADirectoryError(
+                'MultiplexedPath only supports directories')
+
+    def iterdir(self):
+        visited = []
+        for path in self._paths:
+            for file in path.iterdir():
+                if file.name in visited:
+                    continue
+                visited.append(file.name)
+                yield file
+
+    def read_bytes(self):
+        return self.open(mode='rb').read()
+
+    def read_text(self, *args, **kwargs):
+        return self.open(mode='r', *args, **kwargs).read()
+
+    def is_dir(self):
+        return True
+
+    def is_file(self):
+        return False
+
+    def joinpath(self, child):
+        # first try to find child in current paths
+        for file in self.iterdir():
+            if file.name == child:
+                return file
+        # if it does not exist, construct it with the first path
+        return Path(os.path.join(self._paths[0], child))
+
+    __truediv__ = joinpath
+
+    def open(self, *args, **kwargs):
+        for path in self._paths[:-1]:
+            with suppress(Exception):
+                return path.open(*args, **kwargs)
+        return self._paths[-1].open(*args, **kwargs)
+
+    def name(self):
+        return os.path.basename(self._paths[0])
+
+    def __repr__(self):
+        return 'MultiplexedPath({})'.format(
+            ', '.join("'{}'".format(path) for path in self._paths))
+
+
+class NamespaceReader(abc.TraversableResources):
+    def __init__(self, namespace_path):
+        if 'NamespacePath' not in str(namespace_path):
+            raise ValueError('Invalid path')
+        self.path = MultiplexedPath(*list(namespace_path))
+
+    def resource_path(self, resource):
+        """
+        Return the file system path to prevent
+        `resources.path()` from creating a temporary
+        copy.
+        """
+        return str(self.path.joinpath(resource))
+
+    def files(self):
+        return self.path

importlib_resources/tests/namespacedata01/utf-8.file

@@ -0,0 +1 @@
+Hello, UTF-8 world!

importlib_resources/tests/test_open.py

@@ -1,3 +1,4 @@
+import sys
 import unittest
 
 import importlib_resources as resources
@@ -65,6 +66,16 @@ def setUp(self):
         self.data = data01
 
 
+@unittest.skipUnless(
+    sys.version_info[0] >= 3,
+    'namespace packages not available on Python 2'
+)
+class OpenDiskNamespaceTests(OpenTests, unittest.TestCase):
+    def setUp(self):
+        from . import namespacedata01
+        self.data = namespacedata01
+
+
 class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase):
     pass
 

importlib_resources/tests/test_resource.py

@@ -1,3 +1,4 @@
+import os.path
 import sys
 import unittest
 import importlib_resources as resources
@@ -140,36 +141,6 @@ def test_unrelated_contents(self):
             {'__init__.py', 'resource2.txt'})
 
 
-@unittest.skipIf(sys.version_info < (3,), 'No namespace packages in Python 2')
-class NamespaceTest(unittest.TestCase):
-    def test_namespaces_cannot_have_resources(self):
-        contents = resources.contents(
-            'importlib_resources.tests.data03.namespace')
-        self.assertFalse(list(contents))
-        # Even though there is a file in the namespace directory, it is not
-        # considered a resource, since namespace packages can't have them.
-        self.assertFalse(resources.is_resource(
-            'importlib_resources.tests.data03.namespace',
-            'resource1.txt'))
-        # We should get an exception if we try to read it or open it.
-        self.assertRaises(
-            FileNotFoundError,
-            resources.open_text,
-            'importlib_resources.tests.data03.namespace', 'resource1.txt')
-        self.assertRaises(
-            FileNotFoundError,
-            resources.open_binary,
-            'importlib_resources.tests.data03.namespace', 'resource1.txt')
-        self.assertRaises(
-            FileNotFoundError,
-            resources.read_text,
-            'importlib_resources.tests.data03.namespace', 'resource1.txt')
-        self.assertRaises(
-            FileNotFoundError,
-            resources.read_binary,
-            'importlib_resources.tests.data03.namespace', 'resource1.txt')
-
-
 class DeletingZipsTest(unittest.TestCase):
     """Having accessed resources in a zip file should not keep an open
     reference to the zip.
@@ -245,5 +216,39 @@ def test_read_text_does_not_keep_open(self):
         del c
 
 
+@unittest.skipUnless(
+    sys.version_info[0] >= 3,
+    'namespace packages not available on Python 2'
+)
+class ResourceFromNamespaceTest01(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        sys.path.append(os.path.abspath(os.path.join(__file__, '..')))
+
+    def test_is_submodule_resource(self):
+        self.assertTrue(
+            resources.is_resource(import_module('namespacedata01'), 'binary.file'))
+
+    def test_read_submodule_resource_by_name(self):
+        self.assertTrue(
+            resources.is_resource('namespacedata01', 'binary.file'))
+
+    def test_submodule_contents(self):
+        contents = set(resources.contents(import_module('namespacedata01')))
+        try:
+            contents.remove('__pycache__')
+        except KeyError:
+            pass
+        self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'})
+
+    def test_submodule_contents_by_name(self):
+        contents = set(resources.contents('namespacedata01'))
+        try:
+            contents.remove('__pycache__')
+        except KeyError:
+            pass
+        self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'})
+
+
 if __name__ == '__main__':
     unittest.main()