support git subfolder (#755)

docs/docs/cli.md

@@ -212,7 +212,14 @@ poetry add git+https://github.com/sdispater/pendulum.git#develop
 poetry add git+https://github.com/sdispater/pendulum.git#2.0.5
 ```
 
-or make them point to a local directory or file:
+If the package is located within a subdirectory of a git dependency use:
+
+```bash
+poetry add "git+https://github.com/demo/project_in_subdirectory.git?subdirectory=pyproject-demo"
+poetry add "git+https://github.com/demo/project_in_subdirectory.git#develop?subdirectory=pyproject-demo"
+```
+
+You can also point to a local directory or file:
 
 ```bash
 poetry add ./my-package/

poetry/console/commands/init.py

@@ -377,11 +377,17 @@ def _parse_requirements(
                     if parsed.rev:
                         pair["rev"] = url.revision
 
+                    if parsed.subdirectory:
+                        pair["subdirectory"] = parsed.subdirectory
+
                     if extras:
                         pair["extras"] = extras
 
                     package = Provider.get_package_from_vcs(
-                        "git", url.url, reference=pair.get("rev")
+                        "git",
+                        url.url,
+                        reference=pair.get("rev"),
+                        subdirectory=parsed.subdirectory,
                     )
                     pair["name"] = package.name
                     result.append(pair)

poetry/installation/pip_installer.py

@@ -1,16 +1,18 @@
 import os
+import shutil
 import tempfile
 
 from io import open
 from subprocess import CalledProcessError
 
 from clikit.api.io import IO
-from clikit.io import NullIO
 
 from poetry.repositories.pool import Pool
+from poetry.utils._compat import Path
 from poetry.utils._compat import encode
 from poetry.utils.env import Env
 from poetry.utils.helpers import safe_rmtree
+from poetry.utils.temp import DownloadTmpDir
 
 from .base_installer import BaseInstaller
 
@@ -179,7 +181,6 @@ def install_directory(self, package):
         from poetry.core.masonry.builder import SdistBuilder
         from poetry.factory import Factory
         from poetry.utils._compat import decode
-        from poetry.utils.env import NullEnv
         from poetry.utils.toml_file import TomlFile
 
         if package.root_dir:
@@ -210,9 +211,7 @@ def install_directory(self, package):
             # file since pip, as of this comment, does not support
             # build-system for editable packages
             # We also need it for non-PEP-517 packages
-            builder = SdistBuilder(
-                Factory().create_poetry(pyproject.parent), NullEnv(), NullIO()
-            )
+            builder = SdistBuilder(Factory().create_poetry(pyproject.parent))
 
             with open(setup, "w", encoding="utf-8") as f:
                 f.write(decode(builder.build_setup()))
@@ -233,19 +232,29 @@ def install_git(self, package):
         from poetry.core.vcs import Git
 
         src_dir = self._env.path / "src" / package.name
+        tmp_dir = Path(
+            DownloadTmpDir.mkcache(package.source_url, prefix="pypoetry-git")
+        )
+
         if src_dir.exists():
             safe_rmtree(str(src_dir))
 
         src_dir.parent.mkdir(exist_ok=True)
 
         git = Git()
-        git.clone(package.source_url, src_dir)
-        git.checkout(package.source_reference, src_dir)
+
+        if not any(tmp_dir.glob("*")):
+            git.clone(package.source_url, tmp_dir)
+
+        git.checkout(package.source_reference, tmp_dir)
+
+        shutil.copytree(str(tmp_dir / package.source_subdirectory), str(src_dir))
 
         # Now we just need to install from the source directory
         pkg = Package(package.name, package.version)
         pkg.source_type = "directory"
         pkg.source_url = str(src_dir)
         pkg.develop = package.develop
+        pkg.source_subdirectory = package.source_subdirectory
 
         self.install_directory(pkg)

poetry/json/schemas/poetry-schema.json

@@ -296,6 +296,10 @@
                     "type": "string",
                     "description": "The revision to checkout."
                 },
+                "subdirectory": {
+                    "type": "string",
+                    "description": "path relative to repositories root, where package is located"
+                },
                 "python": {
                     "type": "string",
                     "description": "The python versions for which the dependency should be installed."

poetry/packages/locker.py

@@ -142,6 +142,7 @@ def locked_repository(
                 package.source_type = info["source"].get("type", "")
                 package.source_url = info["source"]["url"]
                 package.source_reference = info["source"]["reference"]
+                package.source_subdirectory = info["source"].get("subdirectory", "")
 
             packages.add_package(package)
 
@@ -304,4 +305,7 @@ def _dump_package(self, package):  # type: (Package) -> dict
             if package.source_type == "directory":
                 data["develop"] = package.develop
 
+            if package.source_subdirectory:
+                data["source"]["subdirectory"] = package.source_subdirectory
+
         return data

poetry/puzzle/provider.py

@@ -5,7 +5,6 @@
 import time
 
 from contextlib import contextmanager
-from tempfile import mkdtemp
 from typing import Any
 from typing import List
 from typing import Optional
@@ -40,10 +39,10 @@
 from poetry.utils.env import EnvManager
 from poetry.utils.env import VirtualEnv
 from poetry.utils.helpers import parse_requires
-from poetry.utils.helpers import safe_rmtree
 from poetry.utils.helpers import temporary_directory
 from poetry.utils.inspector import Inspector
 from poetry.utils.setup_reader import SetupReader
+from poetry.utils.temp import DownloadTmpDir
 from poetry.utils.toml_file import TomlFile
 
 from .exceptions import CompatibilityError
@@ -60,7 +59,6 @@ def _formatter_elapsed(self):
 
 
 class Provider:
-
     UNSAFE_PACKAGES = {"setuptools", "distribute", "pip"}
 
     def __init__(self, package, pool, io):  # type: (Package, Pool, Any) -> None
@@ -169,6 +167,7 @@ def search_for_vcs(self, dependency):  # type: (VCSDependency) -> List[Package]
             dependency.source,
             dependency.reference,
             name=dependency.name,
+            subdirectory=dependency.subdirectory,
         )
 
         for extra in dependency.extras:
@@ -182,34 +181,34 @@ def search_for_vcs(self, dependency):  # type: (VCSDependency) -> List[Package]
 
     @classmethod
     def get_package_from_vcs(
-        cls, vcs, url, reference=None, name=None
-    ):  # type: (str, str, Optional[str], Optional[str]) -> Package
+        cls, vcs, url, reference=None, name=None, subdirectory=None
+    ):  # type: (str, str, Optional[str], Optional[str], Optional[str]) -> Package
         if vcs != "git":
             raise ValueError("Unsupported VCS dependency {}".format(vcs))
 
-        tmp_dir = Path(
-            mkdtemp(prefix="pypoetry-git-{}".format(url.split("/")[-1].rstrip(".git")))
-        )
+        tmp_dir = Path(DownloadTmpDir.mkcache(url, prefix="pypoetry-git"))
 
-        try:
-            git = Git()
+        git = Git()
+
+        if not any(tmp_dir.glob("*")):
             git.clone(url, tmp_dir)
-            if reference is not None:
-                git.checkout(reference, tmp_dir)
-            else:
-                reference = "HEAD"
 
-            revision = git.rev_parse(reference, tmp_dir).strip()
+        if reference is not None:
+            git.checkout(reference, tmp_dir)
+        else:
+            reference = "HEAD"
+
+        revision = git.rev_parse(reference, tmp_dir).strip()
 
+        if subdirectory is None:
             package = cls.get_package_from_directory(tmp_dir, name=name)
+        else:
+            package = cls.get_package_from_directory(tmp_dir / subdirectory, name=name)
+            package.source_subdirectory = Path(subdirectory).as_posix()
 
-            package.source_type = "git"
-            package.source_url = url
-            package.source_reference = revision
-        except Exception:
-            raise
-        finally:
-            safe_rmtree(str(tmp_dir))
+        package.source_type = "git"
+        package.source_url = url
+        package.source_reference = revision
 
         return package
 

poetry/repositories/installed_repository.py

@@ -1,3 +1,5 @@
+from subprocess import CalledProcessError
+
 from poetry.core.packages import Package
 from poetry.utils._compat import Path
 from poetry.utils._compat import metadata
@@ -69,7 +71,7 @@ def load(cls, env):  # type: (Env) -> InstalledRepository
                     package.source_type = "git"
                     package.source_url = url
                     package.source_reference = revision
-                except ValueError:
+                except (ValueError, CalledProcessError):
                     package.source_type = "directory"
                     package.source_url = str(path.parent)
 

poetry/utils/exporter.py

@@ -81,6 +81,9 @@ def _export_requirements_txt(
                 line = "-e git+{}@{}#egg={}".format(
                     package.source_url, package.source_reference, package.name
                 )
+                if package.source_subdirectory:
+                    line += "?subdirectory={}".format(package.source_subdirectory)
+
             elif package.source_type in ["directory", "file", "url"]:
                 if package.source_type == "file":
                     dependency = FileDependency(package.name, Path(package.source_url))

poetry/utils/temp.py

@@ -0,0 +1,27 @@
+import atexit
+import shutil
+
+from tempfile import mkdtemp
+
+from poetry.utils._compat import Path
+
+
+def force_rm(action, name, exc):
+    Path(name).unlink()
+
+
+@atexit.register
+def cleanup_tmp():
+    for source, cache in DownloadTmpDir.tmp_dirs.items():
+        shutil.rmtree(cache, onerror=force_rm)
+
+
+class DownloadTmpDir:
+    tmp_dirs = {}
+
+    @classmethod
+    def mkcache(cls, source, suffix="", prefix="", dir=""):
+        if source not in cls.tmp_dirs:
+            cls.tmp_dirs[source] = mkdtemp(suffix, prefix, dir)
+
+        return cls.tmp_dirs[source]

tests/console/commands/test_add.py

@@ -200,6 +200,28 @@ def test_add_git_constraint(app, repo, installer):
     }
 
 
+# As this test needs changes in poetry_core at the time of writing it will fail
+# in Github's checks. Local testing is possible.
+#
+# def test_add_git_constraint_with_subdir(app, repo, installer):
+#     command = app.find("add")
+#     tester = CommandTester(command)
+#
+#     tester.execute(
+#         "git+https://github.com/demo/project_in_subdirectory.git?subdirectory=pyproject-demo"
+#     )
+#
+#     assert len(installer.installs) == 1
+#
+#     content = app.poetry.file.read()["tool"]["poetry"]
+#
+#     assert "demo" in content["dependencies"]
+#     assert content["dependencies"]["demo"] == {
+#         "git": "https://github.com/demo/project_in_subdirectory.git",
+#         "subdirectory": "pyproject-demo",
+#     }
+
+
 def test_add_git_constraint_with_poetry(app, repo, installer):
     command = app.find("add")
     tester = CommandTester(command)

tests/console/commands/test_export.py

@@ -14,6 +14,8 @@
 from ..conftest import Path
 
 
+fixtures_dir = Path(__file__).parent.parent.parent / "fixtures"
+
 PYPROJECT_CONTENT = """\
 [tool.poetry]
 name = "simple-project"
@@ -71,6 +73,35 @@ def app(poetry):
     return Application(poetry)
 
 
+def test_export_with_vcs_subdirectory(app_project):
+    project = fixtures_dir / "project_with_git_subdirectory_dependency"
+    app = app_project(project)
+
+    command = app.find("lock")
+    tester = CommandTester(command)
+    tester.execute()
+
+    assert app.poetry.locker.lock.exists()
+
+    command = app.find("export")
+    tester = CommandTester(command)
+    tester.execute("--format requirements.txt --output requirements.txt")
+
+    requirements = app.poetry.file.parent / "requirements.txt"
+    assert requirements.exists()
+
+    with requirements.open(encoding="utf-8") as f:
+        content = f.read().strip()
+
+    expected = (
+        "-e "
+        "git+https://github.com/demo/project_in_subdirectory.git"
+        "@9cf87a285a2d3fbb0b9fa621997b3acc3631ed24#egg=demo?subdirectory=pyproject-demo"
+    )
+
+    assert expected == content
+
+
 def test_export_exports_requirements_txt_file_locks_if_no_lock_file(app, repo):
     command = app.find("export")
     tester = CommandTester(command)

tests/console/commands/test_lock.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from ..conftest import Path
+
+
+# rom cleo.testers import CommandTester
+
+
+fixtures_dir = Path(__file__).parent.parent.parent / "fixtures"
+
+# As this test needs changes in poetry_core at the time of writing it will fail
+# in Github's checks. Local testing is possible.
+#
+# def test_export_with_vcs_subdirectory(app_project):
+#     project = fixtures_dir / "project_with_git_subdirectory_dependency"
+#     app = app_project(project)
+#
+#     command = app.find("lock")
+#     tester = CommandTester(command)
+#     tester.execute()
+#
+#     assert app.poetry.locker.lock.exists()
+#
+#     with app.poetry.locker.lock.open(encoding="utf-8") as f:
+#         content = f.read()
+#     expected = """\
+# [[package]]
+# category = "main"
+# description = ""
+# name = "demo"
+# optional = false
+# python-versions = "~2.7 || ^3.4"
+# version = "0.1.2"
+#
+# [package.source]
+# reference = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24"
+# subdirectory = "pyproject-demo"
+# type = "git"
+# url = "https://github.com/demo/project_in_subdirectory.git"
+#
+# [metadata]
+# content-hash = "1394d1b3da3a2a62939852f9b1671ad0b6a7dbb0c2e9f1017a0df9b30c8b151d"
+# python-versions = "~2.7 || ^3.4"
+#
+# [metadata.files]
+# demo = []
+# """
+#     assert content == expected