Issue #2731: Constraints files.

This adds constraints files. Like requirements files constraints files
control what version of a package is installed, but unlike
requirements files this doesn't itself choose to install the package.
This allows things that aren't explicitly desired to be constrained if
and only if they are installed.

CHANGES.txt

@@ -1,5 +1,7 @@
 **7.1.0 (unreleased)**
 
+* Allow constraining versions globally without having to know exactly what will
+  be installed by the pip command. :issue:`2731`.
 
 **7.0.3 (2015-06-01)**
 

docs/reference/pip_install.rst

@@ -120,10 +120,14 @@ For example, to specify :ref:`--no-index <--no-index>` and 2 :ref:`--find-links
 --find-links http://some.archives.com/archives
 
 
-Lastly, if you wish, you can refer to other requirements files, like this::
+If you wish, you can refer to other requirements files, like this::
 
     -r more_requirements.txt
 
+You can also refer to constraints files, like this::
+
+    -c some_constraints.txt
+
 .. _`Requirement Specifiers`:
 
 Requirement Specifiers

docs/user_guide.rst

@@ -110,6 +110,39 @@ See also:
   <https://caremad.io/blog/setup-vs-requirement/>`_
 
 
+.. _`Constraints Files`:
+
+Constraints Files
+*****************
+
+Constraints files are requirements files that only control which version of a
+requirement is installed, not whether it is installed or not. Their syntax and
+contents is nearly identical to :ref:`Requirements Files`. There is one key
+difference: Including a package in a constraints file does not trigger
+installation of the package.
+
+Use a constraints file like so:
+
+ ::
+
+   pip install -c constraints.txt
+
+Constraints files are used for exactly the same reason as requirements files
+when you don't know exactly what things you want to install. For instance, say
+that the "helloworld" package doesn't work in your environment, so you have a
+local patched version. Some things you install depend on "helloworld", and some
+don't.
+
+One way to ensure that the patched version is used consistently is to
+manually audit the dependencies of everything you install, and if "helloworld"
+is present, write a requirements file to use when installing that thing.
+
+Constraints files offer a better way: write a single constraints file for your
+organisation and use that everywhere. If the thing being installed requires
+"helloworld" to be installed, your fixed version specified in your constraints
+file will be used.
+
+Constraints file support was added in pip 7.1.
 
 .. _`Installing from Wheels`:
 

pip/basecommand.py

@@ -261,6 +261,13 @@ def populate_requirement_set(requirement_set, args, options, finder,
         """
         Marshal cmd line args into a requirement set.
         """
+        for filename in options.constraints:
+            for req in parse_requirements(
+                    filename,
+                    constraint=True, finder=finder, options=options,
+                    session=session, wheel_cache=wheel_cache):
+                requirement_set.add_requirement(req)
+
         for req in args:
             requirement_set.add_requirement(
                 InstallRequirement.from_line(

pip/cmdoptions.py

@@ -337,6 +337,17 @@ def allow_unsafe():
 )
 
 
+def constraints():
+    return Option(
+        '-c', '--constraint',
+        dest='constraints',
+        action='append',
+        default=[],
+        metavar='file',
+        help='Constrain versions using the given constraints file. '
+        'This option can be used multiple times.')
+
+
 def requirements():
     return Option(
         '-r', '--requirement',

pip/commands/install.py

@@ -56,6 +56,7 @@ def __init__(self, *args, **kw):
 
         cmd_opts = self.cmd_opts
 
+        cmd_opts.add_option(cmdoptions.constraints())
         cmd_opts.add_option(cmdoptions.editable())
         cmd_opts.add_option(cmdoptions.requirements())
         cmd_opts.add_option(cmdoptions.build_dir())

pip/commands/wheel.py

@@ -69,6 +69,7 @@ def __init__(self, *args, **kw):
             metavar='options',
             action='append',
             help="Extra arguments to be supplied to 'setup.py bdist_wheel'.")
+        cmd_opts.add_option(cmdoptions.constraints())
         cmd_opts.add_option(cmdoptions.editable())
         cmd_opts.add_option(cmdoptions.requirements())
         cmd_opts.add_option(cmdoptions.download_cache())

pip/req/req_file.py

@@ -25,6 +25,7 @@
 COMMENT_RE = re.compile(r'(^|\s)+#.*$')
 
 SUPPORTED_OPTIONS = [
+    cmdoptions.constraints,
     cmdoptions.editable,
     cmdoptions.requirements,
     cmdoptions.no_index,
@@ -54,15 +55,16 @@
 
 
 def parse_requirements(filename, finder=None, comes_from=None, options=None,
-                       session=None, wheel_cache=None):
-    """
-    Parse a requirements file and yield InstallRequirement instances.
+                       session=None, constraint=False, wheel_cache=None):
+    """Parse a requirements file and yield InstallRequirement instances.
 
     :param filename:    Path or url of requirements file.
     :param finder:      Instance of pip.index.PackageFinder.
     :param comes_from:  Origin description of requirements.
     :param options:     Global options.
     :param session:     Instance of pip.download.PipSession.
+    :param constraint:  If true, parsing a constraint file rather than
+        requirements file.
     :param wheel_cache: Instance of pip.wheel.WheelCache
     """
     if session is None:
@@ -82,13 +84,15 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None,
 
     for line_number, line in enumerate(lines, 1):
         req_iter = process_line(line, filename, line_number, finder,
-                                comes_from, options, session, wheel_cache)
+                                comes_from, options, session, wheel_cache,
+                                constraint=constraint)
         for req in req_iter:
             yield req
 
 
 def process_line(line, filename, line_number, finder=None, comes_from=None,
-                 options=None, session=None, wheel_cache=None):
+                 options=None, session=None, wheel_cache=None,
+                 constraint=False):
     """Process a single requirements line; This can result in creating/yielding
     requirements, or updating the finder.
 
@@ -103,8 +107,8 @@ def process_line(line, filename, line_number, finder=None, comes_from=None,
     (although our docs imply only one is supported), and all our parsed and
     affect the finder.
 
+    :param constraint: If True, parsing a constraints file.
     """
-
     parser = build_parser()
     defaults = parser.get_default_values()
     defaults.index_url = None
@@ -114,9 +118,12 @@ def process_line(line, filename, line_number, finder=None, comes_from=None,
     args_str, options_str = break_args_options(line)
     opts, _ = parser.parse_args(shlex.split(options_str), defaults)
 
+    # preserve for the nested code path
+    line_comes_from = '%s %s (line %s)' % (
+        '-c' if constraint else '-r', filename, line_number)
+
     # yield a line requirement
     if args_str:
-        comes_from = '-r %s (line %s)' % (filename, line_number)
         isolated = options.isolated_mode if options else False
         if options:
             cmdoptions.check_install_build_global(options, opts)
@@ -126,24 +133,28 @@ def process_line(line, filename, line_number, finder=None, comes_from=None,
             if dest in opts.__dict__ and opts.__dict__[dest]:
                 req_options[dest] = opts.__dict__[dest]
         yield InstallRequirement.from_line(
-            args_str, comes_from, isolated=isolated, options=req_options,
-            wheel_cache=wheel_cache
+            args_str, line_comes_from, constraint=constraint,
+            isolated=isolated, options=req_options, wheel_cache=wheel_cache
         )
 
     # yield an editable requirement
     elif opts.editables:
-        comes_from = '-r %s (line %s)' % (filename, line_number)
         isolated = options.isolated_mode if options else False
         default_vcs = options.default_vcs if options else None
         yield InstallRequirement.from_editable(
-            opts.editables[0], comes_from=comes_from,
-            default_vcs=default_vcs, isolated=isolated,
+            opts.editables[0], comes_from=line_comes_from,
+            constraint=constraint, default_vcs=default_vcs, isolated=isolated,
             wheel_cache=wheel_cache
         )
 
     # parse a nested requirements file
-    elif opts.requirements:
-        req_path = opts.requirements[0]
+    elif opts.requirements or opts.constraints:
+        if opts.requirements:
+            req_path = opts.requirements[0]
+            nested_constraint = False
+        else:
+            req_path = opts.constraints[0]
+            nested_constraint = True
         # original file is over http
         if SCHEME_RE.search(filename):
             # do a url join so relative paths work
@@ -156,7 +167,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None,
         # TODO: Why not use `comes_from='-r {} (line {})'` here as well?
         parser = parse_requirements(
             req_path, finder, comes_from, options, session,
-            wheel_cache=wheel_cache
+            constraint=nested_constraint, wheel_cache=wheel_cache
         )
         for req in parser:
             yield req

pip/req/req_install.py

@@ -60,14 +60,15 @@ class InstallRequirement(object):
     def __init__(self, req, comes_from, source_dir=None, editable=False,
                  link=None, as_egg=False, update=True, editable_options=None,
                  pycompile=True, markers=None, isolated=False, options=None,
-                 wheel_cache=None):
+                 wheel_cache=None, constraint=False):
         self.extras = ()
         if isinstance(req, six.string_types):
             req = pkg_resources.Requirement.parse(req)
             self.extras = req.extras
 
         self.req = req
         self.comes_from = comes_from
+        self.constraint = constraint
         self.source_dir = source_dir
         self.editable = editable
 
@@ -106,7 +107,8 @@ def __init__(self, req, comes_from, source_dir=None, editable=False,
 
     @classmethod
     def from_editable(cls, editable_req, comes_from=None, default_vcs=None,
-                      isolated=False, options=None, wheel_cache=None):
+                      isolated=False, options=None, wheel_cache=None,
+                      constraint=False):
         from pip.index import Link
 
         name, url, extras_override, editable_options = parse_editable(
@@ -119,6 +121,7 @@ def from_editable(cls, editable_req, comes_from=None, default_vcs=None,
         res = cls(name, comes_from, source_dir=source_dir,
                   editable=True,
                   link=Link(url),
+                  constraint=constraint,
                   editable_options=editable_options,
                   isolated=isolated,
                   options=options if options else {},
@@ -132,7 +135,7 @@ def from_editable(cls, editable_req, comes_from=None, default_vcs=None,
     @classmethod
     def from_line(
             cls, name, comes_from=None, isolated=False, options=None,
-            wheel_cache=None):
+            wheel_cache=None, constraint=False):
         """Creates an InstallRequirement from a name, which might be a
         requirement, directory containing 'setup.py', filename, or URL.
         """
@@ -204,7 +207,7 @@ def from_line(
         options = options if options else {}
         res = cls(req, comes_from, link=link, markers=markers,
                   isolated=isolated, options=options,
-                  wheel_cache=wheel_cache)
+                  wheel_cache=wheel_cache, constraint=constraint)
 
         if extras:
             res.extras = pkg_resources.Requirement.parse('__placeholder__' +

pip/req/req_set.py

@@ -231,22 +231,36 @@ def add_requirement(self, install_req, parent_req_name=None):
             self.unnamed_requirements.append(install_req)
             return [install_req]
         else:
-            if parent_req_name is None and self.has_requirement(name):
+            try:
+                existing_req = self.get_requirement(name)
+            except KeyError:
+                existing_req = None
+            if (parent_req_name is None and existing_req and not
+                    existing_req.constraint):
                 raise InstallationError(
                     'Double requirement given: %s (already in %s, name=%r)'
-                    % (install_req, self.get_requirement(name), name))
-            if not self.has_requirement(name):
+                    % (install_req, existing_req, name))
+            if not existing_req:
                 # Add requirement
                 self.requirements[name] = install_req
                 # FIXME: what about other normalizations?  E.g., _ vs. -?
                 if name.lower() != name:
                     self.requirement_aliases[name.lower()] = name
                 result = [install_req]
             else:
-                # Canonicalise to the already-added object
-                install_req = self.get_requirement(name)
-                # No need to scan, this is a duplicate requirement.
-                result = []
+                if not existing_req.constraint:
+                    # No need to scan, we've already encountered this for
+                    # scanning.
+                    result = []
+                elif not install_req.constraint:
+                    # If we're now installing a constraint, mark the existing
+                    # object for real installation.
+                    existing_req.constraint = False
+                    # And now we need to scan this.
+                    result = [existing_req]
+                # Canonicalise to the already-added object for the backref
+                # check below.
+                install_req = existing_req
             if parent_req_name:
                 parent_req = self.get_requirement(parent_req_name)
                 self._dependencies[parent_req].append(install_req)
@@ -260,7 +274,8 @@ def has_requirement(self, project_name):
 
     @property
     def has_requirements(self):
-        return list(self.requirements.values()) or self.unnamed_requirements
+        return list(req for req in self.requirements.values() if not
+                    req.constraint) or self.unnamed_requirements
 
     @property
     def is_download(self):
@@ -285,6 +300,8 @@ def get_requirement(self, project_name):
 
     def uninstall(self, auto_confirm=False):
         for req in self.requirements.values():
+            if req.constraint:
+                continue
             req.uninstall(auto_confirm=auto_confirm)
             req.commit_uninstall()
 
@@ -376,6 +393,9 @@ def _prepare_file(self, finder, req_to_install):
         # Tell user what we are doing for this requirement:
         # obtain (editable), skipping, processing (local url), collecting
         # (remote url or package name)
+        if req_to_install.constraint:
+            return []
+
         if req_to_install.editable:
             logger.info('Obtaining %s', req_to_install)
         else:
@@ -584,6 +604,8 @@ def _to_install(self):
         def schedule(req):
             if req.satisfied_by or req in ordered_reqs:
                 return
+            if req.constraint:
+                return
             ordered_reqs.add(req)
             for dep in self._dependencies[req]:
                 schedule(dep)

pip/wheel.py

@@ -708,6 +708,8 @@ def build(self, autobuilding=False):
 
         buildset = []
         for req in reqset:
+            if req.constraint:
+                continue
             if req.is_wheel:
                 if not autobuilding:
                     logger.info(