Rewrite backtracking and state management

The "forward" part of resolution now works like this:

* A "root state" is populated. This state does not produce a "pin" but
  only populate the root requirements, so subsequence rounds can use it
  as the basis.
* Each subsequent round pushes a state, pin a candidate for it, and
  check whether the pin results in a conflict.
* If no conflcits occur, the round ends, and a new round is started.

When a conflict happens, the backtracking process discards the current
round's state, and try to modify the previous state so the conflict can
be avoided. If this modification is unsuccessful, the still previous
state is tried, until a workable state is found (or we run out of
things to try).

This makes each round and its associating state's lifeline easier to
reason with. Each successful forward round would create a state with a
pin in the stack. A round ending with backtracking will not result in a
new state in the stack, but have a previously-pinned state unpinned
(and modified), with everything newer than it removed.

src/resolvelib/resolvers.py

@@ -99,16 +99,15 @@ def merged_with(self, provider, requirement, parent):
             raise RequirementsConflicted(criterion)
         return criterion
 
-    def excluded_of(self, candidate):
-        """Build a new instance from this, but excluding specified candidate.
+    def excluded_of(self, candidates):
+        """Build a new instance from this, but excluding specified candidates.
 
         Returns the new instance, or None if we still have no valid candidates.
         """
-        cands = self.candidates.excluding([candidate])
+        cands = self.candidates.excluding(candidates)
         if not cands:
             return None
-        incompats = list(self.incompatibilities)
-        incompats.append(candidate)
+        incompats = self.incompatibilities + candidates
         return type(self)(cands, list(self.information), incompats)
 
 
@@ -235,43 +234,76 @@ def _attempt_to_pin_criterion(self, name, criterion):
         return causes
 
     def _backtrack(self):
-        # Drop the current state, it's known not to work.
-        del self._states[-1]
-
-        # We need at least 2 states here:
-        # (a) One to backtrack to.
-        # (b) One to restore state (a) to its state prior to candidate-pinning,
-        #     so we can pin another one instead.
+        """Perform backtracking.
+
+        When we enter here, the stack is like this::
+
+            [ state Z ]
+            [ state Y ]
+            [ state X ]
+            .... earlier states are irrelevant.
+
+        1. No pins worked for Z, so it does not have a pin.
+        2. We want to reset state Y to unpinned, and pin another candidate.
+        3. State X holds what state Y was before the pin, but does not
+           have the incompatibility information gathered in state Y.
+
+        Each iteration of the loop will:
+
+        1.  Discard Z.
+        2.  Discard Y but remember its incompatibility information gathered
+            previously, and the failure we're dealing with right now.
+        3.  Push a new state Y' based on X, and apply the incompatibility
+            information from Y to Y'.
+        4a. If this causes Y' to conflict, we need to backtrack again. Make Y'
+            the new Z and go back to step 2.
+        4b. If the incompatibilites apply cleanly, end backtracking.
+        """
+        while len(self._states) >= 3:
+            # Remove the state that triggered backtracking.
+            del self._states[-1]
+
+            # Retrieve the last candidate pin and known incompatibilities.
+            broken_state = self._states.pop()
+            name, candidate = broken_state.mapping.popitem()
+            incompatibilities_from_broken = [
+                (k, v.incompatibilities)
+                for k, v in broken_state.criteria.items()
+            ]
 
-        while len(self._states) >= 2:
-            # Retract the last candidate pin.
-            prev_state = self._states.pop()
-            try:
-                name, candidate = prev_state.mapping.popitem()
-            except KeyError:
-                continue
             self._r.backtracking(candidate)
 
-            # Create a new state to work on, with the newly known not-working
-            # candidate excluded.
+            # Create a new state from the last known-to-work one, and apply
+            # the previously gathered incompatibility information.
             self._push_new_state()
+            for k, incompatibilities in incompatibilities_from_broken:
+                try:
+                    crit = self.state.criteria[k]
+                except KeyError:
+                    continue
+                self.state.criteria[k] = crit.excluded_of(incompatibilities)
 
-            # Mark the retracted candidate as incompatible.
-            criterion = self.state.criteria[name].excluded_of(candidate)
-            if criterion is None:
-                # This state still does not work. Try the still previous state.
-                del self._states[-1]
-                continue
-            self.state.criteria[name] = criterion
+            # Mark the newly known incompatibility.
+            criterion = self.state.criteria[name].excluded_of([candidate])
 
-            return True
+            # It works! Let's work on this new state.
+            if criterion:
+                self.state.criteria[name] = criterion
+                return True
 
+            # State does not work after adding the new incompatibility
+            # information. Try the still previous state.
+
+        # No way to backtrack anymore.
         return False
 
     def resolve(self, requirements, max_rounds):
         if self._states:
             raise RuntimeError("already resolved")
 
+        self._r.starting()
+
+        # Initialize the root state.
         self._states = [State(mapping=collections.OrderedDict(), criteria={})]
         for r in requirements:
             try:
@@ -280,14 +312,14 @@ def resolve(self, requirements, max_rounds):
                 raise ResolutionImpossible(e.criterion.information)
             self.state.criteria[name] = crit
 
-        self._r.starting()
+        # The root state is saved as a sentinel so the first ever pin can have
+        # something to backtrack to if it fails. The root state is basically
+        # pinning the virtual "root" package in the graph.
+        self._push_new_state()
 
         for round_index in range(max_rounds):
             self._r.starting_round(round_index)
 
-            self._push_new_state()
-            curr = self.state
-
             unsatisfied_criterion_items = [
                 item
                 for item in self.state.criteria.items()
@@ -296,8 +328,7 @@ def resolve(self, requirements, max_rounds):
 
             # All criteria are accounted for. Nothing more to pin, we are done!
             if not unsatisfied_criterion_items:
-                del self._states[-1]
-                self._r.ending(curr)
+                self._r.ending(self.state)
                 return self.state
 
             # Choose the most preferred unpinned criterion to try.
@@ -307,16 +338,20 @@ def resolve(self, requirements, max_rounds):
             )
             failure_causes = self._attempt_to_pin_criterion(name, criterion)
 
-            # Backtrack if pinning fails.
             if failure_causes:
-                result = self._backtrack()
-                if not result:
-                    causes = [
-                        i for crit in failure_causes for i in crit.information
-                    ]
+                # Backtrack if pinning fails. The backtrack process puts us in
+                # an unpinned state, so we can work on it in the next round.
+                success = self._backtrack()
+
+                # Dead ends everywhere. Give up.
+                if not success:
+                    causes = [i for c in failure_causes for i in c.information]
                     raise ResolutionImpossible(causes)
+            else:
+                # Pinning was successful. Push a new state to do another pin.
+                self._push_new_state()
 
-            self._r.ending_round(round_index, curr)
+            self._r.ending_round(round_index, self.state)
 
         raise ResolutionTooDeep(max_rounds)