Verify hooks after collection completes

CHANGELOG.rst

@@ -53,11 +53,16 @@ Changes
 * Change exception raised by ``capture.DontReadFromInput.fileno()`` from ``ValueError``
   to ``io.UnsupportedOperation``. Thanks `@vlad-dragos`_ for the PR.
 
-* fix `#2013`_: turn RecordedWarning into namedtupe,
-  to give it a comprehensible repr while preventing unwarranted modification
+* fix `#2013`_: turn RecordedWarning into ``namedtuple``,
+  to give it a comprehensible repr while preventing unwarranted modification.
 
 * fix `#2208`_: ensure a iteration limit for _pytest.compat.get_real_func.
-  Thanks `@RonnyPfannschmidt`_ for the Report and PR
+  Thanks `@RonnyPfannschmidt`_ for the report and PR.
+
+* Hooks are now verified after collection is complete, rather than right after loading installed plugins. This
+  makes it easy to write hooks for plugins which will be loaded during collection, for example using the
+  ``pytest_plugins`` special variable (`#1821`_).
+  Thanks `@nicoddemus`_ for the PR.
 
 * Modify ``pytest_make_parametrize_id()`` hook to accept ``argname`` as an
   additional parameter.
@@ -96,6 +101,7 @@ Bug Fixes
 
 .. _#1407: https://github.com/pytest-dev/pytest/issues/1407
 .. _#1512: https://github.com/pytest-dev/pytest/issues/1512
+.. _#1821: https://github.com/pytest-dev/pytest/issues/1821
 .. _#1874: https://github.com/pytest-dev/pytest/pull/1874
 .. _#1952: https://github.com/pytest-dev/pytest/pull/1952
 .. _#2007: https://github.com/pytest-dev/pytest/issues/2007

_pytest/config.py

@@ -54,7 +54,6 @@ def main(args=None, plugins=None):
             return 4
         else:
             try:
-                config.pluginmanager.check_pending()
                 return config.hook.pytest_cmdline_main(config=config)
             finally:
                 config._ensure_unconfigure()

_pytest/main.py

@@ -596,6 +596,7 @@ def perform_collect(self, args=None, genitems=True):
         hook = self.config.hook
         try:
             items = self._perform_collect(args, genitems)
+            self.config.pluginmanager.check_pending()
             hook.pytest_collection_modifyitems(session=self,
                 config=self.config, items=items)
         finally:

_pytest/pytester.py

@@ -88,7 +88,7 @@ def matching_platform(self):
             return True
 
     @pytest.hookimpl(hookwrapper=True, tryfirst=True)
-    def pytest_runtest_item(self, item):
+    def pytest_runtest_protocol(self, item):
         lines1 = self.get_open_files()
         yield
         if hasattr(sys, "pypy_version_info"):

testing/acceptance_test.py

@@ -807,3 +807,31 @@ def test(): pass
     """)
     r = testdir.runpytest()
     assert r.ret == 0
+
+
+def test_deferred_hook_checking(testdir):
+    """
+    Check hooks as late as possible (#1821).
+    """
+    testdir.syspathinsert()
+    testdir.makepyfile(**{
+        'plugin.py': """
+        class Hooks:
+            def pytest_my_hook(self, config):
+                pass
+
+        def pytest_configure(config):
+            config.pluginmanager.add_hookspecs(Hooks)
+        """,
+        'conftest.py': """
+            pytest_plugins = ['plugin']
+            def pytest_my_hook(config):
+                return 40
+        """,
+        'test_foo.py': """
+            def test(request):
+                assert request.config.hook.pytest_my_hook(config=request.config) == [40]
+        """
+    })
+    result = testdir.runpytest()
+    result.stdout.fnmatch_lines(['* 1 passed *'])

testing/test_helpconfig.py

@@ -33,7 +33,7 @@ def pytest_hello(xyz):
     """)
     result = testdir.runpytest()
     assert result.ret != 0
-    result.stderr.fnmatch_lines([
+    result.stdout.fnmatch_lines([
         '*unknown hook*pytest_hello*'
     ])