Tests: unroll 2 player gen, add parametrization helper, add docs (#4648)

* Tests: unroll test_multiworlds.TestTwoPlayerMulti

Also adds a helper function that other tests can use to unroll tests.

* Docs: add more details to docs/tests.md

* Explain parametrization, subtests and link to the new helper
* Mention some performance details and work-arounds
* Mention multithreading / pytest-xdist

* Tests: make param.classvar_matrix accept sets

* CI: add test/param.py to type checking

* Tests: add missing typing to test/param.py

* Tests: fix typo in test/param.py doc comment

Co-authored-by: qwint <qwint.42@gmail.com>

* update docs

* Docs: reword note on performance

---------

Co-authored-by: qwint <qwint.42@gmail.com>

Author: black-sliver

.github/pyright-config.json

@@ -2,6 +2,7 @@
   "include": [
     "../BizHawkClient.py",
     "../Patch.py",
+    "../test/param.py",
     "../test/general/test_groups.py",
     "../test/general/test_helpers.py",
     "../test/general/test_memory.py",

docs/tests.md

@@ -82,6 +82,38 @@ Unit tests can also be created using [TestBase](/test/bases.py#L16) or
 may be useful for generating a multiworld under very specific constraints without using the generic world setup, or for
 testing portions of your code that can be tested without relying on a multiworld to be created first.
 
+#### Parametrization
+
+When defining a test that needs to cover a range of inputs it is useful to parameterize (to run the same test
+for multiple inputs) the base test. Some important things to consider when attempting to parametrize your test are:
+
+* [Subtests](https://docs.python.org/3/library/unittest.html#distinguishing-test-iterations-using-subtests)
+  can be used to have parametrized assertions that show up similar to individual tests but without the overhead
+  of needing to instantiate multiple tests; however, subtests can not be multithreaded and do not have individual
+  timing data, so they are not suitable for slow tests.
+
+* Archipelago's tests are test-runner-agnostic. That means tests are not allowed to use e.g. `@pytest.mark.parametrize`.
+  Instead, we define our own parametrization helpers in [test.param](/test/param.py).
+
+* Classes inheriting from `WorldTestBase`, including those created by the helpers in `test.param`, will run all
+  base tests by default, make sure the produced tests actually do what you aim for and do not waste a lot of
+  extra CPU time. Consider using `TestBase` or `unittest.TestCase` directly
+  or setting `WorldTestBase.run_default_tests` to False.
+
+#### Performance Considerations
+
+Archipelago is big enough that the runtime of unittests can have an impact on productivity.
+
+Individual tests should take less than a second, so they can be properly multithreaded.
+
+Ideally, thorough tests are directed at actual code/functionality. Do not just create and/or fill a ton of individual
+Multiworlds that spend most of the test time outside what you actually want to test.
+
+Consider generating/validating "random" games as part of your APWorld release workflow rather than having that be part
+of continuous integration, and add minimal reproducers to the "normal" tests for problems that were found.
+You can use [@unittest.skipIf](https://docs.python.org/3/library/unittest.html#unittest.skipIf) with an environment
+variable to keep all the benefits of the test framework while not running the marked tests by default.
+
 ## Running Tests
 
 #### Using Pycharm
@@ -100,3 +132,11 @@ next to the run and debug buttons.
 #### Running Tests without Pycharm
 
 Run `pip install pytest pytest-subtests`, then use your IDE to run tests or run `pytest` from the source folder.
+
+#### Running Tests Multithreaded
+
+pytest can run multiple test runners in parallel with the pytest-xdist extension.
+
+Install with `pip install pytest-xdist`.
+
+Run with `pytest -n12` to spawn 12 process that each run 1/12th of the tests.

test/multiworld/test_multiworlds.py

@@ -1,12 +1,13 @@
 import unittest
-from typing import List, Tuple
+from typing import ClassVar, List, Tuple
 from unittest import TestCase
 
 from BaseClasses import CollectionState, Location, MultiWorld
 from Fill import distribute_items_restrictive
 from Options import Accessibility
 from worlds.AutoWorld import AutoWorldRegister, call_all, call_single
 from ..general import gen_steps, setup_multiworld
+from ..param import classvar_matrix
 
 
 class MultiworldTestBase(TestCase):
@@ -63,15 +64,18 @@ def test_fills(self) -> None:
             self.assertTrue(self.fulfills_accessibility(), "Collected all locations, but can't beat the game")
 
 
+@classvar_matrix(game=AutoWorldRegister.world_types.keys())
 class TestTwoPlayerMulti(MultiworldTestBase):
+    game: ClassVar[str]
+
     def test_two_player_single_game_fills(self) -> None:
         """Tests that a multiworld of two players for each registered game world can generate."""
-        for world_type in AutoWorldRegister.world_types.values():
-            self.multiworld = setup_multiworld([world_type, world_type], ())
-            for world in self.multiworld.worlds.values():
-                world.options.accessibility.value = Accessibility.option_full
-            self.assertSteps(gen_steps)
-            with self.subTest("filling multiworld", games=world_type.game, seed=self.multiworld.seed):
-                distribute_items_restrictive(self.multiworld)
-                call_all(self.multiworld, "post_fill")
-                self.assertTrue(self.fulfills_accessibility(), "Collected all locations, but can't beat the game")
+        world_type = AutoWorldRegister.world_types[self.game]
+        self.multiworld = setup_multiworld([world_type, world_type], ())
+        for world in self.multiworld.worlds.values():
+            world.options.accessibility.value = Accessibility.option_full
+        self.assertSteps(gen_steps)
+        with self.subTest("filling multiworld", games=world_type.game, seed=self.multiworld.seed):
+            distribute_items_restrictive(self.multiworld)
+            call_all(self.multiworld, "post_fill")
+            self.assertTrue(self.fulfills_accessibility(), "Collected all locations, but can't beat the game")

test/param.py

@@ -0,0 +1,46 @@
+import itertools
+import sys
+from typing import Any, Callable, Iterable
+
+
+def classvar_matrix(**kwargs: Iterable[Any]) -> Callable[[type], None]:
+    """
+    Create a new class for each variation of input, allowing to generate a TestCase matrix / parametrization that
+    supports multi-threading and has better reporting for ``unittest --durations=...`` and ``pytest --durations=...``
+    than subtests.
+
+    The kwargs will be set as ClassVars in the newly created classes. Use as ::
+
+        @classvar_matrix(var_name=[value1, value2])
+        class MyTestCase(unittest.TestCase):
+            var_name: typing.ClassVar[...]
+
+    :param kwargs: A dict of ClassVars to set, where key is the variable name and value is a list of all values.
+    :return: A decorator to be applied to a class.
+    """
+    keys: tuple[str]
+    values: Iterable[Iterable[Any]]
+    keys, values = zip(*kwargs.items())
+    values = map(lambda v: sorted(v) if isinstance(v, (set, frozenset)) else v, values)
+    permutations_dicts = [dict(zip(keys, v)) for v in itertools.product(*values)]
+
+    def decorator(cls: type) -> None:
+        mod = sys.modules[cls.__module__]
+
+        for permutation in permutations_dicts:
+
+            class Unrolled(cls):  # type: ignore
+                pass
+
+            for k, v in permutation.items():
+                setattr(Unrolled, k, v)
+            params = ", ".join([f"{k}={repr(v)}" for k, v in permutation.items()])
+            params = f"{{{params}}}"
+
+            Unrolled.__module__ = cls.__module__
+            Unrolled.__qualname__ = f"{cls.__qualname__}{params}"
+            setattr(mod, f"{cls.__name__}{params}", Unrolled)
+
+        return None
+
+    return decorator