Overhaul tests to use py.test

Author: akx

.github/workflows/run_tests.yaml

@@ -18,18 +18,53 @@ jobs:
           cache-dependency-path: |
             **/requirements*txt
             launch.py
-      - name: Run tests
-        run: python launch.py --tests test --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test
+      - name: Install test dependencies
+        run: pip install wait-for-it -r requirements-test.txt
+        env:
+          PIP_DISABLE_PIP_VERSION_CHECK: "1"
+          PIP_PROGRESS_BAR: "off"
+      - name: Setup environment
+        run: python launch.py --skip-torch-cuda-test --exit
         env:
           PIP_DISABLE_PIP_VERSION_CHECK: "1"
           PIP_PROGRESS_BAR: "off"
           TORCH_INDEX_URL: https://download.pytorch.org/whl/cpu
           WEBUI_LAUNCH_LIVE_OUTPUT: "1"
-      - name: Upload main app stdout-stderr
+          PYTHONUNBUFFERED: "1"
+      - name: Start test server
+        run: >
+          python -m coverage run
+          --data-file=.coverage.server
+          launch.py
+          --skip-prepare-environment
+          --skip-torch-cuda-test
+          --test-server
+          --no-half
+          --disable-opt-split-attention
+          --use-cpu all
+          --add-stop-route
+          2>&1 | tee output.txt &
+      - name: Run tests
+        run: |
+          wait-for-it --service 127.0.0.1:7860 -t 600
+          python -m pytest -vv --junitxml=test/results.xml --cov . --cov-report=xml --verify-base-url test
+      - name: Kill test server
+        if: always()
+        run: curl -vv -XPOST http://127.0.0.1:7860/_stop && sleep 10
+      - name: Show coverage
+        run: |
+          python -m coverage combine .coverage*
+          python -m coverage report -i
+          python -m coverage html -i
+      - name: Upload main app output
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: output
+          path: output.txt
+      - name: Upload coverage HTML
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: stdout-stderr
-          path: |
-            test/stdout.txt
-            test/stderr.txt
+          name: htmlcov
+          path: htmlcov

.gitignore

@@ -35,4 +35,5 @@ notification.mp3
 /cache.json*
 /config_states/
 /node_modules
-/package-lock.json
+/package-lock.json
+/.coverage*

launch.py

@@ -310,12 +310,8 @@ def prepare_environment():
         print("Exiting because of --exit argument")
         exit(0)
 
-    if args.tests and not args.no_tests:
-        exitcode = tests(args.tests)
-        exit(exitcode)
 
-
-def tests(test_dir):
+def configure_for_tests():
     if "--api" not in sys.argv:
         sys.argv.append("--api")
     if "--ckpt" not in sys.argv:
@@ -325,21 +321,8 @@ def tests(test_dir):
         sys.argv.append("--skip-torch-cuda-test")
     if "--disable-nan-check" not in sys.argv:
         sys.argv.append("--disable-nan-check")
-    if "--no-tests" not in sys.argv:
-        sys.argv.append("--no-tests")
-
-    print(f"Launching Web UI in another process for testing with arguments: {' '.join(sys.argv[1:])}")
 
     os.environ['COMMANDLINE_ARGS'] = ""
-    with open(os.path.join(script_path, 'test/stdout.txt'), "w", encoding="utf8") as stdout, open(os.path.join(script_path, 'test/stderr.txt'), "w", encoding="utf8") as stderr:
-        proc = subprocess.Popen([sys.executable, *sys.argv], stdout=stdout, stderr=stderr)
-
-    import test.server_poll
-    exitcode = test.server_poll.run_tests(proc, test_dir)
-
-    print(f"Stopping Web UI process with id {proc.pid}")
-    proc.kill()
-    return exitcode
 
 
 def start():
@@ -351,6 +334,15 @@ def start():
         webui.webui()
 
 
-if __name__ == "__main__":
-    prepare_environment()
+def main():
+    if not args.skip_prepare_environment:
+        prepare_environment()
+
+    if args.test_server:
+        configure_for_tests()
+
     start()
+
+
+if __name__ == "__main__":
+    main()

modules/cmd_args.py

@@ -12,8 +12,8 @@
 parser.add_argument("--reinstall-xformers", action='store_true', help="launch.py argument: install the appropriate version of xformers even if you have some version already installed")
 parser.add_argument("--reinstall-torch", action='store_true', help="launch.py argument: install the appropriate version of torch even if you have some version already installed")
 parser.add_argument("--update-check", action='store_true', help="launch.py argument: chck for updates at startup")
-parser.add_argument("--tests", type=str, default=None, help="launch.py argument: run tests in the specified directory")
-parser.add_argument("--no-tests", action='store_true', help="launch.py argument: do not run tests even if --tests option is specified")
+parser.add_argument("--test-server", action='store_true', help="launch.py argument: configure server for testing")
+parser.add_argument("--skip-prepare-environment", action='store_true', help="launch.py argument: skip all environment preparation")
 parser.add_argument("--skip-install", action='store_true', help="launch.py argument: skip installation of packages")
 parser.add_argument("--data-dir", type=str, default=os.path.dirname(os.path.dirname(os.path.realpath(__file__))), help="base path where all user data is stored")
 parser.add_argument("--config", type=str, default=sd_default_config, help="path to config which constructs model",)

pyproject.toml

@@ -30,3 +30,6 @@ ignore = [
 [tool.ruff.flake8-bugbear]
 # Allow default arguments like, e.g., `data: List[str] = fastapi.Query(None)`.
 extend-immutable-calls = ["fastapi.Depends", "fastapi.security.HTTPBasic"]
+
+[tool.pytest.ini_options]
+base_url = "http://127.0.0.1:7860"

requirements-test.txt

@@ -0,0 +1,3 @@
+pytest-base-url~=2.0
+pytest-cov~=4.0
+pytest~=7.3