test: Run the examples and check their output

.github/workflows/continuous-integration.yml

@@ -40,7 +40,7 @@ jobs:
         run: python3 -m pip install .
 
       - name: Test with pytest
-        run: python3 -m pytest --cov=staged_script test/
+        run: python3 -m pytest --cov=staged_script example/ test/
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4

example/ex_0_the_basics.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """A very basic staged script."""
 
 # © 2024 National Technology & Engineering Solutions of Sandia, LLC

example/ex_1_removing_the_retry_arguments.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """A staged script without retry arguments."""
 
 # © 2024 National Technology & Engineering Solutions of Sandia, LLC

example/ex_2_running_certain_stages_by_default.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """A staged script that runs certain stages by default."""
 
 # © 2024 National Technology & Engineering Solutions of Sandia, LLC

example/ex_3_adding_arguments.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """A staged script with additional arguments."""
 
 # © 2024 National Technology & Engineering Solutions of Sandia, LLC

example/ex_4_customizing_stage_behavior.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """A staged script with custom stage behavior."""
 
 # © 2024 National Technology & Engineering Solutions of Sandia, LLC

example/ex_5_customizing_individual_stages.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """A staged script with phases customized per stage."""
 
 # © 2024 National Technology & Engineering Solutions of Sandia, LLC

example/ex_6_creating_retryable_stages.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """A staged script with a retryable stage."""
 
 # © 2024 National Technology & Engineering Solutions of Sandia, LLC

example/ex_7_customizing_the_summary.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """A staged script with a retryable stage."""
 
 # © 2024 National Technology & Engineering Solutions of Sandia, LLC

example/test_examples.py

@@ -0,0 +1,329 @@
+#!/usr/bin/env python3
+"""Run all the examples and ensure their output is correct."""
+
+# © 2024 National Technology & Engineering Solutions of Sandia, LLC
+# (NTESS).  Under the terms of Contract DE-NA0003525 with NTESS, the
+# U.S. Government retains certain rights in this software.
+
+# SPDX-License-Identifier: BSD-3-Clause
+
+import subprocess
+from pathlib import Path
+from typing import List
+
+
+def assert_output_in_order(stdout: str, output: List[str]) -> None:
+    """
+    Ensure the output appears in the correct order.
+
+    Args:
+        stdout:  The ``stdout`` of the command that was run.
+        output:  The list of terms that should appear in the output in
+            the given order.
+
+    Raises:
+        ValueError:  If any of the terms in the output list cannot be
+            found.
+    """
+    index = 0
+    for term in output:
+        index = stdout.find(term, index) + len(term)
+
+
+def test_ex_0_the_basics_help() -> None:
+    example = Path(__file__).parent / "ex_0_the_basics.py"
+    result = subprocess.run(
+        [example, "--help"],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+    assert_output_in_order(
+        result.stdout,
+        [
+            "the ArgumentParser for the StagedScript base class.",
+            "--stage {goodbye,hello}",
+            "--goodbye-retry-attempts",
+            "--goodbye-retry-delay",
+            "--goodbye-retry-timeout",
+            "--hello-retry-attempts",
+            "--hello-retry-delay",
+            "--hello-retry-timeout",
+        ],
+    )
+
+
+def test_ex_0_the_basics() -> None:
+    example = Path(__file__).parent / "ex_0_the_basics.py"
+    result = subprocess.run(
+        [example, "--stage", "hello"],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+    assert_output_in_order(
+        result.stdout,
+        [
+            "Greeting the user",
+            "Executing:  echo 'Hello World'",
+            "`hello` stage duration",
+            "Bidding farewell",
+            "Skipping this stage",
+            "`goodbye` stage duration",
+            "Ran the following",
+            "--hello-retry-attempts 0",
+            "--hello-retry-delay 0",
+            "--hello-retry-timeout 60",
+            "--goodbye-retry-attempts 0",
+            "--goodbye-retry-delay 0",
+            "--goodbye-retry-timeout 60",
+            "Commands executed",
+            "Timing results",
+            "Script result",
+        ],
+    )
+
+
+def test_ex_1_removing_the_retry_arguments_help() -> None:
+    example = Path(__file__).parent / "ex_1_removing_the_retry_arguments.py"
+    result = subprocess.run(
+        [example, "--help"],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+    assert "Demonstrate removing the retry arguments" in result.stdout
+    for _ in [
+        "--goodbye-retry-attempts",
+        "--goodbye-retry-delay",
+        "--goodbye-retry-timeout",
+        "--hello-retry-attempts",
+        "--hello-retry-delay",
+        "--hello-retry-timeout",
+    ]:
+        assert _ not in result.stdout
+
+
+def test_ex_1_removing_the_retry_arguments() -> None:
+    example = Path(__file__).parent / "ex_1_removing_the_retry_arguments.py"
+    result = subprocess.run(
+        [example, "--stage", "hello"],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+    for _ in [
+        "--goodbye-retry-attempts",
+        "--goodbye-retry-delay",
+        "--goodbye-retry-timeout",
+        "--hello-retry-attempts",
+        "--hello-retry-delay",
+        "--hello-retry-timeout",
+    ]:
+        assert _ not in result.stdout
+
+
+def test_ex_2_running_certain_stages_by_default() -> None:
+    example = (
+        Path(__file__).parent / "ex_2_running_certain_stages_by_default.py"
+    )
+    result = subprocess.run(
+        [example],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+    assert_output_in_order(
+        result.stdout,
+        [
+            "Greeting the user",
+            "Executing:  echo 'Hello World'",
+            "`hello` stage duration",
+            "Bidding farewell",
+            "Executing:  echo 'Goodbye World'",
+            "`goodbye` stage duration",
+        ],
+    )
+
+
+def test_ex_3_adding_arguments() -> None:
+    example = Path(__file__).parent / "ex_3_adding_arguments.py"
+    result = subprocess.run(
+        [example, "--some-file", "foo.txt"],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+    assert_output_in_order(
+        result.stdout,
+        [
+            "Greeting the user",
+            "Executing:  echo 'Hello World'",
+            "Processing file",
+            "/foo.txt",
+            "`hello` stage duration",
+            "Bidding farewell",
+            "Executing:  echo 'Goodbye World'",
+            "Some flag was not set!",
+            "`goodbye` stage duration",
+        ],
+    )
+
+
+def test_ex_4_customizing_stage_behavior() -> None:
+    example = Path(__file__).parent / "ex_4_customizing_stage_behavior.py"
+    result = subprocess.run(
+        [
+            example,
+            "--some-file",
+            "foo.txt",
+            "--some-flag",
+            "--stage",
+            "goodbye",
+        ],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+    assert_output_in_order(
+        result.stdout,
+        [
+            "Checking to make sure it's safe to run a stage",
+            "Greeting the user",
+            "You didn't tell me to run the 'hello' stage",
+            "`hello` stage duration",
+            "Finished stage 'hello'",
+            "Checking to make sure all is well after running",
+            "Checking to make sure it's safe to run a stage",
+            "Bidding farewell",
+            "Executing:  echo 'Goodbye World'",
+            "Some flag was set!",
+            "`goodbye` stage duration",
+            "Finished stage 'goodbye'",
+            "Checking to make sure all is well after running",
+        ],
+    )
+
+
+def test_ex_5_customizing_individual_stages() -> None:
+    example = Path(__file__).parent / "ex_5_customizing_individual_stages.py"
+    result = subprocess.run(
+        [example, "--some-file", "foo.txt"],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+    assert_output_in_order(
+        result.stdout,
+        [
+            "Checking to make sure it's safe to run a stage",
+            "Greeting the user",
+            "The first stage is underway",
+            "Executing:  echo 'Hello World'",
+            "Processing file",
+            "/foo.txt",
+            "`hello` stage duration",
+            "Finished stage 'hello'",
+            "Checking to make sure all is well after running",
+            "Checking to make sure it's safe to run a stage",
+            "Bidding farewell",
+            "Executing:  echo 'Goodbye World'",
+            "Some flag was not set!",
+            "`goodbye` stage duration",
+            "Finished the final stage:  goodbye",
+            "Checking to make sure all is well after running",
+        ],
+    )
+
+
+def test_ex_6_creating_retryable_stages_help() -> None:
+    example = Path(__file__).parent / "ex_6_creating_retryable_stages.py"
+    result = subprocess.run(
+        [example, "--help"],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+    assert_output_in_order(
+        result.stdout,
+        [
+            "retry",
+            "Additional options for retrying stages",
+            "--flaky-retry-attempts",
+            "--flaky-retry-delay",
+            "--flaky-retry-timeout",
+        ],
+    )
+
+
+def test_ex_6_creating_retryable_stages() -> None:
+    example = Path(__file__).parent / "ex_6_creating_retryable_stages.py"
+    result = subprocess.run(
+        [example, "--some-file", "foo.txt"],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+    flaky_output = [
+        "Trying an error-prone operation",
+        "Oh no!  Something went horribly wrong!",
+        "`flaky` stage duration",
+        "Finished stage 'flaky'",
+        "Preparing to retry the 'flaky' stage",
+        "RetryCallState",
+    ]
+    assert_output_in_order(
+        result.stdout,
+        [
+            *flaky_output,
+            *flaky_output,
+            "Trying an error-prone operation",
+            "Thank goodness, everything worked this time",
+            "`flaky` stage duration",
+            "Finished stage 'flaky'",
+        ],
+    )
+
+
+def test_ex_6_creating_retryable_stages_no_retries() -> None:
+    example = Path(__file__).parent / "ex_6_creating_retryable_stages.py"
+    result = subprocess.run(
+        [example, "--some-file", "foo.txt", "--flaky-retry-attempts", "0"],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+    assert_output_in_order(
+        result.stdout,
+        [
+            "Trying an error-prone operation",
+            "Oh no!  Something went horribly wrong!",
+            "`flaky` stage duration",
+            "Finished stage 'flaky'",
+            "Checking to make sure all is well after running",
+            "Checking to make sure it's safe to run a stage",
+            "Bidding farewell",
+            "Script result",
+            "Failure",
+        ],
+    )
+
+
+def test_ex_7_customizing_the_summary() -> None:
+    example = Path(__file__).parent / "ex_7_customizing_the_summary.py"
+    result = subprocess.run(
+        [example, "--some-file", "foo.txt", "--stage", "hello"],
+        capture_output=True,
+        check=True,
+        text=True,
+    )
+    assert_output_in_order(
+        result.stdout,
+        [
+            "Script result",
+            "Success",
+            "Machine details",
+            "hostname",
+            "platform",
+        ],
+    )

pyproject.toml

@@ -104,10 +104,12 @@ ignore = [
 
 [tool.ruff.lint.per-file-ignores]
 "**/test_*.py" = ["S101"]
-"example/ex_*.py" = [
+"example/*.py" = [
     "D101",
     "D102",
+    "D103",
     "D107",
+    "S603",
     "S604",
 ]