Improve testing with pytest.

developer/conan/requirements.txt

@@ -1,4 +1,10 @@
 conan
-packaging
 python-dateutil
 rich
+setuptools
+wheel
+twine
+requests
+packaging
+pytest
+pytest-xdist

python/SetupPython.cmake

@@ -12,6 +12,29 @@ else()
   find_package(Python COMPONENTS Interpreter Development REQUIRED)
 endif()
 
+execute_process(COMMAND ${Python_EXECUTABLE} -m pytest --version
+  RESULT_VARIABLE _Pytest_STATUS
+      OUTPUT_VARIABLE Pytest_Version
+      ERROR_QUIET
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+if(_Pytest_STATUS AND NOT _Pytest_STATUS EQUAL 0)
+  message(AUTHOR_WARNING "Pytest isn't installed on your system python, so some tests won't be run. Run `pip install pytest`")
+  set(Pytest_AVAILABLE OFF)
+else()
+  message("Found Pytest: ${Pytest_Version}")
+  set(Pytest_AVAILABLE ON)
+  execute_process(COMMAND ${Python_EXECUTABLE} -m pip show pytest-xdist -q
+    RESULT_VARIABLE _Pytest_xdist_STATUS
+    ERROR_QUIET
+  )
+  if(_Pytest_xdist_STATUS AND NOT _Pytest_xdist_STATUS EQUAL 0)
+    message(AUTHOR_WARNING "Pytest-xdist isn't installed on your system python, so the pytest runs will not be parallelized")
+  else()
+    set(Pytest_XDIST_OPTS -n auto)
+  endif()
+endif()
+
 get_filename_component(Python_PROGRAM_NAME ${Python_EXECUTABLE} NAME)
 
 get_filename_component(RESOLVED_PYTHON_LIBRARY "${Python_LIBRARIES}" REALPATH)

src/cli/CMakeLists.txt

@@ -230,17 +230,24 @@ if(BUILD_TESTING)
   )
   set_tests_properties(OpenStudioCLI.Labs.Run_RubyPython PROPERTIES RESOURCE_LOCK "compact_osw")
 
-  add_test(NAME OpenStudioCLI.test_logger_rb
-    COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/run_test_logger.py" $<TARGET_FILE:openstudio> ${CMAKE_CURRENT_SOURCE_DIR}/test/logger_test.rb
-  )
+  if (Pytest_AVAILABLE)
+    add_test(NAME OpenStudioCLI.Labs.test_loglevel
+      COMMAND ${Python_EXECUTABLE} -m pytest --verbose ${Pytest_XDIST_OPTS} --os-cli-path $<TARGET_FILE:openstudio> "${CMAKE_CURRENT_SOURCE_DIR}/test/test_loglevel.py"
+    )
+  else()
+    # TODO: Remove. Fallback on these for now, as I don't know if CI has pytest installed
+    add_test(NAME OpenStudioCLI.test_logger_rb
+      COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/run_test_logger.py" $<TARGET_FILE:openstudio> ${CMAKE_CURRENT_SOURCE_DIR}/test/logger_test.rb
+    )
 
-  add_test(NAME OpenStudioCLI.Labs.test_logger_rb
-    COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/run_test_logger.py" $<TARGET_FILE:openstudio> --labs ${CMAKE_CURRENT_SOURCE_DIR}/test/logger_test.rb
-  )
+    add_test(NAME OpenStudioCLI.Labs.test_logger_rb
+      COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/run_test_logger.py" $<TARGET_FILE:openstudio> --labs ${CMAKE_CURRENT_SOURCE_DIR}/test/logger_test.rb
+    )
 
-  add_test(NAME OpenStudioCLI.Labs.test_logger_py
-    COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/run_test_logger.py" $<TARGET_FILE:openstudio> --labs ${CMAKE_CURRENT_SOURCE_DIR}/test/logger_test.py
-  )
+    add_test(NAME OpenStudioCLI.Labs.test_logger_py
+      COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/test/run_test_logger.py" $<TARGET_FILE:openstudio> --labs ${CMAKE_CURRENT_SOURCE_DIR}/test/logger_test.py
+    )
+  endif()
 
 
   # ============ #4856 - Forward a Path properly no matter the slashes employed ============

src/cli/main.cpp

@@ -124,7 +124,7 @@ int main(int argc, char* argv[]) {
       ->add_option_function<LogLevel>(
         "-l,--loglevel",
         [](const LogLevel& level) {
-          fmt::print("Setting log Level to {} ({})\n", logLevelStrs[static_cast<size_t>(level) - static_cast<size_t>(LogLevel::Trace)], level);
+          fmt::print("Setting Log Level to {} ({})\n", logLevelStrs[static_cast<size_t>(level) - static_cast<size_t>(LogLevel::Trace)], level);
           openstudio::Logger::instance().standardOutLogger().setLogLevel(level);
         },
         "LogLevel settings: One of {Trace, Debug, Info, Warn, Error, Fatal} [Default: Warn] Excludes: --verbose")

src/cli/test/conftest.py

@@ -0,0 +1,21 @@
+import pytest
+from pathlib import Path
+
+def validate_file(arg):
+    if (filepath := Path(arg)).is_file():
+        return filepath
+    else:
+        raise FileNotFoundError(arg)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--os-cli-path", type=validate_file, help="Path to the OS CLI" #, required=True
+    )
+
+@pytest.fixture
+def osclipath(request):
+    cli_path = request.config.getoption("--os-cli-path")
+    if cli_path is None:
+        raise ValueError("You must supply --os-cli-path [Path]")
+    return cli_path

src/cli/test/test_loglevel.py

@@ -0,0 +1,86 @@
+import subprocess
+from pathlib import Path
+from typing import List
+
+import pytest
+
+LOG_LEVELS = ["Trace", "Debug", "Info", "Warn", "Error", "Fatal"]
+
+RUBY_PROGRAM = (
+    f"{LOG_LEVELS=}; LOG_LEVELS.each_with_index {{ |s, i| OpenStudio::logFree(OpenStudio::Trace + i, 'test', s) }}"
+)
+PYTHON_PROGRAM = (
+    f"{LOG_LEVELS=}; [openstudio.logFree(openstudio.Trace + i, 'test', s) for i, s in enumerate(LOG_LEVELS)]"
+)
+
+
+def remove_labs_box(lines: List[str]) -> List[str]:
+    """Pop the labs box."""
+    i_warn = 0
+    for i, line in enumerate(lines):
+        if "The `labs` command is experimental - Do not use in production" in line:
+            i_warn = i
+            break
+    lines = lines[: (i_warn - 1)] + lines[(i_warn + 2) :]
+    return lines
+
+
+def get_loglevel_list_index(logLevel: str) -> int:
+    return LOG_LEVELS.index(logLevel)
+
+
+def get_loglevel_int_value(logLevel: str) -> int:
+    return LOG_LEVELS.index(logLevel) - 3
+
+
+@pytest.mark.parametrize("language", ["ruby", "python"])
+@pytest.mark.parametrize("logLevel", LOG_LEVELS)
+def test_loglevel(osclipath: Path, language: str, logLevel: str):
+    list_index = get_loglevel_list_index(logLevel=logLevel)
+    loglevel_int_value = get_loglevel_int_value(logLevel=logLevel)
+    args = [str(osclipath), "labs", "--loglevel", logLevel]
+    if language == "ruby":
+        args += ["-e", RUBY_PROGRAM]
+    else:
+        args += ["-c", PYTHON_PROGRAM]
+    lines = subprocess.check_output(args, encoding="utf-8").splitlines()
+    lines = remove_labs_box(lines)
+    assert f"Setting Log Level to {logLevel} ({loglevel_int_value})" in lines
+    for i, msgLevel in enumerate(LOG_LEVELS):
+        msg = f"[test] <{get_loglevel_int_value(msgLevel)}> {msgLevel}"
+
+        if i < list_index:
+            assert msg not in lines
+        else:
+            assert msg in lines
+
+
+@pytest.mark.parametrize(
+    "language, is_labs",
+    [
+        ["ruby", False],
+        ["ruby", True],
+        # ["python", False], // Not possible
+        ["python", True],
+    ],
+)
+def test_run_logger_file(osclipath, language: str, is_labs: bool):
+    command = [str(osclipath)]
+    if is_labs:
+        command.append("labs")
+    if language == "ruby":
+        command.append("execute_ruby_script")
+        command.append(Path(__file__).parent / "logger_test.rb")
+    elif language == "python":
+        command.append("execute_python_script")
+        command.append(Path(__file__).parent / "logger_test.py")
+    else:
+        raise ValueError("Can only use ruby or python")
+
+    r = subprocess.check_output(command, encoding="utf-8")
+    lines = r.splitlines()
+    lines = remove_labs_box(lines)
+
+    # Ruby when called this way has the openstudio logger messages first instead of last, so just sort
+    lines.sort()
+    assert lines == ["LOGGER - STDOUT Error", "LOGGER - STDOUT Warn", "[test] <1> Error"]