From 4579be27915c9fd5ae81b5e253b698527595f730 Mon Sep 17 00:00:00 2001 From: Frick Date: Wed, 14 Jan 2026 01:54:31 +0000 Subject: [PATCH] Test suite modernization: pytest wrapper and runner fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add LD_LIBRARY_PATH auto-configuration in run_tests.py - Add --timeout and --quiet command-line flags - Create pytest wrapper (conftest.py, test_mcrogueface.py) for IDE integration - Configure pytest.ini to avoid importing mcrfpy modules - Document known issues: 120/179 passing, 40 timeouts, 19 failures 🤖 Generated with Claude Code (https://claude.ai/code) Co-Authored-By: Claude --- tests/KNOWN_ISSUES.md | 62 +++++++++++++ tests/conftest.py | 190 ++++++++++++++++++++++++++++++++++++++ tests/pytest.ini | 14 +++ tests/run_tests.py | 42 ++++++--- tests/test_mcrogueface.py | 116 +++++++++++++++++++++++ 5 files changed, 413 insertions(+), 11 deletions(-) create mode 100644 tests/KNOWN_ISSUES.md create mode 100644 tests/conftest.py create mode 100644 tests/pytest.ini create mode 100644 tests/test_mcrogueface.py diff --git a/tests/KNOWN_ISSUES.md b/tests/KNOWN_ISSUES.md new file mode 100644 index 0000000..215efef --- /dev/null +++ b/tests/KNOWN_ISSUES.md @@ -0,0 +1,62 @@ +# McRogueFace Test Suite - Known Issues + +## Test Results Summary + +As of 2026-01-14, with `--mcrf-timeout=5`: +- **120 passed** (67%) +- **59 failed** (33%) + - 40 timeout failures (tests requiring timers/animations) + - 19 actual failures (API changes, missing features, or bugs) + +## Timeout Failures (40 tests) + +These tests require timers, animations, or callbacks that don't complete within the 5s timeout. +Run with `--mcrf-timeout=30` for a more permissive test run. + +**Animation/Timer tests:** +- WORKING_automation_test_example.py +- benchmark_logging_test.py +- keypress_scene_validation_test.py +- simple_timer_screenshot_test.py +- test_animation_callback_simple.py +- test_animation_property_locking.py +- test_animation_raii.py +- test_animation_removal.py +- test_empty_animation_manager.py +- test_simple_callback.py + +**Headless mode tests:** +- test_headless_detection.py +- test_headless_modes.py + +**Other timing-dependent:** +- test_color_helpers.py +- test_frame_clipping.py +- test_frame_clipping_advanced.py +- test_grid_children.py +- test_no_arg_constructors.py +- test_properties_quick.py +- test_python_object_cache.py +- test_simple_drawable.py + +## Running Tests + +```bash +# Quick run (5s timeout, many timeouts expected) +pytest tests/ -q --mcrf-timeout=5 + +# Full run (30s timeout, should pass most timing tests) +pytest tests/ -q --mcrf-timeout=30 + +# Filter by pattern +pytest tests/ -k "bsp" -q + +# Run original runner +python3 tests/run_tests.py -q +``` + +## Recommendations + +1. **For CI:** Use `--mcrf-timeout=10` and accept ~30% timeout failures +2. **For local dev:** Use `--mcrf-timeout=30` for comprehensive testing +3. **For quick validation:** Use `-k "not animation and not timer"` to skip slow tests diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..aeac22c --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,190 @@ +""" +Pytest configuration for McRogueFace tests. + +Provides fixtures for running McRogueFace scripts in headless mode. + +Usage: + pytest tests/ -q # Run all tests quietly + pytest tests/ -k "bsp" # Run tests matching "bsp" + pytest tests/ -x # Stop on first failure + pytest tests/ --tb=short # Short tracebacks +""" +import os +import subprocess +import pytest +from pathlib import Path + +# Paths +TESTS_DIR = Path(__file__).parent +BUILD_DIR = TESTS_DIR.parent / "build" +LIB_DIR = TESTS_DIR.parent / "__lib" +MCROGUEFACE = BUILD_DIR / "mcrogueface" + +# Default timeout for tests (can be overridden with --timeout) +DEFAULT_TIMEOUT = 10 + + +def pytest_addoption(parser): + """Add custom command line options.""" + parser.addoption( + "--mcrf-timeout", + action="store", + default=DEFAULT_TIMEOUT, + type=int, + help="Timeout in seconds for each McRogueFace test" + ) + + +@pytest.fixture +def mcrf_timeout(request): + """Get the configured timeout.""" + return request.config.getoption("--mcrf-timeout") + + +@pytest.fixture +def mcrf_env(): + """Environment with LD_LIBRARY_PATH set for McRogueFace.""" + env = os.environ.copy() + existing_ld = env.get('LD_LIBRARY_PATH', '') + env['LD_LIBRARY_PATH'] = f"{LIB_DIR}:{existing_ld}" if existing_ld else str(LIB_DIR) + return env + + +@pytest.fixture +def mcrf_exec(mcrf_env, mcrf_timeout): + """ + Fixture that returns a function to execute McRogueFace scripts. + + Usage in tests: + def test_something(mcrf_exec): + passed, output = mcrf_exec("unit/my_test.py") + assert passed + """ + def _exec(script_path, timeout=None): + """ + Execute a McRogueFace script in headless mode. + + Args: + script_path: Path relative to tests/ directory + timeout: Override default timeout + + Returns: + (passed: bool, output: str) + """ + if timeout is None: + timeout = mcrf_timeout + + full_path = TESTS_DIR / script_path + + try: + result = subprocess.run( + [str(MCROGUEFACE), '--headless', '--exec', str(full_path)], + capture_output=True, + text=True, + timeout=timeout, + cwd=str(BUILD_DIR), + env=mcrf_env + ) + + output = result.stdout + result.stderr + passed = result.returncode == 0 + + # Check for PASS/FAIL in output + if 'FAIL' in output and 'PASS' not in output.split('FAIL')[-1]: + passed = False + + return passed, output + + except subprocess.TimeoutExpired: + return False, "TIMEOUT" + except Exception as e: + return False, str(e) + + return _exec + + +def pytest_collect_file(parent, file_path): + """Auto-discover McRogueFace test scripts.""" + # Only collect from unit/, integration/, regression/ directories + try: + rel_path = file_path.relative_to(TESTS_DIR) + except ValueError: + return None + + if rel_path.parts and rel_path.parts[0] in ('unit', 'integration', 'regression'): + if file_path.suffix == '.py' and file_path.name not in ('__init__.py', 'conftest.py'): + return McRFTestFile.from_parent(parent, path=file_path) + return None + + +def pytest_ignore_collect(collection_path, config): + """Prevent pytest from trying to import test scripts as Python modules.""" + try: + rel_path = collection_path.relative_to(TESTS_DIR) + if rel_path.parts and rel_path.parts[0] in ('unit', 'integration', 'regression'): + # Let our custom collector handle these, don't import them + return False # Don't ignore - we'll collect them our way + except ValueError: + pass + return None + + +class McRFTestFile(pytest.File): + """Custom test file for McRogueFace scripts.""" + + def collect(self): + """Yield a single test item for this script.""" + yield McRFTestItem.from_parent(self, name=self.path.stem) + + +class McRFTestItem(pytest.Item): + """Test item that runs a McRogueFace script.""" + + def runtest(self): + """Run the McRogueFace script.""" + timeout = self.config.getoption("--mcrf-timeout") + + env = os.environ.copy() + existing_ld = env.get('LD_LIBRARY_PATH', '') + env['LD_LIBRARY_PATH'] = f"{LIB_DIR}:{existing_ld}" if existing_ld else str(LIB_DIR) + + try: + result = subprocess.run( + [str(MCROGUEFACE), '--headless', '--exec', str(self.path)], + capture_output=True, + text=True, + timeout=timeout, + cwd=str(BUILD_DIR), + env=env + ) + + self.output = result.stdout + result.stderr + passed = result.returncode == 0 + + if 'FAIL' in self.output and 'PASS' not in self.output.split('FAIL')[-1]: + passed = False + + if not passed: + raise McRFTestException(self.output) + + except subprocess.TimeoutExpired: + self.output = "TIMEOUT" + raise McRFTestException("TIMEOUT") + + def repr_failure(self, excinfo): + """Format failure output.""" + if isinstance(excinfo.value, McRFTestException): + output = str(excinfo.value) + # Show last 10 lines + lines = output.strip().split('\n')[-10:] + return '\n'.join(lines) + return super().repr_failure(excinfo) + + def reportinfo(self): + """Report test location.""" + return self.path, None, f"mcrf:{self.path.relative_to(TESTS_DIR)}" + + +class McRFTestException(Exception): + """Exception raised when a McRogueFace test fails.""" + pass diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 0000000..75a0668 --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,14 @@ +[pytest] +# McRogueFace test scripts run via subprocess, not as Python modules +# They contain `import mcrfpy` which is only available inside McRogueFace + +# Don't try to import test scripts from these directories +norecursedirs = unit integration regression benchmarks demo geometry_demo notes vllm_demo + +# Run test_*.py files in tests/ root that are pytest-native wrappers +python_files = test_*.py +python_classes = Test* +python_functions = test_* + +# Custom option for timeout +addopts = -v diff --git a/tests/run_tests.py b/tests/run_tests.py index f51f3a9..e9e6035 100644 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -7,6 +7,8 @@ Usage: python3 tests/run_tests.py # Run all tests python3 tests/run_tests.py unit # Run only unit tests python3 tests/run_tests.py -v # Verbose output + python3 tests/run_tests.py -q # Quiet (no checksums) + python3 tests/run_tests.py --timeout=30 # Custom timeout """ import os import subprocess @@ -18,8 +20,9 @@ from pathlib import Path # Configuration TESTS_DIR = Path(__file__).parent BUILD_DIR = TESTS_DIR.parent / "build" +LIB_DIR = TESTS_DIR.parent / "__lib" MCROGUEFACE = BUILD_DIR / "mcrogueface" -TIMEOUT = 10 # seconds per test +DEFAULT_TIMEOUT = 10 # seconds per test # Test directories to run (in order) TEST_DIRS = ['unit', 'integration', 'regression'] @@ -39,7 +42,7 @@ def get_screenshot_checksum(test_dir): checksums[png.name] = hashlib.md5(f.read()).hexdigest()[:8] return checksums -def run_test(test_path, verbose=False): +def run_test(test_path, verbose=False, timeout=DEFAULT_TIMEOUT): """Run a single test and return (passed, duration, output).""" start = time.time() @@ -47,13 +50,19 @@ def run_test(test_path, verbose=False): for png in BUILD_DIR.glob("test_*.png"): png.unlink() + # Set up environment with library path + env = os.environ.copy() + existing_ld = env.get('LD_LIBRARY_PATH', '') + env['LD_LIBRARY_PATH'] = f"{LIB_DIR}:{existing_ld}" if existing_ld else str(LIB_DIR) + try: result = subprocess.run( [str(MCROGUEFACE), '--headless', '--exec', str(test_path)], capture_output=True, text=True, - timeout=TIMEOUT, - cwd=str(BUILD_DIR) + timeout=timeout, + cwd=str(BUILD_DIR), + env=env ) duration = time.time() - start passed = result.returncode == 0 @@ -66,7 +75,7 @@ def run_test(test_path, verbose=False): return passed, duration, output except subprocess.TimeoutExpired: - return False, TIMEOUT, "TIMEOUT" + return False, timeout, "TIMEOUT" except Exception as e: return False, 0, str(e) @@ -79,6 +88,16 @@ def find_tests(directory): def main(): verbose = '-v' in sys.argv or '--verbose' in sys.argv + quiet = '-q' in sys.argv or '--quiet' in sys.argv + + # Parse --timeout=N + timeout = DEFAULT_TIMEOUT + for arg in sys.argv[1:]: + if arg.startswith('--timeout='): + try: + timeout = int(arg.split('=')[1]) + except ValueError: + pass # Determine which directories to test dirs_to_test = [] @@ -89,7 +108,7 @@ def main(): dirs_to_test = TEST_DIRS print(f"{BOLD}McRogueFace Test Runner{RESET}") - print(f"Testing: {', '.join(dirs_to_test)}") + print(f"Testing: {', '.join(dirs_to_test)} (timeout: {timeout}s)") print("=" * 60) results = {'pass': 0, 'fail': 0, 'total_time': 0} @@ -104,7 +123,7 @@ def main(): for test_path in tests: test_name = test_path.name - passed, duration, output = run_test(test_path, verbose) + passed, duration, output = run_test(test_path, verbose, timeout) results['total_time'] += duration if passed: @@ -115,11 +134,12 @@ def main(): status = f"{RED}FAIL{RESET}" failures.append((test_dir, test_name, output)) - # Get screenshot checksums if any were generated - checksums = get_screenshot_checksum(BUILD_DIR) + # Get screenshot checksums if any were generated (skip in quiet mode) checksum_str = "" - if checksums: - checksum_str = f" [{', '.join(f'{k}:{v}' for k,v in checksums.items())}]" + if not quiet: + checksums = get_screenshot_checksum(BUILD_DIR) + if checksums: + checksum_str = f" [{', '.join(f'{k}:{v}' for k,v in checksums.items())}]" print(f" {status} {test_name} ({duration:.2f}s){checksum_str}") diff --git a/tests/test_mcrogueface.py b/tests/test_mcrogueface.py new file mode 100644 index 0000000..c165b5b --- /dev/null +++ b/tests/test_mcrogueface.py @@ -0,0 +1,116 @@ +""" +Pytest wrapper for McRogueFace test scripts. + +This file discovers and runs all McRogueFace test scripts in unit/, integration/, +and regression/ directories via subprocess. + +Usage: + pytest tests/test_mcrogueface.py -q # Quiet output + pytest tests/test_mcrogueface.py -k "bsp" # Filter by name + pytest tests/test_mcrogueface.py --mcrf-timeout=30 # Custom timeout + pytest tests/test_mcrogueface.py -x # Stop on first failure +""" +import os +import subprocess +import pytest +from pathlib import Path + +# Paths +TESTS_DIR = Path(__file__).parent +BUILD_DIR = TESTS_DIR.parent / "build" +LIB_DIR = TESTS_DIR.parent / "__lib" +MCROGUEFACE = BUILD_DIR / "mcrogueface" + +# Test directories +TEST_DIRS = ['unit', 'integration', 'regression'] + +# Default timeout +DEFAULT_TIMEOUT = 10 + + +def discover_tests(): + """Find all test scripts in test directories.""" + tests = [] + for test_dir in TEST_DIRS: + dir_path = TESTS_DIR / test_dir + if dir_path.exists(): + for test_file in sorted(dir_path.glob("*.py")): + if test_file.name != '__init__.py': + rel_path = f"{test_dir}/{test_file.name}" + tests.append(rel_path) + return tests + + +def get_env(): + """Get environment with LD_LIBRARY_PATH set.""" + env = os.environ.copy() + existing_ld = env.get('LD_LIBRARY_PATH', '') + env['LD_LIBRARY_PATH'] = f"{LIB_DIR}:{existing_ld}" if existing_ld else str(LIB_DIR) + return env + + +def run_mcrf_test(script_path, timeout=DEFAULT_TIMEOUT): + """Run a McRogueFace test script and return (passed, output).""" + full_path = TESTS_DIR / script_path + env = get_env() + + try: + result = subprocess.run( + [str(MCROGUEFACE), '--headless', '--exec', str(full_path)], + capture_output=True, + text=True, + timeout=timeout, + cwd=str(BUILD_DIR), + env=env + ) + + output = result.stdout + result.stderr + passed = result.returncode == 0 + + # Check for PASS/FAIL in output + if 'FAIL' in output and 'PASS' not in output.split('FAIL')[-1]: + passed = False + + return passed, output + + except subprocess.TimeoutExpired: + return False, "TIMEOUT" + except Exception as e: + return False, str(e) + + +# Discover tests at module load time +ALL_TESTS = discover_tests() + + +@pytest.fixture +def mcrf_timeout(request): + """Get timeout from command line or default.""" + return request.config.getoption("--mcrf-timeout", default=DEFAULT_TIMEOUT) + + +def pytest_addoption(parser): + """Add --mcrf-timeout option.""" + try: + parser.addoption( + "--mcrf-timeout", + action="store", + default=DEFAULT_TIMEOUT, + type=int, + help="Timeout in seconds for McRogueFace tests" + ) + except ValueError: + # Option already added + pass + + +@pytest.mark.parametrize("script_path", ALL_TESTS, ids=lambda x: x.replace('/', '::')) +def test_mcrogueface_script(script_path, request): + """Run a McRogueFace test script.""" + timeout = request.config.getoption("--mcrf-timeout", default=DEFAULT_TIMEOUT) + passed, output = run_mcrf_test(script_path, timeout=timeout) + + if not passed: + # Show last 15 lines of output on failure + lines = output.strip().split('\n')[-15:] + pytest.fail('\n'.join(lines))