diff --git a/.claude/commands/upgrade-pylib.md b/.claude/commands/upgrade-pylib.md index 67603ce2dfb..ba4cef525ab 100644 --- a/.claude/commands/upgrade-pylib.md +++ b/.claude/commands/upgrade-pylib.md @@ -14,8 +14,8 @@ If during the upgrade process you encounter any of the following issues with `sc **STOP the upgrade and report the issue first.** Describe: 1. What you were trying to do - - Library name - - The full command executed (e.g. python scripts/update_lib quick cpython/Lib/$ARGUMENTS.py) + - Library name + - The full command executed (e.g. python scripts/update_lib quick cpython/Lib/$ARGUMENTS.py) 2. What went wrong or what's missing 3. Expected vs actual behavior diff --git a/scripts/update_lib/auto_mark.py b/scripts/update_lib/auto_mark.py index 5a15df365a8..faf7abec664 100644 --- a/scripts/update_lib/auto_mark.py +++ b/scripts/update_lib/auto_mark.py @@ -22,6 +22,12 @@ from update_lib.path import test_name_from_path +class TestRunError(Exception): + """Raised when test run fails entirely (e.g., import error, crash).""" + + pass + + @dataclass class Test: name: str = "" @@ -58,7 +64,8 @@ def run_test(test_name: str, skip_build: bool = False) -> TestResult: result = subprocess.run( cmd + ["-m", "test", "-v", "-u", "all", "--slowest", test_name], - capture_output=True, + stdout=subprocess.PIPE, # Capture stdout for parsing + stderr=None, # Let stderr pass through to terminal text=True, ) return parse_results(result) @@ -236,6 +243,9 @@ def _is_super_call_only(func_node: ast.FunctionDef | ast.AsyncFunctionDef) -> bo return False if not isinstance(call.func, ast.Attribute): return False + # Verify the method name matches + if call.func.attr != func_node.name: + return False super_call = call.func.value if not isinstance(super_call, ast.Call): return False @@ -487,6 +497,14 @@ def auto_mark_file( print(f"Running test: {test_name}") results = run_test(test_name, skip_build=skip_build) + + # Check if test run failed entirely (e.g., import error, crash) + if not results.tests_result: + raise TestRunError( + f"Test run failed for {test_name}. " + f"Output: {results.stdout[-500:] if results.stdout else '(no output)'}" + ) + contents = test_path.read_text(encoding="utf-8") all_failing_tests, unexpected_successes, error_messages = collect_test_changes( @@ -577,6 +595,13 @@ def auto_mark_directory( results = run_test(test_name, skip_build=skip_build) + # Check if test run failed entirely (e.g., import error, crash) + if not results.tests_result: + raise TestRunError( + f"Test run failed for {test_name}. " + f"Output: {results.stdout[-500:] if results.stdout else '(no output)'}" + ) + total_added = 0 total_removed = 0 total_regressions = 0 diff --git a/scripts/update_lib/copy_lib.py b/scripts/update_lib/copy_lib.py index 5af0cab85d9..098c8d61163 100644 --- a/scripts/update_lib/copy_lib.py +++ b/scripts/update_lib/copy_lib.py @@ -16,21 +16,12 @@ import sys -def copy_lib( +def _copy_single( src_path: pathlib.Path, + lib_path: pathlib.Path, verbose: bool = True, ) -> None: - """ - Copy library file or directory from CPython. - - Args: - src_path: Source path (e.g., cpython/Lib/dataclasses.py or cpython/Lib/json) - verbose: Print progress messages - """ - from update_lib.path import parse_lib_path - - lib_path = parse_lib_path(src_path) - + """Copy a single file or directory.""" # Remove existing file/directory if lib_path.exists(): if lib_path.is_dir(): @@ -46,6 +37,7 @@ def copy_lib( if src_path.is_dir(): if verbose: print(f"Copying directory: {src_path} -> {lib_path}") + lib_path.parent.mkdir(parents=True, exist_ok=True) shutil.copytree(src_path, lib_path) else: if verbose: @@ -54,6 +46,46 @@ def copy_lib( shutil.copy2(src_path, lib_path) +def copy_lib( + src_path: pathlib.Path, + verbose: bool = True, +) -> None: + """ + Copy library file or directory from CPython. + + Also copies additional files if defined in DEPENDENCIES table. + + Args: + src_path: Source path (e.g., cpython/Lib/dataclasses.py or cpython/Lib/json) + verbose: Print progress messages + """ + from update_lib.deps import get_lib_paths + from update_lib.path import parse_lib_path + + # Extract module name and cpython prefix from path + path_str = str(src_path).replace("\\", "/") + if "/Lib/" in path_str: + cpython_prefix, after_lib = path_str.split("/Lib/", 1) + # Get module name (first component, without .py) + name = after_lib.split("/")[0] + if name.endswith(".py"): + name = name[:-3] + else: + # Fallback: just copy the single file + lib_path = parse_lib_path(src_path) + _copy_single(src_path, lib_path, verbose) + return + + # Get all paths to copy from DEPENDENCIES table + all_src_paths = get_lib_paths(name, cpython_prefix) + + # Copy each file + for src in all_src_paths: + if src.exists(): + lib_path = parse_lib_path(src) + _copy_single(src, lib_path, verbose) + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description=__doc__, diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py new file mode 100644 index 00000000000..a0b6d121f59 --- /dev/null +++ b/scripts/update_lib/deps.py @@ -0,0 +1,366 @@ +""" +Dependency resolution for library updates. + +Handles: +- Irregular library paths (e.g., libregrtest at Lib/test/libregrtest/) +- Library dependencies (e.g., datetime requires _pydatetime) +- Test dependencies (auto-detected from 'from test import ...') +""" + +import ast +import pathlib + +# Manual dependency table for irregular cases +# Format: "name" -> {"lib": [...], "test": [...], "data": [...], "hard_deps": [...]} +# - lib: override default path (default: name.py or name/) +# - hard_deps: additional files to copy alongside the main module +DEPENDENCIES = { + # regrtest is in Lib/test/libregrtest/, not Lib/libregrtest/ + "regrtest": { + "lib": ["test/libregrtest"], + "test": ["test/test_regrtest"], + "data": ["test/regrtestdata"], + }, + # Rust-implemented modules (no lib file, only test) + "int": { + "lib": [], # No Python lib (Rust implementation) + "hard_deps": ["_pylong.py"], + }, + # Pure Python implementations + "abc": { + "hard_deps": ["_py_abc.py"], + }, + "codecs": { + "hard_deps": ["_pycodecs.py"], + }, + "datetime": { + "hard_deps": ["_pydatetime.py"], + }, + "decimal": { + "hard_deps": ["_pydecimal.py"], + }, + "io": { + "hard_deps": ["_pyio.py"], + }, + "warnings": { + "hard_deps": ["_py_warnings.py"], + }, + # Data directories + "pydoc": { + "hard_deps": ["pydoc_data"], + }, + "turtle": { + "hard_deps": ["turtledemo"], + }, + # Test support library (like regrtest) + "support": { + "lib": ["test/support"], + "data": ["test/wheeldata"], + }, +} + +# Test-specific dependencies (only when auto-detection isn't enough) +# - hard_deps: files to migrate (tightly coupled, must be migrated together) +# - data: directories to copy without migration +TEST_DEPENDENCIES = { + # Audio tests + "test_winsound": { + "data": ["audiodata"], + }, + "test_wave": { + "data": ["audiodata"], + }, + "audiotests": { + "data": ["audiodata"], + }, + # Archive tests + "test_tarfile": { + "data": ["archivetestdata"], + }, + "test_zipfile": { + "data": ["archivetestdata"], + }, + # Config tests + "test_configparser": { + "data": ["configdata"], + }, + "test_config": { + "data": ["configdata"], + }, + # Other data directories + "test_decimal": { + "data": ["decimaltestdata"], + }, + "test_dtrace": { + "data": ["dtracedata"], + }, + "test_math": { + "data": ["mathdata"], + }, + "test_ssl": { + "data": ["certdata"], + }, + "test_subprocess": { + "data": ["subprocessdata"], + }, + "test_tkinter": { + "data": ["tkinterdata"], + }, + "test_tokenize": { + "data": ["tokenizedata"], + }, + "test_type_annotations": { + "data": ["typinganndata"], + }, + "test_zipimport": { + "data": ["zipimport_data"], + }, + # XML tests share xmltestdata + "test_xml_etree": { + "data": ["xmltestdata"], + }, + "test_pulldom": { + "data": ["xmltestdata"], + }, + "test_sax": { + "data": ["xmltestdata"], + }, + "test_minidom": { + "data": ["xmltestdata"], + }, + # Multibytecodec support needs cjkencodings + "multibytecodec_support": { + "data": ["cjkencodings"], + }, + # i18n + "i18n_helper": { + "data": ["translationdata"], + }, + # wheeldata is used by test_makefile and support + "test_makefile": { + "data": ["wheeldata"], + }, +} + + +def get_lib_paths(name: str, cpython_prefix: str = "cpython") -> list[pathlib.Path]: + """Get all library paths for a module. + + Args: + name: Module name (e.g., "datetime", "libregrtest") + cpython_prefix: CPython directory prefix + + Returns: + List of paths to copy + """ + paths = [] + dep_info = DEPENDENCIES.get(name, {}) + + # Get main lib path (override or default) + if "lib" in dep_info: + paths = [pathlib.Path(f"{cpython_prefix}/Lib/{p}") for p in dep_info["lib"]] + else: + # Default: try file first, then directory + file_path = pathlib.Path(f"{cpython_prefix}/Lib/{name}.py") + if file_path.exists(): + paths = [file_path] + else: + dir_path = pathlib.Path(f"{cpython_prefix}/Lib/{name}") + if dir_path.exists(): + paths = [dir_path] + else: + paths = [file_path] # Default to file path + + # Add hard_deps + if "hard_deps" in dep_info: + for dep in dep_info["hard_deps"]: + paths.append(pathlib.Path(f"{cpython_prefix}/Lib/{dep}")) + + return paths + + +def get_test_paths(name: str, cpython_prefix: str = "cpython") -> list[pathlib.Path]: + """Get all test paths for a module. + + Args: + name: Module name (e.g., "datetime", "libregrtest") + cpython_prefix: CPython directory prefix + + Returns: + List of test paths + """ + if name in DEPENDENCIES and "test" in DEPENDENCIES[name]: + return [ + pathlib.Path(f"{cpython_prefix}/Lib/{p}") + for p in DEPENDENCIES[name]["test"] + ] + + # Default: try directory first, then file + dir_path = pathlib.Path(f"{cpython_prefix}/Lib/test/test_{name}") + if dir_path.exists(): + return [dir_path] + file_path = pathlib.Path(f"{cpython_prefix}/Lib/test/test_{name}.py") + if file_path.exists(): + return [file_path] + return [dir_path] # Default to directory path + + +def get_data_paths(name: str, cpython_prefix: str = "cpython") -> list[pathlib.Path]: + """Get additional data paths for a module. + + Args: + name: Module name + cpython_prefix: CPython directory prefix + + Returns: + List of data paths (may be empty) + """ + if name in DEPENDENCIES and "data" in DEPENDENCIES[name]: + return [ + pathlib.Path(f"{cpython_prefix}/Lib/{p}") + for p in DEPENDENCIES[name]["data"] + ] + return [] + + +def parse_test_imports(content: str) -> set[str]: + """Parse test file content and extract 'from test import ...' dependencies. + + Args: + content: Python file content + + Returns: + Set of module names imported from test package + """ + try: + tree = ast.parse(content) + except SyntaxError: + return set() + + imports = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + if node.module == "test": + # from test import foo, bar + for alias in node.names: + name = alias.name + # Skip support modules and common imports + if name not in ("support", "__init__"): + imports.add(name) + elif node.module and node.module.startswith("test."): + # from test.foo import bar -> depends on foo + parts = node.module.split(".") + if len(parts) >= 2: + dep = parts[1] + if dep not in ("support", "__init__"): + imports.add(dep) + + return imports + + +def get_test_dependencies( + test_path: pathlib.Path, +) -> dict[str, list[pathlib.Path]]: + """Get test dependencies by parsing imports. + + Args: + test_path: Path to test file or directory + + Returns: + Dict with "hard_deps" (files to migrate) and "data" (dirs to copy) + """ + result = {"hard_deps": [], "data": []} + + if not test_path.exists(): + return result + + # Collect all test files + if test_path.is_file(): + files = [test_path] + else: + files = list(test_path.glob("**/*.py")) + + # Parse all files for imports (auto-detect deps) + all_imports = set() + for f in files: + try: + content = f.read_text(encoding="utf-8") + all_imports.update(parse_test_imports(content)) + except (OSError, UnicodeDecodeError): + continue + + # Also add manual dependencies from TEST_DEPENDENCIES + test_name = test_path.stem if test_path.is_file() else test_path.name + manual_deps = TEST_DEPENDENCIES.get(test_name, {}) + if "hard_deps" in manual_deps: + all_imports.update(manual_deps["hard_deps"]) + + # Convert imports to paths (deps) + for imp in all_imports: + # Check if it's a test file (test_*) or support module + if imp.startswith("test_"): + # It's a test, resolve to test path + dep_path = test_path.parent / f"{imp}.py" + if not dep_path.exists(): + dep_path = test_path.parent / imp + else: + # Support module like string_tests, lock_tests, encoded_modules + # Check file first, then directory + dep_path = test_path.parent / f"{imp}.py" + if not dep_path.exists(): + dep_path = test_path.parent / imp + + if dep_path.exists() and dep_path not in result["hard_deps"]: + result["hard_deps"].append(dep_path) + + # Add data paths from manual table (for the test file itself) + if "data" in manual_deps: + for data_name in manual_deps["data"]: + data_path = test_path.parent / data_name + if data_path.exists() and data_path not in result["data"]: + result["data"].append(data_path) + + # Also add data from auto-detected deps' TEST_DEPENDENCIES + # e.g., test_codecencodings_kr -> multibytecodec_support -> cjkencodings + for imp in all_imports: + dep_info = TEST_DEPENDENCIES.get(imp, {}) + if "data" in dep_info: + for data_name in dep_info["data"]: + data_path = test_path.parent / data_name + if data_path.exists() and data_path not in result["data"]: + result["data"].append(data_path) + + return result + + +def resolve_all_paths( + name: str, + cpython_prefix: str = "cpython", + include_deps: bool = True, +) -> dict[str, list[pathlib.Path]]: + """Resolve all paths for a module update. + + Args: + name: Module name + cpython_prefix: CPython directory prefix + include_deps: Whether to include auto-detected dependencies + + Returns: + Dict with "lib", "test", "data", "test_deps" keys + """ + result = { + "lib": get_lib_paths(name, cpython_prefix), + "test": get_test_paths(name, cpython_prefix), + "data": get_data_paths(name, cpython_prefix), + "test_deps": [], + } + + if include_deps: + # Auto-detect test dependencies + for test_path in result["test"]: + deps = get_test_dependencies(test_path) + for dep in deps: + if dep not in result["test_deps"]: + result["test_deps"].append(dep) + + return result diff --git a/scripts/update_lib/patch_spec.py b/scripts/update_lib/patch_spec.py index 1f88c06c1f4..410525794f0 100644 --- a/scripts/update_lib/patch_spec.py +++ b/scripts/update_lib/patch_spec.py @@ -298,8 +298,17 @@ def _find_import_insert_line(tree: ast.Module) -> int: for node in tree.body: if isinstance(node, (ast.Import, ast.ImportFrom)): last_import_line = node.end_lineno or node.lineno - assert last_import_line is not None - return last_import_line + if last_import_line is not None: + return last_import_line + # No imports found - insert after module docstring if present, else at top + if ( + tree.body + and isinstance(tree.body[0], ast.Expr) + and isinstance(tree.body[0].value, ast.Constant) + and isinstance(tree.body[0].value.value, str) + ): + return tree.body[0].end_lineno or tree.body[0].lineno + return 0 def apply_patches(contents: str, patches: Patches) -> str: diff --git a/scripts/update_lib/path.py b/scripts/update_lib/path.py index 7a3535fd2ae..3096ec2bebe 100644 --- a/scripts/update_lib/path.py +++ b/scripts/update_lib/path.py @@ -47,6 +47,7 @@ def lib_to_test_path(src_path: pathlib.Path) -> pathlib.Path: cpython/Lib/dataclasses.py -> cpython/Lib/test/test_dataclasses/ (if dir exists) cpython/Lib/typing.py -> cpython/Lib/test/test_typing.py (if file exists) cpython/Lib/json/ -> cpython/Lib/test/test_json/ + cpython/Lib/json/__init__.py -> cpython/Lib/test/test_json/ Lib/dataclasses.py -> Lib/test/test_dataclasses/ """ path_str = str(src_path).replace("\\", "/") @@ -55,6 +56,9 @@ def lib_to_test_path(src_path: pathlib.Path) -> pathlib.Path: if lib_marker in path_str: lib_path = parse_lib_path(src_path) lib_name = lib_path.stem if lib_path.suffix == ".py" else lib_path.name + # Handle __init__.py: use parent directory name + if lib_name == "__init__": + lib_name = lib_path.parent.name prefix = path_str[: path_str.index(lib_marker)] # Try directory first, then file dir_path = pathlib.Path(f"{prefix}/Lib/test/test_{lib_name}/") @@ -68,6 +72,9 @@ def lib_to_test_path(src_path: pathlib.Path) -> pathlib.Path: else: # Path starts with Lib/ - extract name directly lib_name = src_path.stem if src_path.suffix == ".py" else src_path.name + # Handle __init__.py: use parent directory name + if lib_name == "__init__": + lib_name = src_path.parent.name # Try directory first, then file dir_path = pathlib.Path(f"Lib/test/test_{lib_name}/") if dir_path.exists(): diff --git a/scripts/update_lib/quick.py b/scripts/update_lib/quick.py index f7d0b6b8683..e770e0dedca 100644 --- a/scripts/update_lib/quick.py +++ b/scripts/update_lib/quick.py @@ -105,6 +105,32 @@ def quick( else: patch_file(src_path, lib_path, verbose=verbose) + # Step 1.5: Handle test dependencies + from update_lib.deps import get_test_dependencies + + test_deps = get_test_dependencies(src_path) + + # Migrate dependency files + for dep_src in test_deps["hard_deps"]: + dep_lib = parse_lib_path(dep_src) + if verbose: + print(f"Migrating dependency: {dep_src.name}") + if dep_src.is_dir(): + patch_directory(dep_src, dep_lib, verbose=False) + else: + patch_file(dep_src, dep_lib, verbose=False) + + # Copy data directories (no migration) + import shutil + + for data_src in test_deps["data"]: + data_lib = parse_lib_path(data_src) + if verbose: + print(f"Copying data: {data_src.name}") + if data_lib.exists(): + shutil.rmtree(data_lib) + shutil.copytree(data_src, data_lib) + # Step 2: Auto-mark if not no_auto_mark: if not lib_path.exists(): @@ -232,6 +258,7 @@ def _expand_shortcut(path: pathlib.Path) -> pathlib.Path: dataclasses -> cpython/Lib/dataclasses.py (if exists) json -> cpython/Lib/json/ (if exists) test_types -> cpython/Lib/test/test_types.py (if exists) + regrtest -> cpython/Lib/test/libregrtest (from DEPENDENCIES) """ # Only expand if it's a simple name (no path separators) and doesn't exist if "/" in str(path) or path.exists(): @@ -239,6 +266,16 @@ def _expand_shortcut(path: pathlib.Path) -> pathlib.Path: name = str(path) + # Check DEPENDENCIES table for path overrides (e.g., regrtest) + from update_lib.deps import DEPENDENCIES + + if name in DEPENDENCIES and "lib" in DEPENDENCIES[name]: + lib_paths = DEPENDENCIES[name]["lib"] + if lib_paths: + override_path = pathlib.Path(f"cpython/Lib/{lib_paths[0]}") + if override_path.exists(): + return override_path + # Test shortcut: test_foo -> cpython/Lib/test/test_foo if name.startswith("test_"): dir_path = pathlib.Path(f"cpython/Lib/test/{name}") @@ -363,6 +400,14 @@ def main(argv: list[str] | None = None) -> int: except FileNotFoundError as e: print(f"Error: {e}", file=sys.stderr) return 1 + except Exception as e: + # Handle TestRunError with a clean message + from update_lib.auto_mark import TestRunError + + if isinstance(e, TestRunError): + print(f"Error: {e}", file=sys.stderr) + return 1 + raise if __name__ == "__main__": diff --git a/scripts/update_lib/tests/test_auto_mark.py b/scripts/update_lib/tests/test_auto_mark.py index e155b40addd..c62a8af888e 100644 --- a/scripts/update_lib/tests/test_auto_mark.py +++ b/scripts/update_lib/tests/test_auto_mark.py @@ -6,6 +6,7 @@ from update_lib.auto_mark import ( Test, TestResult, + _is_super_call_only, apply_test_changes, collect_test_changes, extract_test_methods, @@ -589,5 +590,60 @@ def test_all_regressions(self): self.assertEqual(regressions, {("TestFoo", "test_one")}) +class TestIsSuperCallOnly(unittest.TestCase): + """Tests for _is_super_call_only function.""" + + def _parse_method(self, code: str): + """Parse code and return the first method.""" + import ast + + tree = ast.parse(code) + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return node + return None + + def test_matching_super_call(self): + """Test method that calls super().same_name().""" + code = """ +class Foo: + def test_one(self): + return super().test_one() +""" + method = self._parse_method(code) + self.assertTrue(_is_super_call_only(method)) + + def test_mismatched_super_call(self): + """Test method that calls super().different_name().""" + code = """ +class Foo: + def test_one(self): + return super().test_two() +""" + method = self._parse_method(code) + self.assertFalse(_is_super_call_only(method)) + + def test_not_super_call(self): + """Test method with regular body.""" + code = """ +class Foo: + def test_one(self): + pass +""" + method = self._parse_method(code) + self.assertFalse(_is_super_call_only(method)) + + def test_multiple_statements(self): + """Test method with multiple statements.""" + code = """ +class Foo: + def test_one(self): + x = 1 + return super().test_one() +""" + method = self._parse_method(code) + self.assertFalse(_is_super_call_only(method)) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/update_lib/tests/test_copy_lib.py b/scripts/update_lib/tests/test_copy_lib.py new file mode 100644 index 00000000000..0b3f60b77b8 --- /dev/null +++ b/scripts/update_lib/tests/test_copy_lib.py @@ -0,0 +1,62 @@ +"""Tests for copy_lib.py - library copying with dependencies.""" + +import pathlib +import tempfile +import unittest + + +class TestCopySingle(unittest.TestCase): + """Tests for _copy_single helper function.""" + + def test_copies_file(self): + """Test copying a single file.""" + from update_lib.copy_lib import _copy_single + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + src = tmpdir / "source.py" + src.write_text("content") + dst = tmpdir / "dest.py" + + _copy_single(src, dst, verbose=False) + + self.assertTrue(dst.exists()) + self.assertEqual(dst.read_text(), "content") + + def test_copies_directory(self): + """Test copying a directory.""" + from update_lib.copy_lib import _copy_single + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + src = tmpdir / "source_dir" + src.mkdir() + (src / "file.py").write_text("content") + dst = tmpdir / "dest_dir" + + _copy_single(src, dst, verbose=False) + + self.assertTrue(dst.exists()) + self.assertTrue((dst / "file.py").exists()) + + def test_removes_existing_before_copy(self): + """Test that existing destination is removed before copy.""" + from update_lib.copy_lib import _copy_single + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + src = tmpdir / "source.py" + src.write_text("new content") + dst = tmpdir / "dest.py" + dst.write_text("old content") + + _copy_single(src, dst, verbose=False) + + self.assertEqual(dst.read_text(), "new content") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py new file mode 100644 index 00000000000..1ff45b703c1 --- /dev/null +++ b/scripts/update_lib/tests/test_deps.py @@ -0,0 +1,235 @@ +"""Tests for deps.py - dependency resolution.""" + +import pathlib +import tempfile +import unittest + +from update_lib.deps import ( + get_data_paths, + get_lib_paths, + get_test_dependencies, + get_test_paths, + parse_test_imports, + resolve_all_paths, +) + + +class TestParseTestImports(unittest.TestCase): + """Tests for parse_test_imports function.""" + + def test_from_test_import(self): + """Test parsing 'from test import foo'.""" + code = """ +from test import string_tests +from test import lock_tests, other_tests +""" + imports = parse_test_imports(code) + self.assertEqual(imports, {"string_tests", "lock_tests", "other_tests"}) + + def test_from_test_dot_module(self): + """Test parsing 'from test.foo import bar'.""" + code = """ +from test.string_tests import CommonTest +from test.support import verbose +""" + imports = parse_test_imports(code) + self.assertEqual(imports, {"string_tests"}) # support is excluded + + def test_excludes_support(self): + """Test that 'support' is excluded.""" + code = """ +from test import support +from test.support import verbose +""" + imports = parse_test_imports(code) + self.assertEqual(imports, set()) + + def test_regular_imports_ignored(self): + """Test that regular imports are ignored.""" + code = """ +import os +from collections import defaultdict +from . import helper +""" + imports = parse_test_imports(code) + self.assertEqual(imports, set()) + + def test_syntax_error_returns_empty(self): + """Test that syntax errors return empty set.""" + code = "this is not valid python {" + imports = parse_test_imports(code) + self.assertEqual(imports, set()) + + +class TestGetLibPaths(unittest.TestCase): + """Tests for get_lib_paths function.""" + + def test_known_dependency(self): + """Test library with known dependencies.""" + paths = get_lib_paths("datetime", "cpython") + self.assertEqual(len(paths), 2) + self.assertIn(pathlib.Path("cpython/Lib/datetime.py"), paths) + self.assertIn(pathlib.Path("cpython/Lib/_pydatetime.py"), paths) + + def test_default_file(self): + """Test default to .py file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "foo.py").write_text("# foo") + + paths = get_lib_paths("foo", str(tmpdir)) + self.assertEqual(paths, [tmpdir / "Lib" / "foo.py"]) + + def test_default_directory(self): + """Test default to directory when file doesn't exist.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "foo").mkdir() + + paths = get_lib_paths("foo", str(tmpdir)) + self.assertEqual(paths, [tmpdir / "Lib" / "foo"]) + + +class TestGetTestPaths(unittest.TestCase): + """Tests for get_test_paths function.""" + + def test_known_dependency(self): + """Test test with known path override.""" + paths = get_test_paths("regrtest", "cpython") + self.assertEqual(len(paths), 1) + self.assertEqual(paths[0], pathlib.Path("cpython/Lib/test/test_regrtest")) + + def test_default_directory(self): + """Test default to test_name/ directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_dir = tmpdir / "Lib" / "test" + test_dir.mkdir(parents=True) + (test_dir / "test_foo").mkdir() + + paths = get_test_paths("foo", str(tmpdir)) + self.assertEqual(paths, [tmpdir / "Lib" / "test" / "test_foo"]) + + def test_default_file(self): + """Test fallback to test_name.py file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_dir = tmpdir / "Lib" / "test" + test_dir.mkdir(parents=True) + (test_dir / "test_foo.py").write_text("# test") + + paths = get_test_paths("foo", str(tmpdir)) + self.assertEqual(paths, [tmpdir / "Lib" / "test" / "test_foo.py"]) + + +class TestGetDataPaths(unittest.TestCase): + """Tests for get_data_paths function.""" + + def test_known_data(self): + """Test module with known data paths.""" + paths = get_data_paths("regrtest", "cpython") + self.assertEqual(len(paths), 1) + self.assertEqual(paths[0], pathlib.Path("cpython/Lib/test/regrtestdata")) + + def test_no_data(self): + """Test module without data paths.""" + paths = get_data_paths("datetime", "cpython") + self.assertEqual(paths, []) + + +class TestGetTestDependencies(unittest.TestCase): + """Tests for get_test_dependencies function.""" + + def test_parse_file_imports(self): + """Test parsing imports from test file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_dir = tmpdir / "test" + test_dir.mkdir() + + # Create test file with import + test_file = test_dir / "test_foo.py" + test_file.write_text(""" +from test import string_tests + +class TestFoo: + pass +""") + # Create the dependency file + (test_dir / "string_tests.py").write_text("# string tests") + + result = get_test_dependencies(test_file) + self.assertEqual(len(result["hard_deps"]), 1) + self.assertEqual(result["hard_deps"][0], test_dir / "string_tests.py") + self.assertEqual(result["data"], []) + + def test_nonexistent_path(self): + """Test nonexistent path returns empty.""" + result = get_test_dependencies(pathlib.Path("/nonexistent/path")) + self.assertEqual(result, {"hard_deps": [], "data": []}) + + def test_transitive_data_dependency(self): + """Test that data deps are resolved transitively. + + Chain: test_codecencodings_kr -> multibytecodec_support -> cjkencodings + """ + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_dir = tmpdir / "test" + test_dir.mkdir() + + # Create test_codecencodings_kr.py that imports multibytecodec_support + test_file = test_dir / "test_codecencodings_kr.py" + test_file.write_text(""" +from test import multibytecodec_support + +class TestKR: + pass +""") + # Create multibytecodec_support.py (the intermediate dependency) + (test_dir / "multibytecodec_support.py").write_text("# support module") + + # Create cjkencodings directory (the data dependency of multibytecodec_support) + (test_dir / "cjkencodings").mkdir() + + result = get_test_dependencies(test_file) + + # Should find multibytecodec_support.py as a hard_dep + self.assertEqual(len(result["hard_deps"]), 1) + self.assertEqual( + result["hard_deps"][0], test_dir / "multibytecodec_support.py" + ) + + # Should find cjkencodings as data (from multibytecodec_support's TEST_DEPENDENCIES) + self.assertEqual(len(result["data"]), 1) + self.assertEqual(result["data"][0], test_dir / "cjkencodings") + + +class TestResolveAllPaths(unittest.TestCase): + """Tests for resolve_all_paths function.""" + + def test_datetime(self): + """Test resolving datetime module.""" + result = resolve_all_paths("datetime", include_deps=False) + self.assertEqual(len(result["lib"]), 2) + self.assertIn(pathlib.Path("cpython/Lib/datetime.py"), result["lib"]) + self.assertIn(pathlib.Path("cpython/Lib/_pydatetime.py"), result["lib"]) + + def test_regrtest(self): + """Test resolving regrtest module.""" + result = resolve_all_paths("regrtest", include_deps=False) + self.assertEqual(result["lib"], [pathlib.Path("cpython/Lib/test/libregrtest")]) + self.assertEqual( + result["test"], [pathlib.Path("cpython/Lib/test/test_regrtest")] + ) + self.assertEqual( + result["data"], [pathlib.Path("cpython/Lib/test/regrtestdata")] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_patch_spec.py b/scripts/update_lib/tests/test_patch_spec.py index ec52aa6d284..798bd851b3c 100644 --- a/scripts/update_lib/tests/test_patch_spec.py +++ b/scripts/update_lib/tests/test_patch_spec.py @@ -7,6 +7,7 @@ COMMENT, PatchSpec, UtMethod, + _find_import_insert_line, apply_patches, extract_patches, iter_tests, @@ -321,5 +322,41 @@ def test_one(self): self.assertIn(COMMENT, result) +class TestFindImportInsertLine(unittest.TestCase): + """Tests for _find_import_insert_line function.""" + + def test_with_imports(self): + """Test finding line after imports.""" + code = """import os +import sys + +class Foo: + pass +""" + tree = ast.parse(code) + line = _find_import_insert_line(tree) + self.assertEqual(line, 2) + + def test_no_imports_with_docstring(self): + """Test fallback to after docstring when no imports.""" + code = '''"""Module docstring.""" + +class Foo: + pass +''' + tree = ast.parse(code) + line = _find_import_insert_line(tree) + self.assertEqual(line, 1) + + def test_no_imports_no_docstring(self): + """Test fallback to line 0 when no imports and no docstring.""" + code = """class Foo: + pass +""" + tree = ast.parse(code) + line = _find_import_insert_line(tree) + self.assertEqual(line, 0) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/update_lib/tests/test_path.py b/scripts/update_lib/tests/test_path.py index 06d10e2f10b..affa1c8913a 100644 --- a/scripts/update_lib/tests/test_path.py +++ b/scripts/update_lib/tests/test_path.py @@ -136,6 +136,29 @@ def test_lib_path_prefers_directory(self): result = lib_to_test_path(pathlib.Path("Lib/nonexistent_module.py")) self.assertEqual(result, pathlib.Path("Lib/test/test_nonexistent_module/")) + def test_init_py_uses_parent_name(self): + """Test __init__.py uses parent directory name.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + # Create structure: tmpdir/Lib/json/__init__.py + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + json_dir = lib_dir / "json" + json_dir.mkdir() + (json_dir / "__init__.py").write_text("# json init") + test_dir = lib_dir / "test" + test_dir.mkdir() + + result = lib_to_test_path(tmpdir / "Lib" / "json" / "__init__.py") + # Should use "json" not "__init__" + self.assertEqual(result, tmpdir / "Lib" / "test" / "test_json/") + + def test_init_py_lib_path_uses_parent_name(self): + """Test __init__.py with Lib/ path uses parent directory name.""" + result = lib_to_test_path(pathlib.Path("Lib/json/__init__.py")) + # Should use "json" not "__init__" + self.assertEqual(result, pathlib.Path("Lib/test/test_json/")) + class TestGetTestFiles(unittest.TestCase): """Tests for get_test_files function.""" diff --git a/scripts/update_lib/tests/test_quick.py b/scripts/update_lib/tests/test_quick.py index d4e679e8a16..f02ca21e186 100644 --- a/scripts/update_lib/tests/test_quick.py +++ b/scripts/update_lib/tests/test_quick.py @@ -92,6 +92,25 @@ def test_expand_shortcut_nonexistent(self): result = _expand_shortcut(path) self.assertEqual(result, path) + def test_expand_shortcut_uses_dependencies_table(self): + """Test that _expand_shortcut uses DEPENDENCIES table for overrides.""" + from update_lib.deps import DEPENDENCIES + + # regrtest has lib override in DEPENDENCIES + self.assertIn("regrtest", DEPENDENCIES) + self.assertIn("lib", DEPENDENCIES["regrtest"]) + + # _expand_shortcut should use this override when path exists + path = pathlib.Path("regrtest") + expected = pathlib.Path("cpython/Lib/test/libregrtest") + + # Only test expansion if cpython checkout exists + if expected.exists(): + result = _expand_shortcut(path) + self.assertEqual( + result, expected, "_expand_shortcut should expand 'regrtest'" + ) + class TestCollectOriginalMethods(unittest.TestCase): """Tests for collect_original_methods function.""" @@ -181,5 +200,30 @@ def test_both_none_returns_false(self): self.assertFalse(result) +class TestQuickTestRunFailure(unittest.TestCase): + """Tests for quick() behavior when test run fails.""" + + @patch("update_lib.auto_mark.run_test") + def test_auto_mark_raises_on_test_run_failure(self, mock_run_test): + """Test that auto_mark_file raises when test run fails entirely.""" + from update_lib.auto_mark import TestResult, TestRunError, auto_mark_file + + # Simulate test runner crash (empty tests_result) + mock_run_test.return_value = TestResult( + tests_result="", tests=[], stdout="crash" + ) + + with tempfile.TemporaryDirectory() as tmpdir: + # Create a fake test file with Lib/test structure + lib_test_dir = pathlib.Path(tmpdir) / "Lib" / "test" + lib_test_dir.mkdir(parents=True) + test_file = lib_test_dir / "test_foo.py" + test_file.write_text("import unittest\nclass Test(unittest.TestCase): pass") + + # auto_mark_file should raise TestRunError + with self.assertRaises(TestRunError): + auto_mark_file(test_file) + + if __name__ == "__main__": unittest.main()