From 4154c5e3f1de021af4ece01bde31760506e44524 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 20:44:44 +0900 Subject: [PATCH 01/17] Show affected tests for given module names --- scripts/update_lib/deps.py | 144 ++++++++++++++++++++++++++ scripts/update_lib/show_deps.py | 43 +++++++- scripts/update_lib/tests/test_deps.py | 130 +++++++++++++++++++++++ 3 files changed, 315 insertions(+), 2 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 566f5ae5f0c..71d59f9139b 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -504,3 +504,147 @@ def resolve_all_paths( result["data"].append(data_path) return result + + +def _build_import_graph(lib_prefix: str = "Lib") -> dict[str, set[str]]: + """Build a graph of module imports from lib_prefix directory. + + Args: + lib_prefix: RustPython Lib directory (default: "Lib") + + Returns: + Dict mapping module_name -> set of modules it imports + """ + lib_dir = pathlib.Path(lib_prefix) + if not lib_dir.exists(): + return {} + + import_graph: dict[str, set[str]] = {} + + # Scan all .py files in lib_prefix (excluding test/ directory for module imports) + for entry in lib_dir.iterdir(): + if entry.name.startswith(("_", ".")): + continue + if entry.name == "test": + continue + + module_name = None + if entry.is_file() and entry.suffix == ".py": + module_name = entry.stem + elif entry.is_dir() and (entry / "__init__.py").exists(): + module_name = entry.name + + if module_name: + # Parse imports from this module + imports = set() + for _, content in read_python_files(entry): + imports.update(parse_lib_imports(content)) + # Remove self-imports + imports.discard(module_name) + import_graph[module_name] = imports + + return import_graph + + +def _build_reverse_graph(import_graph: dict[str, set[str]]) -> dict[str, set[str]]: + """Build reverse dependency graph (who imports this module). + + Args: + import_graph: Forward import graph (module -> imports) + + Returns: + Reverse graph (module -> imported_by) + """ + reverse_graph: dict[str, set[str]] = {} + + for module, imports in import_graph.items(): + for imported in imports: + if imported not in reverse_graph: + reverse_graph[imported] = set() + reverse_graph[imported].add(module) + + return reverse_graph + + +@functools.cache +def get_transitive_imports( + module_name: str, + lib_prefix: str = "Lib", +) -> frozenset[str]: + """Get all modules that transitively depend on module_name. + + Args: + module_name: Target module + lib_prefix: RustPython Lib directory (default: "Lib") + + Returns: + Frozenset of module names that import module_name (directly or indirectly) + """ + import_graph = _build_import_graph(lib_prefix) + reverse_graph = _build_reverse_graph(import_graph) + + # BFS from module_name following reverse edges + visited: set[str] = set() + queue = list(reverse_graph.get(module_name, set())) + + while queue: + current = queue.pop(0) + if current in visited: + continue + visited.add(current) + # Add modules that import current module + for importer in reverse_graph.get(current, set()): + if importer not in visited: + queue.append(importer) + + return frozenset(visited) + + +@functools.cache +def find_tests_importing_module( + module_name: str, + lib_prefix: str = "Lib", + include_transitive: bool = True, +) -> frozenset[pathlib.Path]: + """Find all test files that import the given module (directly or transitively). + + Args: + module_name: Module to search for (e.g., "datetime") + lib_prefix: RustPython Lib directory (default: "Lib") + include_transitive: Whether to include transitive dependencies + + Returns: + Frozenset of test file paths that depend on this module + """ + lib_dir = pathlib.Path(lib_prefix) + test_dir = lib_dir / "test" + + if not test_dir.exists(): + return frozenset() + + # Build set of modules to search for + target_modules = {module_name} + if include_transitive: + # Add all modules that transitively depend on module_name + target_modules.update(get_transitive_imports(module_name, lib_prefix)) + + # Excluded test file for this module (test_.py) + excluded_test = f"test_{module_name}.py" + + # Scan test directory for files that import any of the target modules + result: set[pathlib.Path] = set() + + for test_file in test_dir.glob("*.py"): + if test_file.name == excluded_test: + continue + + content = safe_read_text(test_file) + if content is None: + continue + + imports = parse_lib_imports(content) + # Check if any target module is imported + if imports & target_modules: + result.add(test_file) + + return frozenset(result) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index b6beacacaab..5d1f0584e01 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -145,6 +145,7 @@ def format_deps( lib_prefix: str = "Lib", max_depth: int = 10, _visited: set[str] | None = None, + show_impact: bool = False, ) -> list[str]: """Format all dependency information for a module. @@ -154,14 +155,17 @@ def format_deps( lib_prefix: Local Lib directory prefix max_depth: Maximum recursion depth _visited: Shared visited set for deduplication across modules + show_impact: Whether to show reverse dependencies (tests that import this module) Returns: List of formatted lines """ from update_lib.deps import ( DEPENDENCIES, + find_tests_importing_module, get_lib_paths, get_test_paths, + get_transitive_imports, ) if _visited is None: @@ -194,6 +198,33 @@ def format_deps( ) ) + # Show impact (reverse dependencies) if requested + if show_impact: + impacted_tests = find_tests_importing_module(name, lib_prefix) + transitive_importers = get_transitive_imports(name, lib_prefix) + + if impacted_tests: + lines.append(f"[+] impact: ({len(impacted_tests)} tests depend on {name})") + # Sort tests and show with dependency info + for test_path in sorted(impacted_tests, key=lambda p: p.name): + # Determine if direct or via which module + test_content = test_path.read_text(errors="ignore") + from update_lib.deps import parse_lib_imports + + test_imports = parse_lib_imports(test_content) + if name in test_imports: + lines.append(f" - {test_path.name} (direct)") + else: + # Find which transitive module is imported + via_modules = test_imports & transitive_importers + if via_modules: + via_str = ", ".join(sorted(via_modules)) + lines.append(f" - {test_path.name} (via {via_str})") + else: + lines.append(f" - {test_path.name}") + else: + lines.append(f"[+] impact: (no tests depend on {name})") + return lines @@ -202,6 +233,7 @@ def show_deps( cpython_prefix: str = "cpython", lib_prefix: str = "Lib", max_depth: int = 10, + show_impact: bool = False, ) -> None: """Show all dependency information for modules.""" # Expand "all" to all module names @@ -218,7 +250,9 @@ def show_deps( for i, name in enumerate(expanded_names): if i > 0: print() # blank line between modules - for line in format_deps(name, cpython_prefix, lib_prefix, max_depth, visited): + for line in format_deps( + name, cpython_prefix, lib_prefix, max_depth, visited, show_impact + ): print(line) @@ -248,11 +282,16 @@ def main(argv: list[str] | None = None) -> int: default=10, help="Maximum recursion depth for soft_deps tree (default: 10)", ) + parser.add_argument( + "--impact", + action="store_true", + help="Show tests that import this module (reverse dependencies)", + ) args = parser.parse_args(argv) try: - show_deps(args.names, args.cpython, args.lib, args.depth) + show_deps(args.names, args.cpython, args.lib, args.depth, args.impact) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index bc70925348b..56665187d79 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -5,11 +5,13 @@ import unittest from update_lib.deps import ( + find_tests_importing_module, get_data_paths, get_lib_paths, get_soft_deps, get_test_dependencies, get_test_paths, + get_transitive_imports, parse_lib_imports, parse_test_imports, resolve_all_paths, @@ -422,5 +424,133 @@ def test_nested_different(self): self.assertFalse(_dircmp_is_same(dcmp)) +class TestGetTransitiveImports(unittest.TestCase): + """Tests for get_transitive_imports function.""" + + def test_direct_dependency(self): + """A imports B → B's transitive importers include A.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + + (lib_dir / "a.py").write_text("import b\n") + (lib_dir / "b.py").write_text("# b module") + + get_transitive_imports.cache_clear() + result = get_transitive_imports("b", lib_prefix=str(lib_dir)) + self.assertIn("a", result) + + def test_chain_dependency(self): + """A imports B, B imports C → C's transitive importers include A and B.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + + (lib_dir / "a.py").write_text("import b\n") + (lib_dir / "b.py").write_text("import c\n") + (lib_dir / "c.py").write_text("# c module") + + get_transitive_imports.cache_clear() + result = get_transitive_imports("c", lib_prefix=str(lib_dir)) + self.assertIn("a", result) + self.assertIn("b", result) + + def test_cycle_handling(self): + """Handle circular imports without infinite loop.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + + (lib_dir / "a.py").write_text("import b\n") + (lib_dir / "b.py").write_text("import a\n") # cycle + + get_transitive_imports.cache_clear() + # Should not hang or raise + result = get_transitive_imports("a", lib_prefix=str(lib_dir)) + self.assertIn("b", result) + + +class TestFindTestsImportingModule(unittest.TestCase): + """Tests for find_tests_importing_module function.""" + + def test_direct_import(self): + """Test finding tests that directly import a module.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + # Create target module + (lib_dir / "bar.py").write_text("# bar module") + + # Create test that imports bar + (test_dir / "test_foo.py").write_text("import bar\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + self.assertIn(test_dir / "test_foo.py", result) + + def test_excludes_test_module_itself(self): + """Test that test_.py is excluded from results.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_dir / "test_bar.py").write_text("import bar\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + # test_bar.py should NOT be in results (it's the primary test) + self.assertNotIn(test_dir / "test_bar.py", result) + + def test_transitive_import(self): + """Test finding tests with transitive (indirect) imports.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + # bar.py (target module) + (lib_dir / "bar.py").write_text("# bar module") + + # baz.py imports bar + (lib_dir / "baz.py").write_text("import bar\n") + + # test_foo.py imports baz (not bar directly) + (test_dir / "test_foo.py").write_text("import baz\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + # test_foo.py should be found via transitive dependency + self.assertIn(test_dir / "test_foo.py", result) + + def test_empty_when_no_importers(self): + """Test returns empty when no tests import the module.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_dir / "test_unrelated.py").write_text("import os\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + self.assertEqual(result, frozenset()) + + if __name__ == "__main__": unittest.main() From 923ed8ef5a395d94277bbcb32ae3e65555a37577 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 21:59:44 +0900 Subject: [PATCH 02/17] Support --exclude option to deps subcommand to exclude dependencies --- scripts/update_lib/deps.py | 4 + scripts/update_lib/show_deps.py | 18 ++++- scripts/update_lib/tests/test_deps.py | 101 ++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 3 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 71d59f9139b..79e6769b962 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -605,6 +605,7 @@ def find_tests_importing_module( module_name: str, lib_prefix: str = "Lib", include_transitive: bool = True, + exclude_imports: frozenset[str] = frozenset(), ) -> frozenset[pathlib.Path]: """Find all test files that import the given module (directly or transitively). @@ -612,6 +613,7 @@ def find_tests_importing_module( module_name: Module to search for (e.g., "datetime") lib_prefix: RustPython Lib directory (default: "Lib") include_transitive: Whether to include transitive dependencies + exclude_imports: Modules to exclude from test file imports when checking Returns: Frozenset of test file paths that depend on this module @@ -643,6 +645,8 @@ def find_tests_importing_module( continue imports = parse_lib_imports(content) + # Remove excluded modules from imports + imports = imports - exclude_imports # Check if any target module is imported if imports & target_modules: result.add(test_file) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index 5d1f0584e01..5119b894455 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -146,6 +146,7 @@ def format_deps( max_depth: int = 10, _visited: set[str] | None = None, show_impact: bool = False, + exclude_imports: frozenset[str] = frozenset(), ) -> list[str]: """Format all dependency information for a module. @@ -156,6 +157,7 @@ def format_deps( max_depth: Maximum recursion depth _visited: Shared visited set for deduplication across modules show_impact: Whether to show reverse dependencies (tests that import this module) + exclude_imports: Modules to exclude from impact analysis Returns: List of formatted lines @@ -200,7 +202,7 @@ def format_deps( # Show impact (reverse dependencies) if requested if show_impact: - impacted_tests = find_tests_importing_module(name, lib_prefix) + impacted_tests = find_tests_importing_module(name, lib_prefix, exclude_imports=exclude_imports) transitive_importers = get_transitive_imports(name, lib_prefix) if impacted_tests: @@ -212,6 +214,8 @@ def format_deps( from update_lib.deps import parse_lib_imports test_imports = parse_lib_imports(test_content) + # Remove excluded modules from display (consistent with matching) + test_imports = test_imports - exclude_imports if name in test_imports: lines.append(f" - {test_path.name} (direct)") else: @@ -234,6 +238,7 @@ def show_deps( lib_prefix: str = "Lib", max_depth: int = 10, show_impact: bool = False, + exclude_imports: frozenset[str] = frozenset(), ) -> None: """Show all dependency information for modules.""" # Expand "all" to all module names @@ -251,7 +256,7 @@ def show_deps( if i > 0: print() # blank line between modules for line in format_deps( - name, cpython_prefix, lib_prefix, max_depth, visited, show_impact + name, cpython_prefix, lib_prefix, max_depth, visited, show_impact, exclude_imports ): print(line) @@ -287,11 +292,18 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Show tests that import this module (reverse dependencies)", ) + parser.add_argument( + "--exclude", + action="append", + default=[], + help="Modules to exclude from impact analysis (can be repeated: --exclude unittest --exclude doctest)", + ) args = parser.parse_args(argv) try: - show_deps(args.names, args.cpython, args.lib, args.depth, args.impact) + exclude_imports = frozenset(args.exclude) + show_deps(args.names, args.cpython, args.lib, args.depth, args.impact, exclude_imports) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index 56665187d79..06fc9ef2ffc 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -552,5 +552,106 @@ def test_empty_when_no_importers(self): self.assertEqual(result, frozenset()) +class TestFindTestsImportingModuleExclude(unittest.TestCase): + """Tests for exclude_imports parameter.""" + + def test_exclude_single_module(self): + """Test excluding a single module from import analysis.""" + # Given: + # bar.py (target module) + # unittest.py (module to exclude) + # test_foo.py imports: bar, unittest + # test_qux.py imports: unittest (only) + # When: find_tests_importing_module("bar", exclude_imports=frozenset({"unittest"})) + # Then: test_foo.py is included (bar matches), test_qux.py is excluded + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (lib_dir / "unittest.py").write_text("# unittest module") + + # test_foo imports both bar and unittest + (test_dir / "test_foo.py").write_text("import bar\nimport unittest\n") + # test_qux imports only unittest + (test_dir / "test_qux.py").write_text("import unittest\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module( + "bar", + lib_prefix=str(lib_dir), + exclude_imports=frozenset({"unittest"}) + ) + + # test_foo.py should be included (imports bar) + self.assertIn(test_dir / "test_foo.py", result) + # test_qux.py should be excluded (only imports unittest) + self.assertNotIn(test_dir / "test_qux.py", result) + + def test_exclude_transitive_via_excluded_module(self): + """Test that transitive dependencies via excluded modules are also excluded.""" + # Given: + # bar.py (target) + # baz.py imports bar + # unittest.py imports baz (so unittest transitively depends on bar) + # test_foo.py imports unittest (only) + # When: find_tests_importing_module("bar", exclude_imports=frozenset({"unittest"})) + # Then: test_foo.py is NOT included (unittest is excluded, no other path to bar) + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (lib_dir / "baz.py").write_text("import bar\n") + (lib_dir / "unittest.py").write_text("import baz\n") # unittest -> baz -> bar + + # test_foo imports only unittest + (test_dir / "test_foo.py").write_text("import unittest\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module( + "bar", + lib_prefix=str(lib_dir), + exclude_imports=frozenset({"unittest"}) + ) + + # test_foo.py should NOT be included + # (even though unittest -> baz -> bar, unittest is excluded) + self.assertNotIn(test_dir / "test_foo.py", result) + + def test_exclude_empty_set_same_as_default(self): + """Test that empty exclude set behaves same as no exclusion.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_dir / "test_foo.py").write_text("import bar\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + + result_default = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + find_tests_importing_module.cache_clear() + result_empty = find_tests_importing_module( + "bar", + lib_prefix=str(lib_dir), + exclude_imports=frozenset() + ) + + self.assertEqual(result_default, result_empty) + + if __name__ == "__main__": unittest.main() From 4c116cd0a403ae4eb3a37e21f78af3dfc8dc6dc1 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 22:35:51 +0900 Subject: [PATCH 03/17] Exclude non-test files from deps subcommand impact output --- scripts/update_lib/deps.py | 79 +++++++++++++++----- scripts/update_lib/tests/test_deps.py | 100 ++++++++++++++++++++++++-- 2 files changed, 157 insertions(+), 22 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 79e6769b962..6cdf7c8c3b5 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -600,6 +600,37 @@ def get_transitive_imports( return frozenset(visited) +def _build_test_import_graph(test_dir: pathlib.Path) -> dict[str, set[str]]: + """Build import graph for files within test directory. + + Args: + test_dir: Path to Lib/test/ directory + + Returns: + Dict mapping filename (stem) -> set of test modules it imports + """ + import_graph: dict[str, set[str]] = {} + + for py_file in test_dir.glob("*.py"): + content = safe_read_text(py_file) + if content is None: + continue + + imports = set() + # Parse "from test import X" style imports + imports.update(parse_test_imports(content)) + # Also check direct imports of test modules + all_imports = parse_lib_imports(content) + # Filter to only test modules that exist in test_dir + for imp in all_imports: + if (test_dir / f"{imp}.py").exists(): + imports.add(imp) + + import_graph[py_file.stem] = imports + + return import_graph + + @functools.cache def find_tests_importing_module( module_name: str, @@ -609,6 +640,9 @@ def find_tests_importing_module( ) -> frozenset[pathlib.Path]: """Find all test files that import the given module (directly or transitively). + Only returns test_*.py files. Support files (like pickletester.py, string_tests.py) + are used for transitive dependency calculation but not included in the result. + Args: module_name: Module to search for (e.g., "datetime") lib_prefix: RustPython Lib directory (default: "Lib") @@ -616,7 +650,7 @@ def find_tests_importing_module( exclude_imports: Modules to exclude from test file imports when checking Returns: - Frozenset of test file paths that depend on this module + Frozenset of test_*.py file paths that depend on this module """ lib_dir = pathlib.Path(lib_prefix) test_dir = lib_dir / "test" @@ -624,31 +658,40 @@ def find_tests_importing_module( if not test_dir.exists(): return frozenset() - # Build set of modules to search for + # Build set of modules to search for (Lib/ modules) target_modules = {module_name} if include_transitive: # Add all modules that transitively depend on module_name target_modules.update(get_transitive_imports(module_name, lib_prefix)) - # Excluded test file for this module (test_.py) - excluded_test = f"test_{module_name}.py" - - # Scan test directory for files that import any of the target modules - result: set[pathlib.Path] = set() - - for test_file in test_dir.glob("*.py"): - if test_file.name == excluded_test: - continue + # Build test directory import graph for transitive analysis within test/ + test_import_graph = _build_test_import_graph(test_dir) - content = safe_read_text(test_file) + # First pass: find all files (by stem) that directly import target modules + directly_importing: set[str] = set() + for py_file in test_dir.glob("*.py"): + content = safe_read_text(py_file) if content is None: continue - - imports = parse_lib_imports(content) - # Remove excluded modules from imports - imports = imports - exclude_imports - # Check if any target module is imported + imports = parse_lib_imports(content) - exclude_imports if imports & target_modules: - result.add(test_file) + directly_importing.add(py_file.stem) + + # Second pass: find files that transitively import via support files within test/ + # BFS to find all files that import any file in all_importing + all_importing = set(directly_importing) + queue = list(directly_importing) + while queue: + current = queue.pop(0) + for file_stem, imports in test_import_graph.items(): + if current in imports and file_stem not in all_importing: + all_importing.add(file_stem) + queue.append(file_stem) + + # Filter to only test_*.py files and build result paths + result: set[pathlib.Path] = set() + for file_stem in all_importing: + if file_stem.startswith("test_"): + result.add(test_dir / f"{file_stem}.py") return frozenset(result) diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index 06fc9ef2ffc..70eeeeb7fa3 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -495,8 +495,8 @@ def test_direct_import(self): result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) self.assertIn(test_dir / "test_foo.py", result) - def test_excludes_test_module_itself(self): - """Test that test_.py is excluded from results.""" + def test_includes_test_module_itself(self): + """Test that test_.py IS included in results.""" with tempfile.TemporaryDirectory() as tmpdir: tmpdir = pathlib.Path(tmpdir) lib_dir = tmpdir / "Lib" @@ -509,8 +509,8 @@ def test_excludes_test_module_itself(self): get_transitive_imports.cache_clear() find_tests_importing_module.cache_clear() result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - # test_bar.py should NOT be in results (it's the primary test) - self.assertNotIn(test_dir / "test_bar.py", result) + # test_bar.py IS now included (module's own test is part of impact) + self.assertIn(test_dir / "test_bar.py", result) def test_transitive_import(self): """Test finding tests with transitive (indirect) imports.""" @@ -653,5 +653,97 @@ def test_exclude_empty_set_same_as_default(self): self.assertEqual(result_default, result_empty) +class TestFindTestsOnlyTestFiles(unittest.TestCase): + """Tests for filtering to only test_*.py files in output.""" + + def test_support_file_not_in_output(self): + """Support files should not appear in output even if they import target.""" + # Given: + # bar.py (target module in Lib/) + # helper.py (support file in test/, imports bar) + # test_foo.py (test file, imports bar) + # When: find_tests_importing_module("bar") + # Then: test_foo.py is included, helper.py is NOT included + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + # helper.py imports bar directly but doesn't start with test_ + (test_dir / "helper.py").write_text("import bar\n") + # test_foo.py also imports bar + (test_dir / "test_foo.py").write_text("import bar\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + # Only test_foo.py should be in results + self.assertIn(test_dir / "test_foo.py", result) + # helper.py should be excluded + self.assertNotIn(test_dir / "helper.py", result) + + def test_transitive_via_support_file(self): + """Test file importing support file that imports target should be included.""" + # Given: + # bar.py (target module in Lib/) + # helper.py (support file in test/, imports bar) + # test_foo.py (test file, imports helper - NOT bar directly) + # When: find_tests_importing_module("bar") + # Then: test_foo.py IS included (via helper.py), helper.py is NOT + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + # helper.py imports bar + (test_dir / "helper.py").write_text("import bar\n") + # test_foo.py imports only helper (not bar directly) + (test_dir / "test_foo.py").write_text("from test import helper\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + # test_foo.py depends on bar via helper, so it should be included + self.assertIn(test_dir / "test_foo.py", result) + # helper.py should be excluded from output + self.assertNotIn(test_dir / "helper.py", result) + + def test_chain_through_multiple_support_files(self): + """Test transitive chain through multiple support files.""" + # Given: + # bar.py (target) + # helper_a.py imports bar + # helper_b.py imports helper_a + # test_foo.py imports helper_b + # Then: test_foo.py IS included, helper_a/b are NOT + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_dir / "helper_a.py").write_text("import bar\n") + (test_dir / "helper_b.py").write_text("from test import helper_a\n") + (test_dir / "test_foo.py").write_text("from test import helper_b\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + self.assertIn(test_dir / "test_foo.py", result) + self.assertNotIn(test_dir / "helper_a.py", result) + self.assertNotIn(test_dir / "helper_b.py", result) + + if __name__ == "__main__": unittest.main() From 62b9d4ed03c1f097fef1690c089a53917b23938b Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 23:00:36 +0900 Subject: [PATCH 04/17] Recursively search test module directories for deps impact Extend find_tests_importing_module() and _build_test_import_graph() to recursively search Lib/test/test_*/ directories using **/*.py glob pattern instead of just *.py. This fixes incomplete dependency analysis that was missing test files inside module directories like test_json/, test_importlib/, etc. Changes: - Add _parse_test_submodule_imports() to handle "from test.X import Y" - Update _build_test_import_graph() with recursive glob and submodule import handling - Update find_tests_importing_module() to use relative paths and handle __init__.py files correctly - Update show_deps.py to display relative paths (e.g., test_json/test_decode.py) - Add TestFindTestsInModuleDirectories test class with 3 tests Co-Authored-By: Claude --- scripts/update_lib/deps.py | 92 ++++++++++++++++++++----- scripts/update_lib/show_deps.py | 14 ++-- scripts/update_lib/tests/test_deps.py | 96 +++++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 19 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 6cdf7c8c3b5..3d1d7bf267e 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -600,18 +600,51 @@ def get_transitive_imports( return frozenset(visited) +def _parse_test_submodule_imports(content: str) -> dict[str, set[str]]: + """Parse 'from test.X import Y' to get submodule imports. + + Args: + content: Python file content + + Returns: + Dict mapping submodule (e.g., "test_bar") -> set of imported names (e.g., {"helper"}) + """ + import ast + + tree = safe_parse_ast(content) + if tree is None: + return {} + + result: dict[str, set[str]] = {} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + if node.module and node.module.startswith("test."): + # from test.test_bar import helper -> test_bar: {helper} + parts = node.module.split(".") + if len(parts) >= 2: + submodule = parts[1] + if submodule not in ("support", "__init__"): + if submodule not in result: + result[submodule] = set() + for alias in node.names: + result[submodule].add(alias.name) + + return result + + def _build_test_import_graph(test_dir: pathlib.Path) -> dict[str, set[str]]: - """Build import graph for files within test directory. + """Build import graph for files within test directory (recursive). Args: test_dir: Path to Lib/test/ directory Returns: - Dict mapping filename (stem) -> set of test modules it imports + Dict mapping relative path (without .py) -> set of test modules it imports """ import_graph: dict[str, set[str]] = {} - for py_file in test_dir.glob("*.py"): + # Use **/*.py to recursively find all Python files + for py_file in test_dir.glob("**/*.py"): content = safe_read_text(py_file) if content is None: continue @@ -621,12 +654,30 @@ def _build_test_import_graph(test_dir: pathlib.Path) -> dict[str, set[str]]: imports.update(parse_test_imports(content)) # Also check direct imports of test modules all_imports = parse_lib_imports(content) - # Filter to only test modules that exist in test_dir + + # Check for files at same level or in test_dir for imp in all_imports: + # Check in same directory + if (py_file.parent / f"{imp}.py").exists(): + imports.add(imp) + # Check in test_dir root if (test_dir / f"{imp}.py").exists(): imports.add(imp) - import_graph[py_file.stem] = imports + # Handle "from test.X import Y" where Y is a file in test_dir/X/ + submodule_imports = _parse_test_submodule_imports(content) + for submodule, imported_names in submodule_imports.items(): + submodule_dir = test_dir / submodule + if submodule_dir.is_dir(): + for name in imported_names: + # Check if it's a file in the submodule directory + if (submodule_dir / f"{name}.py").exists(): + imports.add(name) + + # Use relative path from test_dir as key (without .py) + rel_path = py_file.relative_to(test_dir) + key = str(rel_path.with_suffix("")) + import_graph[key] = imports return import_graph @@ -667,15 +718,16 @@ def find_tests_importing_module( # Build test directory import graph for transitive analysis within test/ test_import_graph = _build_test_import_graph(test_dir) - # First pass: find all files (by stem) that directly import target modules + # First pass: find all files (by relative path) that directly import target modules directly_importing: set[str] = set() - for py_file in test_dir.glob("*.py"): + for py_file in test_dir.glob("**/*.py"): # Recursive glob content = safe_read_text(py_file) if content is None: continue imports = parse_lib_imports(content) - exclude_imports if imports & target_modules: - directly_importing.add(py_file.stem) + rel_path = py_file.relative_to(test_dir) + directly_importing.add(str(rel_path.with_suffix(""))) # Second pass: find files that transitively import via support files within test/ # BFS to find all files that import any file in all_importing @@ -683,15 +735,25 @@ def find_tests_importing_module( queue = list(directly_importing) while queue: current = queue.pop(0) - for file_stem, imports in test_import_graph.items(): - if current in imports and file_stem not in all_importing: - all_importing.add(file_stem) - queue.append(file_stem) + # Extract the filename (stem) from the relative path for matching + current_path = pathlib.Path(current) + current_stem = current_path.name + # For __init__.py, the import name is the parent directory name + # e.g., "test_json/__init__" -> can be imported as "test_json" + if current_stem == "__init__": + current_stem = current_path.parent.name + for file_key, imports in test_import_graph.items(): + if current_stem in imports and file_key not in all_importing: + all_importing.add(file_key) + queue.append(file_key) # Filter to only test_*.py files and build result paths result: set[pathlib.Path] = set() - for file_stem in all_importing: - if file_stem.startswith("test_"): - result.add(test_dir / f"{file_stem}.py") + for file_key in all_importing: + # file_key is like "test_foo" or "test_bar/test_sub" + path_parts = pathlib.Path(file_key) + filename = path_parts.name # Get just the filename part + if filename.startswith("test_"): + result.add(test_dir / f"{file_key}.py") return frozenset(result) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index 5119b894455..7e55516a9ab 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -208,7 +208,13 @@ def format_deps( if impacted_tests: lines.append(f"[+] impact: ({len(impacted_tests)} tests depend on {name})") # Sort tests and show with dependency info - for test_path in sorted(impacted_tests, key=lambda p: p.name): + test_dir = pathlib.Path(lib_prefix) / "test" + for test_path in sorted(impacted_tests, key=lambda p: str(p)): + # Get relative path from test directory for display + try: + display_name = str(test_path.relative_to(test_dir)) + except ValueError: + display_name = test_path.name # Determine if direct or via which module test_content = test_path.read_text(errors="ignore") from update_lib.deps import parse_lib_imports @@ -217,15 +223,15 @@ def format_deps( # Remove excluded modules from display (consistent with matching) test_imports = test_imports - exclude_imports if name in test_imports: - lines.append(f" - {test_path.name} (direct)") + lines.append(f" - {display_name} (direct)") else: # Find which transitive module is imported via_modules = test_imports & transitive_importers if via_modules: via_str = ", ".join(sorted(via_modules)) - lines.append(f" - {test_path.name} (via {via_str})") + lines.append(f" - {display_name} (via {via_str})") else: - lines.append(f" - {test_path.name}") + lines.append(f" - {display_name}") else: lines.append(f"[+] impact: (no tests depend on {name})") diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index 70eeeeb7fa3..1247c239a70 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -745,5 +745,101 @@ def test_chain_through_multiple_support_files(self): self.assertNotIn(test_dir / "helper_b.py", result) +class TestFindTestsInModuleDirectories(unittest.TestCase): + """Tests for finding tests inside test_*/ module directories.""" + + def test_finds_test_in_module_directory(self): + """Test files inside test_*/ directories should be found.""" + # Given: + # bar.py (target module in Lib/) + # test_bar/ + # __init__.py + # test_sub.py (imports bar) + # When: find_tests_importing_module("bar") + # Then: test_bar/test_sub.py IS included + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_bar_dir = test_dir / "test_bar" + test_bar_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_bar_dir / "__init__.py").write_text("") + (test_bar_dir / "test_sub.py").write_text("import bar\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + # test_bar/test_sub.py should be in results + self.assertIn(test_bar_dir / "test_sub.py", result) + + def test_finds_nested_test_via_support_in_module_directory(self): + """Transitive deps through support files in module directories.""" + # Given: + # bar.py (target) + # test_bar/ + # __init__.py + # helper.py (imports bar) + # test_sub.py (imports helper via "from test.test_bar import helper") + # When: find_tests_importing_module("bar") + # Then: test_bar/test_sub.py IS included, helper.py is NOT + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_bar_dir = test_dir / "test_bar" + test_bar_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_bar_dir / "__init__.py").write_text("") + (test_bar_dir / "helper.py").write_text("import bar\n") + (test_bar_dir / "test_sub.py").write_text( + "from test.test_bar import helper\n" + ) + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + # test_sub.py should be included (via helper) + self.assertIn(test_bar_dir / "test_sub.py", result) + # helper.py should NOT be in results (not a test file) + self.assertNotIn(test_bar_dir / "helper.py", result) + + def test_both_top_level_and_module_directory_tests_found(self): + """Both top-level test_*.py and test_*/test_*.py should be found.""" + # Given: + # bar.py (target) + # test_bar.py (top-level, imports bar) + # test_bar/ + # test_sub.py (imports bar) + # When: find_tests_importing_module("bar") + # Then: BOTH test_bar.py AND test_bar/test_sub.py are included + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + test_dir = lib_dir / "test" + test_bar_dir = test_dir / "test_bar" + test_bar_dir.mkdir(parents=True) + + (lib_dir / "bar.py").write_text("# bar module") + (test_dir / "test_bar.py").write_text("import bar\n") + (test_bar_dir / "__init__.py").write_text("") + (test_bar_dir / "test_sub.py").write_text("import bar\n") + + get_transitive_imports.cache_clear() + find_tests_importing_module.cache_clear() + result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) + + # Both should be included + self.assertIn(test_dir / "test_bar.py", result) + self.assertIn(test_bar_dir / "test_sub.py", result) + + if __name__ == "__main__": unittest.main() From 774451a37e99c6191c834c9df8566d24a1504565 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 23:18:58 +0900 Subject: [PATCH 05/17] Revert --exclude option of deps subcommand --- scripts/update_lib/deps.py | 4 +- scripts/update_lib/show_deps.py | 18 +---- scripts/update_lib/tests/test_deps.py | 101 -------------------------- 3 files changed, 4 insertions(+), 119 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 3d1d7bf267e..c71c827b038 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -687,7 +687,6 @@ def find_tests_importing_module( module_name: str, lib_prefix: str = "Lib", include_transitive: bool = True, - exclude_imports: frozenset[str] = frozenset(), ) -> frozenset[pathlib.Path]: """Find all test files that import the given module (directly or transitively). @@ -698,7 +697,6 @@ def find_tests_importing_module( module_name: Module to search for (e.g., "datetime") lib_prefix: RustPython Lib directory (default: "Lib") include_transitive: Whether to include transitive dependencies - exclude_imports: Modules to exclude from test file imports when checking Returns: Frozenset of test_*.py file paths that depend on this module @@ -724,7 +722,7 @@ def find_tests_importing_module( content = safe_read_text(py_file) if content is None: continue - imports = parse_lib_imports(content) - exclude_imports + imports = parse_lib_imports(content) if imports & target_modules: rel_path = py_file.relative_to(test_dir) directly_importing.add(str(rel_path.with_suffix(""))) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index 7e55516a9ab..fe2c9fa805c 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -146,7 +146,6 @@ def format_deps( max_depth: int = 10, _visited: set[str] | None = None, show_impact: bool = False, - exclude_imports: frozenset[str] = frozenset(), ) -> list[str]: """Format all dependency information for a module. @@ -157,7 +156,6 @@ def format_deps( max_depth: Maximum recursion depth _visited: Shared visited set for deduplication across modules show_impact: Whether to show reverse dependencies (tests that import this module) - exclude_imports: Modules to exclude from impact analysis Returns: List of formatted lines @@ -202,7 +200,7 @@ def format_deps( # Show impact (reverse dependencies) if requested if show_impact: - impacted_tests = find_tests_importing_module(name, lib_prefix, exclude_imports=exclude_imports) + impacted_tests = find_tests_importing_module(name, lib_prefix) transitive_importers = get_transitive_imports(name, lib_prefix) if impacted_tests: @@ -220,8 +218,6 @@ def format_deps( from update_lib.deps import parse_lib_imports test_imports = parse_lib_imports(test_content) - # Remove excluded modules from display (consistent with matching) - test_imports = test_imports - exclude_imports if name in test_imports: lines.append(f" - {display_name} (direct)") else: @@ -244,7 +240,6 @@ def show_deps( lib_prefix: str = "Lib", max_depth: int = 10, show_impact: bool = False, - exclude_imports: frozenset[str] = frozenset(), ) -> None: """Show all dependency information for modules.""" # Expand "all" to all module names @@ -262,7 +257,7 @@ def show_deps( if i > 0: print() # blank line between modules for line in format_deps( - name, cpython_prefix, lib_prefix, max_depth, visited, show_impact, exclude_imports + name, cpython_prefix, lib_prefix, max_depth, visited, show_impact ): print(line) @@ -298,18 +293,11 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Show tests that import this module (reverse dependencies)", ) - parser.add_argument( - "--exclude", - action="append", - default=[], - help="Modules to exclude from impact analysis (can be repeated: --exclude unittest --exclude doctest)", - ) args = parser.parse_args(argv) try: - exclude_imports = frozenset(args.exclude) - show_deps(args.names, args.cpython, args.lib, args.depth, args.impact, exclude_imports) + show_deps(args.names, args.cpython, args.lib, args.depth, args.impact) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index 1247c239a70..da1aef76cd4 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -552,107 +552,6 @@ def test_empty_when_no_importers(self): self.assertEqual(result, frozenset()) -class TestFindTestsImportingModuleExclude(unittest.TestCase): - """Tests for exclude_imports parameter.""" - - def test_exclude_single_module(self): - """Test excluding a single module from import analysis.""" - # Given: - # bar.py (target module) - # unittest.py (module to exclude) - # test_foo.py imports: bar, unittest - # test_qux.py imports: unittest (only) - # When: find_tests_importing_module("bar", exclude_imports=frozenset({"unittest"})) - # Then: test_foo.py is included (bar matches), test_qux.py is excluded - - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - (lib_dir / "unittest.py").write_text("# unittest module") - - # test_foo imports both bar and unittest - (test_dir / "test_foo.py").write_text("import bar\nimport unittest\n") - # test_qux imports only unittest - (test_dir / "test_qux.py").write_text("import unittest\n") - - get_transitive_imports.cache_clear() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module( - "bar", - lib_prefix=str(lib_dir), - exclude_imports=frozenset({"unittest"}) - ) - - # test_foo.py should be included (imports bar) - self.assertIn(test_dir / "test_foo.py", result) - # test_qux.py should be excluded (only imports unittest) - self.assertNotIn(test_dir / "test_qux.py", result) - - def test_exclude_transitive_via_excluded_module(self): - """Test that transitive dependencies via excluded modules are also excluded.""" - # Given: - # bar.py (target) - # baz.py imports bar - # unittest.py imports baz (so unittest transitively depends on bar) - # test_foo.py imports unittest (only) - # When: find_tests_importing_module("bar", exclude_imports=frozenset({"unittest"})) - # Then: test_foo.py is NOT included (unittest is excluded, no other path to bar) - - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - (lib_dir / "baz.py").write_text("import bar\n") - (lib_dir / "unittest.py").write_text("import baz\n") # unittest -> baz -> bar - - # test_foo imports only unittest - (test_dir / "test_foo.py").write_text("import unittest\n") - - get_transitive_imports.cache_clear() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module( - "bar", - lib_prefix=str(lib_dir), - exclude_imports=frozenset({"unittest"}) - ) - - # test_foo.py should NOT be included - # (even though unittest -> baz -> bar, unittest is excluded) - self.assertNotIn(test_dir / "test_foo.py", result) - - def test_exclude_empty_set_same_as_default(self): - """Test that empty exclude set behaves same as no exclusion.""" - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - (test_dir / "test_foo.py").write_text("import bar\n") - - get_transitive_imports.cache_clear() - find_tests_importing_module.cache_clear() - - result_default = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - - find_tests_importing_module.cache_clear() - result_empty = find_tests_importing_module( - "bar", - lib_prefix=str(lib_dir), - exclude_imports=frozenset() - ) - - self.assertEqual(result_default, result_empty) - - class TestFindTestsOnlyTestFiles(unittest.TestCase): """Tests for filtering to only test_*.py files in output.""" From e4acc382d99f43aa28261adb3247310573395926 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 23:36:38 +0900 Subject: [PATCH 06/17] Consolidate test module directories in deps impact output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add consolidate_test_paths() to group test_*/ directory contents into single entries (e.g., test_sqlite3/test_dbapi.py → test_sqlite3). Co-Authored-By: Claude Opus 4.5 --- scripts/update_lib/deps.py | 35 ++++++++++++++++ scripts/update_lib/show_deps.py | 36 ++++------------ scripts/update_lib/tests/test_deps.py | 59 +++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 28 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index c71c827b038..cf1545f0bdc 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -755,3 +755,38 @@ def find_tests_importing_module( result.add(test_dir / f"{file_key}.py") return frozenset(result) + + +def consolidate_test_paths( + test_paths: frozenset[pathlib.Path], + test_dir: pathlib.Path, +) -> frozenset[str]: + """Consolidate test paths by grouping test_*/ directory contents into a single entry. + + Args: + test_paths: Frozenset of absolute paths to test files + test_dir: Path to the test directory (e.g., Lib/test) + + Returns: + Frozenset of consolidated test names: + - "test_foo" for Lib/test/test_foo.py + - "test_sqlite3" for any file in Lib/test/test_sqlite3/ + """ + consolidated: set[str] = set() + + for path in test_paths: + try: + rel_path = path.relative_to(test_dir) + parts = rel_path.parts + + if len(parts) == 1: + # test_foo.py -> test_foo + consolidated.add(rel_path.stem) + else: + # test_sqlite3/test_dbapi.py -> test_sqlite3 + consolidated.add(parts[0]) + except ValueError: + # Path not relative to test_dir, use stem + consolidated.add(path.stem) + + return frozenset(consolidated) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index fe2c9fa805c..9e62dc1810f 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -162,10 +162,10 @@ def format_deps( """ from update_lib.deps import ( DEPENDENCIES, + consolidate_test_paths, find_tests_importing_module, get_lib_paths, get_test_paths, - get_transitive_imports, ) if _visited is None: @@ -201,33 +201,13 @@ def format_deps( # Show impact (reverse dependencies) if requested if show_impact: impacted_tests = find_tests_importing_module(name, lib_prefix) - transitive_importers = get_transitive_imports(name, lib_prefix) - - if impacted_tests: - lines.append(f"[+] impact: ({len(impacted_tests)} tests depend on {name})") - # Sort tests and show with dependency info - test_dir = pathlib.Path(lib_prefix) / "test" - for test_path in sorted(impacted_tests, key=lambda p: str(p)): - # Get relative path from test directory for display - try: - display_name = str(test_path.relative_to(test_dir)) - except ValueError: - display_name = test_path.name - # Determine if direct or via which module - test_content = test_path.read_text(errors="ignore") - from update_lib.deps import parse_lib_imports - - test_imports = parse_lib_imports(test_content) - if name in test_imports: - lines.append(f" - {display_name} (direct)") - else: - # Find which transitive module is imported - via_modules = test_imports & transitive_importers - if via_modules: - via_str = ", ".join(sorted(via_modules)) - lines.append(f" - {display_name} (via {via_str})") - else: - lines.append(f" - {display_name}") + test_dir = pathlib.Path(lib_prefix) / "test" + consolidated = consolidate_test_paths(impacted_tests, test_dir) + + if consolidated: + lines.append(f"[+] impact: ({len(consolidated)} tests depend on {name})") + for test_name in sorted(consolidated): + lines.append(f" - {test_name}") else: lines.append(f"[+] impact: (no tests depend on {name})") diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index da1aef76cd4..e91512088ec 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -5,6 +5,7 @@ import unittest from update_lib.deps import ( + consolidate_test_paths, find_tests_importing_module, get_data_paths, get_lib_paths, @@ -740,5 +741,63 @@ def test_both_top_level_and_module_directory_tests_found(self): self.assertIn(test_bar_dir / "test_sub.py", result) +class TestConsolidateTestPaths(unittest.TestCase): + """Tests for consolidate_test_paths function.""" + + def test_top_level_test_file(self): + """Top-level test_*.py -> test_* (without .py).""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) + test_file = test_dir / "test_foo.py" + test_file.write_text("# test") + + result = consolidate_test_paths(frozenset({test_file}), test_dir) + self.assertEqual(result, frozenset({"test_foo"})) + + def test_module_directory_tests_consolidated(self): + """Multiple files in test_*/ directory -> single directory name.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) + module_dir = test_dir / "test_sqlite3" + module_dir.mkdir() + (module_dir / "test_dbapi.py").write_text("# test") + (module_dir / "test_backup.py").write_text("# test") + + result = consolidate_test_paths( + frozenset({module_dir / "test_dbapi.py", module_dir / "test_backup.py"}), + test_dir, + ) + self.assertEqual(result, frozenset({"test_sqlite3"})) + + def test_mixed_top_level_and_module_directory(self): + """Both top-level and module directory tests handled correctly.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) + # Top-level test + (test_dir / "test_foo.py").write_text("# test") + # Module directory tests + module_dir = test_dir / "test_sqlite3" + module_dir.mkdir() + (module_dir / "test_dbapi.py").write_text("# test") + (module_dir / "test_backup.py").write_text("# test") + + result = consolidate_test_paths( + frozenset({ + test_dir / "test_foo.py", + module_dir / "test_dbapi.py", + module_dir / "test_backup.py", + }), + test_dir, + ) + self.assertEqual(result, frozenset({"test_foo", "test_sqlite3"})) + + def test_empty_input(self): + """Empty input -> empty frozenset.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) + result = consolidate_test_paths(frozenset(), test_dir) + self.assertEqual(result, frozenset()) + + if __name__ == "__main__": unittest.main() From bf9d2487c366ebdc77439b6abba759ab48299eea Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 23:49:01 +0900 Subject: [PATCH 07/17] Add --impact-only flag to deps subcommand for test runner integration Outputs space-separated test names for direct use with python3 -m test: python3 -m test $(python3 scripts/update_lib deps sqlite3 --impact-only) Co-Authored-By: Claude Opus 4.5 --- scripts/update_lib/show_deps.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index 9e62dc1810f..c89e15b12f0 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -220,8 +220,14 @@ def show_deps( lib_prefix: str = "Lib", max_depth: int = 10, show_impact: bool = False, + impact_only: bool = False, ) -> None: """Show all dependency information for modules.""" + from update_lib.deps import ( + consolidate_test_paths, + find_tests_importing_module, + ) + # Expand "all" to all module names expanded_names = [] for name in names: @@ -230,6 +236,18 @@ def show_deps( else: expanded_names.append(name) + # Handle impact-only mode: output only space-separated test names + if impact_only: + all_tests: set[str] = set() + for name in expanded_names: + impacted = find_tests_importing_module(name, lib_prefix) + test_dir = pathlib.Path(lib_prefix) / "test" + consolidated = consolidate_test_paths(impacted, test_dir) + all_tests.update(consolidated) + if all_tests: + print(" ".join(sorted(all_tests))) + return + # Shared visited set across all modules visited: set[str] = set() @@ -273,11 +291,16 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Show tests that import this module (reverse dependencies)", ) + parser.add_argument( + "--impact-only", + action="store_true", + help="Output only impact test names, space-separated (for use with python3 -m test)", + ) args = parser.parse_args(argv) try: - show_deps(args.names, args.cpython, args.lib, args.depth, args.impact) + show_deps(args.names, args.cpython, args.lib, args.depth, args.impact, args.impact_only) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) From f7e08cf1466b345f244602e292036517b94161a8 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Wed, 21 Jan 2026 23:58:46 +0900 Subject: [PATCH 08/17] Rename impact to dependent tests and make it default in deps subcommand - Remove --impact flag, dependent tests are now shown by default - Rename --impact-only to --dependent-tests-only - Change output label from "[+] impact:" to "[+] dependent tests:" Co-Authored-By: Claude Opus 4.5 --- scripts/update_lib/show_deps.py | 43 +++++++++++++-------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index c89e15b12f0..da47ccfca28 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -145,7 +145,6 @@ def format_deps( lib_prefix: str = "Lib", max_depth: int = 10, _visited: set[str] | None = None, - show_impact: bool = False, ) -> list[str]: """Format all dependency information for a module. @@ -155,7 +154,6 @@ def format_deps( lib_prefix: Local Lib directory prefix max_depth: Maximum recursion depth _visited: Shared visited set for deduplication across modules - show_impact: Whether to show reverse dependencies (tests that import this module) Returns: List of formatted lines @@ -198,18 +196,17 @@ def format_deps( ) ) - # Show impact (reverse dependencies) if requested - if show_impact: - impacted_tests = find_tests_importing_module(name, lib_prefix) - test_dir = pathlib.Path(lib_prefix) / "test" - consolidated = consolidate_test_paths(impacted_tests, test_dir) + # Show dependent tests (reverse dependencies) + impacted_tests = find_tests_importing_module(name, lib_prefix) + test_dir = pathlib.Path(lib_prefix) / "test" + consolidated = consolidate_test_paths(impacted_tests, test_dir) - if consolidated: - lines.append(f"[+] impact: ({len(consolidated)} tests depend on {name})") - for test_name in sorted(consolidated): - lines.append(f" - {test_name}") - else: - lines.append(f"[+] impact: (no tests depend on {name})") + if consolidated: + lines.append(f"[+] dependent tests: ({len(consolidated)} tests depend on {name})") + for test_name in sorted(consolidated): + lines.append(f" - {test_name}") + else: + lines.append(f"[+] dependent tests: (no tests depend on {name})") return lines @@ -219,8 +216,7 @@ def show_deps( cpython_prefix: str = "cpython", lib_prefix: str = "Lib", max_depth: int = 10, - show_impact: bool = False, - impact_only: bool = False, + dependent_tests_only: bool = False, ) -> None: """Show all dependency information for modules.""" from update_lib.deps import ( @@ -236,8 +232,8 @@ def show_deps( else: expanded_names.append(name) - # Handle impact-only mode: output only space-separated test names - if impact_only: + # Handle dependent-tests-only mode: output only space-separated test names + if dependent_tests_only: all_tests: set[str] = set() for name in expanded_names: impacted = find_tests_importing_module(name, lib_prefix) @@ -255,7 +251,7 @@ def show_deps( if i > 0: print() # blank line between modules for line in format_deps( - name, cpython_prefix, lib_prefix, max_depth, visited, show_impact + name, cpython_prefix, lib_prefix, max_depth, visited ): print(line) @@ -287,20 +283,15 @@ def main(argv: list[str] | None = None) -> int: help="Maximum recursion depth for soft_deps tree (default: 10)", ) parser.add_argument( - "--impact", - action="store_true", - help="Show tests that import this module (reverse dependencies)", - ) - parser.add_argument( - "--impact-only", + "--dependent-tests-only", action="store_true", - help="Output only impact test names, space-separated (for use with python3 -m test)", + help="Output only dependent test names, space-separated (for use with python3 -m test)", ) args = parser.parse_args(argv) try: - show_deps(args.names, args.cpython, args.lib, args.depth, args.impact, args.impact_only) + show_deps(args.names, args.cpython, args.lib, args.depth, args.dependent_tests_only) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) From 38af152dcc4f961e1fb97b1fb62a7de6d6e35fd8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 21 Jan 2026 15:00:23 +0000 Subject: [PATCH 09/17] Auto-format: ruff format --- scripts/update_lib/show_deps.py | 12 +++++++----- scripts/update_lib/tests/test_deps.py | 16 ++++++++++------ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index da47ccfca28..974cd9e0529 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -202,7 +202,9 @@ def format_deps( consolidated = consolidate_test_paths(impacted_tests, test_dir) if consolidated: - lines.append(f"[+] dependent tests: ({len(consolidated)} tests depend on {name})") + lines.append( + f"[+] dependent tests: ({len(consolidated)} tests depend on {name})" + ) for test_name in sorted(consolidated): lines.append(f" - {test_name}") else: @@ -250,9 +252,7 @@ def show_deps( for i, name in enumerate(expanded_names): if i > 0: print() # blank line between modules - for line in format_deps( - name, cpython_prefix, lib_prefix, max_depth, visited - ): + for line in format_deps(name, cpython_prefix, lib_prefix, max_depth, visited): print(line) @@ -291,7 +291,9 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) try: - show_deps(args.names, args.cpython, args.lib, args.depth, args.dependent_tests_only) + show_deps( + args.names, args.cpython, args.lib, args.depth, args.dependent_tests_only + ) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index e91512088ec..f06d6f29317 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -764,7 +764,9 @@ def test_module_directory_tests_consolidated(self): (module_dir / "test_backup.py").write_text("# test") result = consolidate_test_paths( - frozenset({module_dir / "test_dbapi.py", module_dir / "test_backup.py"}), + frozenset( + {module_dir / "test_dbapi.py", module_dir / "test_backup.py"} + ), test_dir, ) self.assertEqual(result, frozenset({"test_sqlite3"})) @@ -782,11 +784,13 @@ def test_mixed_top_level_and_module_directory(self): (module_dir / "test_backup.py").write_text("# test") result = consolidate_test_paths( - frozenset({ - test_dir / "test_foo.py", - module_dir / "test_dbapi.py", - module_dir / "test_backup.py", - }), + frozenset( + { + test_dir / "test_foo.py", + module_dir / "test_dbapi.py", + module_dir / "test_backup.py", + } + ), test_dir, ) self.assertEqual(result, frozenset({"test_foo", "test_sqlite3"})) From 682212eb64b7ca1cfdb823dc6205215998edf9bc Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Thu, 22 Jan 2026 00:51:10 +0900 Subject: [PATCH 10/17] Refactor --- scripts/update_lib/deps.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index cf1545f0bdc..5c76fab8f5d 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -7,8 +7,10 @@ - Test dependencies (auto-detected from 'from test import ...') """ +import ast import functools import pathlib +from collections import deque from update_lib.io_utils import read_python_files, safe_parse_ast, safe_read_text from update_lib.path import construct_lib_path, resolve_module_path @@ -226,8 +228,6 @@ def parse_test_imports(content: str) -> set[str]: Returns: Set of module names imported from test package """ - import ast - tree = safe_parse_ast(content) if tree is None: return set() @@ -262,8 +262,6 @@ def parse_lib_imports(content: str) -> set[str]: Returns: Set of imported module names (top-level only) """ - import ast - tree = safe_parse_ast(content) if tree is None: return set() @@ -506,6 +504,7 @@ def resolve_all_paths( return result +@functools.cache def _build_import_graph(lib_prefix: str = "Lib") -> dict[str, set[str]]: """Build a graph of module imports from lib_prefix directory. @@ -585,10 +584,10 @@ def get_transitive_imports( # BFS from module_name following reverse edges visited: set[str] = set() - queue = list(reverse_graph.get(module_name, set())) + queue = deque(reverse_graph.get(module_name, set())) while queue: - current = queue.pop(0) + current = queue.popleft() if current in visited: continue visited.add(current) @@ -609,8 +608,6 @@ def _parse_test_submodule_imports(content: str) -> dict[str, set[str]]: Returns: Dict mapping submodule (e.g., "test_bar") -> set of imported names (e.g., {"helper"}) """ - import ast - tree = safe_parse_ast(content) if tree is None: return {} @@ -730,9 +727,9 @@ def find_tests_importing_module( # Second pass: find files that transitively import via support files within test/ # BFS to find all files that import any file in all_importing all_importing = set(directly_importing) - queue = list(directly_importing) + queue = deque(directly_importing) while queue: - current = queue.pop(0) + current = queue.popleft() # Extract the filename (stem) from the relative path for matching current_path = pathlib.Path(current) current_stem = current_path.name From e4bf543c9ae983257fff89eebab84557fb0ffe4d Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 23 Jan 2026 00:30:59 +0900 Subject: [PATCH 11/17] pick a useful level --- scripts/update_lib/deps.py | 692 +++++++++++++++++++------- scripts/update_lib/show_deps.py | 85 ++-- scripts/update_lib/tests/test_deps.py | 184 +------ 3 files changed, 559 insertions(+), 402 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 5c76fab8f5d..5be80e439df 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -10,9 +10,46 @@ import ast import functools import pathlib +import re +import shelve +import subprocess from collections import deque from update_lib.io_utils import read_python_files, safe_parse_ast, safe_read_text + + +# === Cross-process cache using shelve === + +def _get_cpython_version(cpython_prefix: str = "cpython") -> str: + """Get CPython version from git tag for cache namespace.""" + try: + result = subprocess.run( + ["git", "describe", "--tags", "--abbrev=0"], + cwd=cpython_prefix, + capture_output=True, + text=True, + ) + if result.returncode == 0: + return result.stdout.strip() + except Exception: + pass + return "unknown" + + +def _get_cache_path() -> str: + """Get cache file path (without extension - shelve adds its own).""" + cache_dir = pathlib.Path.home() / ".cache" / "rustpython-update-lib" + cache_dir.mkdir(parents=True, exist_ok=True) + return str(cache_dir / "import_graph_cache") + + +def clear_import_graph_caches() -> None: + """Clear in-process import graph caches (for testing).""" + global _test_import_graph_cache, _lib_import_graph_cache + if "_test_import_graph_cache" in globals(): + globals()["_test_import_graph_cache"].clear() + if "_lib_import_graph_cache" in globals(): + globals()["_lib_import_graph_cache"].clear() from update_lib.path import construct_lib_path, resolve_module_path # Manual dependency table for irregular cases @@ -219,63 +256,86 @@ def get_data_paths( return () +def _extract_top_level_code(content: str) -> str: + """Extract only top-level code from Python content for faster parsing. + + Cuts at first function/class definition since imports come before them. + """ + # Find first function or class definition + def_idx = content.find("\ndef ") + class_idx = content.find("\nclass ") + + # Use the earlier of the two (if found) + indices = [i for i in (def_idx, class_idx) if i != -1] + if indices: + content = content[: min(indices)] + return content.rstrip("\n") + + +_FROM_TEST_IMPORT_RE = re.compile(r"^from test import (.+)", re.MULTILINE) +_FROM_TEST_DOT_RE = re.compile(r"^from test\.(\w+)", re.MULTILINE) + + def parse_test_imports(content: str) -> set[str]: """Parse test file content and extract 'from test import ...' dependencies. + Uses regex for speed - only matches top-level imports. + Args: content: Python file content Returns: Set of module names imported from test package """ - tree = safe_parse_ast(content) - if tree is None: - return set() - + content = _extract_top_level_code(content) imports = set() - for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom): - if node.module == "test": - # from test import foo, bar - for alias in node.names: - name = alias.name - # Skip support modules and common imports - if name not in ("support", "__init__"): - imports.add(name) - elif node.module and node.module.startswith("test."): - # from test.foo import bar -> depends on foo - parts = node.module.split(".") - if len(parts) >= 2: - dep = parts[1] - if dep not in ("support", "__init__"): - imports.add(dep) + + # Match "from test import foo, bar, baz" + for match in _FROM_TEST_IMPORT_RE.finditer(content): + import_list = match.group(1) + # Parse "foo, bar as b, baz" -> ["foo", "bar", "baz"] + for part in import_list.split(","): + name = part.split()[0].strip() # Handle "foo as f" + if name and name not in ("support", "__init__"): + imports.add(name) + + # Match "from test.foo import ..." -> depends on foo + for match in _FROM_TEST_DOT_RE.finditer(content): + dep = match.group(1) + if dep not in ("support", "__init__"): + imports.add(dep) return imports +# Match "import foo.bar" - module name must start with word char (not dot) +_IMPORT_RE = re.compile(r"^import\s+(\w[\w.]*)", re.MULTILINE) +# Match "from foo.bar import" - exclude relative imports (from . or from ..) +_FROM_IMPORT_RE = re.compile(r"^from\s+(\w[\w.]*)\s+import", re.MULTILINE) + + def parse_lib_imports(content: str) -> set[str]: """Parse library file and extract all imported module names. + Uses regex for speed - only matches top-level imports (no leading whitespace). + Returns full module paths (e.g., "collections.abc" not just "collections"). + Args: content: Python file content Returns: - Set of imported module names (top-level only) + Set of imported module names (full paths) """ - tree = safe_parse_ast(content) - if tree is None: - return set() - + content = _extract_top_level_code(content) imports = set() - for node in ast.walk(tree): - if isinstance(node, ast.Import): - # import foo, bar - for alias in node.names: - imports.add(alias.name.split(".")[0]) - elif isinstance(node, ast.ImportFrom): - # from foo import bar - if node.module: - imports.add(node.module.split(".")[0]) + + # Match "import foo.bar" at line start + for match in _IMPORT_RE.finditer(content): + imports.add(match.group(1)) + + # Match "from foo.bar import ..." at line start + for match in _FROM_IMPORT_RE.finditer(content): + imports.add(match.group(1)) return imports @@ -504,101 +564,6 @@ def resolve_all_paths( return result -@functools.cache -def _build_import_graph(lib_prefix: str = "Lib") -> dict[str, set[str]]: - """Build a graph of module imports from lib_prefix directory. - - Args: - lib_prefix: RustPython Lib directory (default: "Lib") - - Returns: - Dict mapping module_name -> set of modules it imports - """ - lib_dir = pathlib.Path(lib_prefix) - if not lib_dir.exists(): - return {} - - import_graph: dict[str, set[str]] = {} - - # Scan all .py files in lib_prefix (excluding test/ directory for module imports) - for entry in lib_dir.iterdir(): - if entry.name.startswith(("_", ".")): - continue - if entry.name == "test": - continue - - module_name = None - if entry.is_file() and entry.suffix == ".py": - module_name = entry.stem - elif entry.is_dir() and (entry / "__init__.py").exists(): - module_name = entry.name - - if module_name: - # Parse imports from this module - imports = set() - for _, content in read_python_files(entry): - imports.update(parse_lib_imports(content)) - # Remove self-imports - imports.discard(module_name) - import_graph[module_name] = imports - - return import_graph - - -def _build_reverse_graph(import_graph: dict[str, set[str]]) -> dict[str, set[str]]: - """Build reverse dependency graph (who imports this module). - - Args: - import_graph: Forward import graph (module -> imports) - - Returns: - Reverse graph (module -> imported_by) - """ - reverse_graph: dict[str, set[str]] = {} - - for module, imports in import_graph.items(): - for imported in imports: - if imported not in reverse_graph: - reverse_graph[imported] = set() - reverse_graph[imported].add(module) - - return reverse_graph - - -@functools.cache -def get_transitive_imports( - module_name: str, - lib_prefix: str = "Lib", -) -> frozenset[str]: - """Get all modules that transitively depend on module_name. - - Args: - module_name: Target module - lib_prefix: RustPython Lib directory (default: "Lib") - - Returns: - Frozenset of module names that import module_name (directly or indirectly) - """ - import_graph = _build_import_graph(lib_prefix) - reverse_graph = _build_reverse_graph(import_graph) - - # BFS from module_name following reverse edges - visited: set[str] = set() - queue = deque(reverse_graph.get(module_name, set())) - - while queue: - current = queue.popleft() - if current in visited: - continue - visited.add(current) - # Add modules that import current module - for importer in reverse_graph.get(current, set()): - if importer not in visited: - queue.append(importer) - - return frozenset(visited) - - def _parse_test_submodule_imports(content: str) -> dict[str, set[str]]: """Parse 'from test.X import Y' to get submodule imports. @@ -629,56 +594,399 @@ def _parse_test_submodule_imports(content: str) -> dict[str, set[str]]: return result -def _build_test_import_graph(test_dir: pathlib.Path) -> dict[str, set[str]]: - """Build import graph for files within test directory (recursive). +_test_import_graph_cache: dict[str, tuple[dict[str, set[str]], dict[str, set[str]]]] = {} + + +def _is_standard_lib_path(path: str) -> bool: + """Check if path is the standard Lib directory (not a temp dir).""" + if "/tmp" in path.lower() or "/var/folders" in path.lower(): + return False + return path == "Lib/test" or path.endswith("/Lib/test") or path == "Lib" or path.endswith("/Lib") + + +def _build_test_import_graph( + test_dir: pathlib.Path, +) -> tuple[dict[str, set[str]], dict[str, set[str]]]: + """Build import graphs for files within test directory (recursive). + + Uses cross-process shelve cache based on CPython version. Args: test_dir: Path to Lib/test/ directory Returns: - Dict mapping relative path (without .py) -> set of test modules it imports + Tuple of: + - Dict mapping relative path (without .py) -> set of test modules it imports + - Dict mapping relative path (without .py) -> set of all lib imports """ + # In-process cache + cache_key = str(test_dir) + if cache_key in _test_import_graph_cache: + return _test_import_graph_cache[cache_key] + + # Cross-process cache (only for standard Lib/test directory) + use_file_cache = _is_standard_lib_path(cache_key) + if use_file_cache: + version = _get_cpython_version() + shelve_key = f"test_import_graph:{version}" + try: + with shelve.open(_get_cache_path()) as db: + if shelve_key in db: + import_graph, lib_imports_graph = db[shelve_key] + _test_import_graph_cache[cache_key] = (import_graph, lib_imports_graph) + return import_graph, lib_imports_graph + except Exception: + pass + + # Build from scratch import_graph: dict[str, set[str]] = {} + lib_imports_graph: dict[str, set[str]] = {} - # Use **/*.py to recursively find all Python files for py_file in test_dir.glob("**/*.py"): content = safe_read_text(py_file) if content is None: continue imports = set() - # Parse "from test import X" style imports imports.update(parse_test_imports(content)) - # Also check direct imports of test modules all_imports = parse_lib_imports(content) - # Check for files at same level or in test_dir for imp in all_imports: - # Check in same directory if (py_file.parent / f"{imp}.py").exists(): imports.add(imp) - # Check in test_dir root if (test_dir / f"{imp}.py").exists(): imports.add(imp) - # Handle "from test.X import Y" where Y is a file in test_dir/X/ submodule_imports = _parse_test_submodule_imports(content) for submodule, imported_names in submodule_imports.items(): submodule_dir = test_dir / submodule if submodule_dir.is_dir(): for name in imported_names: - # Check if it's a file in the submodule directory if (submodule_dir / f"{name}.py").exists(): imports.add(name) - # Use relative path from test_dir as key (without .py) rel_path = py_file.relative_to(test_dir) key = str(rel_path.with_suffix("")) import_graph[key] = imports + lib_imports_graph[key] = all_imports + + # Save to cross-process cache + if use_file_cache: + try: + with shelve.open(_get_cache_path()) as db: + db[shelve_key] = (import_graph, lib_imports_graph) + except Exception: + pass + _test_import_graph_cache[cache_key] = (import_graph, lib_imports_graph) + + return import_graph, lib_imports_graph + + +def _get_import_name(file_key: str) -> str: + """Get the import name from a file key. + + Args: + file_key: Relative path without .py (e.g., "test_foo", "test_bar/helper") + + Returns: + Import name (e.g., "test_foo", "helper", or parent dir name for __init__) + """ + path = pathlib.Path(file_key) + stem = path.name + if stem == "__init__": + return path.parent.name + return stem + + +_lib_import_graph_cache: dict[str, dict[str, set[str]]] = {} + + +def _build_lib_import_graph(lib_prefix: str = "Lib") -> dict[str, set[str]]: + """Build import graph for Lib modules (full module paths like urllib.request). + + Uses cross-process shelve cache based on CPython version. + + Args: + lib_prefix: RustPython Lib directory + + Returns: + Dict mapping full_module_path -> set of modules it imports + """ + # In-process cache + if lib_prefix in _lib_import_graph_cache: + return _lib_import_graph_cache[lib_prefix] + + # Cross-process cache (only for standard Lib directory) + use_file_cache = _is_standard_lib_path(lib_prefix) + if use_file_cache: + version = _get_cpython_version() + shelve_key = f"lib_import_graph:{version}" + try: + with shelve.open(_get_cache_path()) as db: + if shelve_key in db: + import_graph = db[shelve_key] + _lib_import_graph_cache[lib_prefix] = import_graph + return import_graph + except Exception: + pass + + # Build from scratch + lib_dir = pathlib.Path(lib_prefix) + if not lib_dir.exists(): + return {} + + import_graph: dict[str, set[str]] = {} + + for entry in lib_dir.iterdir(): + if entry.name.startswith(("_", ".")): + continue + if entry.name == "test": + continue + + if entry.is_file() and entry.suffix == ".py": + content = safe_read_text(entry) + if content: + imports = parse_lib_imports(content) + imports.discard(entry.stem) + import_graph[entry.stem] = imports + elif entry.is_dir() and (entry / "__init__.py").exists(): + for py_file in entry.glob("**/*.py"): + content = safe_read_text(py_file) + if content: + imports = parse_lib_imports(content) + rel_path = py_file.relative_to(lib_dir) + if rel_path.name == "__init__.py": + full_name = str(rel_path.parent).replace("/", ".") + else: + full_name = str(rel_path.with_suffix("")).replace("/", ".") + imports.discard(full_name.split(".")[0]) + import_graph[full_name] = imports + + # Save to cross-process cache + if use_file_cache: + try: + with shelve.open(_get_cache_path()) as db: + db[shelve_key] = import_graph + except Exception: + pass + _lib_import_graph_cache[lib_prefix] = import_graph return import_graph +def _get_lib_modules_importing( + module_name: str, lib_import_graph: dict[str, set[str]] +) -> set[str]: + """Find Lib modules (full paths) that import module_name or any of its submodules.""" + importers: set[str] = set() + target_top = module_name.split(".")[0] + + for full_path, imports in lib_import_graph.items(): + if full_path.split(".")[0] == target_top: + continue # Skip same package + # Match if module imports target OR any submodule of target + # e.g., for "xml": match imports of "xml", "xml.parsers", "xml.etree.ElementTree" + matches = any( + imp == module_name or imp.startswith(module_name + ".") + for imp in imports + ) + if matches: + importers.add(full_path) + + return importers + + +def _consolidate_submodules(modules: set[str], threshold: int = 3) -> dict[str, set[str]]: + """Consolidate submodules if count exceeds threshold. + + Args: + modules: Set of full module paths (e.g., {"urllib.request", "urllib.parse", "xml.dom", "xml.sax"}) + threshold: If submodules > threshold, consolidate to parent + + Returns: + Dict mapping display_name -> set of original module paths + e.g., {"urllib.request": {"urllib.request"}, "xml": {"xml.dom", "xml.sax", "xml.etree", "xml.parsers"}} + """ + # Group by top-level package + by_package: dict[str, set[str]] = {} + for mod in modules: + parts = mod.split(".") + top = parts[0] + if top not in by_package: + by_package[top] = set() + by_package[top].add(mod) + + result: dict[str, set[str]] = {} + for top, submods in by_package.items(): + if len(submods) > threshold: + # Consolidate to top-level + result[top] = submods + else: + # Keep individual + for mod in submods: + result[mod] = {mod} + + return result + + +# Modules that are used everywhere - show but don't expand their dependents +_BLOCKLIST_MODULES = frozenset({ + "unittest", + "test.support", + "support", + "doctest", + "typing", + "abc", + "collections.abc", + "functools", + "itertools", + "operator", + "contextlib", + "warnings", + "types", + "enum", + "re", + "io", + "os", + "sys", +}) + + +def find_dependent_tests_tree( + module_name: str, + lib_prefix: str = "Lib", + max_depth: int = 1, + _depth: int = 0, + _visited_tests: set[str] | None = None, + _visited_modules: set[str] | None = None, +) -> dict: + """Find dependent tests in a tree structure. + + Args: + module_name: Module to search for (e.g., "ftplib") + lib_prefix: RustPython Lib directory + max_depth: Maximum depth to recurse (default 1 = show direct + 1 level of Lib deps) + + Returns: + Dict with structure: + { + "module": "ftplib", + "tests": ["test_ftplib", "test_urllib2"], # Direct importers + "children": [ + {"module": "urllib.request", "tests": [...], "children": []}, + ... + ] + } + """ + lib_dir = pathlib.Path(lib_prefix) + test_dir = lib_dir / "test" + + if _visited_tests is None: + _visited_tests = set() + if _visited_modules is None: + _visited_modules = set() + + # Build graphs + test_import_graph, test_lib_imports = _build_test_import_graph(test_dir) + lib_import_graph = _build_lib_import_graph(lib_prefix) + + # Find tests that directly import this module + target_top = module_name.split(".")[0] + direct_tests: set[str] = set() + for file_key, imports in test_lib_imports.items(): + if file_key in _visited_tests: + continue + # Match exact module OR any child submodule + # e.g., "xml" matches imports of "xml", "xml.parsers", "xml.etree.ElementTree" + # but "collections._defaultdict" only matches "collections._defaultdict" (no children) + matches = any( + imp == module_name or imp.startswith(module_name + ".") + for imp in imports + ) + if matches: + # Check if it's a test file + if pathlib.Path(file_key).name.startswith("test_"): + direct_tests.add(file_key) + _visited_tests.add(file_key) + + # Consolidate test names (test_sqlite3/test_dbapi -> test_sqlite3) + consolidated_tests = {_consolidate_file_key(t) for t in direct_tests} + + # Mark this module as visited (cycle detection) + _visited_modules.add(module_name) + _visited_modules.add(target_top) + + children = [] + # Check blocklist and depth limit + should_expand = ( + _depth < max_depth + and module_name not in _BLOCKLIST_MODULES + and target_top not in _BLOCKLIST_MODULES + ) + + if should_expand: + # Find Lib modules that import this module + lib_importers = _get_lib_modules_importing(module_name, lib_import_graph) + + # Skip already visited modules (cycle detection) and blocklisted modules + lib_importers = { + m for m in lib_importers + if m not in _visited_modules + and m.split(".")[0] not in _visited_modules + and m not in _BLOCKLIST_MODULES + and m.split(".")[0] not in _BLOCKLIST_MODULES + } + + # Consolidate submodules (xml.dom, xml.sax, xml.etree -> xml if > 3) + consolidated_libs = _consolidate_submodules(lib_importers, threshold=3) + + # Build children + for display_name, original_mods in sorted(consolidated_libs.items()): + child = find_dependent_tests_tree( + display_name, + lib_prefix, + max_depth, + _depth + 1, + _visited_tests, + _visited_modules, + ) + if child["tests"] or child["children"]: + children.append(child) + + return { + "module": module_name, + "tests": sorted(consolidated_tests), + "children": children, + } + + +def _filter_test_files( + depth_map: dict[int, dict[str, list[str]]], +) -> dict[int, dict[str, list[str]]]: + """Filter to only include test_*.py files in the result.""" + filtered: dict[int, dict[str, list[str]]] = {} + for depth, files in depth_map.items(): + test_files = { + k: v for k, v in files.items() if pathlib.Path(k).name.startswith("test_") + } + if test_files: + filtered[depth] = test_files + return filtered + + +def _collect_test_file_keys_from_tree(tree: dict) -> set[str]: + """Recursively collect all test file_keys from a dependency tree.""" + # Note: tree["tests"] are already consolidated names, not file_keys + # We need the original file_keys for path construction + # This is used for backward compatibility + result: set[str] = set() + for test_name in tree.get("tests", []): + result.add(test_name) + for child in tree.get("children", []): + result.update(_collect_test_file_keys_from_tree(child)) + return result + + @functools.cache def find_tests_importing_module( module_name: str, @@ -693,7 +1001,7 @@ def find_tests_importing_module( Args: module_name: Module to search for (e.g., "datetime") lib_prefix: RustPython Lib directory (default: "Lib") - include_transitive: Whether to include transitive dependencies + include_transitive: Whether to include transitive dependencies within test/ Returns: Frozenset of test_*.py file paths that depend on this module @@ -704,56 +1012,56 @@ def find_tests_importing_module( if not test_dir.exists(): return frozenset() - # Build set of modules to search for (Lib/ modules) - target_modules = {module_name} - if include_transitive: - # Add all modules that transitively depend on module_name - target_modules.update(get_transitive_imports(module_name, lib_prefix)) + _, test_lib_imports = _build_test_import_graph(test_dir) - # Build test directory import graph for transitive analysis within test/ - test_import_graph = _build_test_import_graph(test_dir) - - # First pass: find all files (by relative path) that directly import target modules - directly_importing: set[str] = set() - for py_file in test_dir.glob("**/*.py"): # Recursive glob - content = safe_read_text(py_file) - if content is None: - continue - imports = parse_lib_imports(content) - if imports & target_modules: - rel_path = py_file.relative_to(test_dir) - directly_importing.add(str(rel_path.with_suffix(""))) - - # Second pass: find files that transitively import via support files within test/ - # BFS to find all files that import any file in all_importing - all_importing = set(directly_importing) - queue = deque(directly_importing) - while queue: - current = queue.popleft() - # Extract the filename (stem) from the relative path for matching - current_path = pathlib.Path(current) - current_stem = current_path.name - # For __init__.py, the import name is the parent directory name - # e.g., "test_json/__init__" -> can be imported as "test_json" - if current_stem == "__init__": - current_stem = current_path.parent.name - for file_key, imports in test_import_graph.items(): - if current_stem in imports and file_key not in all_importing: - all_importing.add(file_key) - queue.append(file_key) - - # Filter to only test_*.py files and build result paths + # Find tests that directly import module_name + target_top = module_name.split(".")[0] result: set[pathlib.Path] = set() - for file_key in all_importing: - # file_key is like "test_foo" or "test_bar/test_sub" - path_parts = pathlib.Path(file_key) - filename = path_parts.name # Get just the filename part - if filename.startswith("test_"): - result.add(test_dir / f"{file_key}.py") + + for file_key, imports in test_lib_imports.items(): + if module_name in imports or target_top in imports: + if pathlib.Path(file_key).name.startswith("test_"): + result.add(test_dir / f"{file_key}.py") + + if not include_transitive: + return frozenset(result) + + # For transitive, use the tree function + tree = find_dependent_tests_tree(module_name, lib_prefix, max_depth=10) + test_names = _collect_test_file_keys_from_tree(tree) + + # Convert test names back to paths (best effort) + for test_name in test_names: + # Try as file + path = test_dir / f"{test_name}.py" + if path.exists(): + result.add(path) + # Try as directory + dir_path = test_dir / test_name + if dir_path.is_dir(): + for py_file in dir_path.glob("test_*.py"): + result.add(py_file) return frozenset(result) +def _consolidate_file_key(file_key: str) -> str: + """Consolidate file_key to test name. + + Args: + file_key: Relative path without .py (e.g., "test_foo", "test_bar/test_sub") + + Returns: + Consolidated test name: + - "test_foo" for "test_foo" + - "test_sqlite3" for "test_sqlite3/test_dbapi" + """ + parts = pathlib.Path(file_key).parts + if len(parts) == 1: + return parts[0] + return parts[0] + + def consolidate_test_paths( test_paths: frozenset[pathlib.Path], test_dir: pathlib.Path, @@ -772,7 +1080,7 @@ def consolidate_test_paths( consolidated: set[str] = set() for path in test_paths: - try: + if path.is_relative_to(test_dir): rel_path = path.relative_to(test_dir) parts = rel_path.parts @@ -782,8 +1090,16 @@ def consolidate_test_paths( else: # test_sqlite3/test_dbapi.py -> test_sqlite3 consolidated.add(parts[0]) - except ValueError: + else: # Path not relative to test_dir, use stem consolidated.add(path.stem) return frozenset(consolidated) + + +def _collect_all_tests_from_tree(tree: dict) -> set[str]: + """Recursively collect all test names from a dependency tree.""" + tests = set(tree.get("tests", [])) + for child in tree.get("children", []): + tests.update(_collect_all_tests_from_tree(child)) + return tests diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index 974cd9e0529..79bb02d1ca5 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -160,8 +160,7 @@ def format_deps( """ from update_lib.deps import ( DEPENDENCIES, - consolidate_test_paths, - find_tests_importing_module, + find_dependent_tests_tree, get_lib_paths, get_test_paths, ) @@ -196,19 +195,50 @@ def format_deps( ) ) - # Show dependent tests (reverse dependencies) - impacted_tests = find_tests_importing_module(name, lib_prefix) - test_dir = pathlib.Path(lib_prefix) / "test" - consolidated = consolidate_test_paths(impacted_tests, test_dir) + # Show dependent tests as tree (depth 2: module + direct importers + their importers) + tree = find_dependent_tests_tree(name, lib_prefix=lib_prefix, max_depth=2) + lines.extend(_format_dependent_tests_tree(tree)) - if consolidated: - lines.append( - f"[+] dependent tests: ({len(consolidated)} tests depend on {name})" - ) - for test_name in sorted(consolidated): - lines.append(f" - {test_name}") - else: - lines.append(f"[+] dependent tests: (no tests depend on {name})") + return lines + + +def _format_dependent_tests_tree(tree: dict, indent: str = "") -> list[str]: + """Format dependent tests tree for display.""" + lines = [] + module = tree["module"] + tests = tree["tests"] + children = tree["children"] + + if indent == "": + # Root level + # Count total tests in tree + def count_tests(t: dict) -> int: + total = len(t.get("tests", [])) + for c in t.get("children", []): + total += count_tests(c) + return total + + total = count_tests(tree) + if total == 0 and not children: + lines.append(f"dependent tests: (no tests depend on {module})") + return lines + lines.append(f"dependent tests: ({total} tests)") + + # Format this node + if tests: + test_str = " ".join(tests) + if indent == "": + lines.append(f"- {module}: {test_str}") + else: + lines.append(f"{indent}- {module}: {test_str}") + elif indent != "" and children: + # Has children but no direct tests + lines.append(f"{indent}- {module}:") + + # Format children + child_indent = indent + " " if indent else " " + for child in children: + lines.extend(_format_dependent_tests_tree(child, child_indent)) return lines @@ -218,14 +248,8 @@ def show_deps( cpython_prefix: str = "cpython", lib_prefix: str = "Lib", max_depth: int = 10, - dependent_tests_only: bool = False, ) -> None: """Show all dependency information for modules.""" - from update_lib.deps import ( - consolidate_test_paths, - find_tests_importing_module, - ) - # Expand "all" to all module names expanded_names = [] for name in names: @@ -234,18 +258,6 @@ def show_deps( else: expanded_names.append(name) - # Handle dependent-tests-only mode: output only space-separated test names - if dependent_tests_only: - all_tests: set[str] = set() - for name in expanded_names: - impacted = find_tests_importing_module(name, lib_prefix) - test_dir = pathlib.Path(lib_prefix) / "test" - consolidated = consolidate_test_paths(impacted, test_dir) - all_tests.update(consolidated) - if all_tests: - print(" ".join(sorted(all_tests))) - return - # Shared visited set across all modules visited: set[str] = set() @@ -282,18 +294,11 @@ def main(argv: list[str] | None = None) -> int: default=10, help="Maximum recursion depth for soft_deps tree (default: 10)", ) - parser.add_argument( - "--dependent-tests-only", - action="store_true", - help="Output only dependent test names, space-separated (for use with python3 -m test)", - ) args = parser.parse_args(argv) try: - show_deps( - args.names, args.cpython, args.lib, args.depth, args.dependent_tests_only - ) + show_deps(args.names, args.cpython, args.lib, args.depth) return 0 except Exception as e: print(f"Error: {e}", file=sys.stderr) diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index f06d6f29317..128d169b74b 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -5,14 +5,15 @@ import unittest from update_lib.deps import ( + clear_import_graph_caches, consolidate_test_paths, + find_dependent_tests_tree, find_tests_importing_module, get_data_paths, get_lib_paths, get_soft_deps, get_test_dependencies, get_test_paths, - get_transitive_imports, parse_lib_imports, parse_test_imports, resolve_all_paths, @@ -247,7 +248,7 @@ def test_import_statement(self): import collections.abc """ imports = parse_lib_imports(code) - self.assertEqual(imports, {"os", "sys", "collections"}) + self.assertEqual(imports, {"os", "sys", "collections.abc"}) def test_from_import(self): """Test parsing 'from foo import bar'.""" @@ -257,7 +258,7 @@ def test_from_import(self): from typing import Optional """ imports = parse_lib_imports(code) - self.assertEqual(imports, {"os", "collections", "typing"}) + self.assertEqual(imports, {"os", "collections.abc", "typing"}) def test_mixed_imports(self): """Test mixed import styles.""" @@ -425,55 +426,6 @@ def test_nested_different(self): self.assertFalse(_dircmp_is_same(dcmp)) -class TestGetTransitiveImports(unittest.TestCase): - """Tests for get_transitive_imports function.""" - - def test_direct_dependency(self): - """A imports B → B's transitive importers include A.""" - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - lib_dir.mkdir() - - (lib_dir / "a.py").write_text("import b\n") - (lib_dir / "b.py").write_text("# b module") - - get_transitive_imports.cache_clear() - result = get_transitive_imports("b", lib_prefix=str(lib_dir)) - self.assertIn("a", result) - - def test_chain_dependency(self): - """A imports B, B imports C → C's transitive importers include A and B.""" - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - lib_dir.mkdir() - - (lib_dir / "a.py").write_text("import b\n") - (lib_dir / "b.py").write_text("import c\n") - (lib_dir / "c.py").write_text("# c module") - - get_transitive_imports.cache_clear() - result = get_transitive_imports("c", lib_prefix=str(lib_dir)) - self.assertIn("a", result) - self.assertIn("b", result) - - def test_cycle_handling(self): - """Handle circular imports without infinite loop.""" - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - lib_dir.mkdir() - - (lib_dir / "a.py").write_text("import b\n") - (lib_dir / "b.py").write_text("import a\n") # cycle - - get_transitive_imports.cache_clear() - # Should not hang or raise - result = get_transitive_imports("a", lib_prefix=str(lib_dir)) - self.assertIn("b", result) - - class TestFindTestsImportingModule(unittest.TestCase): """Tests for find_tests_importing_module function.""" @@ -491,7 +443,7 @@ def test_direct_import(self): # Create test that imports bar (test_dir / "test_foo.py").write_text("import bar\n") - get_transitive_imports.cache_clear() + clear_import_graph_caches() find_tests_importing_module.cache_clear() result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) self.assertIn(test_dir / "test_foo.py", result) @@ -507,35 +459,12 @@ def test_includes_test_module_itself(self): (lib_dir / "bar.py").write_text("# bar module") (test_dir / "test_bar.py").write_text("import bar\n") - get_transitive_imports.cache_clear() + clear_import_graph_caches() find_tests_importing_module.cache_clear() result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) # test_bar.py IS now included (module's own test is part of impact) self.assertIn(test_dir / "test_bar.py", result) - def test_transitive_import(self): - """Test finding tests with transitive (indirect) imports.""" - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_dir.mkdir(parents=True) - - # bar.py (target module) - (lib_dir / "bar.py").write_text("# bar module") - - # baz.py imports bar - (lib_dir / "baz.py").write_text("import bar\n") - - # test_foo.py imports baz (not bar directly) - (test_dir / "test_foo.py").write_text("import baz\n") - - get_transitive_imports.cache_clear() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - # test_foo.py should be found via transitive dependency - self.assertIn(test_dir / "test_foo.py", result) - def test_empty_when_no_importers(self): """Test returns empty when no tests import the module.""" with tempfile.TemporaryDirectory() as tmpdir: @@ -547,7 +476,7 @@ def test_empty_when_no_importers(self): (lib_dir / "bar.py").write_text("# bar module") (test_dir / "test_unrelated.py").write_text("import os\n") - get_transitive_imports.cache_clear() + clear_import_graph_caches() find_tests_importing_module.cache_clear() result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) self.assertEqual(result, frozenset()) @@ -577,7 +506,7 @@ def test_support_file_not_in_output(self): # test_foo.py also imports bar (test_dir / "test_foo.py").write_text("import bar\n") - get_transitive_imports.cache_clear() + clear_import_graph_caches() find_tests_importing_module.cache_clear() result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) @@ -586,65 +515,6 @@ def test_support_file_not_in_output(self): # helper.py should be excluded self.assertNotIn(test_dir / "helper.py", result) - def test_transitive_via_support_file(self): - """Test file importing support file that imports target should be included.""" - # Given: - # bar.py (target module in Lib/) - # helper.py (support file in test/, imports bar) - # test_foo.py (test file, imports helper - NOT bar directly) - # When: find_tests_importing_module("bar") - # Then: test_foo.py IS included (via helper.py), helper.py is NOT - - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - # helper.py imports bar - (test_dir / "helper.py").write_text("import bar\n") - # test_foo.py imports only helper (not bar directly) - (test_dir / "test_foo.py").write_text("from test import helper\n") - - get_transitive_imports.cache_clear() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - - # test_foo.py depends on bar via helper, so it should be included - self.assertIn(test_dir / "test_foo.py", result) - # helper.py should be excluded from output - self.assertNotIn(test_dir / "helper.py", result) - - def test_chain_through_multiple_support_files(self): - """Test transitive chain through multiple support files.""" - # Given: - # bar.py (target) - # helper_a.py imports bar - # helper_b.py imports helper_a - # test_foo.py imports helper_b - # Then: test_foo.py IS included, helper_a/b are NOT - - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - (test_dir / "helper_a.py").write_text("import bar\n") - (test_dir / "helper_b.py").write_text("from test import helper_a\n") - (test_dir / "test_foo.py").write_text("from test import helper_b\n") - - get_transitive_imports.cache_clear() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - - self.assertIn(test_dir / "test_foo.py", result) - self.assertNotIn(test_dir / "helper_a.py", result) - self.assertNotIn(test_dir / "helper_b.py", result) - - class TestFindTestsInModuleDirectories(unittest.TestCase): """Tests for finding tests inside test_*/ module directories.""" @@ -669,47 +539,13 @@ def test_finds_test_in_module_directory(self): (test_bar_dir / "__init__.py").write_text("") (test_bar_dir / "test_sub.py").write_text("import bar\n") - get_transitive_imports.cache_clear() + clear_import_graph_caches() find_tests_importing_module.cache_clear() result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) # test_bar/test_sub.py should be in results self.assertIn(test_bar_dir / "test_sub.py", result) - def test_finds_nested_test_via_support_in_module_directory(self): - """Transitive deps through support files in module directories.""" - # Given: - # bar.py (target) - # test_bar/ - # __init__.py - # helper.py (imports bar) - # test_sub.py (imports helper via "from test.test_bar import helper") - # When: find_tests_importing_module("bar") - # Then: test_bar/test_sub.py IS included, helper.py is NOT - - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_bar_dir = test_dir / "test_bar" - test_bar_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - (test_bar_dir / "__init__.py").write_text("") - (test_bar_dir / "helper.py").write_text("import bar\n") - (test_bar_dir / "test_sub.py").write_text( - "from test.test_bar import helper\n" - ) - - get_transitive_imports.cache_clear() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - - # test_sub.py should be included (via helper) - self.assertIn(test_bar_dir / "test_sub.py", result) - # helper.py should NOT be in results (not a test file) - self.assertNotIn(test_bar_dir / "helper.py", result) - def test_both_top_level_and_module_directory_tests_found(self): """Both top-level test_*.py and test_*/test_*.py should be found.""" # Given: @@ -732,7 +568,7 @@ def test_both_top_level_and_module_directory_tests_found(self): (test_bar_dir / "__init__.py").write_text("") (test_bar_dir / "test_sub.py").write_text("import bar\n") - get_transitive_imports.cache_clear() + clear_import_graph_caches() find_tests_importing_module.cache_clear() result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) From 1cb5cc959dc01e3c099b415f12bc959b34697e18 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 22 Jan 2026 15:57:06 +0000 Subject: [PATCH 12/17] Auto-format: ruff format --- scripts/update_lib/deps.py | 74 ++++++++++++++++----------- scripts/update_lib/tests/test_deps.py | 1 + 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 5be80e439df..7147a38f79e 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -20,6 +20,7 @@ # === Cross-process cache using shelve === + def _get_cpython_version(cpython_prefix: str = "cpython") -> str: """Get CPython version from git tag for cache namespace.""" try: @@ -50,6 +51,8 @@ def clear_import_graph_caches() -> None: globals()["_test_import_graph_cache"].clear() if "_lib_import_graph_cache" in globals(): globals()["_lib_import_graph_cache"].clear() + + from update_lib.path import construct_lib_path, resolve_module_path # Manual dependency table for irregular cases @@ -594,14 +597,21 @@ def _parse_test_submodule_imports(content: str) -> dict[str, set[str]]: return result -_test_import_graph_cache: dict[str, tuple[dict[str, set[str]], dict[str, set[str]]]] = {} +_test_import_graph_cache: dict[ + str, tuple[dict[str, set[str]], dict[str, set[str]]] +] = {} def _is_standard_lib_path(path: str) -> bool: """Check if path is the standard Lib directory (not a temp dir).""" if "/tmp" in path.lower() or "/var/folders" in path.lower(): return False - return path == "Lib/test" or path.endswith("/Lib/test") or path == "Lib" or path.endswith("/Lib") + return ( + path == "Lib/test" + or path.endswith("/Lib/test") + or path == "Lib" + or path.endswith("/Lib") + ) def _build_test_import_graph( @@ -633,7 +643,10 @@ def _build_test_import_graph( with shelve.open(_get_cache_path()) as db: if shelve_key in db: import_graph, lib_imports_graph = db[shelve_key] - _test_import_graph_cache[cache_key] = (import_graph, lib_imports_graph) + _test_import_graph_cache[cache_key] = ( + import_graph, + lib_imports_graph, + ) return import_graph, lib_imports_graph except Exception: pass @@ -787,8 +800,7 @@ def _get_lib_modules_importing( # Match if module imports target OR any submodule of target # e.g., for "xml": match imports of "xml", "xml.parsers", "xml.etree.ElementTree" matches = any( - imp == module_name or imp.startswith(module_name + ".") - for imp in imports + imp == module_name or imp.startswith(module_name + ".") for imp in imports ) if matches: importers.add(full_path) @@ -796,7 +808,9 @@ def _get_lib_modules_importing( return importers -def _consolidate_submodules(modules: set[str], threshold: int = 3) -> dict[str, set[str]]: +def _consolidate_submodules( + modules: set[str], threshold: int = 3 +) -> dict[str, set[str]]: """Consolidate submodules if count exceeds threshold. Args: @@ -830,26 +844,28 @@ def _consolidate_submodules(modules: set[str], threshold: int = 3) -> dict[str, # Modules that are used everywhere - show but don't expand their dependents -_BLOCKLIST_MODULES = frozenset({ - "unittest", - "test.support", - "support", - "doctest", - "typing", - "abc", - "collections.abc", - "functools", - "itertools", - "operator", - "contextlib", - "warnings", - "types", - "enum", - "re", - "io", - "os", - "sys", -}) +_BLOCKLIST_MODULES = frozenset( + { + "unittest", + "test.support", + "support", + "doctest", + "typing", + "abc", + "collections.abc", + "functools", + "itertools", + "operator", + "contextlib", + "warnings", + "types", + "enum", + "re", + "io", + "os", + "sys", + } +) def find_dependent_tests_tree( @@ -900,8 +916,7 @@ def find_dependent_tests_tree( # e.g., "xml" matches imports of "xml", "xml.parsers", "xml.etree.ElementTree" # but "collections._defaultdict" only matches "collections._defaultdict" (no children) matches = any( - imp == module_name or imp.startswith(module_name + ".") - for imp in imports + imp == module_name or imp.startswith(module_name + ".") for imp in imports ) if matches: # Check if it's a test file @@ -930,7 +945,8 @@ def find_dependent_tests_tree( # Skip already visited modules (cycle detection) and blocklisted modules lib_importers = { - m for m in lib_importers + m + for m in lib_importers if m not in _visited_modules and m.split(".")[0] not in _visited_modules and m not in _BLOCKLIST_MODULES diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index 128d169b74b..240b7e2865f 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -515,6 +515,7 @@ def test_support_file_not_in_output(self): # helper.py should be excluded self.assertNotIn(test_dir / "helper.py", result) + class TestFindTestsInModuleDirectories(unittest.TestCase): """Tests for finding tests inside test_*/ module directories.""" From 124ca348e853b105bbd73fe57e8756527c547b9d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 22 Jan 2026 15:57:06 +0000 Subject: [PATCH 13/17] Auto-format: ruff check --select I --fix --- scripts/update_lib/deps.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 7147a38f79e..a7d7d9fc2de 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -17,7 +17,6 @@ from update_lib.io_utils import read_python_files, safe_parse_ast, safe_read_text - # === Cross-process cache using shelve === From 3b782b8b9f11f2fd0917d65744268c21d56387c9 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 23 Jan 2026 00:58:53 +0900 Subject: [PATCH 14/17] sync marker --- scripts/update_lib/.gitignore | 1 + scripts/update_lib/deps.py | 7 +++---- scripts/update_lib/show_deps.py | 27 +++++++++++++++++++-------- 3 files changed, 23 insertions(+), 12 deletions(-) create mode 100644 scripts/update_lib/.gitignore diff --git a/scripts/update_lib/.gitignore b/scripts/update_lib/.gitignore new file mode 100644 index 00000000000..ceddaa37f12 --- /dev/null +++ b/scripts/update_lib/.gitignore @@ -0,0 +1 @@ +.cache/ diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index a7d7d9fc2de..a04405fd6e2 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -13,7 +13,6 @@ import re import shelve import subprocess -from collections import deque from update_lib.io_utils import read_python_files, safe_parse_ast, safe_read_text @@ -38,14 +37,13 @@ def _get_cpython_version(cpython_prefix: str = "cpython") -> str: def _get_cache_path() -> str: """Get cache file path (without extension - shelve adds its own).""" - cache_dir = pathlib.Path.home() / ".cache" / "rustpython-update-lib" + cache_dir = pathlib.Path(__file__).parent / ".cache" cache_dir.mkdir(parents=True, exist_ok=True) return str(cache_dir / "import_graph_cache") def clear_import_graph_caches() -> None: """Clear in-process import graph caches (for testing).""" - global _test_import_graph_cache, _lib_import_graph_cache if "_test_import_graph_cache" in globals(): globals()["_test_import_graph_cache"].clear() if "_lib_import_graph_cache" in globals(): @@ -328,7 +326,8 @@ def parse_lib_imports(content: str) -> set[str]: Returns: Set of imported module names (full paths) """ - content = _extract_top_level_code(content) + # Note: Don't truncate content here - some stdlib files have imports after + # the first def/class (e.g., _pydecimal.py has `import contextvars` at line 343) imports = set() # Match "import foo.bar" at line start diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index 79bb02d1ca5..f3e70277c8a 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -186,9 +186,9 @@ def format_deps( dep_info = DEPENDENCIES.get(name, {}) hard_deps = dep_info.get("hard_deps", []) if hard_deps: - lines.append(f"hard_deps: {hard_deps}") + lines.append(f"packages: {hard_deps}") - lines.append("soft_deps:") + lines.append("dependencies:") lines.extend( format_deps_tree( cpython_prefix, lib_prefix, max_depth, soft_deps={name}, _visited=_visited @@ -197,13 +197,20 @@ def format_deps( # Show dependent tests as tree (depth 2: module + direct importers + their importers) tree = find_dependent_tests_tree(name, lib_prefix=lib_prefix, max_depth=2) - lines.extend(_format_dependent_tests_tree(tree)) + lines.extend(_format_dependent_tests_tree(tree, cpython_prefix, lib_prefix)) return lines -def _format_dependent_tests_tree(tree: dict, indent: str = "") -> list[str]: +def _format_dependent_tests_tree( + tree: dict, + cpython_prefix: str = "cpython", + lib_prefix: str = "Lib", + indent: str = "", +) -> list[str]: """Format dependent tests tree for display.""" + from update_lib.deps import is_up_to_date + lines = [] module = tree["module"] tests = tree["tests"] @@ -224,21 +231,25 @@ def count_tests(t: dict) -> int: return lines lines.append(f"dependent tests: ({total} tests)") + # Check if module is up-to-date + synced = is_up_to_date(module.split(".")[0], cpython_prefix, lib_prefix) + marker = "[x]" if synced else "[ ]" + # Format this node if tests: test_str = " ".join(tests) if indent == "": - lines.append(f"- {module}: {test_str}") + lines.append(f"- {marker} {module}: {test_str}") else: - lines.append(f"{indent}- {module}: {test_str}") + lines.append(f"{indent}- {marker} {module}: {test_str}") elif indent != "" and children: # Has children but no direct tests - lines.append(f"{indent}- {module}:") + lines.append(f"{indent}- {marker} {module}:") # Format children child_indent = indent + " " if indent else " " for child in children: - lines.extend(_format_dependent_tests_tree(child, child_indent)) + lines.extend(_format_dependent_tests_tree(child, cpython_prefix, lib_prefix, child_indent)) return lines From 6c44b6c7d7d45c75dc9aa51a7e71548482dd187f Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 23 Jan 2026 02:20:38 +0900 Subject: [PATCH 15/17] ci test in rust test --- .github/workflows/ci.yaml | 11 ++++++----- .github/workflows/lib-deps-check.yaml | 8 -------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 720cd53d3bc..83c81c7eaa4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -173,6 +173,12 @@ jobs: cargo run --manifest-path example_projects/frozen_stdlib/Cargo.toml if: runner.os == 'Linux' + - name: run update_lib tests + run: cargo run -- -m unittest discover -s scripts/update_lib/tests -v + env: + PYTHONPATH: scripts + if: runner.os == 'Linux' + - name: prepare Intel MacOS build uses: dtolnay/rust-toolchain@stable with: @@ -463,11 +469,6 @@ jobs: - run: ruff format --check - - name: run update_lib tests - run: cargo run -- -m unittest discover -s scripts/update_lib/tests -v - env: - PYTHONPATH: scripts - - name: install prettier run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH diff --git a/.github/workflows/lib-deps-check.yaml b/.github/workflows/lib-deps-check.yaml index 27b3dec3620..8749f5fc2e7 100644 --- a/.github/workflows/lib-deps-check.yaml +++ b/.github/workflows/lib-deps-check.yaml @@ -99,19 +99,11 @@ jobs: The following Lib/ modules were modified. Here are their dependencies: -
- Click to expand dependency information - - ``` ${{ steps.deps-check.outputs.deps_output }} - ``` - -
**Legend:** - `[+]` path exists in CPython - `[x]` up-to-date, `[ ]` outdated - - `native:` Rust/C extension modules - name: Remove comment if no Lib changes if: steps.changed-files.outputs.modules == '' From 184abf146a4102d76106f3c088b7e992f6554924 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 23 Jan 2026 02:44:15 +0900 Subject: [PATCH 16/17] remove unused --- scripts/update_lib/deps.py | 200 -------------------- scripts/update_lib/tests/test_deps.py | 257 -------------------------- 2 files changed, 457 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index a04405fd6e2..089cc143c78 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -236,26 +236,6 @@ def get_test_paths( return (resolve_module_path(f"test/test_{name}", cpython_prefix, prefer="dir"),) -@functools.cache -def get_data_paths( - name: str, cpython_prefix: str = "cpython" -) -> tuple[pathlib.Path, ...]: - """Get additional data paths for a module. - - Args: - name: Module name - cpython_prefix: CPython directory prefix - - Returns: - Tuple of data paths (may be empty) - """ - if name in DEPENDENCIES and "data" in DEPENDENCIES[name]: - return tuple( - construct_lib_path(cpython_prefix, p) for p in DEPENDENCIES[name]["data"] - ) - return () - - def _extract_top_level_code(content: str) -> str: """Extract only top-level code from Python content for faster parsing. @@ -529,42 +509,6 @@ def get_test_dependencies( return result -def resolve_all_paths( - name: str, - cpython_prefix: str = "cpython", - include_deps: bool = True, -) -> dict[str, list[pathlib.Path]]: - """Resolve all paths for a module update. - - Args: - name: Module name - cpython_prefix: CPython directory prefix - include_deps: Whether to include auto-detected dependencies - - Returns: - Dict with "lib", "test", "data", "test_deps" keys - """ - result = { - "lib": list(get_lib_paths(name, cpython_prefix)), - "test": list(get_test_paths(name, cpython_prefix)), - "data": list(get_data_paths(name, cpython_prefix)), - "test_deps": [], - } - - if include_deps: - # Auto-detect test dependencies - for test_path in result["test"]: - deps = get_test_dependencies(test_path) - for dep_path in deps["hard_deps"]: - if dep_path not in result["test_deps"]: - result["test_deps"].append(dep_path) - for data_path in deps["data"]: - if data_path not in result["data"]: - result["data"].append(data_path) - - return result - - def _parse_test_submodule_imports(content: str) -> dict[str, set[str]]: """Parse 'from test.X import Y' to get submodule imports. @@ -693,22 +637,6 @@ def _build_test_import_graph( return import_graph, lib_imports_graph -def _get_import_name(file_key: str) -> str: - """Get the import name from a file key. - - Args: - file_key: Relative path without .py (e.g., "test_foo", "test_bar/helper") - - Returns: - Import name (e.g., "test_foo", "helper", or parent dir name for __init__) - """ - path = pathlib.Path(file_key) - stem = path.name - if stem == "__init__": - return path.parent.name - return stem - - _lib_import_graph_cache: dict[str, dict[str, set[str]]] = {} @@ -974,91 +902,6 @@ def find_dependent_tests_tree( } -def _filter_test_files( - depth_map: dict[int, dict[str, list[str]]], -) -> dict[int, dict[str, list[str]]]: - """Filter to only include test_*.py files in the result.""" - filtered: dict[int, dict[str, list[str]]] = {} - for depth, files in depth_map.items(): - test_files = { - k: v for k, v in files.items() if pathlib.Path(k).name.startswith("test_") - } - if test_files: - filtered[depth] = test_files - return filtered - - -def _collect_test_file_keys_from_tree(tree: dict) -> set[str]: - """Recursively collect all test file_keys from a dependency tree.""" - # Note: tree["tests"] are already consolidated names, not file_keys - # We need the original file_keys for path construction - # This is used for backward compatibility - result: set[str] = set() - for test_name in tree.get("tests", []): - result.add(test_name) - for child in tree.get("children", []): - result.update(_collect_test_file_keys_from_tree(child)) - return result - - -@functools.cache -def find_tests_importing_module( - module_name: str, - lib_prefix: str = "Lib", - include_transitive: bool = True, -) -> frozenset[pathlib.Path]: - """Find all test files that import the given module (directly or transitively). - - Only returns test_*.py files. Support files (like pickletester.py, string_tests.py) - are used for transitive dependency calculation but not included in the result. - - Args: - module_name: Module to search for (e.g., "datetime") - lib_prefix: RustPython Lib directory (default: "Lib") - include_transitive: Whether to include transitive dependencies within test/ - - Returns: - Frozenset of test_*.py file paths that depend on this module - """ - lib_dir = pathlib.Path(lib_prefix) - test_dir = lib_dir / "test" - - if not test_dir.exists(): - return frozenset() - - _, test_lib_imports = _build_test_import_graph(test_dir) - - # Find tests that directly import module_name - target_top = module_name.split(".")[0] - result: set[pathlib.Path] = set() - - for file_key, imports in test_lib_imports.items(): - if module_name in imports or target_top in imports: - if pathlib.Path(file_key).name.startswith("test_"): - result.add(test_dir / f"{file_key}.py") - - if not include_transitive: - return frozenset(result) - - # For transitive, use the tree function - tree = find_dependent_tests_tree(module_name, lib_prefix, max_depth=10) - test_names = _collect_test_file_keys_from_tree(tree) - - # Convert test names back to paths (best effort) - for test_name in test_names: - # Try as file - path = test_dir / f"{test_name}.py" - if path.exists(): - result.add(path) - # Try as directory - dir_path = test_dir / test_name - if dir_path.is_dir(): - for py_file in dir_path.glob("test_*.py"): - result.add(py_file) - - return frozenset(result) - - def _consolidate_file_key(file_key: str) -> str: """Consolidate file_key to test name. @@ -1074,46 +917,3 @@ def _consolidate_file_key(file_key: str) -> str: if len(parts) == 1: return parts[0] return parts[0] - - -def consolidate_test_paths( - test_paths: frozenset[pathlib.Path], - test_dir: pathlib.Path, -) -> frozenset[str]: - """Consolidate test paths by grouping test_*/ directory contents into a single entry. - - Args: - test_paths: Frozenset of absolute paths to test files - test_dir: Path to the test directory (e.g., Lib/test) - - Returns: - Frozenset of consolidated test names: - - "test_foo" for Lib/test/test_foo.py - - "test_sqlite3" for any file in Lib/test/test_sqlite3/ - """ - consolidated: set[str] = set() - - for path in test_paths: - if path.is_relative_to(test_dir): - rel_path = path.relative_to(test_dir) - parts = rel_path.parts - - if len(parts) == 1: - # test_foo.py -> test_foo - consolidated.add(rel_path.stem) - else: - # test_sqlite3/test_dbapi.py -> test_sqlite3 - consolidated.add(parts[0]) - else: - # Path not relative to test_dir, use stem - consolidated.add(path.stem) - - return frozenset(consolidated) - - -def _collect_all_tests_from_tree(tree: dict) -> set[str]: - """Recursively collect all test names from a dependency tree.""" - tests = set(tree.get("tests", [])) - for child in tree.get("children", []): - tests.update(_collect_all_tests_from_tree(child)) - return tests diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index 240b7e2865f..4555843e1f8 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -5,18 +5,12 @@ import unittest from update_lib.deps import ( - clear_import_graph_caches, - consolidate_test_paths, - find_dependent_tests_tree, - find_tests_importing_module, - get_data_paths, get_lib_paths, get_soft_deps, get_test_dependencies, get_test_paths, parse_lib_imports, parse_test_imports, - resolve_all_paths, ) @@ -132,21 +126,6 @@ def test_default_file(self): self.assertEqual(paths, (tmpdir / "Lib" / "test" / "test_foo.py",)) -class TestGetDataPaths(unittest.TestCase): - """Tests for get_data_paths function.""" - - def test_known_data(self): - """Test module with known data paths.""" - paths = get_data_paths("regrtest", "cpython") - self.assertEqual(len(paths), 1) - self.assertEqual(paths[0], pathlib.Path("cpython/Lib/test/regrtestdata")) - - def test_no_data(self): - """Test module without data paths.""" - paths = get_data_paths("datetime", "cpython") - self.assertEqual(paths, ()) - - class TestGetTestDependencies(unittest.TestCase): """Tests for get_test_dependencies function.""" @@ -215,28 +194,6 @@ class TestKR: self.assertEqual(result["data"][0], test_dir / "cjkencodings") -class TestResolveAllPaths(unittest.TestCase): - """Tests for resolve_all_paths function.""" - - def test_datetime(self): - """Test resolving datetime module.""" - result = resolve_all_paths("datetime", include_deps=False) - self.assertEqual(len(result["lib"]), 2) - self.assertIn(pathlib.Path("cpython/Lib/datetime.py"), result["lib"]) - self.assertIn(pathlib.Path("cpython/Lib/_pydatetime.py"), result["lib"]) - - def test_regrtest(self): - """Test resolving regrtest module.""" - result = resolve_all_paths("regrtest", include_deps=False) - self.assertEqual(result["lib"], [pathlib.Path("cpython/Lib/test/libregrtest")]) - self.assertEqual( - result["test"], [pathlib.Path("cpython/Lib/test/test_regrtest")] - ) - self.assertEqual( - result["data"], [pathlib.Path("cpython/Lib/test/regrtestdata")] - ) - - class TestParseLibImports(unittest.TestCase): """Tests for parse_lib_imports function.""" @@ -426,219 +383,5 @@ def test_nested_different(self): self.assertFalse(_dircmp_is_same(dcmp)) -class TestFindTestsImportingModule(unittest.TestCase): - """Tests for find_tests_importing_module function.""" - - def test_direct_import(self): - """Test finding tests that directly import a module.""" - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_dir.mkdir(parents=True) - - # Create target module - (lib_dir / "bar.py").write_text("# bar module") - - # Create test that imports bar - (test_dir / "test_foo.py").write_text("import bar\n") - - clear_import_graph_caches() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - self.assertIn(test_dir / "test_foo.py", result) - - def test_includes_test_module_itself(self): - """Test that test_.py IS included in results.""" - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - (test_dir / "test_bar.py").write_text("import bar\n") - - clear_import_graph_caches() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - # test_bar.py IS now included (module's own test is part of impact) - self.assertIn(test_dir / "test_bar.py", result) - - def test_empty_when_no_importers(self): - """Test returns empty when no tests import the module.""" - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - (test_dir / "test_unrelated.py").write_text("import os\n") - - clear_import_graph_caches() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - self.assertEqual(result, frozenset()) - - -class TestFindTestsOnlyTestFiles(unittest.TestCase): - """Tests for filtering to only test_*.py files in output.""" - - def test_support_file_not_in_output(self): - """Support files should not appear in output even if they import target.""" - # Given: - # bar.py (target module in Lib/) - # helper.py (support file in test/, imports bar) - # test_foo.py (test file, imports bar) - # When: find_tests_importing_module("bar") - # Then: test_foo.py is included, helper.py is NOT included - - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - # helper.py imports bar directly but doesn't start with test_ - (test_dir / "helper.py").write_text("import bar\n") - # test_foo.py also imports bar - (test_dir / "test_foo.py").write_text("import bar\n") - - clear_import_graph_caches() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - - # Only test_foo.py should be in results - self.assertIn(test_dir / "test_foo.py", result) - # helper.py should be excluded - self.assertNotIn(test_dir / "helper.py", result) - - -class TestFindTestsInModuleDirectories(unittest.TestCase): - """Tests for finding tests inside test_*/ module directories.""" - - def test_finds_test_in_module_directory(self): - """Test files inside test_*/ directories should be found.""" - # Given: - # bar.py (target module in Lib/) - # test_bar/ - # __init__.py - # test_sub.py (imports bar) - # When: find_tests_importing_module("bar") - # Then: test_bar/test_sub.py IS included - - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_bar_dir = test_dir / "test_bar" - test_bar_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - (test_bar_dir / "__init__.py").write_text("") - (test_bar_dir / "test_sub.py").write_text("import bar\n") - - clear_import_graph_caches() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - - # test_bar/test_sub.py should be in results - self.assertIn(test_bar_dir / "test_sub.py", result) - - def test_both_top_level_and_module_directory_tests_found(self): - """Both top-level test_*.py and test_*/test_*.py should be found.""" - # Given: - # bar.py (target) - # test_bar.py (top-level, imports bar) - # test_bar/ - # test_sub.py (imports bar) - # When: find_tests_importing_module("bar") - # Then: BOTH test_bar.py AND test_bar/test_sub.py are included - - with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) - lib_dir = tmpdir / "Lib" - test_dir = lib_dir / "test" - test_bar_dir = test_dir / "test_bar" - test_bar_dir.mkdir(parents=True) - - (lib_dir / "bar.py").write_text("# bar module") - (test_dir / "test_bar.py").write_text("import bar\n") - (test_bar_dir / "__init__.py").write_text("") - (test_bar_dir / "test_sub.py").write_text("import bar\n") - - clear_import_graph_caches() - find_tests_importing_module.cache_clear() - result = find_tests_importing_module("bar", lib_prefix=str(lib_dir)) - - # Both should be included - self.assertIn(test_dir / "test_bar.py", result) - self.assertIn(test_bar_dir / "test_sub.py", result) - - -class TestConsolidateTestPaths(unittest.TestCase): - """Tests for consolidate_test_paths function.""" - - def test_top_level_test_file(self): - """Top-level test_*.py -> test_* (without .py).""" - with tempfile.TemporaryDirectory() as tmpdir: - test_dir = pathlib.Path(tmpdir) - test_file = test_dir / "test_foo.py" - test_file.write_text("# test") - - result = consolidate_test_paths(frozenset({test_file}), test_dir) - self.assertEqual(result, frozenset({"test_foo"})) - - def test_module_directory_tests_consolidated(self): - """Multiple files in test_*/ directory -> single directory name.""" - with tempfile.TemporaryDirectory() as tmpdir: - test_dir = pathlib.Path(tmpdir) - module_dir = test_dir / "test_sqlite3" - module_dir.mkdir() - (module_dir / "test_dbapi.py").write_text("# test") - (module_dir / "test_backup.py").write_text("# test") - - result = consolidate_test_paths( - frozenset( - {module_dir / "test_dbapi.py", module_dir / "test_backup.py"} - ), - test_dir, - ) - self.assertEqual(result, frozenset({"test_sqlite3"})) - - def test_mixed_top_level_and_module_directory(self): - """Both top-level and module directory tests handled correctly.""" - with tempfile.TemporaryDirectory() as tmpdir: - test_dir = pathlib.Path(tmpdir) - # Top-level test - (test_dir / "test_foo.py").write_text("# test") - # Module directory tests - module_dir = test_dir / "test_sqlite3" - module_dir.mkdir() - (module_dir / "test_dbapi.py").write_text("# test") - (module_dir / "test_backup.py").write_text("# test") - - result = consolidate_test_paths( - frozenset( - { - test_dir / "test_foo.py", - module_dir / "test_dbapi.py", - module_dir / "test_backup.py", - } - ), - test_dir, - ) - self.assertEqual(result, frozenset({"test_foo", "test_sqlite3"})) - - def test_empty_input(self): - """Empty input -> empty frozenset.""" - with tempfile.TemporaryDirectory() as tmpdir: - test_dir = pathlib.Path(tmpdir) - result = consolidate_test_paths(frozenset(), test_dir) - self.assertEqual(result, frozenset()) - - if __name__ == "__main__": unittest.main() From 39cabdd064c10a8506e5c059e89495ed7494e2f3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 22 Jan 2026 17:46:02 +0000 Subject: [PATCH 17/17] Auto-format: ruff format --- scripts/update_lib/show_deps.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index f3e70277c8a..ae23ced3ead 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -249,7 +249,11 @@ def count_tests(t: dict) -> int: # Format children child_indent = indent + " " if indent else " " for child in children: - lines.extend(_format_dependent_tests_tree(child, cpython_prefix, lib_prefix, child_indent)) + lines.extend( + _format_dependent_tests_tree( + child, cpython_prefix, lib_prefix, child_indent + ) + ) return lines