From 6d726d421f56610e6a61856cb0f4692d30096b75 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Fri, 16 Jan 2026 20:14:55 +0900 Subject: [PATCH 1/5] Insert unittest import line if it doesn't exist --- scripts/lib_updater.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index 4e2f7999ec1..dfb5896e578 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -270,11 +270,41 @@ def {test_name}(self): yield (lineno, textwrap.indent(patch_lines, DEFAULT_INDENT)) +def has_unittest_import(tree: ast.Module) -> bool: + """Check if 'import unittest' is already present in the file.""" + for node in tree.body: + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == UT and alias.asname is None: + return True + return False + + +def find_import_insert_line(tree: ast.Module) -> int: + """Find the line number after the last import statement.""" + last_import_line = 0 + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + last_import_line = node.end_lineno or node.lineno + return last_import_line + + def apply_patches(contents: str, patches: Patches) -> str: tree = ast.parse(contents) lines = contents.splitlines() modifications = list(iter_patch_lines(tree, patches)) + + # If we have modifications and unittest is not imported, add it + if modifications and not has_unittest_import(tree): + import_line = find_import_insert_line(tree) + modifications.append( + ( + import_line, + "\nimport unittest # XXX: RUSTPYTHON; importing to be able to skip tests", + ) + ) + # Going in reverse to not distrupt the line offset for lineno, patch in sorted(modifications, reverse=True): lines.insert(lineno, patch) From 7a3744630e3926ff33a1ed5c5ae65b82ed456770 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Fri, 16 Jan 2026 20:15:22 +0900 Subject: [PATCH 2/5] Complement upgrade-pylib Claude Code command --- .claude/commands/upgrade-pylib.md | 86 +++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 5 deletions(-) diff --git a/.claude/commands/upgrade-pylib.md b/.claude/commands/upgrade-pylib.md index 520d5deb7ba..02ec7c0d376 100644 --- a/.claude/commands/upgrade-pylib.md +++ b/.claude/commands/upgrade-pylib.md @@ -15,9 +15,62 @@ Upgrade a Python standard library module from CPython to RustPython. - If `cpython/Lib/$ARGUMENTS.py` exists, copy it to `Lib/$ARGUMENTS.py` - If `cpython/Lib/$ARGUMENTS/` directory exists, copy it to `Lib/$ARGUMENTS/` -3. **Upgrade tests** - - Run: `python lib_updater.py --quick-upgrade cpython/Lib/test/test_$ARGUMENTS` - - This will update the test files with appropriate RustPython markers +3. **Upgrade tests (quick upgrade with lib_updater)** + - If `cpython/Lib/test/test_$ARGUMENTS.py` is a single file: + - Run: `python3 scripts/lib_updater.py --quick-upgrade cpython/Lib/test/test_$ARGUMENTS.py` + - If `cpython/Lib/test/test_$ARGUMENTS/` is a directory: + - Run the script for each `.py` file in the directory: + ```bash + for f in cpython/Lib/test/test_$ARGUMENTS/*.py; do + python3 scripts/lib_updater.py --quick-upgrade "$f" + done + ``` + - This will update the test files with basic RustPython markers (`@unittest.expectedFailure`, `@unittest.skip`, etc.) + - **Handle lib_updater warnings**: If you see warnings like `WARNING: TestCFoo does not exist in remote file`, it means the class structure changed between versions and markers couldn't be transferred automatically. These need to be manually restored in step 4 or added in step 5. + +4. **Review git diff and restore RUSTPYTHON-specific changes** + - Run `git diff Lib/test/test_$ARGUMENTS` to review all changes + - **Only restore changes that have explicit `RUSTPYTHON` comments**. Look for: + - `# XXX: RUSTPYTHON` or `# XXX RUSTPYTHON` - Comments marking RustPython-specific code modifications + - `# TODO: RUSTPYTHON` - Comments marking tests that need work + - Code changes with inline `# ... RUSTPYTHON` comments + - **Do NOT restore other diff changes** - these are likely upstream CPython changes, not RustPython-specific modifications + - When restoring, preserve the original context and formatting + +5. **Verify tests** + - Run: `cargo run --release -- -m test test_$ARGUMENTS -v` + - The `-v` flag shows detailed output to identify which tests fail and why + - For each new failure, add appropriate markers based on the failure type: + - **Test assertion failure** → `@unittest.expectedFailure` with `# TODO: RUSTPYTHON` comment + - **Panic/crash** → `@unittest.skip("TODO: RUSTPYTHON; ")` + - **Class-specific markers**: If a test fails only in the C implementation (TestCFoo) but passes in the Python implementation (TestPyFoo), or vice versa, add the marker to the specific subclass, not the base class: + ```python + # Base class - no marker here + class TestFoo: + def test_something(self): + ... + + class TestPyFoo(TestFoo, PyTest): pass + + class TestCFoo(TestFoo, CTest): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_something(self): + return super().test_something() + ``` + - **New tests from CPython**: The upgrade may bring in entirely new tests that didn't exist before. These won't have any RUSTPYTHON markers in the diff - they just need to be tested and marked if they fail. + - Example markers: + ```python + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_something(self): + ... + + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON; panics with 'index out of bounds'") + def test_crashes(self): + ... + ``` ## Example Usage ``` @@ -26,7 +79,30 @@ Upgrade a Python standard library module from CPython to RustPython. /upgrade-pylib asyncio ``` +## Example: Restoring RUSTPYTHON changes + +When git diff shows removed RUSTPYTHON-specific code like: +```diff +-# XXX RUSTPYTHON: we don't import _json as fresh since... +-cjson = import_helper.import_fresh_module('json') #, fresh=['_json']) ++cjson = import_helper.import_fresh_module('json', fresh=['_json']) +``` + +You should restore the RustPython version: +```python +# XXX RUSTPYTHON: we don't import _json as fresh since... +cjson = import_helper.import_fresh_module('json') #, fresh=['_json']) +``` + ## Notes - The cpython/ directory should contain the CPython source that we're syncing from -- lib_updater.py handles adding `# TODO: RUSTPYTHON` markers and `@unittest.expectedFailure` decorators -- After upgrading, you may need to run tests to verify: `cargo run --release -- -m test test_$ARGUMENTS` +- `scripts/lib_updater.py` handles basic patching: + - Transfers `@unittest.expectedFailure` and `@unittest.skip` decorators with `TODO: RUSTPYTHON` markers + - Adds `import unittest # XXX: RUSTPYTHON` if needed for the decorators + - **Limitation**: If a class was restructured (e.g., method overrides removed), lib_updater will warn and skip those markers +- The script does NOT preserve all RustPython-specific changes - you must review `git diff` and restore them +- Common RustPython markers to look for: + - `# XXX: RUSTPYTHON` or `# XXX RUSTPYTHON` - Inline comments for code modifications + - `# TODO: RUSTPYTHON` - Test skip/failure markers + - Any code with `RUSTPYTHON` in comments that was removed in the diff +- **Important**: Not all changes in the git diff need to be restored. Only restore changes that have explicit `RUSTPYTHON` comments. Other changes are upstream CPython updates. From 95f822db223b83eacaec3a613c79f37d44281c5f Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Fri, 16 Jan 2026 21:01:22 +0900 Subject: [PATCH 3/5] Consider docstring and PEP 263 in lib_updater.py --- scripts/lib_updater.py | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index dfb5896e578..f404dd68557 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -280,13 +280,45 @@ def has_unittest_import(tree: ast.Module) -> bool: return False -def find_import_insert_line(tree: ast.Module) -> int: - """Find the line number after the last import statement.""" +def find_import_insert_line(tree: ast.Module, lines: list[str] | None = None) -> int: + """Find the line number after the last import statement. + + If no imports exist, returns the line after the module docstring + (if present). If there's no docstring either, attempts to skip + shebang and encoding lines to comply with PEP 263. + """ last_import_line = 0 for node in tree.body: if isinstance(node, (ast.Import, ast.ImportFrom)): last_import_line = node.end_lineno or node.lineno - return last_import_line + + # If we found imports, return after the last one + if last_import_line > 0: + return last_import_line + + # No imports found - check for module docstring + if tree.body: + first_node = tree.body[0] + if ( + isinstance(first_node, ast.Expr) + and isinstance(first_node.value, ast.Constant) + and isinstance(first_node.value.value, str) + ): + return first_node.end_lineno or first_node.lineno + + # No imports and no docstring - try to skip shebang/encoding if lines provided + # PEP 263: encoding declaration must be in first or second line + # and match: ^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+) + if lines: + insert_line = 0 + for i, line in enumerate(lines[:2]): # Only check first two lines per PEP 263 + if line.startswith("#!") or re.match( + r"^[ \t\f]*#.*?coding[:=][ \t]*[-_.a-zA-Z0-9]+", line + ): + insert_line = i + 1 + return insert_line + + return 0 def apply_patches(contents: str, patches: Patches) -> str: @@ -297,7 +329,7 @@ def apply_patches(contents: str, patches: Patches) -> str: # If we have modifications and unittest is not imported, add it if modifications and not has_unittest_import(tree): - import_line = find_import_insert_line(tree) + import_line = find_import_insert_line(tree, lines) modifications.append( ( import_line, From 5b54ec9d68c1693ff12f08a38537e9dee77cf28a Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Sat, 17 Jan 2026 20:28:29 +0900 Subject: [PATCH 4/5] Revert "Consider docstring and PEP 263 in lib_updater.py" This reverts commit 95f822db223b83eacaec3a613c79f37d44281c5f. --- scripts/lib_updater.py | 40 ++++------------------------------------ 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index f404dd68557..dfb5896e578 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -280,45 +280,13 @@ def has_unittest_import(tree: ast.Module) -> bool: return False -def find_import_insert_line(tree: ast.Module, lines: list[str] | None = None) -> int: - """Find the line number after the last import statement. - - If no imports exist, returns the line after the module docstring - (if present). If there's no docstring either, attempts to skip - shebang and encoding lines to comply with PEP 263. - """ +def find_import_insert_line(tree: ast.Module) -> int: + """Find the line number after the last import statement.""" last_import_line = 0 for node in tree.body: if isinstance(node, (ast.Import, ast.ImportFrom)): last_import_line = node.end_lineno or node.lineno - - # If we found imports, return after the last one - if last_import_line > 0: - return last_import_line - - # No imports found - check for module docstring - if tree.body: - first_node = tree.body[0] - if ( - isinstance(first_node, ast.Expr) - and isinstance(first_node.value, ast.Constant) - and isinstance(first_node.value.value, str) - ): - return first_node.end_lineno or first_node.lineno - - # No imports and no docstring - try to skip shebang/encoding if lines provided - # PEP 263: encoding declaration must be in first or second line - # and match: ^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+) - if lines: - insert_line = 0 - for i, line in enumerate(lines[:2]): # Only check first two lines per PEP 263 - if line.startswith("#!") or re.match( - r"^[ \t\f]*#.*?coding[:=][ \t]*[-_.a-zA-Z0-9]+", line - ): - insert_line = i + 1 - return insert_line - - return 0 + return last_import_line def apply_patches(contents: str, patches: Patches) -> str: @@ -329,7 +297,7 @@ def apply_patches(contents: str, patches: Patches) -> str: # If we have modifications and unittest is not imported, add it if modifications and not has_unittest_import(tree): - import_line = find_import_insert_line(tree, lines) + import_line = find_import_insert_line(tree) modifications.append( ( import_line, From 80dfec2eb50d30677f458259c29aed3becee529f Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Sat, 17 Jan 2026 20:29:31 +0900 Subject: [PATCH 5/5] Ensure test module having import statement at least one --- scripts/lib_updater.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index dfb5896e578..6176bd54431 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -282,10 +282,11 @@ def has_unittest_import(tree: ast.Module) -> bool: def find_import_insert_line(tree: ast.Module) -> int: """Find the line number after the last import statement.""" - last_import_line = 0 + last_import_line = None for node in tree.body: if isinstance(node, (ast.Import, ast.ImportFrom)): last_import_line = node.end_lineno or node.lineno + assert last_import_line is not None return last_import_line