From 91514b0bf00280f7b0ba2de408b8a6302a23c417 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 13 Apr 2025 23:19:35 -0700 Subject: [PATCH 1/6] initial auto-upgrader --- scripts/fix_test.py | 87 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 scripts/fix_test.py diff --git a/scripts/fix_test.py b/scripts/fix_test.py new file mode 100644 index 00000000000..d69fdb4cb6e --- /dev/null +++ b/scripts/fix_test.py @@ -0,0 +1,87 @@ +import argparse + +def parse_args(): + parser = argparse.ArgumentParser(description="Fix test.") + parser.add_argument("--test", type=str, help="Name of test") + parser.add_argument("--path", type=str, help="Path to test file") + parser.add_argument("--force", action="store_true", help="Force modification") + + args = parser.parse_args() + return args + +class Test: + name: str = "" + path: str = "" + result: str = "" + + def __str__(self): + return f"Test(name={self.name}, path={self.path}, result={self.result})" + +class TestResult: + tests_result: str = "" + tests = [] + + def __str__(self): + return f"TestResult(tests_result={self.tests_result},tests={len(self.tests)})" + + +def parse_results(result): + lines = result.stdout.splitlines() + test_results = TestResult() + in_test_results = False + for line in lines: + if line == "Run tests sequentially": + in_test_results = True + elif line.startswith("-----------"): + in_test_results = False + if in_test_results and not line.startswith("tests") and not line.startswith("["): + line = line.split(" ") + if line != [] and len(line) > 3: + test = Test() + test.name = line[0] + test.path = line[1].strip("(").strip(")") + test.result = " ".join(line[3:]).lower() + test_results.tests.append(test) + else: + if "== Tests result: " in line: + res = line.split("== Tests result: ")[1] + res = res.split(" ")[0] + test_results.tests_result = res + return test_results + +def path_to_test(path) -> list[str]: + return path.split(".")[2:] + +def modify_test(file: str, test: list[str]) -> str: + lines = file.splitlines() + result = [] + for line in lines: + if line.lstrip(" ").startswith("def " + test[-1]): + whitespace = line[:line.index("def ")] + result.append(whitespace + "# TODO: RUSTPYTHON") + result.append(whitespace + "@unittest.expectedFailure") + result.append(line) + return "\n".join(result) + +def run_test(test_name): + print(f"Running test: {test_name}") + rustpython_location = "./target/release/rustpython" + import subprocess + result = subprocess.run([rustpython_location, "-m", "test", "-v", test_name], capture_output=True, text=True) + return parse_results(result) + + +if __name__ == "__main__": + args = parse_args() + test_name = args.test + tests = run_test(test_name) + f = open(args.path).read() + for test in tests.tests: + if test.result == "fail": + print("Modifying test:", test.name) + f = modify_test(f, path_to_test(test.path)) + with open(args.path, "w") as file: + if args.force or run_test().tests_result == "ok": + file.write(f) + else: + raise Exception("Test failed after modification") From 0236917a621ecdbcff8a5d47414c02e6b4673fc5 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 13 Apr 2025 23:33:31 -0700 Subject: [PATCH 2/6] platform support --- scripts/fix_test.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/scripts/fix_test.py b/scripts/fix_test.py index d69fdb4cb6e..6898479d90c 100644 --- a/scripts/fix_test.py +++ b/scripts/fix_test.py @@ -1,10 +1,12 @@ import argparse +import platform def parse_args(): parser = argparse.ArgumentParser(description="Fix test.") parser.add_argument("--test", type=str, help="Name of test") parser.add_argument("--path", type=str, help="Path to test file") parser.add_argument("--force", action="store_true", help="Force modification") + parser.add_argument("--platform", action="store_true", help="Platform specific failure") args = parser.parse_args() return args @@ -20,6 +22,7 @@ def __str__(self): class TestResult: tests_result: str = "" tests = [] + stdout = "" def __str__(self): return f"TestResult(tests_result={self.tests_result},tests={len(self.tests)})" @@ -28,6 +31,7 @@ def __str__(self): def parse_results(result): lines = result.stdout.splitlines() test_results = TestResult() + test_results.stdout = result.stdout in_test_results = False for line in lines: if line == "Run tests sequentially": @@ -52,14 +56,20 @@ def parse_results(result): def path_to_test(path) -> list[str]: return path.split(".")[2:] -def modify_test(file: str, test: list[str]) -> str: +def modify_test(file: str, test: list[str], for_platform: bool = False) -> str: lines = file.splitlines() result = [] + failure_fixture = "expectedFailure" + if for_platform: + if platform.system() == "Windows": + failure_fixture = "expectedFailureIfWindows(\"TODO: RUSTPYTHON: Generated by fix_test script\")" + else: + raise Exception("Platform not supported") for line in lines: if line.lstrip(" ").startswith("def " + test[-1]): whitespace = line[:line.index("def ")] result.append(whitespace + "# TODO: RUSTPYTHON") - result.append(whitespace + "@unittest.expectedFailure") + result.append(whitespace + f"@unittest.{failure_fixture}") result.append(line) return "\n".join(result) @@ -77,11 +87,9 @@ def run_test(test_name): tests = run_test(test_name) f = open(args.path).read() for test in tests.tests: - if test.result == "fail": + if test.result == "fail" or test.result == "error": print("Modifying test:", test.name) - f = modify_test(f, path_to_test(test.path)) + f = modify_test(f, path_to_test(test.path), args.platform) with open(args.path, "w") as file: - if args.force or run_test().tests_result == "ok": - file.write(f) - else: - raise Exception("Test failed after modification") + # TODO: Find validation method, and make --force override it + file.write(f) From 193a7615d9ab376e33d11bb165e4b319c13fa291 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 14 Apr 2025 10:19:12 -0700 Subject: [PATCH 3/6] use ast walking --- scripts/fix_test.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/scripts/fix_test.py b/scripts/fix_test.py index 6898479d90c..2b61cbfde1d 100644 --- a/scripts/fix_test.py +++ b/scripts/fix_test.py @@ -1,4 +1,5 @@ import argparse +import ast import platform def parse_args(): @@ -57,21 +58,18 @@ def path_to_test(path) -> list[str]: return path.split(".")[2:] def modify_test(file: str, test: list[str], for_platform: bool = False) -> str: + a = ast.parse(file) lines = file.splitlines() - result = [] - failure_fixture = "expectedFailure" - if for_platform: - if platform.system() == "Windows": - failure_fixture = "expectedFailureIfWindows(\"TODO: RUSTPYTHON: Generated by fix_test script\")" - else: - raise Exception("Platform not supported") - for line in lines: - if line.lstrip(" ").startswith("def " + test[-1]): - whitespace = line[:line.index("def ")] - result.append(whitespace + "# TODO: RUSTPYTHON") - result.append(whitespace + f"@unittest.{failure_fixture}") - result.append(line) - return "\n".join(result) + fixture = "@unittest.expectedFailure" + for node in ast.walk(a): + if isinstance(node, ast.FunctionDef): + if node.name == test[-1]: + assert not for_platform + indent = " " * node.col_offset + lines.insert(node.lineno - 1, indent + fixture) + lines.insert(node.lineno - 1, indent + "# TODO: RUSTPYTHON") + break + return "\n".join(lines) def run_test(test_name): print(f"Running test: {test_name}") From 5f402f2eb14346da2b6b9bab13a6534e9a83e27a Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 12:16:29 -0700 Subject: [PATCH 4/6] detect testname automatically --- scripts/fix_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/fix_test.py b/scripts/fix_test.py index 2b61cbfde1d..b74fde54400 100644 --- a/scripts/fix_test.py +++ b/scripts/fix_test.py @@ -1,11 +1,11 @@ import argparse import ast import platform +from pathlib import Path def parse_args(): parser = argparse.ArgumentParser(description="Fix test.") - parser.add_argument("--test", type=str, help="Name of test") - parser.add_argument("--path", type=str, help="Path to test file") + parser.add_argument("--path", type=Path, help="Path to test file") parser.add_argument("--force", action="store_true", help="Force modification") parser.add_argument("--platform", action="store_true", help="Platform specific failure") @@ -81,7 +81,7 @@ def run_test(test_name): if __name__ == "__main__": args = parse_args() - test_name = args.test + test_name = args.path.stem tests = run_test(test_name) f = open(args.path).read() for test in tests.tests: From 452e62917b486e67f66a08e1e31d651464576578 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 13:54:22 -0700 Subject: [PATCH 5/6] handle classes properly --- scripts/fix_test.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/fix_test.py b/scripts/fix_test.py index b74fde54400..69401937e27 100644 --- a/scripts/fix_test.py +++ b/scripts/fix_test.py @@ -1,5 +1,6 @@ import argparse import ast +import itertools import platform from pathlib import Path @@ -71,6 +72,37 @@ def modify_test(file: str, test: list[str], for_platform: bool = False) -> str: break return "\n".join(lines) +def modify_test_v2(file: str, test: list[str], for_platform: bool = False) -> str: + a = ast.parse(file) + lines = file.splitlines() + fixture = "@unittest.expectedFailure" + for key, node in ast.iter_fields(a): + if key == "body": + for i, n in enumerate(node): + match n: + case ast.ClassDef(): + if len(test) == 2 and test[0] == n.name: + # look through body for function def + for i, fn in enumerate(n.body): + match fn: + case ast.FunctionDef(): + if fn.name == test[-1]: + assert not for_platform + indent = " " * fn.col_offset + lines.insert(fn.lineno - 1, indent + fixture) + lines.insert(fn.lineno - 1, indent + "# TODO: RUSTPYTHON") + break + case ast.FunctionDef(): + if n.name == test[0] and len(test) == 1: + assert not for_platform + indent = " " * n.col_offset + lines.insert(n.lineno - 1, indent + fixture) + lines.insert(n.lineno - 1, indent + "# TODO: RUSTPYTHON") + break + if i > 500: + exit() + return "\n".join(lines) + def run_test(test_name): print(f"Running test: {test_name}") rustpython_location = "./target/release/rustpython" @@ -87,7 +119,7 @@ def run_test(test_name): for test in tests.tests: if test.result == "fail" or test.result == "error": print("Modifying test:", test.name) - f = modify_test(f, path_to_test(test.path), args.platform) + f = modify_test_v2(f, path_to_test(test.path), args.platform) with open(args.path, "w") as file: # TODO: Find validation method, and make --force override it file.write(f) From 55802b23120b275eb80cae403e72c9f4a2f7bab0 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 23 Apr 2025 20:49:56 -0700 Subject: [PATCH 6/6] add instructions to fix_test.py --- scripts/fix_test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/fix_test.py b/scripts/fix_test.py index 69401937e27..99dfa2699a3 100644 --- a/scripts/fix_test.py +++ b/scripts/fix_test.py @@ -1,3 +1,15 @@ +""" +An automated script to mark failures in python test suite. +It adds @unittest.expectedFailure to the test functions that are failing in RustPython, but not in CPython. +As well as marking the test with a TODO comment. + +How to use: +1. Copy a specific test from the CPython repository to the RustPython repository. +2. Remove all unexpected failures from the test and skip the tests that hang +3. Run python ./scripts/fix_test.py --test test_venv --path ./Lib/test/test_venv.py or equivalent for the test from the project root. +4. Ensure that there are no unexpected successes in the test. +5. Actually fix the test. +""" import argparse import ast import itertools