From 2703f94c3e0ba5697658fbb1f1a77138c06367b6 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Mon, 30 Mar 2026 16:59:48 +0200 Subject: [PATCH 001/834] Split `cargo check` matrix to individual targets. Avoid cache poisoning (#7540) * Split check matrix. Prevent cache poisoning * Use `rustup` * Change name * Align cargo args --- .github/workflows/ci.yaml | 76 ++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fa019069108..ddcc284e0b7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,6 +27,8 @@ env: PYTHON_VERSION: "3.14.3" X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include + CARGO_INCREMENTAL: 0 + CARGO_TERM_COLOR: always jobs: rust_tests: @@ -108,41 +110,39 @@ jobs: cargo_check: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} - name: Ensure compilation on various targets + name: cargo check runs-on: ${{ matrix.os }} strategy: matrix: include: - os: ubuntu-latest - targets: - - aarch64-linux-android - - i686-unknown-linux-gnu - - i686-unknown-linux-musl - - wasm32-wasip2 - - x86_64-unknown-freebsd + target: aarch64-linux-android + - os: ubuntu-latest + target: i686-unknown-linux-gnu dependencies: gcc-multilib: true + - os: ubuntu-latest + target: i686-unknown-linux-musl + dependencies: musl-tools: true - os: ubuntu-latest - targets: - - aarch64-unknown-linux-gnu + target: wasm32-wasip2 + - os: ubuntu-latest + target: x86_64-unknown-freebsd + - os: ubuntu-latest + target: aarch64-unknown-linux-gnu dependencies: - gcc-aarch64-linux-gnu: true # conflict with `gcc-multilib` + gcc-aarch64-linux-gnu: true - os: macos-latest - targets: - - aarch64-apple-ios - - x86_64-apple-darwin + target: aarch64-apple-ios + - os: macos-latest + target: x86_64-apple-darwin fail-fast: false steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - with: - prefix-key: v0-rust-${{ join(matrix.targets, '-') }} - save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Install dependencies uses: ./.github/actions/install-linux-deps # zizmor has an issue with dynamic `with` @@ -152,13 +152,25 @@ jobs: musl-tools: ${{ matrix.dependencies.musl-tools || false }} gcc-aarch64-linux-gnu: ${{ matrix.dependencies.gcc-aarch64-linux-gnu || false }} - - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + - name: Restore cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + if: ${{ github.ref != 'refs/heads/main' }} # Never restore on main with: - targets: ${{ join(matrix.targets, ',') }} - toolchain: stable + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + # key won't match, will rely on restore-keys + key: cargo-check-${{ runner.os }}-${{ matrix.target }} + restore-keys: | + cargo-check-${{ runner.os }}-${{ matrix.target }}- + + - run: rustup toolchain install stable --target "${{ matrix.target }}" - name: Setup Android NDK - if: ${{ contains(matrix.targets, 'aarch64-linux-android') }} + if: ${{ matrix.target == 'aarch64-linux-android' }} id: setup-ndk uses: nttld/setup-ndk@v1 with: @@ -174,18 +186,24 @@ jobs: # args: --ignore-rust-version - name: Check compilation - run: | - for target in ${{ join(matrix.targets, ' ') }} - do - echo "::group::${target}" - cargo check --target $target ${{ env.CARGO_ARGS_NO_SSL }} - echo "::endgroup::" - done + run: cargo check --target "${{ matrix.target }}" ${{ env.CARGO_ARGS_NO_SSL }} env: CC_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang AR_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang + - name: Save cache + if: ${{ github.ref == 'refs/heads/main' }} # only save on main + uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: cargo-check-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('Cargo.lock') }}-${{ github.sha }} + snippets_cpython: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} env: From c45f69977ba4ea67720173622f003d48d43dcf8e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 01:29:35 +0900 Subject: [PATCH 002/834] Bump schannel from 0.1.28 to 0.1.29 (#7542) Bumps [schannel](https://github.com/steffengy/schannel-rs) from 0.1.28 to 0.1.29. - [Release notes](https://github.com/steffengy/schannel-rs/releases) - [Commits](https://github.com/steffengy/schannel-rs/compare/v0.1.28...v0.1.29) --- updated-dependencies: - dependency-name: schannel dependency-version: 0.1.29 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ec08473394e..74cd95662a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3511,9 +3511,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] diff --git a/Cargo.toml b/Cargo.toml index af5d834c03f..3b4069d47a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -214,7 +214,7 @@ rand_core = { version = "0.9", features = ["os_rng"] } rustix = { version = "1.1", features = ["event"] } rustyline = "17.0.1" serde = { package = "serde_core", version = "1.0.225", default-features = false, features = ["alloc"] } -schannel = "0.1.28" +schannel = "0.1.29" scoped-tls = "1" scopeguard = "1" static_assertions = "1.1" From 106f1c9f3710570aef5c8dff0280df8e54ea1a21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 01:29:47 +0900 Subject: [PATCH 003/834] Bump env_logger from 0.11.9 to 0.11.10 (#7543) Bumps [env_logger](https://github.com/rust-cli/env_logger) from 0.11.9 to 0.11.10. - [Release notes](https://github.com/rust-cli/env_logger/releases) - [Changelog](https://github.com/rust-cli/env_logger/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-cli/env_logger/compare/v0.11.9...v0.11.10) --- updated-dependencies: - dependency-name: env_logger dependency-version: 0.11.10 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 74cd95662a4..1045146ee0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,9 +79,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -100,9 +100,9 @@ checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -1143,9 +1143,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.9" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" dependencies = [ "anstream", "anstyle", @@ -1585,9 +1585,9 @@ checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jiff" -version = "0.2.18" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", "log", @@ -1598,9 +1598,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.18" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", From 1a4964b7414345822b842e47b59c2ce492f4ecc1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 01:30:08 +0900 Subject: [PATCH 004/834] Bump dtolnay/rust-toolchain (#7545) Bumps [dtolnay/rust-toolchain](https://github.com/dtolnay/rust-toolchain) from efa25f7f19611383d5b0ccf2d1c8914531636bf9 to 3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9. - [Release notes](https://github.com/dtolnay/rust-toolchain/releases) - [Commits](https://github.com/dtolnay/rust-toolchain/compare/efa25f7f19611383d5b0ccf2d1c8914531636bf9...3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9) --- updated-dependencies: - dependency-name: dtolnay/rust-toolchain dependency-version: 3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 12 ++++++------ .github/workflows/release.yml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ddcc284e0b7..3f5207a12c8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -47,7 +47,7 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 with: components: clippy toolchain: stable @@ -248,7 +248,7 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 with: toolchain: stable @@ -363,7 +363,7 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 with: toolchain: stable components: rustfmt @@ -422,7 +422,7 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 with: toolchain: ${{ env.NIGHTLY_CHANNEL }} components: miri @@ -448,7 +448,7 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 with: components: clippy toolchain: stable @@ -526,7 +526,7 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 with: target: wasm32-wasip1 toolchain: stable diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f8cb5d915d2..f06f0d3adc6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,7 +52,7 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 with: target: ${{ matrix.target }} @@ -89,7 +89,7 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 with: targets: wasm32-wasip1 From 611b122ed724ebcd236c5efdf5ce8ef394773a74 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Mon, 30 Mar 2026 18:30:41 +0200 Subject: [PATCH 005/834] Resolve template injection (#7546) --- .github/workflows/comment-commands.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/comment-commands.yml b/.github/workflows/comment-commands.yml index 3f3402270ea..eb9e2f96d1a 100644 --- a/.github/workflows/comment-commands.yml +++ b/.github/workflows/comment-commands.yml @@ -18,4 +18,6 @@ jobs: steps: # Using REST API and not `gh issue edit`. https://github.com/cli/cli/issues/6235#issuecomment-1243487651 - run: | - curl -H "Authorization: token ${{ github.token }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees + curl -H "Authorization: token ${{ github.token }}" -d '{"assignees": ["${{ env.USER }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees + env: + USER: ${{ github.event.comment.user.login }} From 43851c21b964ea08530267ba3f0e1a42ac509498 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 01:31:08 +0900 Subject: [PATCH 006/834] Bump minimatch and serve in /wasm/demo (#7539) Bumps [minimatch](https://github.com/isaacs/minimatch) to 3.1.5 and updates ancestor dependency [serve](https://github.com/vercel/serve). These dependencies need to be updated together. Updates `minimatch` from 3.1.2 to 3.1.5 - [Changelog](https://github.com/isaacs/minimatch/blob/main/changelog.md) - [Commits](https://github.com/isaacs/minimatch/compare/v3.1.2...v3.1.5) Updates `serve` from 14.2.5 to 14.2.6 - [Release notes](https://github.com/vercel/serve/releases) - [Changelog](https://github.com/vercel/serve/blob/main/CHANGELOG.md) - [Commits](https://github.com/vercel/serve/compare/v14.2.5...v14.2.6) --- updated-dependencies: - dependency-name: minimatch dependency-version: 3.1.5 dependency-type: indirect - dependency-name: serve dependency-version: 14.2.6 dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- wasm/demo/package-lock.json | 75 ++++++++++--------------------------- wasm/demo/package.json | 2 +- 2 files changed, 20 insertions(+), 57 deletions(-) diff --git a/wasm/demo/package-lock.json b/wasm/demo/package-lock.json index 768ba66e5a3..761c3d3575b 100644 --- a/wasm/demo/package-lock.json +++ b/wasm/demo/package-lock.json @@ -21,7 +21,7 @@ "css-loader": "^7.1.2", "html-webpack-plugin": "^5.6.3", "mini-css-extract-plugin": "^2.9.2", - "serve": "^14.2.5", + "serve": "^14.2.6", "webpack": "^5.105.0", "webpack-cli": "^6.0.1", "webpack-dev-server": "^5.2.1" @@ -839,9 +839,9 @@ } }, "node_modules/ajv": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", - "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", + "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "dev": true, "license": "MIT", "dependencies": { @@ -1129,9 +1129,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", + "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", "dev": true, "license": "MIT", "dependencies": { @@ -3448,9 +3448,9 @@ "license": "ISC" }, "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", "dev": true, "license": "ISC", "dependencies": { @@ -4003,16 +4003,6 @@ "node": ">= 0.10" } }, - "node_modules/punycode": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", - "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/qs": { "version": "6.13.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", @@ -4375,14 +4365,14 @@ } }, "node_modules/serve": { - "version": "14.2.5", - "resolved": "https://registry.npmjs.org/serve/-/serve-14.2.5.tgz", - "integrity": "sha512-Qn/qMkzCcMFVPb60E/hQy+iRLpiU8PamOfOSYoAHmmF+fFFmpPpqa6Oci2iWYpTdOUM3VF+TINud7CfbQnsZbA==", + "version": "14.2.6", + "resolved": "https://registry.npmjs.org/serve/-/serve-14.2.6.tgz", + "integrity": "sha512-QEjUSA+sD4Rotm1znR8s50YqA3kYpRGPmtd5GlFxbaL9n/FdUNbqMhxClqdditSk0LlZyA/dhud6XNRTOC9x2Q==", "dev": true, "license": "MIT", "dependencies": { "@zeit/schemas": "2.36.0", - "ajv": "8.12.0", + "ajv": "8.18.0", "arg": "5.0.2", "boxen": "7.0.0", "chalk": "5.0.1", @@ -4390,7 +4380,7 @@ "clipboardy": "3.0.0", "compression": "1.8.1", "is-port-reachable": "4.0.0", - "serve-handler": "6.1.6", + "serve-handler": "6.1.7", "update-check": "1.5.4" }, "bin": { @@ -4401,16 +4391,16 @@ } }, "node_modules/serve-handler": { - "version": "6.1.6", - "resolved": "https://registry.npmjs.org/serve-handler/-/serve-handler-6.1.6.tgz", - "integrity": "sha512-x5RL9Y2p5+Sh3D38Fh9i/iQ5ZK+e4xuXRd/pGbM4D13tgo/MGwbttUk8emytcr1YYzBYs+apnUngBDFYfpjPuQ==", + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/serve-handler/-/serve-handler-6.1.7.tgz", + "integrity": "sha512-CinAq1xWb0vR3twAv9evEU8cNWkXCb9kd5ePAHUKJBkOsUpR1wt/CvGdeca7vqumL1U5cSaeVQ6zZMxiJ3yWsg==", "dev": true, "license": "MIT", "dependencies": { "bytes": "3.0.0", "content-disposition": "0.5.2", "mime-types": "2.1.18", - "minimatch": "3.1.2", + "minimatch": "3.1.5", "path-is-inside": "1.0.2", "path-to-regexp": "3.3.0", "range-parser": "1.2.0" @@ -4524,23 +4514,6 @@ "node": ">= 0.8.0" } }, - "node_modules/serve/node_modules/ajv": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", - "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, "node_modules/serve/node_modules/chalk": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.0.1.tgz", @@ -5203,16 +5176,6 @@ "integrity": "sha512-NtkVvqVCqsJo5U3mYRum2Tw6uCltOxfIJ/AfTZeTmw6U39IB5X23xF+kRZ9aiPaORqeiQQ7Q209/ibhOvxzwHA==", "license": "MIT" }, - "node_modules/uri-js": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", - "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "punycode": "^2.1.0" - } - }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/wasm/demo/package.json b/wasm/demo/package.json index 7954e8cd866..a815b90c70b 100644 --- a/wasm/demo/package.json +++ b/wasm/demo/package.json @@ -16,7 +16,7 @@ "css-loader": "^7.1.2", "html-webpack-plugin": "^5.6.3", "mini-css-extract-plugin": "^2.9.2", - "serve": "^14.2.5", + "serve": "^14.2.6", "webpack": "^5.105.0", "webpack-cli": "^6.0.1", "webpack-dev-server": "^5.2.1" From b275a90cf92f8d97420e38cfb780ae3478ae4178 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 18:38:01 +0200 Subject: [PATCH 007/834] Bump j178/prek-action from 2.0.0 to 2.0.1 (#7544) Bumps [j178/prek-action](https://github.com/j178/prek-action) from 2.0.0 to 2.0.1. - [Release notes](https://github.com/j178/prek-action/releases) - [Commits](https://github.com/j178/prek-action/compare/79f765515bd648eb4d6bb1b17277b7cb22cb6468...53276d8b0d10f8b6672aa85b4588c6921d0370cc) --- updated-dependencies: - dependency-name: j178/prek-action dependency-version: 2.0.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3f5207a12c8..117e4e6af0c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -390,7 +390,7 @@ jobs: - name: prek id: prek - uses: j178/prek-action@79f765515bd648eb4d6bb1b17277b7cb22cb6468 # v2.0.0 + uses: j178/prek-action@53276d8b0d10f8b6672aa85b4588c6921d0370cc # v2.0.1 with: cache: false show-verbose-logs: false From 5cc9eab2dd6afdf982334a956b884b5718e18f92 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Tue, 31 Mar 2026 07:53:08 +0200 Subject: [PATCH 008/834] Resolve `excessive-permissions` warning in `ci.yaml` (#7547) * Resolve `excessive-permissions` warning in `ci.yaml` * Update .github/workflows/ci.yaml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --------- Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 117e4e6af0c..99906707cf3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,6 +8,9 @@ on: name: CI +permissions: + contents: read + # Cancel previous workflows if they are the same workflow on same ref (branch/tags) # with the same event (push/pull_request) even they are in progress. # This setting will help reduce the number of duplicated workflows. From 403c2be01d3b903578e286a51b15125e50dc2a5f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Tue, 31 Mar 2026 15:45:18 +0900 Subject: [PATCH 009/834] Improve codegen bytecode parity (#7541) - Add CFG block splitting, jump threading, backward jump normalization - Add genexpr StopIteration wrapper - Add ConstantData::Slice and constant slice folding - Add duplicate_exits_without_lineno and Block: Clone - Add builtin(genexpr) optimization for tuple/list/set/all/any - Add compile_try_except_no_finally for try-except without finally - Add module_name_declared_global_in_nested_scope - Add constant tuple folding in try_fold_constant_expr - Add fstring literal-only optimization and empty literal elision - Fix duplicate_exits_without_lineno: splice new blocks into linked list --- Lib/test/test_monitoring.py | 1 - Lib/test/test_scope.py | 1 - crates/codegen/src/compile.rs | 779 +++++++++++++++++- crates/codegen/src/ir.rs | 118 ++- ...pile__tests__nested_double_async_with.snap | 158 ++-- crates/compiler-core/src/bytecode/oparg.rs | 6 + crates/vm/src/frame.rs | 14 +- crates/vm/src/version.rs | 4 +- crates/vm/src/vm/mod.rs | 4 + 9 files changed, 933 insertions(+), 152 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index ed28ae07f86..5125701202b 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1261,7 +1261,6 @@ def func2(): ('instruction', 'func2', 46), ('line', 'get_events', 11)]) - @unittest.expectedFailure # TODO: RUSTPYTHON; - instruction offsets differ from CPython def test_try_except(self): def func3(): diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 952afb7e0d3..520fbc1b662 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -692,7 +692,6 @@ def dec(self): self.assertEqual(c.dec(), 1) self.assertEqual(c.dec(), 0) - @unittest.expectedFailure # TODO: RUSTPYTHON; figure out how to communicate that `y = 9` should be stored as a global rather than a STORE_NAME, even when the `global y` is in a nested subscope def testGlobalInParallelNestedFunctions(self): # A symbol table bug leaked the global statement from one # function to other nested functions in the same block. diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 6098c6e2231..fab31076218 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -56,7 +56,7 @@ impl ExprExt for ast::Expr { | ast::Expr::NoneLiteral(_) | ast::Expr::BooleanLiteral(_) | ast::Expr::EllipsisLiteral(_) - ) + ) || matches!(self, ast::Expr::Tuple(ast::ExprTuple { elts, .. }) if elts.iter().all(ExprExt::is_constant)) } fn is_constant_slice(&self) -> bool { @@ -121,6 +121,15 @@ enum SuperCallType<'a> { ZeroArg, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BuiltinGeneratorCallKind { + Tuple, + List, + Set, + All, + Any, +} + #[derive(Debug, Clone)] pub struct FBlockInfo { pub fb_type: FBlockType, @@ -2002,6 +2011,16 @@ impl Compiler { symboltable::maybe_mangle_name(private, mangled_names, name) } + fn module_name_declared_global_in_nested_scope(table: &SymbolTable, name: &str) -> bool { + table.sub_tables.iter().any(|subtable| { + (!subtable.comp_inlined + && subtable + .lookup(name) + .is_some_and(|symbol| symbol.scope == SymbolScope::GlobalExplicit)) + || Self::module_name_declared_global_in_nested_scope(subtable, name) + }) + } + // = compiler_nameop fn compile_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> { enum NameOp { @@ -2088,12 +2107,20 @@ impl Compiler { } }; + let module_global_from_nested_scope = { + let current_table = self.current_symbol_table(); + current_table.typ == CompilerScope::Module + && Self::module_name_declared_global_in_nested_scope(current_table, name.as_ref()) + }; + // Determine operation type based on scope let op_type = match actual_scope { SymbolScope::Free => NameOp::Deref, SymbolScope::Cell => NameOp::Deref, SymbolScope::Local => { - if is_function_like { + if module_global_from_nested_scope { + NameOp::Global + } else if is_function_like { NameOp::Fast } else { NameOp::Name @@ -2111,7 +2138,13 @@ impl Compiler { } } SymbolScope::GlobalExplicit => NameOp::Global, - SymbolScope::Unknown => NameOp::Name, + SymbolScope::Unknown => { + if module_global_from_nested_scope { + NameOp::Global + } else { + NameOp::Name + } + } }; // Generate appropriate instructions based on operation type @@ -2538,9 +2571,15 @@ impl Compiler { statement.range(), )); } - self.compile_expression(v)?; - // Unwind fblock stack with preserve_tos=true (preserve return value) - self.unwind_fblock_stack(true, false)?; + let folded_constant = self.try_fold_constant_expr(v)?; + let preserve_tos = folded_constant.is_none(); + if preserve_tos { + self.compile_expression(v)?; + } + self.unwind_fblock_stack(preserve_tos, false)?; + if let Some(constant) = folded_constant { + self.emit_load_const(constant); + } self.emit_return_value(); } None => { @@ -2987,6 +3026,10 @@ impl Compiler { orelse: &[ast::Stmt], finalbody: &[ast::Stmt], ) -> CompileResult<()> { + if finalbody.is_empty() { + return self.compile_try_except_no_finally(body, handlers, orelse); + } + let handler_block = self.new_block(); let finally_block = self.new_block(); @@ -3397,6 +3440,175 @@ impl Compiler { Ok(()) } + fn compile_try_except_no_finally( + &mut self, + body: &[ast::Stmt], + handlers: &[ast::ExceptHandler], + orelse: &[ast::Stmt], + ) -> CompileResult<()> { + let handler_block = self.new_block(); + let cleanup_block = self.new_block(); + let orelse_block = self.new_block(); + let end_block = self.new_block(); + + emit!(self, Instruction::Nop); + emit!( + self, + PseudoInstruction::SetupFinally { + delta: handler_block + } + ); + + self.push_fblock(FBlockType::TryExcept, handler_block, handler_block)?; + self.compile_statements(body)?; + self.pop_fblock(FBlockType::TryExcept); + emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); + emit!( + self, + PseudoInstruction::JumpNoInterrupt { + delta: orelse_block + } + ); + self.set_no_location(); + + self.switch_to_block(handler_block); + emit!( + self, + PseudoInstruction::SetupCleanup { + delta: cleanup_block + } + ); + self.set_no_location(); + emit!(self, Instruction::PushExcInfo); + self.set_no_location(); + self.push_fblock(FBlockType::ExceptionHandler, cleanup_block, cleanup_block)?; + + for handler in handlers { + let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { + type_, + name, + body, + range: handler_range, + .. + }) = handler; + self.set_source_range(*handler_range); + let next_handler = self.new_block(); + + if let Some(exc_type) = type_ { + self.compile_expression(exc_type)?; + emit!(self, Instruction::CheckExcMatch); + emit!( + self, + Instruction::PopJumpIfFalse { + delta: next_handler + } + ); + } + + if let Some(alias) = name { + self.store_name(alias.as_str())?; + + let cleanup_end = self.new_block(); + let handler_normal_exit = self.new_block(); + emit!(self, PseudoInstruction::SetupCleanup { delta: cleanup_end }); + self.push_fblock_full( + FBlockType::HandlerCleanup, + cleanup_end, + cleanup_end, + FBlockDatum::ExceptionName(alias.as_str().to_owned()), + )?; + + self.compile_statements(body)?; + + self.pop_fblock(FBlockType::HandlerCleanup); + emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); + emit!( + self, + PseudoInstruction::JumpNoInterrupt { + delta: handler_normal_exit + } + ); + self.set_no_location(); + + self.switch_to_block(cleanup_end); + self.emit_load_const(ConstantData::None); + self.set_no_location(); + self.store_name(alias.as_str())?; + self.set_no_location(); + self.compile_name(alias.as_str(), NameUsage::Delete)?; + self.set_no_location(); + emit!(self, Instruction::Reraise { depth: 1 }); + self.set_no_location(); + + self.switch_to_block(handler_normal_exit); + emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); + self.pop_fblock(FBlockType::ExceptionHandler); + emit!(self, Instruction::PopExcept); + self.set_no_location(); + + self.emit_load_const(ConstantData::None); + self.set_no_location(); + self.store_name(alias.as_str())?; + self.set_no_location(); + self.compile_name(alias.as_str(), NameUsage::Delete)?; + self.set_no_location(); + + emit!( + self, + PseudoInstruction::JumpNoInterrupt { delta: end_block } + ); + self.set_no_location(); + } else { + emit!(self, Instruction::PopTop); + self.push_fblock(FBlockType::HandlerCleanup, end_block, end_block)?; + + self.compile_statements(body)?; + + self.pop_fblock(FBlockType::HandlerCleanup); + emit!(self, PseudoInstruction::PopBlock); + self.set_no_location(); + self.pop_fblock(FBlockType::ExceptionHandler); + emit!(self, Instruction::PopExcept); + self.set_no_location(); + emit!( + self, + PseudoInstruction::JumpNoInterrupt { delta: end_block } + ); + self.set_no_location(); + } + + self.push_fblock(FBlockType::ExceptionHandler, cleanup_block, cleanup_block)?; + self.switch_to_block(next_handler); + } + + emit!(self, Instruction::Reraise { depth: 0 }); + self.set_no_location(); + self.pop_fblock(FBlockType::ExceptionHandler); + + self.switch_to_block(cleanup_block); + emit!(self, Instruction::Copy { i: 3 }); + self.set_no_location(); + emit!(self, Instruction::PopExcept); + self.set_no_location(); + emit!(self, Instruction::Reraise { depth: 1 }); + self.set_no_location(); + + self.switch_to_block(orelse_block); + self.set_no_location(); + self.compile_statements(orelse)?; + emit!( + self, + PseudoInstruction::JumpNoInterrupt { delta: end_block } + ); + self.set_no_location(); + + self.switch_to_block(end_block); + Ok(()) + } + fn compile_try_star_except( &mut self, body: &[ast::Stmt], @@ -6801,21 +7013,7 @@ impl Compiler { self.compile_expression(last_comparator)?; self.set_source_range(compare_range); self.compile_addcompare(last_op); - if condition { - emit!( - self, - Instruction::PopJumpIfTrue { - delta: target_block - } - ); - } else { - emit!( - self, - Instruction::PopJumpIfFalse { - delta: target_block, - } - ); - } + self.emit_pop_jump_by_condition(condition, target_block); return Ok(()); } @@ -6834,21 +7032,7 @@ impl Compiler { self.compile_expression(last_comparator)?; self.set_source_range(compare_range); self.compile_addcompare(last_op); - if condition { - emit!( - self, - Instruction::PopJumpIfTrue { - delta: target_block - } - ); - } else { - emit!( - self, - Instruction::PopJumpIfFalse { - delta: target_block, - } - ); - } + self.emit_pop_jump_by_condition(condition, target_block); emit!(self, PseudoInstruction::Jump { delta: end }); self.switch_to_block(cleanup); @@ -6866,6 +7050,24 @@ impl Compiler { Ok(()) } + fn emit_pop_jump_by_condition(&mut self, condition: bool, target_block: BlockIdx) { + if condition { + emit!( + self, + Instruction::PopJumpIfTrue { + delta: target_block + } + ); + } else { + emit!( + self, + Instruction::PopJumpIfFalse { + delta: target_block, + } + ); + } + } + fn compile_annotation(&mut self, annotation: &ast::Expr) -> CompileResult<()> { if self.future_annotations { self.emit_load_const(ConstantData::Str { @@ -7554,7 +7756,26 @@ impl Compiler { | ast::Expr::BooleanLiteral(_) | ast::Expr::NoneLiteral(_) | ast::Expr::EllipsisLiteral(_) - ) + ) || matches!(expr, ast::Expr::FString(fstring) if Self::fstring_value_is_const(&fstring.value)) + } + + fn fstring_value_is_const(fstring: &ast::FStringValue) -> bool { + for part in fstring { + if !Self::fstring_part_is_const(part) { + return false; + } + } + true + } + + fn fstring_part_is_const(part: &ast::FStringPart) -> bool { + match part { + ast::FStringPart::Literal(_) => true, + ast::FStringPart::FString(fstring) => fstring + .elements + .iter() + .all(|element| matches!(element, ast::InterpolatedStringElement::Literal(_))), + } } fn compile_expression(&mut self, expression: &ast::Expr) -> CompileResult<()> { @@ -8081,6 +8302,149 @@ impl Compiler { Ok(()) } + fn detect_builtin_generator_call( + &self, + func: &ast::Expr, + args: &ast::Arguments, + ) -> Option { + let ast::Expr::Name(ast::ExprName { id, .. }) = func else { + return None; + }; + if args.args.len() != 1 + || !args.keywords.is_empty() + || !matches!(args.args[0], ast::Expr::Generator(_)) + { + return None; + } + match id.as_str() { + "tuple" => Some(BuiltinGeneratorCallKind::Tuple), + "list" => Some(BuiltinGeneratorCallKind::List), + "set" => Some(BuiltinGeneratorCallKind::Set), + "all" => Some(BuiltinGeneratorCallKind::All), + "any" => Some(BuiltinGeneratorCallKind::Any), + _ => None, + } + } + + /// Emit the optimized inline loop for builtin(genexpr) calls. + /// + /// Stack on entry: `[func, iter]` where `iter` is the already-compiled + /// generator iterator and `func` is the builtin candidate. + /// On return the compiler is positioned at the fallback block with + /// `[func, iter]` still on the stack (for the normal CALL path). + fn optimize_builtin_generator_call( + &mut self, + kind: BuiltinGeneratorCallKind, + end: BlockIdx, + ) -> CompileResult<()> { + let common_constant = match kind { + BuiltinGeneratorCallKind::Tuple => bytecode::CommonConstant::BuiltinTuple, + BuiltinGeneratorCallKind::List => bytecode::CommonConstant::BuiltinList, + BuiltinGeneratorCallKind::Set => bytecode::CommonConstant::BuiltinSet, + BuiltinGeneratorCallKind::All => bytecode::CommonConstant::BuiltinAll, + BuiltinGeneratorCallKind::Any => bytecode::CommonConstant::BuiltinAny, + }; + + let loop_block = self.new_block(); + let cleanup = self.new_block(); + let fallback = self.new_block(); + let result = matches!( + kind, + BuiltinGeneratorCallKind::All | BuiltinGeneratorCallKind::Any + ) + .then(|| self.new_block()); + + // Stack: [func, iter] — copy func (TOS1) for identity check + emit!(self, Instruction::Copy { i: 2 }); + emit!( + self, + Instruction::LoadCommonConstant { + idx: common_constant + } + ); + emit!(self, Instruction::IsOp { invert: Invert::No }); + emit!(self, Instruction::PopJumpIfFalse { delta: fallback }); + emit!(self, Instruction::NotTaken); + // Remove func from [func, iter] → [iter] + emit!(self, Instruction::Swap { i: 2 }); + emit!(self, Instruction::PopTop); + + if matches!( + kind, + BuiltinGeneratorCallKind::Tuple | BuiltinGeneratorCallKind::List + ) { + // [iter] → [iter, list] → [list, iter] + emit!(self, Instruction::BuildList { count: 0 }); + emit!(self, Instruction::Swap { i: 2 }); + } else if matches!(kind, BuiltinGeneratorCallKind::Set) { + // [iter] → [iter, set] → [set, iter] + emit!(self, Instruction::BuildSet { count: 0 }); + emit!(self, Instruction::Swap { i: 2 }); + } + + self.switch_to_block(loop_block); + emit!(self, Instruction::ForIter { delta: cleanup }); + + match kind { + BuiltinGeneratorCallKind::Tuple | BuiltinGeneratorCallKind::List => { + emit!(self, Instruction::ListAppend { i: 2 }); + emit!(self, PseudoInstruction::Jump { delta: loop_block }); + } + BuiltinGeneratorCallKind::Set => { + emit!(self, Instruction::SetAdd { i: 2 }); + emit!(self, PseudoInstruction::Jump { delta: loop_block }); + } + BuiltinGeneratorCallKind::All => { + let result = result.expect("all() optimization should have a result block"); + emit!(self, Instruction::ToBool); + emit!(self, Instruction::PopJumpIfFalse { delta: result }); + emit!(self, Instruction::NotTaken); + emit!(self, PseudoInstruction::Jump { delta: loop_block }); + } + BuiltinGeneratorCallKind::Any => { + let result = result.expect("any() optimization should have a result block"); + emit!(self, Instruction::ToBool); + emit!(self, Instruction::PopJumpIfTrue { delta: result }); + emit!(self, Instruction::NotTaken); + emit!(self, PseudoInstruction::Jump { delta: loop_block }); + } + } + + if let Some(result_block) = result { + self.switch_to_block(result_block); + emit!(self, Instruction::PopIter); + self.emit_load_const(ConstantData::Boolean { + value: matches!(kind, BuiltinGeneratorCallKind::Any), + }); + emit!(self, PseudoInstruction::Jump { delta: end }); + } + + self.switch_to_block(cleanup); + emit!(self, Instruction::EndFor); + emit!(self, Instruction::PopIter); + match kind { + BuiltinGeneratorCallKind::Tuple => { + emit!( + self, + Instruction::CallIntrinsic1 { + func: IntrinsicFunction1::ListToTuple + } + ); + } + BuiltinGeneratorCallKind::List | BuiltinGeneratorCallKind::Set => {} + BuiltinGeneratorCallKind::All => { + self.emit_load_const(ConstantData::Boolean { value: true }); + } + BuiltinGeneratorCallKind::Any => { + self.emit_load_const(ConstantData::Boolean { value: false }); + } + } + emit!(self, PseudoInstruction::Jump { delta: end }); + + self.switch_to_block(fallback); + Ok(()) + } + fn compile_call(&mut self, func: &ast::Expr, args: &ast::Arguments) -> CompileResult<()> { // Save the call expression's source range so CALL instructions use the // call start line, not the last argument's line. @@ -8131,6 +8495,23 @@ impl Compiler { } self.codegen_call_helper(0, args, call_range)?; } + } else if let Some(kind) = (!uses_ex_call) + .then(|| self.detect_builtin_generator_call(func, args)) + .flatten() + { + // Optimized builtin(genexpr) path: compile the genexpr only once + // so its code object appears exactly once in co_consts. + let end = self.new_block(); + self.compile_expression(func)?; + self.compile_expression(&args.args[0])?; + // Stack: [func, iter] + self.optimize_builtin_generator_call(kind, end)?; + // Fallback block: [func, iter] → [func, null, iter] → CALL + emit!(self, Instruction::PushNull); + emit!(self, Instruction::Swap { i: 2 }); + self.set_source_range(call_range); + emit!(self, Instruction::Call { argc: 1 }); + self.switch_to_block(end); } else { // Regular call: push func, then NULL for self_or_null slot // Stack layout: [func, NULL, args...] - same as method call [func, self, args...] @@ -9101,6 +9482,16 @@ impl Compiler { ast::Expr::BooleanLiteral(b) => ConstantData::Boolean { value: b.value }, ast::Expr::NoneLiteral(_) => ConstantData::None, ast::Expr::EllipsisLiteral(_) => ConstantData::Ellipsis, + ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { + let mut elements = Vec::with_capacity(elts.len()); + for elt in elts { + let Some(constant) = self.try_fold_constant_expr(elt)? else { + return Ok(None); + }; + elements.push(constant); + } + ConstantData::Tuple { elements } + } _ => return Ok(None), })) } @@ -9618,10 +10009,8 @@ impl Compiler { mut pending_literal: Option, mut element_count: u32, ) -> CompileResult<()> { - if let Some(value) = pending_literal.take() { - self.emit_load_const(ConstantData::Str { value }); - element_count += 1; - } + let keep_empty = element_count == 0; + self.emit_pending_fstring_literal(&mut pending_literal, &mut element_count, keep_empty); if element_count == 0 { self.emit_load_const(ConstantData::Str { @@ -9639,6 +10028,27 @@ impl Compiler { Ok(()) } + fn emit_pending_fstring_literal( + &mut self, + pending_literal: &mut Option, + element_count: &mut u32, + keep_empty: bool, + ) { + let Some(value) = pending_literal.take() else { + return; + }; + + // CPython drops empty literal fragments when they are adjacent to + // formatted values, but still emits an empty string for a fully-empty + // f-string. + if value.is_empty() && (!keep_empty || *element_count > 0) { + return; + } + + self.emit_load_const(ConstantData::Str { value }); + *element_count += 1; + } + /// Optimize `'format_str' % (args,)` into f-string bytecode. /// Returns true if optimization was applied, false to fall back to normal BINARY_OP %. /// Matches CPython's codegen.c `compiler_formatted_value` optimization. @@ -9824,10 +10234,7 @@ impl Compiler { } } - if let Some(value) = pending_literal.take() { - self.emit_load_const(ConstantData::Str { value }); - *element_count += 1; - } + self.emit_pending_fstring_literal(pending_literal, element_count, false); self.compile_expression(&fstring_expr.expression)?; @@ -10371,6 +10778,24 @@ mod tests { }) } + fn has_common_constant(code: &CodeObject, expected: bytecode::CommonConstant) -> bool { + code.instructions.iter().any(|unit| match unit.op { + Instruction::LoadCommonConstant { idx } => { + idx.get(OpArg::new(u32::from(u8::from(unit.arg)))) == expected + } + _ => false, + }) + } + + fn has_intrinsic_1(code: &CodeObject, expected: IntrinsicFunction1) -> bool { + code.instructions.iter().any(|unit| match unit.op { + Instruction::CallIntrinsic1 { func } => { + func.get(OpArg::new(u32::from(u8::from(unit.arg)))) == expected + } + _ => false, + }) + } + macro_rules! assert_dis_snapshot { ($value:expr) => { insta::assert_snapshot!( @@ -10528,6 +10953,116 @@ def f(obj, arg): ); } + #[test] + fn test_builtin_any_genexpr_call_is_optimized() { + let code = compile_exec( + "\ +def f(xs): + return any(x for x in xs) +", + ); + let f = find_code(&code, "f").expect("missing function code"); + + assert!(has_common_constant(f, bytecode::CommonConstant::BuiltinAny)); + assert!( + f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::PopJumpIfTrue { .. })) + ); + assert!( + f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::NotTaken)) + ); + assert_eq!( + f.instructions + .iter() + .filter(|unit| matches!(unit.op, Instruction::PushNull)) + .count(), + 1, + "fallback call path should remain for shadowed any()" + ); + } + + #[test] + fn test_builtin_tuple_list_set_genexpr_calls_are_optimized() { + let code = compile_exec( + "\ +def tuple_f(xs): + return tuple(x for x in xs) + +def list_f(xs): + return list(x for x in xs) + +def set_f(xs): + return set(x for x in xs) +", + ); + + let tuple_f = find_code(&code, "tuple_f").expect("missing tuple_f code"); + assert!(has_common_constant( + tuple_f, + bytecode::CommonConstant::BuiltinTuple + )); + assert!(has_intrinsic_1(tuple_f, IntrinsicFunction1::ListToTuple)); + let tuple_list_append = tuple_f + .instructions + .iter() + .find_map(|unit| match unit.op { + Instruction::ListAppend { .. } => Some(u32::from(u8::from(unit.arg))), + _ => None, + }) + .expect("tuple(genexpr) fast path should emit LIST_APPEND"); + assert_eq!(tuple_list_append, 2); + + let list_f = find_code(&code, "list_f").expect("missing list_f code"); + assert!(has_common_constant( + list_f, + bytecode::CommonConstant::BuiltinList + )); + assert!( + list_f + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::ListAppend { .. })) + ); + + let set_f = find_code(&code, "set_f").expect("missing set_f code"); + assert!(has_common_constant( + set_f, + bytecode::CommonConstant::BuiltinSet + )); + assert!( + set_f + .instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::SetAdd { .. })) + ); + } + + #[test] + fn test_module_store_uses_store_global_when_nested_scope_declares_global() { + let code = compile_exec( + "\ +_address_fmt_re = None + +class C: + def f(self): + global _address_fmt_re + if _address_fmt_re is None: + _address_fmt_re = 1 +", + ); + + assert!(code.instructions.iter().any(|unit| match unit.op { + Instruction::StoreGlobal { namei } => { + let idx = namei.get(OpArg::new(u32::from(u8::from(unit.arg)))); + code.names[usize::try_from(idx).unwrap()].as_str() == "_address_fmt_re" + } + _ => false, + })); + } + #[test] fn test_conditional_return_epilogue_is_duplicated() { let code = compile_exec( @@ -10686,6 +11221,160 @@ def f(cls, proto): ); } + #[test] + fn test_literal_only_fstring_statement_is_optimized_away() { + let code = compile_exec( + "\ +def f(): + f'''Not a docstring''' +", + ); + let f = find_code(&code, "f").expect("missing function code"); + + assert!( + !f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::PopTop)), + "literal-only f-string statement should be removed" + ); + assert!( + !f.constants.iter().any(|constant| matches!( + constant, + ConstantData::Str { value } if value.to_string() == "Not a docstring" + )), + "literal-only f-string should not survive in constants" + ); + } + + #[test] + fn test_empty_fstring_literals_are_elided_around_interpolation() { + let code = compile_exec( + "\ +def f(x): + if '' f'{x}': + return 1 + return 2 +", + ); + let f = find_code(&code, "f").expect("missing function code"); + + let empty_string_loads = f + .instructions + .iter() + .filter_map(|unit| match unit.op { + Instruction::LoadConst { consti } => { + Some(&f.constants[consti.get(OpArg::new(u32::from(u8::from(unit.arg))))]) + } + _ => None, + }) + .filter(|constant| { + matches!( + constant, + ConstantData::Str { value } if value.is_empty() + ) + }) + .count(); + let build_string_count = f + .instructions + .iter() + .filter(|unit| matches!(unit.op, Instruction::BuildString { .. })) + .count(); + + assert_eq!(empty_string_loads, 0); + assert_eq!(build_string_count, 0); + } + + #[test] + fn test_large_power_is_not_constant_folded() { + let code = compile_exec("x = 2**100\n"); + + assert!(code.instructions.iter().any(|unit| match unit.op { + Instruction::BinaryOp { op } => { + op.get(OpArg::new(u32::from(u8::from(unit.arg)))) == oparg::BinaryOperator::Power + } + _ => false, + })); + } + + #[test] + fn test_list_of_constant_tuples_uses_list_extend() { + let code = compile_exec( + "\ +deprecated_cases = [('a', 'b'), ('c', 'd'), ('e', 'f'), ('g', 'h'), ('i', 'j')] +", + ); + + assert!( + code.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::ListExtend { .. })), + "expected constant tuple list folding" + ); + } + + #[test] + fn test_constant_list_iterable_uses_tuple() { + let code = compile_exec( + "\ +def f(): + return {x: y for x, y in [(1, 2), ]} +", + ); + let f = find_code(&code, "f").expect("missing function code"); + + assert!( + !f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BuildList { .. })), + "constant list iterable should avoid BUILD_LIST before GET_ITER" + ); + assert!(f.constants.iter().any(|constant| matches!( + constant, + ConstantData::Tuple { elements } + if matches!( + elements.as_slice(), + [ConstantData::Tuple { elements: inner }] + if matches!( + inner.as_slice(), + [ + ConstantData::Integer { .. }, + ConstantData::Integer { .. } + ] + ) + ) + ))); + } + + #[test] + fn test_constant_set_iterable_keeps_runtime_set_build() { + let code = compile_exec( + "\ +def f(): + return [x for x in {1, 2, 3}] +", + ); + let f = find_code(&code, "f").expect("missing function code"); + + assert!( + f.instructions + .iter() + .any(|unit| matches!(unit.op, Instruction::BuildSet { .. })), + "constant set iterable should keep BUILD_SET before GET_ITER" + ); + assert!(f.constants.iter().any(|constant| matches!( + constant, + ConstantData::Tuple { elements } + if matches!( + elements.as_slice(), + [ + ConstantData::Integer { .. }, + ConstantData::Integer { .. }, + ConstantData::Integer { .. } + ] + ) + ))); + } + #[test] fn test_optimized_assert_preserves_nested_scope_order() { compile_exec_optimized( diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index 12c1961e88f..bc709147479 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -25,6 +25,9 @@ struct LineTableLocation { end_col: i32, } +const MAX_INT_SIZE_BITS: u64 = 128; +const MIN_CONST_SEQUENCE_SIZE: usize = 3; + /// Metadata for a code unit // = _PyCompile_CodeUnitMetadata #[derive(Clone, Debug)] @@ -795,7 +798,12 @@ impl CodeInfo { let result = match op { BinOp::Add => l + r, BinOp::Subtract => l - r, - BinOp::Multiply => l * r, + BinOp::Multiply => { + if !l.is_zero() && !r.is_zero() && l.bits() + r.bits() > MAX_INT_SIZE_BITS { + return None; + } + l * r + } BinOp::FloorDivide => { if r.is_zero() { return None; @@ -821,18 +829,22 @@ impl CodeInfo { } } BinOp::Power => { - let exp: u32 = r.try_into().ok()?; - if exp > 128 { + let exp: u64 = r.try_into().ok()?; + let exp_usize = usize::try_from(exp).ok()?; + if !l.is_zero() && exp > 0 && l.bits() > MAX_INT_SIZE_BITS / exp { return None; - } // prevent huge results - num_traits::pow::pow(l.clone(), exp as usize) + } + num_traits::pow::pow(l.clone(), exp_usize) } BinOp::Lshift => { - let shift: u32 = r.try_into().ok()?; - if shift > 128 { + let shift: u64 = r.try_into().ok()?; + let shift_usize = usize::try_from(shift).ok()?; + if shift > MAX_INT_SIZE_BITS + || (!l.is_zero() && l.bits() > MAX_INT_SIZE_BITS - shift) + { return None; } - l << (shift as usize) + l << shift_usize } BinOp::Rshift => { let shift: u32 = r.try_into().ok()?; @@ -1070,7 +1082,7 @@ impl CodeInfo { } } - if !all_const || list_size < 3 { + if !all_const || list_size < MIN_CONST_SEQUENCE_SIZE { i += 1; continue; } @@ -1117,31 +1129,42 @@ impl CodeInfo { } } - /// Convert constant list/set construction before GET_ITER to just LOAD_CONST tuple. + /// Convert constant list construction before GET_ITER to just LOAD_CONST tuple. /// BUILD_LIST 0 + LOAD_CONST (tuple) + LIST_EXTEND 1 + GET_ITER /// → LOAD_CONST (tuple) + GET_ITER - /// Also handles BUILD_SET 0 + LOAD_CONST + SET_UPDATE 1 + GET_ITER. fn fold_const_iterable_for_iter(&mut self) { for block in &mut self.blocks { let mut i = 0; - while i + 3 < block.instructions.len() { + while i + 1 < block.instructions.len() { let is_build = matches!( block.instructions[i].instr.real(), - Some(Instruction::BuildList { .. } | Instruction::BuildSet { .. }) + Some(Instruction::BuildList { .. }) ) && u32::from(block.instructions[i].arg) == 0; let is_const = matches!( - block.instructions[i + 1].instr.real(), + block + .instructions + .get(i + 1) + .and_then(|instr| instr.instr.real()), Some(Instruction::LoadConst { .. }) ); let is_extend = matches!( - block.instructions[i + 2].instr.real(), - Some(Instruction::ListExtend { .. } | Instruction::SetUpdate { .. }) - ) && u32::from(block.instructions[i + 2].arg) == 1; + block + .instructions + .get(i + 2) + .and_then(|instr| instr.instr.real()), + Some(Instruction::ListExtend { .. }) + ) && block + .instructions + .get(i + 2) + .is_some_and(|instr| u32::from(instr.arg) == 1); let is_iter = matches!( - block.instructions[i + 3].instr.real(), + block + .instructions + .get(i + 3) + .and_then(|instr| instr.instr.real()), Some(Instruction::GetIter) ); @@ -1153,6 +1176,56 @@ impl CodeInfo { block.instructions[i + 2].instr = Instruction::Nop.into(); block.instructions[i + 2].location = loc; i += 4; + } else if matches!( + block.instructions[i].instr.real(), + Some(Instruction::BuildList { .. }) + ) && matches!( + block.instructions[i + 1].instr.real(), + Some(Instruction::GetIter) + ) { + let seq_size = u32::from(block.instructions[i].arg) as usize; + + if seq_size != 0 && i >= seq_size { + let start_idx = i - seq_size; + let mut elements = Vec::with_capacity(seq_size); + let mut all_const = true; + + for j in start_idx..i { + match Self::get_const_value_from(&self.metadata, &block.instructions[j]) + { + Some(constant) => elements.push(constant), + None => { + all_const = false; + break; + } + } + } + + if all_const { + let const_data = ConstantData::Tuple { elements }; + let (const_idx, _) = self.metadata.consts.insert_full(const_data); + let folded_loc = block.instructions[i].location; + + for j in start_idx..i { + block.instructions[j].instr = Instruction::Nop.into(); + block.instructions[j].location = folded_loc; + } + + block.instructions[i].instr = Instruction::LoadConst { + consti: Arg::marker(), + } + .into(); + block.instructions[i].arg = OpArg::new(const_idx as u32); + i += 2; + continue; + } + } + + block.instructions[i].instr = Instruction::BuildTuple { + count: Arg::marker(), + } + .into(); + i += 2; } else { i += 1; } @@ -2669,14 +2742,16 @@ fn duplicate_exits_without_lineno(blocks: &mut Vec, predecessors: &mut Ve continue; } - // Copy the exit block + // Copy the exit block and splice it into the linked list after current let new_idx = BlockIdx(blocks.len() as u32); let mut new_block = blocks[target.idx()].clone(); let jump_loc = last.location; let jump_end_loc = last.end_location; propagate_locations_in_block(&mut new_block, jump_loc, jump_end_loc); - new_block.next = blocks[target.idx()].next; + let old_next = blocks[current.idx()].next; + new_block.next = old_next; blocks.push(new_block); + blocks[current.idx()].next = new_idx; // Update the jump target let last_mut = blocks[current.idx()].instructions.last_mut().unwrap(); @@ -2684,7 +2759,8 @@ fn duplicate_exits_without_lineno(blocks: &mut Vec, predecessors: &mut Ve predecessors[target.idx()] -= 1; predecessors.push(1); - current = blocks[current.idx()].next; + // Skip past the newly inserted block + current = old_next; } current = BlockIdx(0); diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap index 02af6cad00b..ba8be3589d3 100644 --- a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap @@ -33,12 +33,12 @@ expression: "compile_exec(\"\\\nasync def test():\n for stop_exc in (StopIter 26 CACHE 27 STORE_FAST (0, stop_exc) - 3 >> 28 LOAD_GLOBAL (4, self) + 3 28 LOAD_GLOBAL (4, self) 29 CACHE 30 CACHE 31 CACHE - 32 CACHE - >> 33 LOAD_ATTR (7, subTest, method=true) + >> 32 CACHE + 33 LOAD_ATTR (7, subTest, method=true) 34 CACHE 35 CACHE 36 CACHE @@ -51,9 +51,9 @@ expression: "compile_exec(\"\\\nasync def test():\n for stop_exc in (StopIter 43 LOAD_GLOBAL (9, NULL + type) 44 CACHE 45 CACHE - >> 46 CACHE + 46 CACHE 47 CACHE - 48 LOAD_FAST (0, stop_exc) + >> 48 LOAD_FAST (0, stop_exc) 49 CALL (1) 50 CACHE 51 CACHE @@ -138,30 +138,30 @@ expression: "compile_exec(\"\\\nasync def test():\n for stop_exc in (StopIter 125 POP_TOP 126 POP_TOP 127 POP_TOP - 128 JUMP_FORWARD (3) + 128 JUMP_FORWARD (48) 129 COPY (3) 130 POP_EXCEPT 131 RERAISE (1) - 132 JUMP_FORWARD (46) - 133 PUSH_EXC_INFO + 132 PUSH_EXC_INFO - 7 134 LOAD_GLOBAL (12, Exception) + 7 133 LOAD_GLOBAL (12, Exception) + 134 CACHE 135 CACHE 136 CACHE 137 CACHE - 138 CACHE - 139 CHECK_EXC_MATCH - 140 POP_JUMP_IF_FALSE (33) - 141 CACHE - 142 NOT_TAKEN - 143 STORE_FAST (1, ex) + 138 CHECK_EXC_MATCH + 139 POP_JUMP_IF_FALSE (32) + 140 CACHE + 141 NOT_TAKEN + 142 STORE_FAST (1, ex) - 8 144 LOAD_GLOBAL (4, self) + 8 143 LOAD_GLOBAL (4, self) + 144 CACHE 145 CACHE 146 CACHE 147 CACHE - 148 CACHE - 149 LOAD_ATTR (15, assertIs, method=true) + 148 LOAD_ATTR (15, assertIs, method=true) + 149 CACHE 150 CACHE 151 CACHE 152 CACHE @@ -170,87 +170,85 @@ expression: "compile_exec(\"\\\nasync def test():\n for stop_exc in (StopIter 155 CACHE 156 CACHE 157 CACHE - 158 CACHE - 159 LOAD_FAST_LOAD_FAST (ex, stop_exc) - 160 CALL (2) + 158 LOAD_FAST_LOAD_FAST (ex, stop_exc) + 159 CALL (2) + 160 CACHE 161 CACHE 162 CACHE - 163 CACHE - 164 POP_TOP - 165 JUMP_FORWARD (4) - 166 LOAD_CONST (None) - 167 STORE_FAST (1, ex) - 168 DELETE_FAST (1, ex) - 169 RERAISE (1) - 170 POP_EXCEPT - 171 LOAD_CONST (None) - 172 STORE_FAST (1, ex) - 173 DELETE_FAST (1, ex) - 174 JUMP_FORWARD (28) - 175 RERAISE (0) - 176 COPY (3) - 177 POP_EXCEPT - 178 RERAISE (1) + 163 POP_TOP + 164 POP_EXCEPT + 165 LOAD_CONST (None) + 166 STORE_FAST (1, ex) + 167 DELETE_FAST (1, ex) + 168 JUMP_FORWARD (32) + 169 LOAD_CONST (None) + 170 STORE_FAST (1, ex) + 171 DELETE_FAST (1, ex) + 172 RERAISE (1) + 173 RERAISE (0) + 174 COPY (3) + 175 POP_EXCEPT + 176 RERAISE (1) - 10 179 LOAD_GLOBAL (4, self) + 10 177 LOAD_GLOBAL (4, self) + 178 CACHE + 179 CACHE 180 CACHE 181 CACHE - 182 CACHE + 182 LOAD_ATTR (17, fail, method=true) 183 CACHE - 184 LOAD_ATTR (17, fail, method=true) + 184 CACHE 185 CACHE 186 CACHE - 187 CACHE + >> 187 CACHE 188 CACHE - >> 189 CACHE + 189 CACHE 190 CACHE 191 CACHE - 192 CACHE - 193 CACHE - 194 LOAD_FAST_BORROW (0, stop_exc) - 195 FORMAT_SIMPLE - 196 LOAD_CONST (" was suppressed") - 197 BUILD_STRING (2) - 198 CALL (1) + 192 LOAD_FAST_BORROW (0, stop_exc) + 193 FORMAT_SIMPLE + 194 LOAD_CONST (" was suppressed") + 195 BUILD_STRING (2) + 196 CALL (1) + 197 CACHE + 198 CACHE 199 CACHE - 200 CACHE - 201 CACHE - 202 POP_TOP - 203 NOP + 200 POP_TOP + 201 NOP - 3 204 LOAD_CONST (None) - 205 LOAD_CONST (None) - >> 206 LOAD_CONST (None) - 207 CALL (3) + 3 202 LOAD_CONST (None) + 203 LOAD_CONST (None) + >> 204 LOAD_CONST (None) + 205 CALL (3) + 206 CACHE + 207 CACHE 208 CACHE - 209 CACHE - 210 CACHE - 211 POP_TOP - 212 JUMP_BACKWARD (189) - 213 CACHE - 214 PUSH_EXC_INFO - 215 WITH_EXCEPT_START - 216 TO_BOOL + 209 POP_TOP + 210 JUMP_BACKWARD (187) + 211 CACHE + 212 PUSH_EXC_INFO + 213 WITH_EXCEPT_START + 214 TO_BOOL + 215 CACHE + 216 CACHE 217 CACHE - 218 CACHE + 218 POP_JUMP_IF_TRUE (2) 219 CACHE - 220 POP_JUMP_IF_TRUE (2) - 221 CACHE - 222 NOT_TAKEN - 223 RERAISE (2) + 220 NOT_TAKEN + 221 RERAISE (2) + 222 POP_TOP + 223 POP_EXCEPT 224 POP_TOP - 225 POP_EXCEPT + 225 POP_TOP 226 POP_TOP - 227 POP_TOP - 228 POP_TOP - 229 JUMP_BACKWARD (206) - 230 CACHE - 231 COPY (3) - 232 POP_EXCEPT - 233 RERAISE (1) + 227 JUMP_BACKWARD (204) + 228 CACHE + 229 COPY (3) + 230 POP_EXCEPT + 231 RERAISE (1) - 2 234 CALL_INTRINSIC_1 (StopIterationError) - 235 RERAISE (1) + 2 232 CALL_INTRINSIC_1 (StopIterationError) + 233 RERAISE (1) 2 MAKE_FUNCTION 3 STORE_NAME (0, test) diff --git a/crates/compiler-core/src/bytecode/oparg.rs b/crates/compiler-core/src/bytecode/oparg.rs index 3c5f78d474c..a0ab45a3abc 100644 --- a/crates/compiler-core/src/bytecode/oparg.rs +++ b/crates/compiler-core/src/bytecode/oparg.rs @@ -645,6 +645,10 @@ oparg_enum!( BuiltinAll = 3, /// Built-in `any` function BuiltinAny = 4, + /// Built-in `list` type + BuiltinList = 5, + /// Built-in `set` type + BuiltinSet = 6, } ); @@ -656,6 +660,8 @@ impl fmt::Display for CommonConstant { Self::BuiltinTuple => "tuple", Self::BuiltinAll => "all", Self::BuiltinAny => "any", + Self::BuiltinList => "list", + Self::BuiltinSet => "set", }; write!(f, "{name}") } diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 5c2312015e9..7cb33ec95d9 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -2786,8 +2786,18 @@ impl ExecutingFrame<'_> { vm.ctx.exceptions.not_implemented_error.to_owned().into() } CommonConstant::BuiltinTuple => vm.ctx.types.tuple_type.to_owned().into(), - CommonConstant::BuiltinAll => vm.builtins.get_attr("all", vm)?, - CommonConstant::BuiltinAny => vm.builtins.get_attr("any", vm)?, + CommonConstant::BuiltinAll => vm + .callable_cache + .builtin_all + .clone() + .expect("builtin_all not initialized"), + CommonConstant::BuiltinAny => vm + .callable_cache + .builtin_any + .clone() + .expect("builtin_any not initialized"), + CommonConstant::BuiltinList => vm.ctx.types.list_type.to_owned().into(), + CommonConstant::BuiltinSet => vm.ctx.types.set_type.to_owned().into(), }; self.push_value(value); Ok(None) diff --git a/crates/vm/src/version.rs b/crates/vm/src/version.rs index 21efecd6c5a..5b80a33322b 100644 --- a/crates/vm/src/version.rs +++ b/crates/vm/src/version.rs @@ -129,8 +129,8 @@ pub fn get_git_datetime() -> String { } // Must be aligned to Lib/importlib/_bootstrap_external.py -// Bumped to 2997 for MAKE_CELL/COPY_FREE_VARS prolog and cell-local merging -pub const PYC_MAGIC_NUMBER: u16 = 2997; +// Bumped to 2994 for new CommonConstant discriminants (BuiltinList, BuiltinSet) +pub const PYC_MAGIC_NUMBER: u16 = 2994; // CPython format: magic_number | ('\r' << 16) | ('\n' << 24) // This protects against text-mode file reads diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index eee2055b751..da9149cc5fc 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -576,6 +576,8 @@ pub(crate) struct CallableCache { pub len: Option, pub isinstance: Option, pub list_append: Option, + pub builtin_all: Option, + pub builtin_any: Option, } pub struct PyGlobalState { @@ -641,6 +643,8 @@ impl VirtualMachine { .get_attr(self.ctx.intern_str("append")) .ok_or_else(|| self.new_runtime_error("failed to cache list.append".to_owned()))?; self.callable_cache.list_append = Some(list_append); + self.callable_cache.builtin_all = Some(self.builtins.get_attr("all", self)?); + self.callable_cache.builtin_any = Some(self.builtins.get_attr("any", self)?); Ok(()) } From d9c4c95369a843131dfb39026942937e51cbc20a Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Tue, 31 Mar 2026 21:05:08 +0900 Subject: [PATCH 010/834] fix crates (#7549) --- crates/compiler-source/Cargo.toml | 2 +- crates/doc/Cargo.toml | 1 + crates/pylib/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/compiler-source/Cargo.toml b/crates/compiler-source/Cargo.toml index 068d31e87ed..cb94c17477f 100644 --- a/crates/compiler-source/Cargo.toml +++ b/crates/compiler-source/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustpython-compiler-source" description = "(DEPRECATED) RustPython Source and Index" -version = "0.5.0+deprecated" +version = "0.4.1+deprecated" authors.workspace = true edition.workspace = true rust-version.workspace = true diff --git a/crates/doc/Cargo.toml b/crates/doc/Cargo.toml index 3435fabc8b5..a053f9a6383 100644 --- a/crates/doc/Cargo.toml +++ b/crates/doc/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "rustpython-doc" +description = "Python __doc__ database for RustPython" version.workspace = true authors.workspace = true edition.workspace = true diff --git a/crates/pylib/Cargo.toml b/crates/pylib/Cargo.toml index dcbb5928599..0a9326640c1 100644 --- a/crates/pylib/Cargo.toml +++ b/crates/pylib/Cargo.toml @@ -2,7 +2,7 @@ name = "rustpython-pylib" description = "A subset of the Python standard library for use with RustPython" license-file = "Lib/PSF-LICENSE" -include = ["Cargo.toml", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"] +include = ["Cargo.toml", "build.rs", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"] authors = ["CPython Developers"] version.workspace = true edition.workspace = true From 344b7a5abd9bf004993b1c05ee4fa89238c2c872 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Thu, 2 Apr 2026 17:44:58 +0200 Subject: [PATCH 011/834] [zizmor] ignore `superfluous-actions` (#7548) --- .github/workflows/ci.yaml | 22 +++++++++------------- .github/workflows/release.yml | 4 ++-- .github/zizmor.yml | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 15 deletions(-) create mode 100644 .github/zizmor.yml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 99906707cf3..d1075152e99 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,10 +50,9 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + - uses: dtolnay/rust-toolchain@stable with: components: clippy - toolchain: stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -170,7 +169,9 @@ jobs: restore-keys: | cargo-check-${{ runner.os }}-${{ matrix.target }}- - - run: rustup toolchain install stable --target "${{ matrix.target }}" + - uses: dtolnay/rust-toolchain@stable + with: + target: ${{ matrix.target }} - name: Setup Android NDK if: ${{ matrix.target == 'aarch64-linux-android' }} @@ -251,9 +252,7 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 - with: - toolchain: stable + - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -366,9 +365,8 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + - uses: dtolnay/rust-toolchain@stable with: - toolchain: stable components: rustfmt - uses: cargo-bins/cargo-binstall@113a77a4ce971c41332f2129c3d995df993cf746 # v1.17.8 @@ -425,7 +423,7 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.NIGHTLY_CHANNEL }} components: miri @@ -451,10 +449,9 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + - uses: dtolnay/rust-toolchain@stable with: components: clippy - toolchain: stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -529,10 +526,9 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasip1 - toolchain: stable - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f06f0d3adc6..ae6c9da9b74 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,7 +52,7 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + - uses: dtolnay/rust-toolchain@stable with: target: ${{ matrix.target }} @@ -89,7 +89,7 @@ jobs: with: persist-credentials: false - - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + - uses: dtolnay/rust-toolchain@stable with: targets: wasm32-wasip1 diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 00000000000..f22f76b70d8 --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,14 @@ +rules: + unpinned-uses: + config: + policies: + # dtolnay/rust-toolchain is a trusted action that uses lightweight branch + # refs (@stable, @nightly, etc.) by design. Pinning to a hash would break + # the intended usage pattern. + # We can remove this once https://github.com/dtolnay/rust-toolchain/issues/180 is resolved + dtolnay/rust-toolchain: any + # dtolnay/rust-toolchain handles component installation, target addition, and + # override configuration beyond what a bare `rustup` invocation provides. + # See: https://github.com/zizmorcore/zizmor/issues/1817 + superfluous-actions: + disable: true From 1adda8a73d826a93fee449e8e8d426158542608c Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Thu, 2 Apr 2026 17:49:55 +0200 Subject: [PATCH 012/834] Update `test_file` & `test_largefile` from 3.14.3 (#7550) --- Cargo.lock | 2 +- Lib/test/test_file.py | 13 +++++++++++-- Lib/test/test_largefile.py | 32 ++++++++++++++++++++++++++------ 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1045146ee0d..ea7eeec661a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3091,7 +3091,7 @@ dependencies = [ [[package]] name = "rustpython-compiler-source" -version = "0.5.0+deprecated" +version = "0.4.1+deprecated" dependencies = [ "rustpython-ruff_source_file", "rustpython-ruff_text_size", diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index 36aea52a5a9..029c903e01a 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -126,7 +126,7 @@ def testMethods(self): # it must also return None if an exception was given try: 1/0 - except: + except ZeroDivisionError: self.assertEqual(self.f.__exit__(*sys.exc_info()), None) def testReadWhenWriting(self): @@ -216,6 +216,16 @@ def testSetBufferSize(self): with self.assertWarnsRegex(RuntimeWarning, 'line buffering'): self._checkBufferSize(1) + def testDefaultBufferSize(self): + with self.open(TESTFN, 'wb') as f: + blksize = f.raw._blksize + f.write(b"\0" * 5_000_000) + + with self.open(TESTFN, 'rb') as f: + data = f.read1() + expected_size = max(min(blksize, 8192 * 1024), io.DEFAULT_BUFFER_SIZE) + self.assertEqual(len(data), expected_size) + def testTruncateOnWindows(self): # SF bug # "file.truncate fault on windows" @@ -344,7 +354,6 @@ def testIteration(self): class COtherFileTests(OtherFileTests, unittest.TestCase): open = io.open - class PyOtherFileTests(OtherFileTests, unittest.TestCase): open = staticmethod(pyio.open) diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 3c11c59baef..41f7b70e5cf 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -2,13 +2,12 @@ """ import os -import stat import sys import unittest import socket import shutil import threading -from test.support import requires, bigmemtest +from test.support import requires, bigmemtest, requires_resource from test.support import SHORT_TIMEOUT from test.support import socket_helper from test.support.os_helper import TESTFN, unlink @@ -29,7 +28,7 @@ def setUp(self): mode = 'w+b' with self.open(TESTFN, mode) as f: - current_size = os.fstat(f.fileno())[stat.ST_SIZE] + current_size = os.fstat(f.fileno()).st_size if current_size == size+1: return @@ -40,13 +39,13 @@ def setUp(self): f.seek(size) f.write(b'a') f.flush() - self.assertEqual(os.fstat(f.fileno())[stat.ST_SIZE], size+1) + self.assertEqual(os.fstat(f.fileno()).st_size, size+1) @classmethod def tearDownClass(cls): with cls.open(TESTFN, 'wb'): pass - if not os.stat(TESTFN)[stat.ST_SIZE] == 0: + if not os.stat(TESTFN).st_size == 0: raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') unlink(TESTFN2) @@ -67,7 +66,7 @@ def test_large_read(self, _size): self.assertEqual(f.tell(), size + 1) def test_osstat(self): - self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1) + self.assertEqual(os.stat(TESTFN).st_size, size+1) def test_seek_read(self): with self.open(TESTFN, 'rb') as f: @@ -142,6 +141,9 @@ def test_truncate(self): f.truncate(1) self.assertEqual(f.tell(), 0) # else pointer moved f.seek(0) + # Verify readall on a truncated file is well behaved. read() + # without a size can be unbounded, this should get just the byte + # that remains. self.assertEqual(len(f.read()), 1) # else wasn't truncated def test_seekable(self): @@ -152,6 +154,22 @@ def test_seekable(self): f.seek(pos) self.assertTrue(f.seekable()) + @bigmemtest(size=size, memuse=2, dry_run=False) + def test_seek_readall(self, _size): + # Seek which doesn't change position should readall successfully. + with self.open(TESTFN, 'rb') as f: + self.assertEqual(f.seek(0, os.SEEK_CUR), 0) + self.assertEqual(len(f.read()), size + 1) + + # Seek which changes (or might change) position should readall + # successfully. + with self.open(TESTFN, 'rb') as f: + self.assertEqual(f.seek(20, os.SEEK_SET), 20) + self.assertEqual(len(f.read()), size - 19) + + with self.open(TESTFN, 'rb') as f: + self.assertEqual(f.seek(-3, os.SEEK_END), size - 2) + self.assertEqual(len(f.read()), 3) def skip_no_disk_space(path, required): def decorator(fun): @@ -173,6 +191,7 @@ class TestCopyfile(LargeFileTest, unittest.TestCase): # Exact required disk space would be (size * 2), but let's give it a # bit more tolerance. @skip_no_disk_space(TESTFN, size * 2.5) + @requires_resource('cpu') def test_it(self): # Internally shutil.copyfile() can use "fast copy" methods like # os.sendfile(). @@ -222,6 +241,7 @@ def run(sock): # Exact required disk space would be (size * 2), but let's give it a # bit more tolerance. @skip_no_disk_space(TESTFN, size * 2.5) + @requires_resource('cpu') def test_it(self): port = socket_helper.find_unused_port() with socket.create_server(("", port)) as sock: From 5afa3493a1a90eac7fbb00833115bb53f1d9421c Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Thu, 2 Apr 2026 17:50:25 +0200 Subject: [PATCH 013/834] Add `test_perfmaps.py` from 3.14.3 (#7551) --- Lib/test/test_perfmaps.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 Lib/test/test_perfmaps.py diff --git a/Lib/test/test_perfmaps.py b/Lib/test/test_perfmaps.py new file mode 100644 index 00000000000..d4c6fe0124a --- /dev/null +++ b/Lib/test/test_perfmaps.py @@ -0,0 +1,23 @@ +import os +import sys +import unittest + +try: + from _testinternalcapi import perf_map_state_teardown, write_perf_map_entry +except ImportError: + raise unittest.SkipTest("requires _testinternalcapi") + + +if sys.platform != 'linux': + raise unittest.SkipTest('Linux only') + + +class TestPerfMapWriting(unittest.TestCase): + def test_write_perf_map_entry(self): + self.assertEqual(write_perf_map_entry(0x1234, 5678, "entry1"), 0) + self.assertEqual(write_perf_map_entry(0x2345, 6789, "entry2"), 0) + with open(f"/tmp/perf-{os.getpid()}.map") as f: + perf_file_contents = f.read() + self.assertIn("1234 162e entry1", perf_file_contents) + self.assertIn("2345 1a85 entry2", perf_file_contents) + perf_map_state_teardown() From 3f49f42702cd6e08e88e594a860bd7994cc838b9 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Thu, 2 Apr 2026 18:33:19 +0200 Subject: [PATCH 014/834] Update `test_module` from 3.14.3 (#7552) * Update `test_module` from 3.14.3 * Unmark passing test --- Lib/test/test_module/__init__.py | 45 +++++++++++++++++++++++++------- Lib/test/test_module/final_a.py | 19 ++++++++++++++ Lib/test/test_module/final_b.py | 19 ++++++++++++++ 3 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 Lib/test/test_module/final_a.py create mode 100644 Lib/test/test_module/final_b.py diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index 59c74fd0d41..4704ad29974 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -1,4 +1,5 @@ # Test the module type +import importlib.machinery import unittest import weakref from test.support import gc_collect @@ -29,7 +30,7 @@ def test_uninitialized(self): self.fail("__name__ = %s" % repr(s)) except AttributeError: pass - self.assertEqual(foo.__doc__, ModuleType.__doc__) + self.assertEqual(foo.__doc__, ModuleType.__doc__ or '') def test_uninitialized_missing_getattr(self): # Issue 8297 @@ -102,8 +103,7 @@ def f(): gc_collect() self.assertEqual(f().__dict__["bar"], 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_clear_dict_in_ref_cycle(self): destroyed = [] m = ModuleType("foo") @@ -152,15 +152,13 @@ def test_module_getattr_errors(self): if 'test.test_module.bad_getattr2' in sys.modules: del sys.modules['test.test_module.bad_getattr2'] - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_module_dir(self): import test.test_module.good_getattr as gga self.assertEqual(dir(gga), ['a', 'b', 'c']) del sys.modules['test.test_module.good_getattr'] - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_module_dir_errors(self): import test.test_module.bad_getattr as bga from test.test_module import bad_getattr2 @@ -270,11 +268,38 @@ def test_module_repr_source(self): self.assertEqual(r[-len(ends_with):], ends_with, '{!r} does not end with {!r}'.format(r, ends_with)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_module_repr_with_namespace_package(self): + m = ModuleType('foo') + loader = importlib.machinery.NamespaceLoader('foo', ['bar'], 'baz') + spec = importlib.machinery.ModuleSpec('foo', loader) + m.__loader__ = loader + m.__spec__ = spec + self.assertEqual(repr(m), "") + + def test_module_repr_with_namespace_package_and_custom_loader(self): + m = ModuleType('foo') + loader = BareLoader() + spec = importlib.machinery.ModuleSpec('foo', loader) + m.__loader__ = loader + m.__spec__ = spec + expected_repr_pattern = r"\)>" + self.assertRegex(repr(m), expected_repr_pattern) + self.assertNotIn('from', repr(m)) + + def test_module_repr_with_fake_namespace_package(self): + m = ModuleType('foo') + loader = BareLoader() + loader._path = ['spam'] + spec = importlib.machinery.ModuleSpec('foo', loader) + m.__loader__ = loader + m.__spec__ = spec + expected_repr_pattern = r"\)>" + self.assertRegex(repr(m), expected_repr_pattern) + self.assertNotIn('from', repr(m)) + def test_module_finalization_at_shutdown(self): # Module globals and builtins should still be available during shutdown - rc, out, err = assert_python_ok("-c", "from test import final_a") + rc, out, err = assert_python_ok("-c", "from test.test_module import final_a") self.assertFalse(err) lines = out.splitlines() self.assertEqual(set(lines), { diff --git a/Lib/test/test_module/final_a.py b/Lib/test/test_module/final_a.py new file mode 100644 index 00000000000..a983f311124 --- /dev/null +++ b/Lib/test/test_module/final_a.py @@ -0,0 +1,19 @@ +""" +Fodder for module finalization tests in test_module. +""" + +import shutil +import test.test_module.final_b + +x = 'a' + +class C: + def __del__(self): + # Inspect module globals and builtins + print("x =", x) + print("final_b.x =", test.test_module.final_b.x) + print("shutil.rmtree =", getattr(shutil.rmtree, '__name__', None)) + print("len =", getattr(len, '__name__', None)) + +c = C() +_underscored = C() diff --git a/Lib/test/test_module/final_b.py b/Lib/test/test_module/final_b.py new file mode 100644 index 00000000000..f3e8d559490 --- /dev/null +++ b/Lib/test/test_module/final_b.py @@ -0,0 +1,19 @@ +""" +Fodder for module finalization tests in test_module. +""" + +import shutil +import test.test_module.final_a + +x = 'b' + +class C: + def __del__(self): + # Inspect module globals and builtins + print("x =", x) + print("final_a.x =", test.test_module.final_a.x) + print("shutil.rmtree =", getattr(shutil.rmtree, '__name__', None)) + print("len =", getattr(len, '__name__', None)) + +c = C() +_underscored = C() From 6d7d74cc0bd3e15a6605df7a098915f2dc3d9554 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Thu, 2 Apr 2026 18:33:34 +0200 Subject: [PATCH 015/834] Update `test_opcache.py` from 3.14.3 (#7553) --- Lib/test/test_opcache.py | 866 +++++++++++++++++++++++++++++++++++---- 1 file changed, 782 insertions(+), 84 deletions(-) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index aeb4de6caa5..2e50c1c62b3 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -4,7 +4,9 @@ import threading import types import unittest -from test.support import threading_helper, check_impl_detail, requires_specialization +from test.support import (threading_helper, check_impl_detail, + requires_specialization, requires_specialization_ft, + cpython_only, requires_jit_disabled, reset_code) from test.support.import_helper import import_module # Skip this module on other interpreters, it is cpython specific: @@ -14,18 +16,14 @@ _testinternalcapi = import_module("_testinternalcapi") -def disabling_optimizer(func): - def wrapper(*args, **kwargs): - if not hasattr(_testinternalcapi, "get_optimizer"): - return func(*args, **kwargs) - old_opt = _testinternalcapi.get_optimizer() - _testinternalcapi.set_optimizer(None) - try: - return func(*args, **kwargs) - finally: - _testinternalcapi.set_optimizer(old_opt) - - return wrapper +def have_dict_key_versions(): + # max version value that can be stored in the load global cache. This is + # determined by the type of module_keys_version and builtin_keys_version + # in _PyLoadGlobalCache, uint16_t. + max_version = 1<<16 + # use a wide safety margin (use only half of what's available) + limit = max_version // 2 + return _testinternalcapi.get_next_dict_keys_version() < limit class TestBase(unittest.TestCase): @@ -34,6 +32,11 @@ def assert_specialized(self, f, opname): opnames = {instruction.opname for instruction in instructions} self.assertIn(opname, opnames) + def assert_no_opcode(self, f, opname): + instructions = dis.get_instructions(f, adaptive=True) + opnames = {instruction.opname for instruction in instructions} + self.assertNotIn(opname, opnames) + class TestLoadSuperAttrCache(unittest.TestCase): def test_descriptor_not_double_executed_on_spec_fail(self): @@ -52,7 +55,8 @@ def f(self): d = D() - self.assertEqual(d.f(), 1) # warmup + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD - 1): + self.assertEqual(d.f(), 1) # warmup calls.clear() self.assertEqual(d.f(), 1) # try to specialize self.assertEqual(calls, [(d, D)]) @@ -72,7 +76,7 @@ def f(o): return o.x o = C() - for i in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): assert f(o) == 1 Descriptor.__get__ = lambda self, instance, value: 2 @@ -99,13 +103,13 @@ def __set__(self, instance, value): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Descriptor.__get__ = __get__ Descriptor.__set__ = __set__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_descriptor_shadows_class_attribute(self): @@ -120,7 +124,7 @@ class Class(metaclass=Metaclass): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) def test_metaclass_set_descriptor_after_optimization(self): @@ -137,12 +141,12 @@ def attribute(self): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Metaclass.attribute = attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_del_descriptor_after_optimization(self): @@ -157,12 +161,12 @@ class Class(metaclass=Metaclass): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) del Metaclass.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_type_descriptor_shadows_attribute_method(self): @@ -172,7 +176,7 @@ class Class: def f(): return Class.mro - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertIsNone(f()) def test_type_descriptor_shadows_attribute_member(self): @@ -182,7 +186,7 @@ class Class: def f(): return Class.__base__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertIs(f(), object) def test_type_descriptor_shadows_attribute_getset(self): @@ -192,7 +196,7 @@ class Class: def f(): return Class.__name__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertEqual(f(), "Class") def test_metaclass_getattribute(self): @@ -206,7 +210,7 @@ class Class(metaclass=Metaclass): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) def test_metaclass_swap(self): @@ -226,12 +230,12 @@ class Class(metaclass=OldMetaclass): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Class.__class__ = NewMetaclass - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_load_shadowing_slot_should_raise_type_error(self): @@ -248,7 +252,7 @@ def f(o): o = Sneaky() o.shadowed = 42 - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): with self.assertRaises(TypeError): f(o) @@ -265,7 +269,7 @@ def f(o): o = Sneaky() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): with self.assertRaises(TypeError): f(o) @@ -281,7 +285,7 @@ def f(o): o = Sneaky() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): with self.assertRaises(TypeError): f(o) @@ -297,7 +301,7 @@ def f(o): o = Sneaky() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): with self.assertRaises(TypeError): f(o) @@ -325,13 +329,13 @@ def attribute(): def f(): return instance.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Descriptor.__get__ = __get__ Descriptor.__set__ = __set__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_descriptor_added_after_optimization(self): @@ -354,13 +358,13 @@ def __set__(self, instance, value): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Descriptor.__get__ = __get__ Descriptor.__set__ = __set__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_descriptor_shadows_class_attribute(self): @@ -376,7 +380,7 @@ def attribute(): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) def test_metaclass_set_descriptor_after_optimization(self): @@ -394,12 +398,12 @@ def attribute(self): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Metaclass.attribute = attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_del_descriptor_after_optimization(self): @@ -415,12 +419,12 @@ def attribute(): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) del Metaclass.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_type_descriptor_shadows_attribute_method(self): @@ -431,7 +435,7 @@ def mro(): def f(): return Class.mro() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertEqual(f(), ["Spam", "eggs"]) def test_type_descriptor_shadows_attribute_member(self): @@ -442,7 +446,7 @@ def __base__(): def f(): return Class.__base__() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertNotEqual(f(), "Spam") def test_metaclass_getattribute(self): @@ -457,7 +461,7 @@ def attribute(): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) def test_metaclass_swap(self): @@ -477,22 +481,27 @@ class Class(metaclass=OldMetaclass): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Class.__class__ = NewMetaclass - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) +class InitTakesArg: + def __init__(self, arg): + self.arg = arg + + class TestCallCache(TestBase): def test_too_many_defaults_0(self): def f(): pass f.__defaults__ = (None,) - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f() def test_too_many_defaults_1(self): @@ -500,7 +509,7 @@ def f(x): pass f.__defaults__ = (None, None) - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f(None) f() @@ -509,13 +518,13 @@ def f(x, y): pass f.__defaults__ = (None, None, None) - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f(None, None) f(None) f() - @disabling_optimizer - @requires_specialization + @requires_jit_disabled + @requires_specialization_ft def test_assign_init_code(self): class MyClass: def __init__(self): @@ -525,30 +534,67 @@ def instantiate(): return MyClass() # Trigger specialization - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): instantiate() self.assert_specialized(instantiate, "CALL_ALLOC_AND_ENTER_INIT") def count_args(self, *args): self.num_args = len(args) - # Set MyClass.__init__.__code__ to a code object that is incompatible - # (uses varargs) with the current specialization + # Set MyClass.__init__.__code__ to a code object that uses different + # args MyClass.__init__.__code__ = count_args.__code__ instantiate() + @requires_jit_disabled + @requires_specialization_ft + def test_push_init_frame_fails(self): + def instantiate(): + return InitTakesArg() + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + with self.assertRaises(TypeError): + instantiate() + self.assert_specialized(instantiate, "CALL_ALLOC_AND_ENTER_INIT") + + with self.assertRaises(TypeError): + instantiate() + + def test_recursion_check_for_general_calls(self): + def test(default=None): + return test() + + with self.assertRaises(RecursionError): + test() + + def test_dont_specialize_custom_vectorcall(self): + def f(): + raise Exception("no way") + + _testinternalcapi.set_vectorcall_nop(f) + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + f() + + +def make_deferred_ref_count_obj(): + """Create an object that uses deferred reference counting. + + Only objects that use deferred refence counting may be stored in inline + caches in free-threaded builds. This constructs a new class named Foo, + which uses deferred reference counting. + """ + return type("Foo", (object,), {}) + @threading_helper.requires_working_threading() -@requires_specialization class TestRacesDoNotCrash(TestBase): # Careful with these. Bigger numbers have a higher chance of catching bugs, # but you can also burn through a *ton* of type/dict/function versions: ITEMS = 1000 LOOPS = 4 - WARMUPS = 2 WRITERS = 2 - @disabling_optimizer + @requires_jit_disabled def assert_races_do_not_crash( self, opname, get_items, read, write, *, check_items=False ): @@ -558,11 +604,11 @@ def assert_races_do_not_crash( # Reset: if check_items: for item in items: - item.__code__ = item.__code__.replace() + reset_code(item) else: - read.__code__ = read.__code__.replace() + reset_code(read) # Specialize: - for _ in range(self.WARMUPS): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): read(items) if check_items: for item in items: @@ -581,6 +627,7 @@ def assert_races_do_not_crash( for writer in writers: writer.join() + @requires_specialization_ft def test_binary_subscr_getitem(self): def get_items(): class C: @@ -607,9 +654,10 @@ def write(items): pass type(item).__getitem__ = lambda self, item: None - opname = "BINARY_SUBSCR_GETITEM" + opname = "BINARY_OP_SUBSCR_GETITEM" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_binary_subscr_list_int(self): def get_items(): items = [] @@ -630,9 +678,10 @@ def write(items): item.clear() item.append(None) - opname = "BINARY_SUBSCR_LIST_INT" + opname = "BINARY_OP_SUBSCR_LIST_INT" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization def test_for_iter_gen(self): def get_items(): def g(): @@ -664,6 +713,7 @@ def write(items): opname = "FOR_ITER_GEN" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization def test_for_iter_list(self): def get_items(): items = [] @@ -685,10 +735,11 @@ def write(items): opname = "FOR_ITER_LIST" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_class(self): def get_items(): class C: - a = object() + a = make_deferred_ref_count_obj() items = [] for _ in range(self.ITEMS): @@ -709,11 +760,45 @@ def write(items): del item.a except AttributeError: pass - item.a = object() + item.a = make_deferred_ref_count_obj() opname = "LOAD_ATTR_CLASS" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft + def test_load_attr_class_with_metaclass_check(self): + def get_items(): + class Meta(type): + pass + + class C(metaclass=Meta): + a = make_deferred_ref_count_obj() + + items = [] + for _ in range(self.ITEMS): + item = C + items.append(item) + return items + + def read(items): + for item in items: + try: + item.a + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.a + except AttributeError: + pass + item.a = make_deferred_ref_count_obj() + + opname = "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK" + self.assert_races_do_not_crash(opname, get_items, read, write) + + @requires_specialization_ft def test_load_attr_getattribute_overridden(self): def get_items(): class C: @@ -743,6 +828,7 @@ def write(items): opname = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_instance_value(self): def get_items(): class C: @@ -766,6 +852,7 @@ def write(items): opname = "LOAD_ATTR_INSTANCE_VALUE" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_method_lazy_dict(self): def get_items(): class C(Exception): @@ -795,6 +882,7 @@ def write(items): opname = "LOAD_ATTR_METHOD_LAZY_DICT" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_method_no_dict(self): def get_items(): class C: @@ -825,6 +913,7 @@ def write(items): opname = "LOAD_ATTR_METHOD_NO_DICT" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_method_with_values(self): def get_items(): class C: @@ -854,6 +943,7 @@ def write(items): opname = "LOAD_ATTR_METHOD_WITH_VALUES" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_module(self): def get_items(): items = [] @@ -878,6 +968,7 @@ def write(items): opname = "LOAD_ATTR_MODULE" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_attr_property(self): def get_items(): class C: @@ -907,6 +998,34 @@ def write(items): opname = "LOAD_ATTR_PROPERTY" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft + def test_load_attr_slot(self): + def get_items(): + class C: + __slots__ = ["a", "b"] + + items = [] + for i in range(self.ITEMS): + item = C() + item.a = i + item.b = i + self.ITEMS + items.append(item) + return items + + def read(items): + for item in items: + item.a + item.b + + def write(items): + for item in items: + item.a = 100 + item.b = 200 + + opname = "LOAD_ATTR_SLOT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + @requires_specialization_ft def test_load_attr_with_hint(self): def get_items(): class C: @@ -917,7 +1036,7 @@ class C: item = C() item.a = None # Resize into a combined unicode dict: - for i in range(29): + for i in range(_testinternalcapi.SHARED_KEYS_MAX_SIZE - 1): setattr(item, f"_{i}", None) items.append(item) return items @@ -933,7 +1052,10 @@ def write(items): opname = "LOAD_ATTR_WITH_HINT" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_load_global_module(self): + if not have_dict_key_versions(): + raise unittest.SkipTest("Low on dict key versions") def get_items(): items = [] for _ in range(self.ITEMS): @@ -954,6 +1076,7 @@ def write(items): opname, get_items, read, write, check_items=True ) + @requires_specialization def test_store_attr_instance_value(self): def get_items(): class C: @@ -976,6 +1099,7 @@ def write(items): opname = "STORE_ATTR_INSTANCE_VALUE" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization def test_store_attr_with_hint(self): def get_items(): class C: @@ -985,7 +1109,7 @@ class C: for _ in range(self.ITEMS): item = C() # Resize into a combined unicode dict: - for i in range(29): + for i in range(_testinternalcapi.SHARED_KEYS_MAX_SIZE - 1): setattr(item, f"_{i}", None) items.append(item) return items @@ -1001,6 +1125,7 @@ def write(items): opname = "STORE_ATTR_WITH_HINT" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_store_subscr_list_int(self): def get_items(): items = [] @@ -1024,6 +1149,7 @@ def write(items): opname = "STORE_SUBSCR_LIST_INT" self.assert_races_do_not_crash(opname, get_items, read, write) + @requires_specialization_ft def test_unpack_sequence_list(self): def get_items(): items = [] @@ -1079,7 +1205,7 @@ def test_dict_dematerialization(self): c.a = 1 c.b = 2 c.__dict__ - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a self.assertEqual( _testinternalcapi.get_object_dict_values(c), @@ -1091,7 +1217,7 @@ def test_dict_dematerialization_multiple_refs(self): c.a = 1 c.b = 2 d = c.__dict__ - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a self.assertIs(c.__dict__, d) @@ -1100,7 +1226,7 @@ def test_dict_dematerialization_copy(self): c.a = 1 c.b = 2 c2 = copy.copy(c) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a c2.a self.assertEqual( @@ -1112,7 +1238,7 @@ def test_dict_dematerialization_copy(self): (1, 2, '') ) c3 = copy.deepcopy(c) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a c3.a self.assertEqual( @@ -1126,7 +1252,7 @@ def test_dict_dematerialization_pickle(self): c.a = 1 c.b = 2 c2 = pickle.loads(pickle.dumps(c)) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a c2.a self.assertEqual( @@ -1144,7 +1270,7 @@ class D(dict): pass c.a = 1 c.b = 2 c.__dict__ = D(c.__dict__) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a self.assertIs( _testinternalcapi.get_object_dict_values(c), @@ -1155,23 +1281,595 @@ class D(dict): pass {'a':1, 'b':2} ) - def test_store_attr_with_hint(self): - # gh-133441: Regression test for STORE_ATTR_WITH_HINT bytecode - class Node: + def test_125868(self): + + def make_special_dict(): + """Create a dictionary an object with a this table: + index | key | value + ----- | --- | ----- + 0 | 'b' | 'value' + 1 | 'b' | NULL + """ + class A: + pass + a = A() + a.a = 1 + a.b = 2 + d = a.__dict__.copy() + del d['a'] + del d['b'] + d['b'] = "value" + return d + + class NoInlineAorB: + pass + for i in range(ord('c'), ord('z')): + setattr(NoInlineAorB(), chr(i), i) + + c = NoInlineAorB() + c.a = 0 + c.b = 1 + self.assertFalse(_testinternalcapi.has_inline_values(c)) + + def f(o, n): + for i in range(n): + o.b = i + # Prime f to store to dict slot 1 + f(c, _testinternalcapi.SPECIALIZATION_THRESHOLD) + + test_obj = NoInlineAorB() + test_obj.__dict__ = make_special_dict() + self.assertEqual(test_obj.b, "value") + + #This should set x.b = 0 + f(test_obj, 1) + self.assertEqual(test_obj.b, 0) + + +class TestSpecializer(TestBase): + + @cpython_only + @requires_specialization_ft + def test_binary_op(self): + def binary_op_add_int(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1, 2 + c = a + b + self.assertEqual(c, 3) + + binary_op_add_int() + self.assert_specialized(binary_op_add_int, "BINARY_OP_ADD_INT") + self.assert_no_opcode(binary_op_add_int, "BINARY_OP") + + def binary_op_add_unicode(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = "foo", "bar" + c = a + b + self.assertEqual(c, "foobar") + + binary_op_add_unicode() + self.assert_specialized(binary_op_add_unicode, "BINARY_OP_ADD_UNICODE") + self.assert_no_opcode(binary_op_add_unicode, "BINARY_OP") + + def binary_op_add_extend(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 6, 3.0 + c = a + b + self.assertEqual(c, 9.0) + c = b + a + self.assertEqual(c, 9.0) + c = a - b + self.assertEqual(c, 3.0) + c = b - a + self.assertEqual(c, -3.0) + c = a * b + self.assertEqual(c, 18.0) + c = b * a + self.assertEqual(c, 18.0) + c = a / b + self.assertEqual(c, 2.0) + c = b / a + self.assertEqual(c, 0.5) + + binary_op_add_extend() + self.assert_specialized(binary_op_add_extend, "BINARY_OP_EXTEND") + self.assert_no_opcode(binary_op_add_extend, "BINARY_OP") + + def binary_op_zero_division(): + def compactlong_lhs(arg): + 42 / arg + def float_lhs(arg): + 42.0 / arg + + with self.assertRaises(ZeroDivisionError): + compactlong_lhs(0) + with self.assertRaises(ZeroDivisionError): + compactlong_lhs(0.0) + with self.assertRaises(ZeroDivisionError): + float_lhs(0.0) + with self.assertRaises(ZeroDivisionError): + float_lhs(0) + + self.assert_no_opcode(compactlong_lhs, "BINARY_OP_EXTEND") + self.assert_no_opcode(float_lhs, "BINARY_OP_EXTEND") + + binary_op_zero_division() + + def binary_op_nan(): + def compactlong_lhs(arg): + return ( + 42 + arg, + 42 - arg, + 42 * arg, + 42 / arg, + ) + def compactlong_rhs(arg): + return ( + arg + 42, + arg - 42, + arg * 2, + arg / 42, + ) + nan = float('nan') + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(compactlong_lhs(1.0), (43.0, 41.0, 42.0, 42.0)) + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): + self.assertTrue(all(filter(lambda x: x is nan, compactlong_lhs(nan)))) + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(compactlong_rhs(42.0), (84.0, 0.0, 84.0, 1.0)) + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): + self.assertTrue(all(filter(lambda x: x is nan, compactlong_rhs(nan)))) + + self.assert_no_opcode(compactlong_lhs, "BINARY_OP_EXTEND") + self.assert_no_opcode(compactlong_rhs, "BINARY_OP_EXTEND") + + binary_op_nan() + + def binary_op_bitwise_extend(): + for _ in range(100): + a, b = 2, 7 + x = a | b + self.assertEqual(x, 7) + y = a & b + self.assertEqual(y, 2) + z = a ^ b + self.assertEqual(z, 5) + a, b = 3, 9 + a |= b + self.assertEqual(a, 11) + a, b = 11, 9 + a &= b + self.assertEqual(a, 9) + a, b = 3, 9 + a ^= b + self.assertEqual(a, 10) + + binary_op_bitwise_extend() + self.assert_specialized(binary_op_bitwise_extend, "BINARY_OP_EXTEND") + self.assert_no_opcode(binary_op_bitwise_extend, "BINARY_OP") + + @cpython_only + @requires_specialization_ft + def test_load_super_attr(self): + """Ensure that LOAD_SUPER_ATTR is specialized as expected.""" + + class A: def __init__(self): - self.parents = {} + meth = super().__init__ + super().__init__() + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + A() + + self.assert_specialized(A.__init__, "LOAD_SUPER_ATTR_ATTR") + self.assert_specialized(A.__init__, "LOAD_SUPER_ATTR_METHOD") + self.assert_no_opcode(A.__init__, "LOAD_SUPER_ATTR") - def __setstate__(self, data_dict): - self.__dict__ = data_dict - self.parents = {} + # Temporarily replace super() with something else. + real_super = super - class Dict(dict): + def fake_super(): + def init(self): + pass + + return init + + # Force unspecialize + globals()['super'] = fake_super + try: + # Should be unspecialized after enough calls. + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): + A() + finally: + globals()['super'] = real_super + + # Ensure the specialized instructions are not present + self.assert_no_opcode(A.__init__, "LOAD_SUPER_ATTR_ATTR") + self.assert_no_opcode(A.__init__, "LOAD_SUPER_ATTR_METHOD") + + @cpython_only + @requires_specialization_ft + def test_contain_op(self): + def contains_op_dict(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1, {1: 2, 2: 5} + self.assertTrue(a in b) + self.assertFalse(3 in b) + + contains_op_dict() + self.assert_specialized(contains_op_dict, "CONTAINS_OP_DICT") + self.assert_no_opcode(contains_op_dict, "CONTAINS_OP") + + def contains_op_set(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1, {1, 2} + self.assertTrue(a in b) + self.assertFalse(3 in b) + + contains_op_set() + self.assert_specialized(contains_op_set, "CONTAINS_OP_SET") + self.assert_no_opcode(contains_op_set, "CONTAINS_OP") + + @cpython_only + @requires_specialization_ft + def test_send_with(self): + def run_async(coro): + while True: + try: + coro.send(None) + except StopIteration: + break + + class CM: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + pass + + async def send_with(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + async with CM(): + x = 1 + + run_async(send_with()) + # Note there are still unspecialized "SEND" opcodes in the + # cleanup paths of the 'with' statement. + self.assert_specialized(send_with, "SEND_GEN") + + @cpython_only + @requires_specialization_ft + def test_send_yield_from(self): + def g(): + yield None + + def send_yield_from(): + yield from g() + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + list(send_yield_from()) + + self.assert_specialized(send_yield_from, "SEND_GEN") + self.assert_no_opcode(send_yield_from, "SEND") + + @cpython_only + @requires_specialization_ft + def test_store_attr_slot(self): + class C: + __slots__ = ['x'] + + def set_slot(n): + c = C() + for i in range(n): + c.x = i + + set_slot(_testinternalcapi.SPECIALIZATION_THRESHOLD) + + self.assert_specialized(set_slot, "STORE_ATTR_SLOT") + self.assert_no_opcode(set_slot, "STORE_ATTR") + + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + set_slot(_testinternalcapi.SPECIALIZATION_COOLDOWN) + self.assert_no_opcode(set_slot, "STORE_ATTR_SLOT") + + @cpython_only + @requires_specialization_ft + def test_store_attr_instance_value(self): + class C: + pass + + @reset_code + def set_value(n): + c = C() + for i in range(n): + c.x = i + + set_value(_testinternalcapi.SPECIALIZATION_THRESHOLD) + + self.assert_specialized(set_value, "STORE_ATTR_INSTANCE_VALUE") + self.assert_no_opcode(set_value, "STORE_ATTR") + + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + set_value(_testinternalcapi.SPECIALIZATION_COOLDOWN) + self.assert_no_opcode(set_value, "STORE_ATTR_INSTANCE_VALUE") + + @cpython_only + @requires_specialization_ft + def test_store_attr_with_hint(self): + class C: pass - obj = Node() - obj.__setstate__({'parents': {}}) - obj.__setstate__({'parents': {}}) - obj.__setstate__(Dict({'parents': {}})) + c = C() + for i in range(_testinternalcapi.SHARED_KEYS_MAX_SIZE - 1): + setattr(c, f"_{i}", None) + + @reset_code + def set_value(n): + for i in range(n): + c.x = i + + set_value(_testinternalcapi.SPECIALIZATION_THRESHOLD) + + self.assert_specialized(set_value, "STORE_ATTR_WITH_HINT") + self.assert_no_opcode(set_value, "STORE_ATTR") + + # Adding a property for 'x' should unspecialize it. + C.x = property(lambda self: None, lambda self, x: None) + set_value(_testinternalcapi.SPECIALIZATION_COOLDOWN) + self.assert_no_opcode(set_value, "STORE_ATTR_WITH_HINT") + + @cpython_only + @requires_specialization_ft + def test_to_bool(self): + def to_bool_bool(): + true_cnt, false_cnt = 0, 0 + elems = [e % 2 == 0 for e in range(_testinternalcapi.SPECIALIZATION_THRESHOLD)] + for e in elems: + if e: + true_cnt += 1 + else: + false_cnt += 1 + d, m = divmod(_testinternalcapi.SPECIALIZATION_THRESHOLD, 2) + self.assertEqual(true_cnt, d + m) + self.assertEqual(false_cnt, d) + + to_bool_bool() + self.assert_specialized(to_bool_bool, "TO_BOOL_BOOL") + self.assert_no_opcode(to_bool_bool, "TO_BOOL") + + def to_bool_int(): + count = 0 + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + if i: + count += 1 + else: + count -= 1 + self.assertEqual(count, _testinternalcapi.SPECIALIZATION_THRESHOLD - 2) + + to_bool_int() + self.assert_specialized(to_bool_int, "TO_BOOL_INT") + self.assert_no_opcode(to_bool_int, "TO_BOOL") + + def to_bool_list(): + count = 0 + elems = list(range(_testinternalcapi.SPECIALIZATION_THRESHOLD)) + while elems: + count += elems.pop() + self.assertEqual(elems, []) + self.assertEqual(count, sum(range(_testinternalcapi.SPECIALIZATION_THRESHOLD))) + + to_bool_list() + self.assert_specialized(to_bool_list, "TO_BOOL_LIST") + self.assert_no_opcode(to_bool_list, "TO_BOOL") + + def to_bool_none(): + count = 0 + elems = [None] * _testinternalcapi.SPECIALIZATION_THRESHOLD + for e in elems: + if not e: + count += 1 + self.assertEqual(count, _testinternalcapi.SPECIALIZATION_THRESHOLD) + + to_bool_none() + self.assert_specialized(to_bool_none, "TO_BOOL_NONE") + self.assert_no_opcode(to_bool_none, "TO_BOOL") + + def to_bool_str(): + count = 0 + elems = [""] + ["foo"] * (_testinternalcapi.SPECIALIZATION_THRESHOLD - 1) + for e in elems: + if e: + count += 1 + self.assertEqual(count, _testinternalcapi.SPECIALIZATION_THRESHOLD - 1) + + to_bool_str() + self.assert_specialized(to_bool_str, "TO_BOOL_STR") + self.assert_no_opcode(to_bool_str, "TO_BOOL") + + @cpython_only + @requires_specialization_ft + def test_unpack_sequence(self): + def unpack_sequence_two_tuple(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + t = 1, 2 + a, b = t + self.assertEqual(a, 1) + self.assertEqual(b, 2) + + unpack_sequence_two_tuple() + self.assert_specialized(unpack_sequence_two_tuple, + "UNPACK_SEQUENCE_TWO_TUPLE") + self.assert_no_opcode(unpack_sequence_two_tuple, "UNPACK_SEQUENCE") + + def unpack_sequence_tuple(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b, c, d = 1, 2, 3, 4 + self.assertEqual(a, 1) + self.assertEqual(b, 2) + self.assertEqual(c, 3) + self.assertEqual(d, 4) + + unpack_sequence_tuple() + self.assert_specialized(unpack_sequence_tuple, "UNPACK_SEQUENCE_TUPLE") + self.assert_no_opcode(unpack_sequence_tuple, "UNPACK_SEQUENCE") + + def unpack_sequence_list(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = [1, 2] + self.assertEqual(a, 1) + self.assertEqual(b, 2) + + unpack_sequence_list() + self.assert_specialized(unpack_sequence_list, "UNPACK_SEQUENCE_LIST") + self.assert_no_opcode(unpack_sequence_list, "UNPACK_SEQUENCE") + + @cpython_only + @requires_specialization_ft + def test_binary_subscr(self): + def binary_subscr_list_int(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = [1, 2, 3] + for idx, expected in enumerate(a): + self.assertEqual(a[idx], expected) + + binary_subscr_list_int() + self.assert_specialized(binary_subscr_list_int, + "BINARY_OP_SUBSCR_LIST_INT") + self.assert_no_opcode(binary_subscr_list_int, "BINARY_OP") + + def binary_subscr_tuple_int(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = (1, 2, 3) + for idx, expected in enumerate(a): + self.assertEqual(a[idx], expected) + + binary_subscr_tuple_int() + self.assert_specialized(binary_subscr_tuple_int, + "BINARY_OP_SUBSCR_TUPLE_INT") + self.assert_no_opcode(binary_subscr_tuple_int, "BINARY_OP") + + def binary_subscr_dict(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = {1: 2, 2: 3} + self.assertEqual(a[1], 2) + self.assertEqual(a[2], 3) + + binary_subscr_dict() + self.assert_specialized(binary_subscr_dict, "BINARY_OP_SUBSCR_DICT") + self.assert_no_opcode(binary_subscr_dict, "BINARY_OP") + + def binary_subscr_str_int(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = "foobar" + for idx, expected in enumerate(a): + self.assertEqual(a[idx], expected) + + binary_subscr_str_int() + self.assert_specialized(binary_subscr_str_int, "BINARY_OP_SUBSCR_STR_INT") + self.assert_no_opcode(binary_subscr_str_int, "BINARY_OP") + + def binary_subscr_getitems(): + class C: + def __init__(self, val): + self.val = val + def __getitem__(self, item): + return self.val + + items = [C(i) for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD)] + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(items[i][i], i) + + binary_subscr_getitems() + self.assert_specialized(binary_subscr_getitems, "BINARY_OP_SUBSCR_GETITEM") + self.assert_no_opcode(binary_subscr_getitems, "BINARY_OP") + + @cpython_only + @requires_specialization_ft + def test_compare_op(self): + def compare_op_int(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1, 2 + c = a == b + self.assertFalse(c) + + compare_op_int() + self.assert_specialized(compare_op_int, "COMPARE_OP_INT") + self.assert_no_opcode(compare_op_int, "COMPARE_OP") + + def compare_op_float(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1.0, 2.0 + c = a == b + self.assertFalse(c) + + compare_op_float() + self.assert_specialized(compare_op_float, "COMPARE_OP_FLOAT") + self.assert_no_opcode(compare_op_float, "COMPARE_OP") + + def compare_op_str(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = "spam", "ham" + c = a == b + self.assertFalse(c) + + compare_op_str() + self.assert_specialized(compare_op_str, "COMPARE_OP_STR") + self.assert_no_opcode(compare_op_str, "COMPARE_OP") + + @cpython_only + @requires_specialization_ft + def test_load_const(self): + def load_const(): + def unused(): pass + # Currently, the empty tuple is immortal, and the otherwise + # unused nested function's code object is mortal. This test will + # have to use different values if either of that changes. + return () + + load_const() + self.assert_specialized(load_const, "LOAD_CONST_IMMORTAL") + self.assert_specialized(load_const, "LOAD_CONST_MORTAL") + self.assert_no_opcode(load_const, "LOAD_CONST") + + @cpython_only + @requires_specialization_ft + def test_for_iter(self): + L = list(range(10)) + def for_iter_list(): + for i in L: + self.assertIn(i, L) + + for_iter_list() + self.assert_specialized(for_iter_list, "FOR_ITER_LIST") + self.assert_no_opcode(for_iter_list, "FOR_ITER") + + t = tuple(range(10)) + def for_iter_tuple(): + for i in t: + self.assertIn(i, t) + + for_iter_tuple() + self.assert_specialized(for_iter_tuple, "FOR_ITER_TUPLE") + self.assert_no_opcode(for_iter_tuple, "FOR_ITER") + + r = range(10) + def for_iter_range(): + for i in r: + self.assertIn(i, r) + + for_iter_range() + self.assert_specialized(for_iter_range, "FOR_ITER_RANGE") + self.assert_no_opcode(for_iter_range, "FOR_ITER") + + def for_iter_generator(): + for i in (i for i in range(10)): + i + 1 + + for_iter_generator() + self.assert_specialized(for_iter_generator, "FOR_ITER_GEN") + self.assert_no_opcode(for_iter_generator, "FOR_ITER") if __name__ == "__main__": From b61dfdc5344fcf7dd583ed87b840c11bef17d99c Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Thu, 2 Apr 2026 18:33:50 +0200 Subject: [PATCH 016/834] Updtae `test_optimizer.py` from 3.14.3 (#7554) --- Lib/test/test_optimizer.py | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 Lib/test/test_optimizer.py diff --git a/Lib/test/test_optimizer.py b/Lib/test/test_optimizer.py new file mode 100644 index 00000000000..fac4d1a4ab4 --- /dev/null +++ b/Lib/test/test_optimizer.py @@ -0,0 +1,90 @@ +import unittest +import types +from test.support import import_helper + + +_testinternalcapi = import_helper.import_module("_testinternalcapi") + + +class TestRareEventCounters(unittest.TestCase): + def setUp(self): + _testinternalcapi.reset_rare_event_counters() + + def test_set_class(self): + class A: + pass + class B: + pass + a = A() + + orig_counter = _testinternalcapi.get_rare_event_counters()["set_class"] + a.__class__ = B + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["set_class"] + ) + + def test_set_bases(self): + class A: + pass + class B: + pass + class C(B): + pass + + orig_counter = _testinternalcapi.get_rare_event_counters()["set_bases"] + C.__bases__ = (A,) + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["set_bases"] + ) + + def test_set_eval_frame_func(self): + orig_counter = _testinternalcapi.get_rare_event_counters()["set_eval_frame_func"] + _testinternalcapi.set_eval_frame_record([]) + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["set_eval_frame_func"] + ) + _testinternalcapi.set_eval_frame_default() + + def test_builtin_dict(self): + orig_counter = _testinternalcapi.get_rare_event_counters()["builtin_dict"] + if isinstance(__builtins__, types.ModuleType): + builtins = __builtins__.__dict__ + else: + builtins = __builtins__ + builtins["FOO"] = 42 + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["builtin_dict"] + ) + del builtins["FOO"] + + def test_func_modification(self): + def func(x=0): + pass + + for attribute in ( + "__code__", + "__defaults__", + "__kwdefaults__" + ): + orig_counter = _testinternalcapi.get_rare_event_counters()["func_modification"] + setattr(func, attribute, getattr(func, attribute)) + self.assertEqual( + orig_counter + 1, + _testinternalcapi.get_rare_event_counters()["func_modification"] + ) + + +class TestOptimizerSymbols(unittest.TestCase): + + @unittest.skipUnless(hasattr(_testinternalcapi, "uop_symbols_test"), + "requires _testinternalcapi.uop_symbols_test") + def test_optimizer_symbols(self): + _testinternalcapi.uop_symbols_test() + + +if __name__ == "__main__": + unittest.main() From 3d9688402a18905cbfbd039ae6fb08daba2470c1 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Thu, 2 Apr 2026 18:43:11 +0200 Subject: [PATCH 017/834] Replace unmaintained `unic` crates (#7555) --- Cargo.lock | 299 ++++++++++++++++++++++--------- Cargo.toml | 8 +- Lib/test/test_str.py | 1 + Lib/test/test_unicodedata.py | 2 +- crates/literal/Cargo.toml | 2 +- crates/literal/src/char.rs | 17 +- crates/stdlib/Cargo.toml | 7 +- crates/stdlib/src/unicodedata.rs | 102 ++++++----- crates/vm/Cargo.toml | 5 +- crates/vm/src/builtins/str.rs | 31 +++- 10 files changed, 320 insertions(+), 154 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ea7eeec661a..5fd981fd135 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1503,6 +1503,90 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "indexmap" version = "2.13.0" @@ -1824,6 +1908,12 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + [[package]] name = "lock_api" version = "0.4.14" @@ -1944,12 +2034,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "md-5" version = "0.10.6" @@ -2479,6 +2563,15 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -3149,12 +3242,12 @@ name = "rustpython-literal" version = "0.5.0" dependencies = [ "hexf-parse", + "icu_properties", "is-macro", "lexical-parse-float", "num-traits", "rand 0.9.2", "rustpython-wtf8", - "unic-ucd-category", ] [[package]] @@ -3275,6 +3368,8 @@ dependencies = [ "gethostname", "hex", "hmac", + "icu_normalizer", + "icu_properties", "indexmap", "itertools 0.14.0", "libc", @@ -3327,12 +3422,7 @@ dependencies = [ "termios", "tk-sys", "ucd", - "unic-char-property", - "unic-normal", "unic-ucd-age", - "unic-ucd-bidi", - "unic-ucd-category", - "unicode-bidi-mirroring", "unicode_names2 2.0.0", "uuid", "webpki-roots", @@ -3368,6 +3458,7 @@ dependencies = [ "glob", "half", "hex", + "icu_properties", "indexmap", "is-macro", "itertools 0.14.0", @@ -3411,9 +3502,6 @@ dependencies = [ "thiserror 2.0.18", "timsort", "uname", - "unic-ucd-bidi", - "unic-ucd-category", - "unic-ucd-ident", "unicode-casing", "wasm-bindgen", "which", @@ -3964,6 +4052,16 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "639ce8ef6d2ba56be0383a94dd13b92138d58de44c62618303bb798fa92bdc00" +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -4106,15 +4204,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" -[[package]] -name = "unic-normal" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09d64d33589a94628bc2aeb037f35c2e25f3f049c7348b5aa5580b48e6bba62" -dependencies = [ - "unic-ucd-normal", -] - [[package]] name = "unic-ucd-age" version = "0.9.0" @@ -4126,61 +4215,6 @@ dependencies = [ "unic-ucd-version", ] -[[package]] -name = "unic-ucd-bidi" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d568b51222484e1f8209ce48caa6b430bf352962b877d592c29ab31fb53d8c" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-category" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8d4591f5fcfe1bd4453baaf803c40e1b1e69ff8455c47620440b46efef91c0" -dependencies = [ - "matches", - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-hangul" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1dc690e19010e1523edb9713224cba5ef55b54894fe33424439ec9a40c0054" -dependencies = [ - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-ident" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - -[[package]] -name = "unic-ucd-normal" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86aed873b8202d22b13859dda5fe7c001d271412c31d411fd9b827e030569410" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-hangul", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-version" version = "0.9.0" @@ -4190,12 +4224,6 @@ dependencies = [ "unic-common", ] -[[package]] -name = "unicode-bidi-mirroring" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfa6e8c60bb66d49db113e0125ee8711b7647b5579dc7f5f19c42357ed039fe" - [[package]] name = "unicode-casing" version = "0.1.1" @@ -4283,6 +4311,18 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -4814,6 +4854,18 @@ version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + [[package]] name = "x509-cert" version = "0.2.5" @@ -4851,6 +4903,29 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.34" @@ -4871,6 +4946,27 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.2" @@ -4891,6 +4987,39 @@ dependencies = [ "syn", ] +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zlib-rs" version = "0.5.5" diff --git a/Cargo.toml b/Cargo.toml index 3b4069d47a1..7bd8b8f3374 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -222,15 +222,11 @@ strum = "0.28" strum_macros = "0.28" syn = "2" thiserror = "2.0" +icu_properties = "2" +icu_normalizer = "2" unicode-casing = "0.1.1" -unic-char-property = "0.9.0" -unic-normal = "0.9.0" unic-ucd-age = "0.9.0" -unic-ucd-bidi = "0.9.0" -unic-ucd-category = "0.9.0" -unic-ucd-ident = "0.9.0" unicode_names2 = "2.0.0" -unicode-bidi-mirroring = "0.4" widestring = "1.2.0" windows-sys = "0.61.2" wasm-bindgen = "0.2.106" diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index 11e2abb82c5..68037923283 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -854,6 +854,7 @@ def test_isprintable(self): self.assertTrue('\U0001F46F'.isprintable()) self.assertFalse('\U000E0020'.isprintable()) + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_resource('cpu') def test_isprintable_invariant(self): for codepoint in range(sys.maxunicode + 1): diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index ceae20e8cb2..bb54818ecb0 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -232,7 +232,6 @@ def test_issue10254(self): b = 'C\u0338' * 20 + '\xC7' self.assertEqual(self.db.normalize('NFC', a), b) - @unittest.expectedFailure # TODO: RUSTPYTHON; ? + def test_issue29456(self): # Fix #29456 u1176_str_a = '\u1100\u1176\u11a8' @@ -389,6 +388,7 @@ def unistr(data): data = [int(x, 16) for x in data.split(" ")] return "".join([chr(x) for x in data]) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true : 13055 @requires_resource('network') @requires_resource('cpu') def test_normalization(self): diff --git a/crates/literal/Cargo.toml b/crates/literal/Cargo.toml index bd6a2699742..3f0bec33c30 100644 --- a/crates/literal/Cargo.toml +++ b/crates/literal/Cargo.toml @@ -15,7 +15,7 @@ hexf-parse = "0.2.1" is-macro.workspace = true lexical-parse-float = { version = "1.0.6", features = ["format"] } num-traits = { workspace = true } -unic-ucd-category = { workspace = true } +icu_properties = { workspace = true } [dev-dependencies] rand = { workspace = true } diff --git a/crates/literal/src/char.rs b/crates/literal/src/char.rs index cd64f6dfa9e..5b446cc1a19 100644 --- a/crates/literal/src/char.rs +++ b/crates/literal/src/char.rs @@ -1,4 +1,4 @@ -use unic_ucd_category::GeneralCategory; +use icu_properties::props::{EnumeratedProperty, GeneralCategory}; /// According to python following categories aren't printable: /// * Cc (Other, Control) @@ -10,6 +10,17 @@ use unic_ucd_category::GeneralCategory; /// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR) /// * Zs (Separator, Space) other than ASCII space('\x20'). pub fn is_printable(c: char) -> bool { - let cat = GeneralCategory::of(c); - !(cat.is_other() || cat.is_separator()) + let cat = GeneralCategory::for_char(c); + + !matches!( + cat, + GeneralCategory::SpaceSeparator + | GeneralCategory::LineSeparator + | GeneralCategory::ParagraphSeparator + | GeneralCategory::Control + | GeneralCategory::Format + | GeneralCategory::Surrogate + | GeneralCategory::PrivateUse + | GeneralCategory::Unassigned + ) } diff --git a/crates/stdlib/Cargo.toml b/crates/stdlib/Cargo.toml index df29f09a5a7..f828507d6cf 100644 --- a/crates/stdlib/Cargo.toml +++ b/crates/stdlib/Cargo.toml @@ -78,13 +78,10 @@ constant_time_eq = { workspace = true } ## unicode stuff unicode_names2 = { workspace = true } # update version all at the same time -unic-char-property = { workspace = true } -unic-normal = { workspace = true } -unic-ucd-bidi = { workspace = true } -unic-ucd-category = { workspace = true } +icu_properties = { workspace = true } +icu_normalizer = { workspace = true } unic-ucd-age = { workspace = true } ucd = "0.1.1" -unicode-bidi-mirroring = { workspace = true } # compression adler32 = "1.2.0" diff --git a/crates/stdlib/src/unicodedata.rs b/crates/stdlib/src/unicodedata.rs index 5664fd0c36e..6ee5b0c2ee8 100644 --- a/crates/stdlib/src/unicodedata.rs +++ b/crates/stdlib/src/unicodedata.rs @@ -40,15 +40,19 @@ mod unicodedata { builtins::{PyModule, PyStrRef}, function::OptionalArg, }; + + use icu_normalizer::{ComposingNormalizerBorrowed, DecomposingNormalizerBorrowed}; + use icu_properties::{ + CodePointSetData, + props::{ + BidiClass, BidiMirrored, CanonicalCombiningClass, EastAsianWidth, EnumeratedProperty, + GeneralCategory, NamedEnumeratedProperty, + }, + }; use itertools::Itertools; use rustpython_common::wtf8::{CodePoint, Wtf8Buf}; - use ucd::{Codepoint, DecompositionType, EastAsianWidth, Number, NumericType}; - use unic_char_property::EnumeratedCharProperty; - use unic_normal::StrNormalForm; + use ucd::{Codepoint, DecompositionType, Number, NumericType}; use unic_ucd_age::{Age, UNICODE_VERSION, UnicodeVersion}; - use unic_ucd_bidi::BidiClass; - use unic_ucd_category::GeneralCategory; - use unicode_bidi_mirroring::is_mirroring; pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { __module_exec(vm, module); @@ -117,9 +121,9 @@ mod unicodedata { .extract_char(character, vm)? .map_or(GeneralCategory::Unassigned, |c| { c.to_char() - .map_or(GeneralCategory::Surrogate, GeneralCategory::of) + .map_or(GeneralCategory::Surrogate, GeneralCategory::for_char) }) - .abbr_name() + .short_name() .to_owned()) } @@ -165,8 +169,8 @@ mod unicodedata { let bidi = match self.extract_char(character, vm)? { Some(c) => c .to_char() - .map_or(BidiClass::LeftToRight, BidiClass::of) - .abbr_name(), + .map_or(BidiClass::LeftToRight, BidiClass::for_char) + .short_name(), None => "", }; Ok(bidi) @@ -182,18 +186,34 @@ mod unicodedata { Ok(self .extract_char(character, vm)? .and_then(|c| c.to_char()) - .map_or(EastAsianWidth::Neutral, |c| c.east_asian_width()) - .abbr_name()) + .map_or(EastAsianWidth::Neutral, EastAsianWidth::for_char) + .short_name()) } #[pymethod] fn normalize(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult { let text = unistr.as_wtf8(); let normalized_text = match form { - Nfc => text.map_utf8(|s| s.nfc()).collect(), - Nfkc => text.map_utf8(|s| s.nfkc()).collect(), - Nfd => text.map_utf8(|s| s.nfd()).collect(), - Nfkd => text.map_utf8(|s| s.nfkd()).collect(), + Nfc => { + let normalizer = ComposingNormalizerBorrowed::new_nfc(); + text.map_utf8(|s| normalizer.normalize_iter(s.chars())) + .collect() + } + Nfkc => { + let normalizer = ComposingNormalizerBorrowed::new_nfkc(); + text.map_utf8(|s| normalizer.normalize_iter(s.chars())) + .collect() + } + Nfd => { + let normalizer = DecomposingNormalizerBorrowed::new_nfd(); + text.map_utf8(|s| normalizer.normalize_iter(s.chars())) + .collect() + } + Nfkd => { + let normalizer = DecomposingNormalizerBorrowed::new_nfkd(); + text.map_utf8(|s| normalizer.normalize_iter(s.chars())) + .collect() + } }; Ok(normalized_text) } @@ -202,10 +222,26 @@ mod unicodedata { fn is_normalized(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult { let text = unistr.as_wtf8(); let normalized: Wtf8Buf = match form { - Nfc => text.map_utf8(|s| s.nfc()).collect(), - Nfkc => text.map_utf8(|s| s.nfkc()).collect(), - Nfd => text.map_utf8(|s| s.nfd()).collect(), - Nfkd => text.map_utf8(|s| s.nfkd()).collect(), + Nfc => { + let normalizer = ComposingNormalizerBorrowed::new_nfc(); + text.map_utf8(|s| normalizer.normalize_iter(s.chars())) + .collect() + } + Nfkc => { + let normalizer = ComposingNormalizerBorrowed::new_nfkc(); + text.map_utf8(|s| normalizer.normalize_iter(s.chars())) + .collect() + } + Nfd => { + let normalizer = DecomposingNormalizerBorrowed::new_nfd(); + text.map_utf8(|s| normalizer.normalize_iter(s.chars())) + .collect() + } + Nfkd => { + let normalizer = DecomposingNormalizerBorrowed::new_nfkd(); + text.map_utf8(|s| normalizer.normalize_iter(s.chars())) + .collect() + } }; Ok(text == &*normalized) } @@ -216,7 +252,8 @@ mod unicodedata { Some(c) => { if let Some(ch) = c.to_char() { // Check if the character is mirrored in bidirectional text using Unicode standard - Ok(if is_mirroring(ch) { 1 } else { 0 }) + let bidi_mirrored = CodePointSetData::new::(); + Ok(if bidi_mirrored.contains(ch) { 1 } else { 0 }) } else { Ok(0) } @@ -226,11 +263,13 @@ mod unicodedata { } #[pymethod] - fn combining(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult { + fn combining(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult { Ok(self .extract_char(character, vm)? .and_then(|c| c.to_char()) - .map_or(0, |ch| ch.canonical_combining_class() as i32)) + .map_or(0, |ch| { + CanonicalCombiningClass::for_char(ch).to_icu4c_value() + })) } #[pymethod] @@ -339,23 +378,6 @@ mod unicodedata { } } - trait EastAsianWidthAbbrName { - fn abbr_name(&self) -> &'static str; - } - - impl EastAsianWidthAbbrName for EastAsianWidth { - fn abbr_name(&self) -> &'static str { - match self { - Self::Narrow => "Na", - Self::Wide => "W", - Self::Neutral => "N", - Self::Ambiguous => "A", - Self::FullWidth => "F", - Self::HalfWidth => "H", - } - } - } - #[pyattr] fn ucd_3_2_0(vm: &VirtualMachine) -> PyRef { Ucd { diff --git a/crates/vm/Cargo.toml b/crates/vm/Cargo.toml index 4eac727c2fa..b721418a4cc 100644 --- a/crates/vm/Cargo.toml +++ b/crates/vm/Cargo.toml @@ -86,10 +86,7 @@ timsort = "0.1.2" # TODO: use unic for this; needed for title case: # https://github.com/RustPython/RustPython/pull/832#discussion_r275428939 unicode-casing = { workspace = true } -# update version all at the same time -unic-ucd-bidi = { workspace = true } -unic-ucd-category = { workspace = true } -unic-ucd-ident = { workspace = true } +icu_properties = { workspace = true } [target.'cfg(unix)'.dependencies] rustix = { workspace = true } diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index 8707c5cf769..b31dc6ccc9d 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -43,9 +43,10 @@ use rustpython_common::{ str::DeduceStrKind, wtf8::{CodePoint, Wtf8, Wtf8Buf, Wtf8Chunk, Wtf8Concat}, }; -use unic_ucd_bidi::BidiClass; -use unic_ucd_category::GeneralCategory; -use unic_ucd_ident::{is_xid_continue, is_xid_start}; + +use icu_properties::props::{ + BidiClass, BinaryProperty, EnumeratedProperty, GeneralCategory, XidContinue, XidStart, +}; use unicode_casing::CharExt; impl<'a> TryFromBorrowedObject<'a> for String { @@ -966,7 +967,9 @@ impl PyStr { #[pymethod] fn isdecimal(&self) -> bool { !self.data.is_empty() - && self.char_all(|c| GeneralCategory::of(c) == GeneralCategory::DecimalNumber) + && self.char_all(|c| { + matches!(GeneralCategory::for_char(c), GeneralCategory::DecimalNumber) + }) } fn __mod__(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -1091,11 +1094,17 @@ impl PyStr { #[pymethod] fn isspace(&self) -> bool { - use unic_ucd_bidi::bidi_class::abbr_names::*; !self.data.is_empty() && self.char_all(|c| { - GeneralCategory::of(c) == GeneralCategory::SpaceSeparator - || matches!(BidiClass::of(c), WS | B | S) + matches!( + GeneralCategory::for_char(c), + GeneralCategory::SpaceSeparator + ) || matches!( + BidiClass::for_char(c), + BidiClass::WhiteSpace + | BidiClass::ParagraphSeparator + | BidiClass::SegmentSeparator + ) }) } @@ -1355,9 +1364,13 @@ impl PyStr { pub fn isidentifier(&self) -> bool { let Some(s) = self.to_str() else { return false }; let mut chars = s.chars(); - let is_identifier_start = chars.next().is_some_and(|c| c == '_' || is_xid_start(c)); + + let is_identifier_start = chars + .next() + .is_some_and(|c| c == '_' || XidStart::for_char(c)); + // a string is not an identifier if it has whitespace or starts with a number - is_identifier_start && chars.all(is_xid_continue) + is_identifier_start && chars.all(XidContinue::for_char) } // https://docs.python.org/3/library/stdtypes.html#str.translate From a09afab9127a124478024fa6e8c2138db00164b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2026 01:44:50 +0900 Subject: [PATCH 018/834] Bump lodash from 4.17.23 to 4.18.1 in /wasm/demo (#7556) Bumps [lodash](https://github.com/lodash/lodash) from 4.17.23 to 4.18.1. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.23...4.18.1) --- updated-dependencies: - dependency-name: lodash dependency-version: 4.18.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- wasm/demo/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wasm/demo/package-lock.json b/wasm/demo/package-lock.json index 761c3d3575b..f766a3282c5 100644 --- a/wasm/demo/package-lock.json +++ b/wasm/demo/package-lock.json @@ -3266,9 +3266,9 @@ } }, "node_modules/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "dev": true, "license": "MIT" }, From 87fc4540c4d3beb0c103422511cf04cc0b71e8a0 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Sat, 4 Apr 2026 02:46:46 +0200 Subject: [PATCH 019/834] Fix VM's infinite recursion crash with musl libc (#7558) * Fix VM's infinite recursion crash with musl libc * Lintfix/cleanup warnings --- crates/vm/src/vm/mod.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index da9149cc5fc..231ba5cde38 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -94,7 +94,7 @@ pub struct VirtualMachine { pub initialized: bool, recursion_depth: Cell, /// C stack soft limit for detecting stack overflow (like c_stack_soft_limit) - #[cfg_attr(miri, allow(dead_code))] + #[cfg_attr(any(miri, target_env = "musl"), allow(dead_code))] c_stack_soft_limit: Cell, /// Async generator firstiter hook (per-thread, set via sys.set_asyncgen_hooks) pub async_gen_firstiter: RefCell>, @@ -1424,12 +1424,12 @@ impl VirtualMachine { /// Stack margin bytes (like _PyOS_STACK_MARGIN_BYTES). /// 2048 * sizeof(void*) = 16KB for 64-bit. - #[cfg_attr(miri, allow(dead_code))] + #[cfg_attr(any(miri, target_env = "musl"), allow(dead_code))] const STACK_MARGIN_BYTES: usize = 2048 * core::mem::size_of::(); /// Get the stack boundaries using platform-specific APIs. /// Returns (base, top) where base is the lowest address and top is the highest. - #[cfg(all(not(miri), windows))] + #[cfg(all(not(miri), not(target_env = "musl"), windows))] fn get_stack_bounds() -> (usize, usize) { use windows_sys::Win32::System::Threading::{ GetCurrentThreadStackLimits, SetThreadStackGuarantee, @@ -1448,7 +1448,7 @@ impl VirtualMachine { /// Get stack boundaries on non-Windows platforms. /// Falls back to estimating based on current stack pointer. - #[cfg(all(not(miri), not(windows)))] + #[cfg(all(not(miri), not(target_env = "musl"), not(windows)))] fn get_stack_bounds() -> (usize, usize) { // Use pthread_attr_getstack on platforms that support it #[cfg(any(target_os = "linux", target_os = "android"))] @@ -1499,15 +1499,16 @@ impl VirtualMachine { /// Calculate the C stack soft limit based on actual stack boundaries. /// soft_limit = base + 2 * margin (for downward-growing stacks) - #[cfg(not(miri))] + #[cfg(all(not(miri), not(target_env = "musl")))] fn calculate_c_stack_soft_limit() -> usize { let (base, _top) = Self::get_stack_bounds(); - // Soft limit is 2 margins above the base base + Self::STACK_MARGIN_BYTES * 2 } - /// Miri doesn't support inline assembly, so disable C stack checking. - #[cfg(miri)] + /// Musl currently reports stack bounds in a way that trips the VM's + /// native stack guard during frozen stdlib bootstrap, so keep the Python + /// recursion limit as the only guard there. + #[cfg(any(miri, target_env = "musl"))] fn calculate_c_stack_soft_limit() -> usize { 0 } @@ -1515,19 +1516,18 @@ impl VirtualMachine { /// Check if we're near the C stack limit (like _Py_MakeRecCheck). /// Returns true only when stack pointer is in the "danger zone" between /// soft_limit and hard_limit (soft_limit - 2*margin). - #[cfg(not(miri))] + #[cfg(all(not(miri), not(target_env = "musl")))] #[inline(always)] fn check_c_stack_overflow(&self) -> bool { let current_sp = psm::stack_pointer() as usize; let soft_limit = self.c_stack_soft_limit.get(); - // Stack grows downward: check if we're below soft limit but above hard limit - // This matches CPython's _Py_MakeRecCheck behavior current_sp < soft_limit && current_sp >= soft_limit.saturating_sub(Self::STACK_MARGIN_BYTES * 2) } - /// Miri doesn't support inline assembly, so always return false. - #[cfg(miri)] + /// Miri does not support the native stack probe, and musl currently trips + /// the probe during stdlib bootstrap. + #[cfg(any(miri, target_env = "musl"))] #[inline(always)] fn check_c_stack_overflow(&self) -> bool { false From eed618d85886ceb3f97f54164c7f53b392d3c5e5 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 4 Apr 2026 14:45:30 +0900 Subject: [PATCH 020/834] Fix `str.zfill()` width calculation for non-ASCII strings (#7534) --- crates/vm/src/anystr.rs | 4 ++++ extra_tests/snippets/builtin_str.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/crates/vm/src/anystr.rs b/crates/vm/src/anystr.rs index 79b62a58abf..a9369bf068e 100644 --- a/crates/vm/src/anystr.rs +++ b/crates/vm/src/anystr.rs @@ -396,6 +396,10 @@ pub trait AnyStr { fn py_zfill(&self, width: isize) -> Vec { let width = width.to_usize().unwrap_or(0); + let char_len = self.elements().count(); + let width = self + .bytes_len() + .saturating_add(width.saturating_sub(char_len)); rustpython_common::str::zfill(self.as_bytes(), width) } diff --git a/extra_tests/snippets/builtin_str.py b/extra_tests/snippets/builtin_str.py index 415156184c4..12f97ac619d 100644 --- a/extra_tests/snippets/builtin_str.py +++ b/extra_tests/snippets/builtin_str.py @@ -61,6 +61,9 @@ assert not a.endswith("on") assert not a.endswith(("A", "ll")) assert a.zfill(8) == "000Hallo" +assert "á".zfill(4) == "000á" +assert "🙂".zfill(5) == "0000🙂" +assert "+あ".zfill(5) == "+000あ" assert a.isalnum() assert not a.isdigit() assert not a.isdecimal() From e009cc0c3bda0550028e1e26a9a316669d8eb1ab Mon Sep 17 00:00:00 2001 From: Joshua Megnauth <48846352+joshuamegnauth54@users.noreply.github.com> Date: Mon, 6 Apr 2026 10:29:30 -0400 Subject: [PATCH 021/834] fix: Swapcase must handle multibyte expansions (#7559) `swapcase` used `to_ascii_lowercase` and uppercase to swap cases. This is fine for ASCII, but code points may expand into multiple bytes which leads to incorrect case swaps for some languages. The fix is to use `to_lowercase` and `to_uppercase` instead. Unfortunately, this leads to a realloc in `swapcase` when bytes are expanded. Part of #7526. --- crates/vm/src/builtins/str.rs | 6 +++--- extra_tests/snippets/builtin_str.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index b31dc6ccc9d..74564278925 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -1045,11 +1045,11 @@ impl PyStr { let mut swapped_str = Wtf8Buf::with_capacity(self.data.len()); for c_orig in self.as_wtf8().code_points() { let c = c_orig.to_char_lossy(); - // to_uppercase returns an iterator, to_ascii_uppercase returns the char + // to_uppercase returns an iterator because case changes may be multiple bytes if c.is_lowercase() { - swapped_str.push_char(c.to_ascii_uppercase()); + swapped_str.extend(c.to_uppercase()); } else if c.is_uppercase() { - swapped_str.push_char(c.to_ascii_lowercase()); + swapped_str.extend(c.to_lowercase()); } else { swapped_str.push(c_orig); } diff --git a/extra_tests/snippets/builtin_str.py b/extra_tests/snippets/builtin_str.py index 12f97ac619d..b852e678ace 100644 --- a/extra_tests/snippets/builtin_str.py +++ b/extra_tests/snippets/builtin_str.py @@ -235,6 +235,8 @@ assert not "😂".isidentifier() assert not "123".isidentifier() +assert "Σίσυφος".swapcase() == "σΊΣΥΦΟΣ" + # String Formatting assert "{} {}".format(1, 2) == "1 2" assert "{0} {1}".format(2, 3) == "2 3" From 72f397c6df7d66f8a28942f245343d73aed438f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 23:29:58 +0900 Subject: [PATCH 022/834] Bump pyo3 from 0.28.2 to 0.28.3 in the pyo3 group (#7563) Bumps the pyo3 group with 1 update: [pyo3](https://github.com/pyo3/pyo3). Updates `pyo3` from 0.28.2 to 0.28.3 - [Release notes](https://github.com/pyo3/pyo3/releases) - [Changelog](https://github.com/PyO3/pyo3/blob/main/CHANGELOG.md) - [Commits](https://github.com/pyo3/pyo3/compare/v0.28.2...v0.28.3) --- updated-dependencies: - dependency-name: pyo3 dependency-version: 0.28.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: pyo3 ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5fd981fd135..20336689a32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2643,9 +2643,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf85e27e86080aafd5a22eae58a162e133a589551542b3e5cee4beb27e54f8e1" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" dependencies = [ "libc", "once_cell", @@ -2657,18 +2657,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf94ee265674bf76c09fa430b0e99c26e319c945d96ca0d5a8215f31bf81cf7" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "491aa5fc66d8059dd44a75f4580a2962c1862a1c2945359db36f6c2818b748dc" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" dependencies = [ "libc", "pyo3-build-config", @@ -2676,9 +2676,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d671734e9d7a43449f8480f8b38115df67bef8d21f76837fa75ee7aaa5e52e" +checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2688,9 +2688,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22faaa1ce6c430a1f71658760497291065e6450d7b5dc2bcf254d49f66ee700a" +checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" dependencies = [ "heck", "proc-macro2", From d5a90e5c1f5d96d55fc197fc953f8a1285f6dde4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 23:30:13 +0900 Subject: [PATCH 023/834] Bump cargo-bins/cargo-binstall from 1.17.8 to 1.17.9 (#7566) Bumps [cargo-bins/cargo-binstall](https://github.com/cargo-bins/cargo-binstall) from 1.17.8 to 1.17.9. - [Release notes](https://github.com/cargo-bins/cargo-binstall/releases) - [Changelog](https://github.com/cargo-bins/cargo-binstall/blob/main/release-plz.toml) - [Commits](https://github.com/cargo-bins/cargo-binstall/compare/113a77a4ce971c41332f2129c3d995df993cf746...0b24824336e2b3800b0f89d9e08b2c08bfa3dcdd) --- updated-dependencies: - dependency-name: cargo-bins/cargo-binstall dependency-version: 1.17.9 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d1075152e99..bdf452b8bcc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -369,7 +369,7 @@ jobs: with: components: rustfmt - - uses: cargo-bins/cargo-binstall@113a77a4ce971c41332f2129c3d995df993cf746 # v1.17.8 + - uses: cargo-bins/cargo-binstall@0b24824336e2b3800b0f89d9e08b2c08bfa3dcdd # v1.17.9 - name: cargo shear run: | From 00dd9a5ed1c0bc60dc87957fa6ca2f121e4ce68f Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Thu, 9 Apr 2026 19:38:14 +0200 Subject: [PATCH 024/834] Unify python version used by CI. Update to `3.14.4` (#7571) * Add `.python-version` file * Modify CI to use `.python-version` file * Use correct path for `.python-version` file --- .github/workflows/ci.yaml | 8 -------- .github/workflows/cron-ci.yaml | 19 +++++++++---------- .github/workflows/lib-deps-check.yaml | 20 +++++++++++++------- .github/workflows/update-libs-status.yaml | 16 +++++++++++----- .github/workflows/upgrade-pylib.md | 2 +- .gitignore | 1 - .python-version | 1 + 7 files changed, 35 insertions(+), 32 deletions(-) create mode 100644 .python-version diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bdf452b8bcc..bb78a825a4c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,8 +26,6 @@ env: # - rustpython-compiler-source: deprecated # - rustpython-venvlauncher: Windows-only WORKSPACE_EXCLUDES: --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher - # Python version targeted by the CI. - PYTHON_VERSION: "3.14.3" X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include CARGO_INCREMENTAL: 0 @@ -259,8 +257,6 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - name: Install macOS dependencies uses: ./.github/actions/install-macos-deps @@ -362,8 +358,6 @@ jobs: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - uses: dtolnay/rust-toolchain@stable with: @@ -469,8 +463,6 @@ jobs: tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - run: python -m pip install -r requirements.txt working-directory: ./wasm/tests diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index a481e71de06..2e648e12cb7 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -13,7 +13,6 @@ name: Periodic checks/tasks env: CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls,jit,host_env - PYTHON_VERSION: "3.14.3" jobs: # codecov collects code coverage data from the rust tests, python snippets and python test suite. @@ -30,9 +29,9 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@cargo-llvm-cov - - uses: actions/setup-python@v6.2.0 - with: - python-version: ${{ env.PYTHON_VERSION }} + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - run: sudo apt-get update && sudo apt-get -y install lcov - name: Run cargo-llvm-cov with Rust tests. run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher --verbose --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls,jit,host_env @@ -96,9 +95,9 @@ jobs: persist-credentials: true - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v6.2.0 - with: - python-version: ${{ env.PYTHON_VERSION }} + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - name: build rustpython run: cargo build --release --verbose - name: Collect what is left data @@ -157,9 +156,9 @@ jobs: persist-credentials: true - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v6.2.0 - with: - python-version: ${{ env.PYTHON_VERSION }} + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - run: cargo install cargo-criterion - name: build benchmarks run: cargo build --release --benches diff --git a/.github/workflows/lib-deps-check.yaml b/.github/workflows/lib-deps-check.yaml index 1b0caea6759..b009c427df7 100644 --- a/.github/workflows/lib-deps-check.yaml +++ b/.github/workflows/lib-deps-check.yaml @@ -10,9 +10,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number }} cancel-in-progress: true -env: - PYTHON_VERSION: "3.14.3" - jobs: check_deps: permissions: @@ -37,9 +34,20 @@ jobs: # Checkout only Lib/ directory from PR head for accurate comparison git checkout ${{ github.event.pull_request.head.sha }} -- Lib/ - - name: Checkout CPython + - name: Get target CPython version + id: cpython-version run: | - git clone --depth 1 --branch "v${{ env.PYTHON_VERSION }}" https://github.com/python/cpython.git cpython + version=$(cat .python-version) + echo "version=${version}" >> "$GITHUB_OUTPUT" + + - name: Checkout CPython + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: python/cpython + path: cpython + ref: "v${{ steps.cpython-version.outputs.version }}" + fetch-depth: 1 + persist-credentials: false - name: Get changed Lib files id: changed-files @@ -75,8 +83,6 @@ jobs: - name: Setup Python if: steps.changed-files.outputs.modules != '' uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: "${{ env.PYTHON_VERSION }}" - name: Run deps check if: steps.changed-files.outputs.modules != '' diff --git a/.github/workflows/update-libs-status.yaml b/.github/workflows/update-libs-status.yaml index 53685f88474..db32c7a3e94 100644 --- a/.github/workflows/update-libs-status.yaml +++ b/.github/workflows/update-libs-status.yaml @@ -13,7 +13,6 @@ permissions: issues: write env: - PYTHON_VERSION: "v3.14.3" ISSUE_ID: "6839" jobs: @@ -29,13 +28,20 @@ jobs: sparse-checkout: |- Lib scripts/update_lib + .python-version - - name: Clone CPython ${{ env.PYTHON_VERSION }} + - name: Get target CPython version + id: cpython-version + run: | + version=$(cat rustpython/.python-version) + echo "version=${version}" >> "$GITHUB_OUTPUT" + + - name: Clone CPython uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: python/cpython path: cpython - ref: ${{ env.PYTHON_VERSION }} + ref: "v${{ steps.cpython-version.outputs.version }}" persist-credentials: false sparse-checkout: | Lib @@ -56,14 +62,14 @@ jobs: ## Summary - Check \`scripts/update_lib\` for tools. As a note, the current latest Python version is \`${{ env.PYTHON_VERSION }}\`. + Check \`scripts/update_lib\` for tools. As a note, the current latest Python version is \`${{ steps.cpython-version.outputs.version }}\`. Previous versions' issues as reference - 3.13: #5529 - """ % (self.OutputString(attrs).replace('"', r'\"')) + """ % (output_string.replace('"', r'\"')) def OutputString(self, attrs=None): # Build up our result From f197699e3c6ff94f6803e70f26dce076b2515cd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 14:36:50 +0000 Subject: [PATCH 048/834] Bump reviewdog/action-actionlint from 1.71.0 to 1.72.0 Bumps [reviewdog/action-actionlint](https://github.com/reviewdog/action-actionlint) from 1.71.0 to 1.72.0. - [Release notes](https://github.com/reviewdog/action-actionlint/releases) - [Commits](https://github.com/reviewdog/action-actionlint/compare/0d952c597ef8459f634d7145b0b044a9699e5e43...6fb7acc99f4a1008869fa8a0f09cfca740837d9d) --- updated-dependencies: - dependency-name: reviewdog/action-actionlint dependency-version: 1.72.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f93d7463485..26e9b85dd44 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -371,7 +371,7 @@ jobs: cargo shear - name: actionlint - uses: reviewdog/action-actionlint@0d952c597ef8459f634d7145b0b044a9699e5e43 # v1.71.0 + uses: reviewdog/action-actionlint@6fb7acc99f4a1008869fa8a0f09cfca740837d9d # v1.72.0 - name: zizmor uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2 From 73218f42d52aa62db6f27e0507d408d6d65dfadb Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 15 Apr 2026 07:15:27 +0200 Subject: [PATCH 049/834] Update `test_range.py` from 3.14.4 (#7605) --- Lib/test/test_range.py | 55 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 4f0a954692b..f69054180be 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -91,8 +91,7 @@ def test_range(self): r = range(-sys.maxsize, sys.maxsize, 2) self.assertEqual(len(r), sys.maxsize) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_range_constructor_error_messages(self): with self.assertRaisesRegex( TypeError, @@ -409,11 +408,7 @@ def test_iterator_pickling_overflowing_index(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto): it = iter(range(2**32 + 2)) - _, _, idx = it.__reduce__() - self.assertEqual(idx, 0) - it.__setstate__(2**32 + 1) # undocumented way to set r->index - _, _, idx = it.__reduce__() - self.assertEqual(idx, 2**32 + 1) + it.__setstate__(2**32 + 1) # undocumented way to advance an iterator d = pickle.dumps(it, proto) it = pickle.loads(d) self.assertEqual(next(it), 2**32 + 1) @@ -444,6 +439,50 @@ def test_large_exhausted_iterator_pickling(self): self.assertEqual(list(i), []) self.assertEqual(list(i2), []) + @unittest.skip("TODO: RUSTPYTHON; crash") + def test_iterator_unpickle_compat(self): + testcases = [ + b'c__builtin__\niter\n(c__builtin__\nxrange\n(I10\nI20\nI2\ntRtRI2\nb.', + b'c__builtin__\niter\n(c__builtin__\nxrange\n(K\nK\x14K\x02tRtRK\x02b.', + b'\x80\x02c__builtin__\niter\nc__builtin__\nxrange\nK\nK\x14K\x02\x87R\x85RK\x02b.', + b'\x80\x03cbuiltins\niter\ncbuiltins\nrange\nK\nK\x14K\x02\x87R\x85RK\x02b.', + b'\x80\x04\x951\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x8c\x04iter\x93\x8c\x08builtins\x8c\x05range\x93K\nK\x14K\x02\x87R\x85RK\x02b.', + + b'c__builtin__\niter\n(c__builtin__\nxrange\n(L-36893488147419103232L\nI20\nI2\ntRtRL18446744073709551623L\nb.', + b'c__builtin__\niter\n(c__builtin__\nxrange\n(L-36893488147419103232L\nK\x14K\x02tRtRL18446744073709551623L\nb.', + b'\x80\x02c__builtin__\niter\nc__builtin__\nxrange\n\x8a\t\x00\x00\x00\x00\x00\x00\x00\x00\xfeK\x14K\x02\x87R\x85R\x8a\t\x07\x00\x00\x00\x00\x00\x00\x00\x01b.', + b'\x80\x03cbuiltins\niter\ncbuiltins\nrange\n\x8a\t\x00\x00\x00\x00\x00\x00\x00\x00\xfeK\x14K\x02\x87R\x85R\x8a\t\x07\x00\x00\x00\x00\x00\x00\x00\x01b.', + b'\x80\x04\x95C\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x8c\x04iter\x93\x8c\x08builtins\x8c\x05range\x93\x8a\t\x00\x00\x00\x00\x00\x00\x00\x00\xfeK\x14K\x02\x87R\x85R\x8a\t\x07\x00\x00\x00\x00\x00\x00\x00\x01b.', + ] + for t in testcases: + it = pickle.loads(t) + self.assertEqual(list(it), [14, 16, 18]) + + @unittest.skip("TODO: RUSTPYTHON; crash") + def test_iterator_setstate(self): + it = iter(range(10, 20, 2)) + it.__setstate__(2) + self.assertEqual(list(it), [14, 16, 18]) + it = reversed(range(10, 20, 2)) + it.__setstate__(3) + self.assertEqual(list(it), [12, 10]) + it = iter(range(-2**65, 20, 2)) + it.__setstate__(2**64 + 7) + self.assertEqual(list(it), [14, 16, 18]) + it = reversed(range(10, 2**65, 2)) + it.__setstate__(2**64 - 7) + self.assertEqual(list(it), [12, 10]) + + def test_iterator_invalid_setstate(self): + for invalid_value in (1.0, ""): + ranges = (('rangeiter', range(10, 100, 2)), + ('longrangeiter', range(10, 2**65, 2))) + for rng_name, rng in ranges: + with self.subTest(invalid_value=invalid_value, range=rng_name): + it = iter(rng) + with self.assertRaises(TypeError): + it.__setstate__(invalid_value) + def test_odd_bug(self): # This used to raise a "SystemError: NULL result without error" # because the range validation step was eating the exception @@ -516,6 +555,7 @@ def test_range_iterators(self): for start in limits for end in limits for step in (-2**63, -2**31, -2, -1, 1, 2)] + test_ranges += [(-2**63, 2**63-2, 1)] # regression test for gh-100810 for start, end, step in test_ranges: iter1 = range(start, end, step) @@ -661,6 +701,7 @@ def test_comparison(self): with self.assertRaises(TypeError): range(0) >= range(0) + def test_attributes(self): # test the start, stop and step attributes of range objects self.assert_attrs(range(0), 0, 0, 1) From 9f1429d95fe39f150419f8ca0276455adae91fc8 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 15 Apr 2026 07:15:48 +0200 Subject: [PATCH 050/834] Update `test_compile.py` from 3.14.4 (#7607) --- Lib/test/test_compile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 0495c58329c..9676aded5d1 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -250,8 +250,8 @@ def test_32_63_bit_values(self): d = -281474976710656 # 1 << 48 e = +4611686018427387904 # 1 << 62 f = -4611686018427387904 # 1 << 62 - g = +9223372036854775807 # 1 << 63 - 1 - h = -9223372036854775807 # 1 << 63 - 1 + g = +9223372036854775807 # (1 << 63) - 1 + h = -9223372036854775807 # (1 << 63) - 1 for variable in self.test_32_63_bit_values.__code__.co_consts: if variable is not None: From c98939a7c1ebcf83dc2f7d3a57a50d9f6c61d054 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 15 Apr 2026 07:17:01 +0200 Subject: [PATCH 051/834] Update seq tests from 3.14.4 (#7606) * Update seq related tests --- Lib/test/seq_tests.py | 31 +++++++++---- Lib/test/test_dict.py | 83 ++++++++++++++++++++++++++++++++++ Lib/test/test_listcomps.py | 12 +++++ Lib/test/test_set.py | 2 - Lib/test/test_userdict.py | 93 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 210 insertions(+), 11 deletions(-) diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py index c7497d09f64..7b2d6521b1e 100644 --- a/Lib/test/seq_tests.py +++ b/Lib/test/seq_tests.py @@ -261,23 +261,20 @@ def test_minmax(self): self.assertEqual(min(u), 0) self.assertEqual(max(u), 2) - def test_addmul(self): + def test_add(self): u1 = self.type2test([0]) u2 = self.type2test([0, 1]) self.assertEqual(u1, u1 + self.type2test()) self.assertEqual(u1, self.type2test() + u1) self.assertEqual(u1 + self.type2test([1]), u2) self.assertEqual(self.type2test([-1]) + u1, self.type2test([-1, 0])) - self.assertEqual(self.type2test(), u2*0) - self.assertEqual(self.type2test(), 0*u2) + + def test_mul(self): + u2 = self.type2test([0, 1]) self.assertEqual(self.type2test(), u2*0) self.assertEqual(self.type2test(), 0*u2) self.assertEqual(u2, u2*1) self.assertEqual(u2, 1*u2) - self.assertEqual(u2, u2*1) - self.assertEqual(u2, 1*u2) - self.assertEqual(u2+u2, u2*2) - self.assertEqual(u2+u2, 2*u2) self.assertEqual(u2+u2, u2*2) self.assertEqual(u2+u2, 2*u2) self.assertEqual(u2+u2+u2, u2*3) @@ -286,8 +283,9 @@ def test_addmul(self): class subclass(self.type2test): pass u3 = subclass([0, 1]) - self.assertEqual(u3, u3*1) - self.assertIsNot(u3, u3*1) + r = u3*1 + self.assertEqual(r, u3) + self.assertIsNot(r, u3) def test_iadd(self): u = self.type2test([0, 1]) @@ -348,6 +346,21 @@ def test_subscript(self): self.assertRaises(ValueError, a.__getitem__, slice(0, 10, 0)) self.assertRaises(TypeError, a.__getitem__, 'x') + def _assert_cmp(self, a, b, r): + self.assertIs(a == b, r == 0) + self.assertIs(a != b, r != 0) + self.assertIs(a > b, r > 0) + self.assertIs(a <= b, r <= 0) + self.assertIs(a < b, r < 0) + self.assertIs(a >= b, r >= 0) + + def test_cmp(self): + a = self.type2test([0, 1]) + self._assert_cmp(a, a, 0) + self._assert_cmp(a, self.type2test([0, 1]), 0) + self._assert_cmp(a, self.type2test([0]), 1) + self._assert_cmp(a, self.type2test([0, 2]), -1) + def test_count(self): a = self.type2test([0, 1, 2])*3 self.assertEqual(a.count(0), 3) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 85d15830dcd..80d9e87d38f 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1574,6 +1574,26 @@ def make_pairs(): self.assertEqual(d.get(key3_3), 44) self.assertGreaterEqual(eq_count, 1) + def test_overwrite_managed_dict(self): + # GH-130327: Overwriting an object's managed dictionary with another object's + # skipped traversal in favor of inline values, causing the GC to believe that + # the __dict__ wasn't reachable. + import gc + + class Shenanigans: + pass + + to_be_deleted = Shenanigans() + to_be_deleted.attr = "whatever" + holds_reference = Shenanigans() + holds_reference.__dict__ = to_be_deleted.__dict__ + holds_reference.ref = {"circular": to_be_deleted, "data": 42} + + del to_be_deleted + gc.collect() + self.assertEqual(holds_reference.ref['data'], 42) + self.assertEqual(holds_reference.attr, "whatever") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_unhashable_key(self): d = {'a': 1} @@ -1686,6 +1706,69 @@ def test_hash_collision_remove_add(self): self.assertEqual(len(d), len(items), d) self.assertEqual(d, dict(items)) + def test_clear_reentrant_embedded(self): + # gh-130555: dict.clear() must be safe when values are embedded + # in an object and a destructor mutates the dict. + class MyObj: pass + class ClearOnDelete: + def __del__(self): + nonlocal x + del x + + x = MyObj() + x.a = ClearOnDelete() + + d = x.__dict__ + d.clear() + + def test_clear_reentrant_cycle(self): + # gh-130555: dict.clear() must be safe for embedded dicts when the + # object is part of a reference cycle and the last reference to the + # dict is via the cycle. + class MyObj: pass + obj = MyObj() + obj.f = obj + obj.attr = "attr" + + d = obj.__dict__ + del obj + + d.clear() + + def test_clear_reentrant_force_combined(self): + # gh-130555: dict.clear() must be safe when a destructor forces the + # dict from embedded/split to combined (setting ma_values to NULL). + class MyObj: pass + class ForceConvert: + def __del__(self): + d[1] = "trigger" + + x = MyObj() + x.a = ForceConvert() + x.b = "other" + + d = x.__dict__ + d.clear() + + def test_clear_reentrant_delete(self): + # gh-130555: dict.clear() must be safe when a destructor deletes + # a key from the same embedded dict. + class MyObj: pass + class DelKey: + def __del__(self): + try: + del d['b'] + except KeyError: + pass + + x = MyObj() + x.a = DelKey() + x.b = "value_b" + x.c = "value_c" + + d = x.__dict__ + d.clear() + class CAPITest(unittest.TestCase): diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 964383966c2..47c27bc3faa 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -180,6 +180,18 @@ def test_references___class___defined(self): code, outputs={"res": [2]}, scopes=["module", "function"]) self._check_in_scopes(code, raises=NameError, scopes=["class"]) + def test_references___classdict__(self): + code = """ + class i: [__classdict__ for x in y] + """ + self._check_in_scopes(code, raises=NameError) + + def test_references___conditional_annotations__(self): + code = """ + class i: [__conditional_annotations__ for x in y] + """ + self._check_in_scopes(code, raises=NameError) + def test_references___class___enclosing(self): code = """ __class__ = 2 diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index c68ba49d916..de36b4525c5 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -1969,7 +1969,6 @@ def check_set_op_does_not_crash(self, function): self.assertIn("changed size during iteration", str(e)) -@unittest.skip("TODO: RUSTPYTHON; segfault") class TestBinaryOpsMutating(TestOperationsMutating): def test_eq_with_mutation(self): @@ -2058,7 +2057,6 @@ class TestBinaryOpsMutating_Subclass_Set(TestBinaryOpsMutating, unittest.TestCas constructor2 = set -@unittest.skip("TODO: RUSTPYTHON; segfault") class TestMethodsMutating(TestOperationsMutating): def test_issubset_with_mutation(self): diff --git a/Lib/test/test_userdict.py b/Lib/test/test_userdict.py index 75de9ea252d..5f996893ee5 100644 --- a/Lib/test/test_userdict.py +++ b/Lib/test/test_userdict.py @@ -1,8 +1,18 @@ # Check every path through every method of UserDict +from collections import UserDict from test import mapping_tests import unittest import collections +import types + + +class UserDictSubclass(UserDict): + pass + +class UserDictSubclass2(UserDict): + pass + d0 = {} d1 = {"one": 1} @@ -155,6 +165,25 @@ def test_init(self): self.assertRaises(TypeError, collections.UserDict, (), ()) self.assertRaises(TypeError, collections.UserDict.__init__) + def test_data(self): + u = UserDict() + self.assertEqual(u.data, {}) + self.assertIs(type(u.data), dict) + d = {'a': 1, 'b': 2} + u = UserDict(d) + self.assertEqual(u.data, d) + self.assertIsNot(u.data, d) + self.assertIs(type(u.data), dict) + u = UserDict(u) + self.assertEqual(u.data, d) + self.assertIs(type(u.data), dict) + u = UserDict([('a', 1), ('b', 2)]) + self.assertEqual(u.data, d) + self.assertIs(type(u.data), dict) + u = UserDict(a=1, b=2) + self.assertEqual(u.data, d) + self.assertIs(type(u.data), dict) + def test_update(self): for kw in 'self', 'dict', 'other', 'iterable': d = collections.UserDict() @@ -215,6 +244,70 @@ class G(collections.UserDict): test_repr_deep = mapping_tests.TestHashMappingProtocol.test_repr_deep + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: unsupported operand type(s) for |: 'UserDict' and 'mappingproxy' + def test_mixed_or(self): + for t in UserDict, dict, types.MappingProxyType: + with self.subTest(t.__name__): + u = UserDict({0: 'a', 1: 'b'}) | t({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + + u = t({0: 'a', 1: 'b'}) | UserDict({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + + u = UserDict({0: 'a', 1: 'b'}) | UserDictSubclass({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + + u = UserDictSubclass({0: 'a', 1: 'b'}) | UserDict({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + + u = UserDictSubclass({0: 'a', 1: 'b'}) | UserDictSubclass2({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + + u = UserDict({1: 'c', 2: 'd'}).__ror__(UserDict({0: 'a', 1: 'b'})) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + + u = UserDictSubclass({1: 'c', 2: 'd'}).__ror__(UserDictSubclass2({0: 'a', 1: 'b'})) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + + def test_mixed_ior(self): + for t in UserDict, dict, types.MappingProxyType: + with self.subTest(t.__name__): + u = u2 = UserDict({0: 'a', 1: 'b'}) + u |= t({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + self.assertIs(u, u2) + + u = dict({0: 'a', 1: 'b'}) + u |= UserDict({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), dict) + + u = u2 = UserDict({0: 'a', 1: 'b'}) + u |= UserDictSubclass({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + self.assertIs(u, u2) + + u = u2 = UserDictSubclass({0: 'a', 1: 'b'}) + u |= UserDict({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + self.assertIs(u, u2) + + u = u2 = UserDictSubclass({0: 'a', 1: 'b'}) + u |= UserDictSubclass2({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + self.assertIs(u, u2) + if __name__ == "__main__": unittest.main() From 63a1c0e95ca444433fb4b7345ccc5d80704091e7 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 15 Apr 2026 07:17:27 +0200 Subject: [PATCH 052/834] Update `test_bz2.py` from 3.14.4 (#7608) * Update `test_bz2.py` from 3.14.4 * Mark failing test --- Lib/test/test_bz2.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 148d8f98c79..cc7122f1516 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -66,18 +66,28 @@ class BaseTest(unittest.TestCase): EMPTY_DATA = b'BZh9\x17rE8P\x90\x00\x00\x00\x00' BAD_DATA = b'this is not a valid bzip2 file' - # Some tests need more than one block of uncompressed data. Since one block - # is at least 100,000 bytes, we gather some data dynamically and compress it. - # Note that this assumes that compression works correctly, so we cannot - # simply use the bigger test data for all tests. + # Some tests need more than one block of data. The bz2 module does not + # support flushing a block during compression, so we must read in data until + # there are at least 2 blocks. Since different orderings of Python files may + # be compressed differently, we need to check the compression output for + # more than one bzip2 block header magic, a hex encoding of Pi + # (0x314159265359) + bz2_block_magic = bytes.fromhex('314159265359') test_size = 0 - BIG_TEXT = bytearray(128*1024) + BIG_TEXT = b'' + BIG_DATA = b'' + compressor = BZ2Compressor(1) for fname in glob.glob(os.path.join(glob.escape(os.path.dirname(__file__)), '*.py')): with open(fname, 'rb') as fh: - test_size += fh.readinto(memoryview(BIG_TEXT)[test_size:]) - if test_size > 128*1024: + data = fh.read() + BIG_DATA += compressor.compress(data) + BIG_TEXT += data + # TODO(emmatyping): if it is impossible for a block header to cross + # multiple outputs, we can just search the output of each compress call + # which should be more efficient + if BIG_DATA.count(bz2_block_magic) > 1: + BIG_DATA += compressor.flush() break - BIG_DATA = bz2.compress(BIG_TEXT, compresslevel=1) def setUp(self): fd, self.filename = tempfile.mkstemp() @@ -927,6 +937,7 @@ def testPickle(self): with self.assertRaises(TypeError): pickle.dumps(BZ2Decompressor(), proto) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 100 def testDecompressorChunksMaxsize(self): bzd = BZ2Decompressor() max_length = 100 From 8108b6a15392349aedb0ece85c6145a6c436d3b8 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 15 Apr 2026 07:17:48 +0200 Subject: [PATCH 053/834] Update `test_bisect.py` from 3.14.4 (#7609) --- Lib/test/test_bisect.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_bisect.py b/Lib/test/test_bisect.py index 97204d4cad3..a7e1f533ff2 100644 --- a/Lib/test/test_bisect.py +++ b/Lib/test/test_bisect.py @@ -391,9 +391,9 @@ class TestErrorHandlingC(TestErrorHandling, unittest.TestCase): class TestDocExample: def test_grades(self): - def grade(score, breakpoints=[60, 70, 80, 90], grades='FDCBA'): - i = self.module.bisect(breakpoints, score) - return grades[i] + def grade(score): + i = self.module.bisect([60, 70, 80, 90], score) + return "FDCBA"[i] result = [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] self.assertEqual(result, ['F', 'A', 'C', 'C', 'B', 'A', 'A']) From a2c3e65b8168dd341b849f1ac80058a1dc39f1eb Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 15 Apr 2026 07:19:17 +0200 Subject: [PATCH 054/834] =?UTF-8?q?Unify=20CI=20caches=20(reduce=20total?= =?UTF-8?q?=20cache=20size=20by=2075%=20=C2=B1)=20(#7610)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Singular job for creating caches * Use shared cache --- .github/workflows/ci.yaml | 96 +++++++++++++++++++++-------- .github/workflows/update-caches.yml | 73 ++++++++++++++++++++++ 2 files changed, 143 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/update-caches.yml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 26e9b85dd44..cab7428e107 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,6 +29,9 @@ env: X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include CARGO_INCREMENTAL: 0 + CARGO_PROFILE_TEST_DEBUG: 0 + CARGO_PROFILE_DEV_DEBUG: 0 + CARGO_PROFILE_RELEASE_DEBUG: 0 CARGO_TERM_COLOR: always jobs: @@ -52,9 +55,19 @@ jobs: with: components: clippy - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - name: Restore cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - save-if: ${{ github.ref == 'refs/heads/main' }} + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}- + restore-keys: | + ${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-- - name: Install macOS dependencies uses: ./.github/actions/install-macos-deps @@ -154,7 +167,6 @@ jobs: - name: Restore cache uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 - if: ${{ github.ref != 'refs/heads/main' }} # Never restore on main with: path: | ~/.cargo/bin/ @@ -163,9 +175,12 @@ jobs: ~/.cargo/git/db/ target/ # key won't match, will rely on restore-keys - key: cargo-check-${{ runner.os }}-${{ matrix.target }} + key: ${{ runner.os }}-${{ matrix.target }} restore-keys: | - cargo-check-${{ runner.os }}-${{ matrix.target }}- + ${{ runner.os }}-stable-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-${{ matrix.target }}- + ${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-- - uses: dtolnay/rust-toolchain@stable with: @@ -194,18 +209,6 @@ jobs: AR_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang - - name: Save cache - if: ${{ github.ref == 'refs/heads/main' }} # only save on main - uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: cargo-check-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('Cargo.lock') }}-${{ github.sha }} - snippets_cpython: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} env: @@ -252,9 +255,19 @@ jobs: - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - name: Restore cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - save-if: ${{ github.ref == 'refs/heads/main' }} + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}- + restore-keys: | + ${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-- - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -377,7 +390,6 @@ jobs: uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2 - name: restore prek cache - if: ${{ github.ref != 'refs/heads/main' }} # never restore on main uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: key: prek-${{ hashFiles('.pre-commit-config.yaml') }} @@ -422,9 +434,18 @@ jobs: toolchain: ${{ env.NIGHTLY_CHANNEL }} components: miri - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - name: Restore cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - save-if: ${{ github.ref == 'refs/heads/main' }} + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }} + restore-keys: | + ${{ runner.os }}- - name: Run tests under miri run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test -p rustpython-vm -- miri_test @@ -447,9 +468,20 @@ jobs: with: components: clippy - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - name: Restore cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - save-if: ${{ github.ref == 'refs/heads/main' }} + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}- + restore-keys: | + ${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-- + ${{ runner.os }}- - name: cargo clippy run: cargo clippy --manifest-path=crates/wasm/Cargo.toml -- -Dwarnings @@ -522,9 +554,21 @@ jobs: with: target: wasm32-wasip1 - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - name: Restore cache + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - save-if: ${{ github.ref == 'refs/heads/main' }} + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}- + restore-keys: | + ${{ runner.os }}-stable-wasm32-wasip1-${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-wasm32-wasip1- + ${{ runner.os }}-stable--${{ hashFiles('**/Cargo.toml') }}- + ${{ runner.os }}-stable-- - name: Setup Wasmer uses: wasmerio/setup-wasmer@v3 diff --git a/.github/workflows/update-caches.yml b/.github/workflows/update-caches.yml new file mode 100644 index 00000000000..73464d06401 --- /dev/null +++ b/.github/workflows/update-caches.yml @@ -0,0 +1,73 @@ +name: Update Actions Caches + +permissions: + contents: read + +on: + workflow_dispatch: + push: + branches: + - main + +env: + CARGO_INCREMENTAL: 0 + CARGO_TERM_COLOR: always + CARGO_PROFILE_TEST_DEBUG: 0 + CARGO_PROFILE_DEV_DEBUG: 0 + CARGO_PROFILE_RELEASE_DEBUG: 0 + CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls,host_env,threading,jit + +jobs: + build-caches: + name: Build Caches + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: macos-latest + toolchain: stable + target: "" + - os: ubuntu-latest + toolchain: stable + target: "" + - os: windows-latest + toolchain: stable + target: "" + steps: + - name: Checkout RustPython main branch + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: RustPython/RustPython + ref: main + persist-credentials: false + + - name: Setup Rust + uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: ${{ matrix.toolchain }} + target: ${{ matrix.target }} + + - name: Install macos dependencies + uses: ./.github/actions/install-macos-deps + with: + openssl: true + + - name: Build dev cache # dev profile used by check & doc + run: cargo build --profile dev ${{ env.CARGO_ARGS }} + + - name: Build test cache + run: cargo build --profile test ${{ env.CARGO_ARGS }} + + - name: Build release cache + run: cargo build --profile release ${{ env.CARGO_ARGS }} + + - name: Save cache + uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-${{ matrix.toolchain }}-${{ matrix.target }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('Cargo.lock') }}-${{ github.sha }} From 330aa084889510721e9fc32b84e2335593bd5352 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 15 Apr 2026 20:18:14 +0200 Subject: [PATCH 055/834] Macro for defining opcode & instruction enums (#7573) * Macro for defining opcode & instruction enums * Convert `Instruction` as well * revert `#[repr(...)]` changes --- crates/codegen/src/ir.rs | 324 ++--- crates/compiler-core/src/bytecode.rs | 3 +- .../compiler-core/src/bytecode/instruction.rs | 1112 +++++++++-------- crates/stdlib/src/_opcode.rs | 222 ++-- crates/vm/src/frame.rs | 25 +- crates/vm/src/vm/context.rs | 6 +- scripts/generate_opcode_metadata.py | 57 +- 7 files changed, 843 insertions(+), 906 deletions(-) diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs index d226cb5757f..61e549199d5 100644 --- a/crates/codegen/src/ir.rs +++ b/crates/codegen/src/ir.rs @@ -8,10 +8,10 @@ use num_traits::{ToPrimitive, Zero}; use rustpython_compiler_core::{ OneIndexed, SourceLocation, bytecode::{ - AnyInstruction, Arg, CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN, CO_FAST_LOCAL, CodeFlags, - CodeObject, CodeUnit, CodeUnits, ConstantData, ExceptionTableEntry, InstrDisplayContext, - Instruction, InstructionMetadata, Label, OpArg, PseudoInstruction, PyCodeLocationInfoKind, - encode_exception_table, oparg, + AnyInstruction, AnyOpcode, CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN, CO_FAST_LOCAL, + CodeFlags, CodeObject, CodeUnit, CodeUnits, ConstantData, ExceptionTableEntry, + InstrDisplayContext, Instruction, InstructionMetadata, Label, OpArg, Opcode, + PseudoInstruction, PseudoOpcode, PyCodeLocationInfoKind, encode_exception_table, oparg, }, varint::{write_signed_varint, write_varint}, }; @@ -449,23 +449,15 @@ impl CodeInfo { // Direction must be based on concrete instruction offsets. // Empty blocks can share offsets, so block-order-based resolution // may classify some jumps incorrectly. - op = match op { - Instruction::JumpForward { .. } if target_offset <= current_offset => { - Instruction::JumpBackward { - delta: Arg::marker(), - } + op = match op.into() { + Opcode::JumpForward if target_offset <= current_offset => { + Opcode::JumpBackward.into() } - Instruction::JumpBackward { .. } if target_offset > current_offset => { - Instruction::JumpForward { - delta: Arg::marker(), - } + Opcode::JumpBackward if target_offset > current_offset => { + Opcode::JumpForward.into() } - Instruction::JumpBackwardNoInterrupt { .. } - if target_offset > current_offset => - { - Instruction::JumpForward { - delta: Arg::marker(), - } + Opcode::JumpBackwardNoInterrupt if target_offset > current_offset => { + Opcode::JumpForward.into() } _ => op, }; @@ -479,9 +471,8 @@ impl CodeInfo { .expect("END_ASYNC_FOR target must be before instruction"); OpArg::new(arg) } else if matches!( - op, - Instruction::JumpBackward { .. } - | Instruction::JumpBackwardNoInterrupt { .. } + op.into(), + Opcode::JumpBackward | Opcode::JumpBackwardNoInterrupt ) { let arg = offset_after .checked_sub(target_offset) @@ -713,10 +704,7 @@ impl CodeInfo { let (const_idx, _) = self.metadata.consts.insert_full(neg_const); // Replace LOAD_CONST/LOAD_SMALL_INT with new LOAD_CONST let load_location = block.instructions[i].location; - block.instructions[i].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); + block.instructions[i].instr = Opcode::LoadConst.into(); block.instructions[i].arg = OpArg::new(const_idx as u32); // Replace UNARY_NEGATIVE with NOP, inheriting the LOAD_CONST // location so that remove_nops can clean it up @@ -772,10 +760,7 @@ impl CodeInfo { } let (const_idx, _) = self.metadata.consts.insert_full(result_const); // Replace first instruction with LOAD_CONST result - block.instructions[i].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); + block.instructions[i].instr = Opcode::LoadConst.into(); block.instructions[i].arg = OpArg::new(const_idx as u32); // NOP out the second and third instructions let loc = block.instructions[i].location; @@ -982,10 +967,7 @@ impl CodeInfo { let (const_idx, _) = self.metadata.consts.insert_full(ConstantData::Tuple { elements: Vec::new(), }); - block.instructions[i].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); + block.instructions[i].instr = Opcode::LoadConst.into(); block.instructions[i].arg = OpArg::new(const_idx as u32); i += 1; continue; @@ -1050,10 +1032,7 @@ impl CodeInfo { } // Replace BUILD_TUPLE with LOAD_CONST - block.instructions[i].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); + block.instructions[i].instr = Opcode::LoadConst.into(); block.instructions[i].arg = OpArg::new(const_idx as u32); i += 1; @@ -1123,20 +1102,14 @@ impl CodeInfo { let eh = block.instructions[i].except_handler; // slot[start_idx] → BUILD_LIST 0 - block.instructions[start_idx].instr = Instruction::BuildList { - count: Arg::marker(), - } - .into(); + block.instructions[start_idx].instr = Opcode::BuildList.into(); block.instructions[start_idx].arg = OpArg::new(0); block.instructions[start_idx].location = folded_loc; block.instructions[start_idx].end_location = end_loc; block.instructions[start_idx].except_handler = eh; // slot[start_idx+1] → LOAD_CONST (tuple) - block.instructions[start_idx + 1].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); + block.instructions[start_idx + 1].instr = Opcode::LoadConst.into(); block.instructions[start_idx + 1].arg = OpArg::new(const_idx as u32); block.instructions[start_idx + 1].location = folded_loc; block.instructions[start_idx + 1].end_location = end_loc; @@ -1149,7 +1122,7 @@ impl CodeInfo { } // slot[i] (was BUILD_LIST) → LIST_EXTEND 1 - block.instructions[i].instr = Instruction::ListExtend { i: Arg::marker() }.into(); + block.instructions[i].instr = Opcode::ListExtend.into(); block.instructions[i].arg = OpArg::new(1); i += 1; @@ -1239,20 +1212,14 @@ impl CodeInfo { block.instructions[j].location = folded_loc; } - block.instructions[i].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); + block.instructions[i].instr = Opcode::LoadConst.into(); block.instructions[i].arg = OpArg::new(const_idx as u32); i += 2; continue; } } - block.instructions[i].instr = Instruction::BuildTuple { - count: Arg::marker(), - } - .into(); + block.instructions[i].instr = Opcode::BuildTuple.into(); i += 2; } else { i += 1; @@ -1323,19 +1290,13 @@ impl CodeInfo { let end_loc = block.instructions[i].end_location; let eh = block.instructions[i].except_handler; - block.instructions[start_idx].instr = Instruction::BuildSet { - count: Arg::marker(), - } - .into(); + block.instructions[start_idx].instr = Opcode::BuildSet.into(); block.instructions[start_idx].arg = OpArg::new(0); block.instructions[start_idx].location = folded_loc; block.instructions[start_idx].end_location = end_loc; block.instructions[start_idx].except_handler = eh; - block.instructions[start_idx + 1].instr = Instruction::LoadConst { - consti: Arg::marker(), - } - .into(); + block.instructions[start_idx + 1].instr = Opcode::LoadConst.into(); block.instructions[start_idx + 1].arg = OpArg::new(const_idx as u32); block.instructions[start_idx + 1].location = folded_loc; block.instructions[start_idx + 1].end_location = end_loc; @@ -1346,7 +1307,7 @@ impl CodeInfo { block.instructions[j].location = folded_loc; } - block.instructions[i].instr = Instruction::SetUpdate { i: Arg::marker() }.into(); + block.instructions[i].instr = Opcode::SetUpdate.into(); block.instructions[i].arg = OpArg::new(1); i += 1; @@ -1377,16 +1338,15 @@ impl CodeInfo { } match n { 1 => { - instructions[i].instr = AnyInstruction::Real(Instruction::Nop); + instructions[i].instr = Opcode::Nop.into(); instructions[i].arg = OpArg::new(0); - instructions[i + 1].instr = AnyInstruction::Real(Instruction::Nop); + instructions[i + 1].instr = Opcode::Nop.into(); instructions[i + 1].arg = OpArg::new(0); } 2 | 3 => { - instructions[i].instr = AnyInstruction::Real(Instruction::Nop); + instructions[i].instr = Opcode::Nop.into(); instructions[i].arg = OpArg::new(0); - instructions[i + 1].instr = - AnyInstruction::Real(Instruction::Swap { i: Arg::marker() }); + instructions[i + 1].instr = Opcode::Swap.into(); instructions[i + 1].arg = OpArg::new(n); } _ => {} @@ -1409,15 +1369,15 @@ impl CodeInfo { /// Instruction classes that are safe to reorder around SWAP. fn is_swappable(instr: &AnyInstruction) -> bool { matches!( - instr, - AnyInstruction::Real(Instruction::StoreFast { .. } | Instruction::PopTop) + (*instr).into(), + AnyOpcode::Real(Opcode::StoreFast | Opcode::PopTop) ) } /// Variable index that a STORE_FAST writes to, or None. fn stores_to(info: &InstructionInfo) -> Option { - match info.instr { - AnyInstruction::Real(Instruction::StoreFast { .. }) => Some(u32::from(info.arg)), + match info.instr.into() { + AnyOpcode::Real(Opcode::StoreFast) => Some(u32::from(info.arg)), _ => None, } } @@ -1517,7 +1477,7 @@ impl CodeInfo { } } // Safe to reorder. SWAP -> NOP, swap j and k. - instructions[i].instr = AnyInstruction::Real(Instruction::Nop); + instructions[i].instr = Opcode::Nop.into(); instructions[i].arg = OpArg::new(0); instructions.swap(j, k); i += 1; @@ -1544,10 +1504,8 @@ impl CodeInfo { while i < len { // Look for UNPACK_SEQUENCE or UNPACK_EX let is_unpack = matches!( - instructions[i].instr, - AnyInstruction::Real( - Instruction::UnpackSequence { .. } | Instruction::UnpackEx { .. } - ) + instructions[i].instr.into(), + AnyOpcode::Real(Opcode::UnpackSequence | Opcode::UnpackEx) ); if !is_unpack { i += 1; @@ -1558,8 +1516,8 @@ impl CodeInfo { let mut run_end = run_start; while run_end < len && matches!( - instructions[run_end].instr, - AnyInstruction::Real(Instruction::StoreFast { .. }) + instructions[run_end].instr.into(), + AnyOpcode::Real(Opcode::StoreFast) ) { run_end += 1; @@ -1574,7 +1532,7 @@ impl CodeInfo { for (j, instr) in instructions[run_start..run_end].iter_mut().enumerate() { let idx = u32::from(instr.arg); if last_occurrence[&idx] != j { - instr.instr = AnyInstruction::Real(Instruction::PopTop); + instr.instr = Opcode::PopTop.into(); instr.arg = OpArg::new(0); } } @@ -1600,9 +1558,9 @@ impl CodeInfo { }; if matches!( - next_instr, - Instruction::PopJumpIfFalse { .. } | Instruction::PopJumpIfTrue { .. } - ) && matches!(curr_instr, Instruction::CompareOp { .. }) + next_instr.into(), + Opcode::PopJumpIfFalse | Opcode::PopJumpIfTrue + ) && matches!(curr_instr.into(), Opcode::CompareOp) { block.instructions[i].arg = OpArg::new( u32::from(block.instructions[i].arg) | oparg::COMPARE_OP_BOOL_MASK, @@ -1624,12 +1582,7 @@ impl CodeInfo { let idx2 = u32::from(next.arg); if idx1 < 16 && idx2 < 16 { let packed = (idx1 << 4) | idx2; - Some(( - Instruction::LoadFastLoadFast { - var_nums: Arg::marker(), - }, - OpArg::new(packed), - )) + Some((Opcode::LoadFastLoadFast.into(), OpArg::new(packed))) } else { None } @@ -1649,12 +1602,7 @@ impl CodeInfo { let idx2 = u32::from(next.arg); if idx1 < 16 && idx2 < 16 { let packed = (idx1 << 4) | idx2; - Some(( - Instruction::StoreFastStoreFast { - var_nums: Arg::marker(), - }, - OpArg::new(packed), - )) + Some((Opcode::StoreFastStoreFast.into(), OpArg::new(packed))) } else { None } @@ -1688,12 +1636,7 @@ impl CodeInfo { .metadata .consts .insert_full(ConstantData::Boolean { value: !value }); - Some(( - (Instruction::LoadConst { - consti: Arg::marker(), - }), - OpArg::new(const_idx as u32), - )) + Some(((Opcode::LoadConst.into()), OpArg::new(const_idx as u32))) } _ => None, } @@ -1767,7 +1710,7 @@ impl CodeInfo { // LOAD_SMALL_INT oparg is unsigned, so only 0..=255 can be encoded if let Some(small) = value.to_i32().filter(|v| (0..=255).contains(v)) { // Convert LOAD_CONST to LOAD_SMALL_INT - instr.instr = Instruction::LoadSmallInt { i: Arg::marker() }.into(); + instr.instr = Opcode::LoadSmallInt.into(); // The arg is the i32 value stored as u32 (two's complement) instr.arg = OpArg::new(small as u32); } @@ -1883,10 +1826,7 @@ impl CodeInfo { let idx2 = u32::from(next.arg); if idx1 < 16 && idx2 < 16 { let packed = (idx1 << 4) | idx2; - block.instructions[i].instr = Instruction::StoreFastLoadFast { - var_nums: Arg::marker(), - } - .into(); + block.instructions[i].instr = Opcode::StoreFastLoadFast.into(); block.instructions[i].arg = OpArg::new(packed); // Replace second instruction with NOP (CPython: INSTR_SET_OP0(inst2, NOP)) set_to_nop(&mut block.instructions[i + 1]); @@ -1943,8 +1883,8 @@ impl CodeInfo { } // Push values to stack with source instruction index - let source = match instr { - Instruction::LoadFast { .. } | Instruction::LoadFastLoadFast { .. } => i, + let source = match instr.into() { + Opcode::LoadFast | Opcode::LoadFastLoadFast => i, _ => NOT_LOCAL, }; for _ in 0..pushes { @@ -1971,18 +1911,12 @@ impl CodeInfo { let Some(instr) = info.instr.real() else { continue; }; - match instr { - Instruction::LoadFast { .. } => { - info.instr = Instruction::LoadFastBorrow { - var_num: Arg::marker(), - } - .into(); + match instr.into() { + Opcode::LoadFast => { + info.instr = Opcode::LoadFastBorrow.into(); } - Instruction::LoadFastLoadFast { .. } => { - info.instr = Instruction::LoadFastBorrowLoadFastBorrow { - var_nums: Arg::marker(), - } - .into(); + Opcode::LoadFastLoadFast => { + info.instr = Opcode::LoadFastBorrowLoadFastBorrow.into(); } _ => {} } @@ -2078,10 +2012,7 @@ impl CodeInfo { Some(Instruction::LoadFast { var_num }) => { let var_idx = usize::from(var_num.get(info.arg)); if var_idx < nlocals && unsafe_mask[var_idx] { - info.instr = Instruction::LoadFastCheck { - var_num: Arg::marker(), - } - .into(); + info.instr = Opcode::LoadFastCheck.into(); changed = true; } if var_idx < nlocals { @@ -2099,28 +2030,19 @@ impl CodeInfo { if needs_check_1 || needs_check_2 { let mut first = info; first.instr = if needs_check_1 { - Instruction::LoadFastCheck { - var_num: Arg::marker(), - } + Opcode::LoadFastCheck } else { - Instruction::LoadFast { - var_num: Arg::marker(), - } + Opcode::LoadFast } .into(); first.arg = OpArg::new(idx1 as u32); let mut second = info; second.instr = if needs_check_2 { - Instruction::LoadFastCheck { - var_num: Arg::marker(), - } + Opcode::LoadFastCheck.into() } else { - Instruction::LoadFast { - var_num: Arg::marker(), - } - } - .into(); + Opcode::LoadFast.into() + }; second.arg = OpArg::new(idx2 as u32); new_instructions.push(first); @@ -2615,10 +2537,7 @@ fn push_cold_blocks_to_end(blocks: &mut Vec) { ..Block::default() }; jump_block.instructions.push(InstructionInfo { - instr: PseudoInstruction::JumpNoInterrupt { - delta: Arg::marker(), - } - .into(), + instr: PseudoOpcode::JumpNoInterrupt.into(), arg: OpArg::new(0), target: warm_next, location: SourceLocation::default(), @@ -2737,16 +2656,13 @@ enum JumpThreadKind { } fn jump_thread_kind(instr: AnyInstruction) -> Option { - match instr { - AnyInstruction::Pseudo(PseudoInstruction::Jump { .. }) - | AnyInstruction::Real(Instruction::JumpForward { .. }) - | AnyInstruction::Real(Instruction::JumpBackward { .. }) => Some(JumpThreadKind::Plain), - AnyInstruction::Pseudo(PseudoInstruction::JumpNoInterrupt { .. }) - | AnyInstruction::Real(Instruction::JumpBackwardNoInterrupt { .. }) => { - Some(JumpThreadKind::NoInterrupt) - } - _ => None, - } + Some(match instr.into() { + AnyOpcode::Pseudo(PseudoOpcode::Jump) + | AnyOpcode::Real(Opcode::JumpForward | Opcode::JumpBackward) => JumpThreadKind::Plain, + AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) + | AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt) => JumpThreadKind::NoInterrupt, + _ => return None, + }) } fn threaded_jump_instr( @@ -2763,19 +2679,11 @@ fn threaded_jump_instr( if source_kind == JumpThreadKind::NoInterrupt { return Some(source); } - Some(match source { - AnyInstruction::Pseudo(_) => PseudoInstruction::Jump { - delta: Arg::marker(), - } - .into(), - AnyInstruction::Real(Instruction::JumpBackwardNoInterrupt { .. }) => { - Instruction::JumpBackward { - delta: Arg::marker(), - } - .into() - } - AnyInstruction::Real(Instruction::JumpForward { .. }) - | AnyInstruction::Real(Instruction::JumpBackward { .. }) => source, + + Some(match source.into() { + AnyOpcode::Pseudo(_) => PseudoOpcode::Jump.into(), + AnyOpcode::Real(Opcode::JumpBackwardNoInterrupt) => Opcode::JumpBackward.into(), + AnyOpcode::Real(Opcode::JumpForward | Opcode::JumpBackward) => source, _ => return None, }) } @@ -2852,35 +2760,23 @@ fn jump_threading_impl(blocks: &mut [Block], include_conditional: bool) { fn is_conditional_jump(instr: &AnyInstruction) -> bool { matches!( - instr.real(), + instr.real().map(Into::into), Some( - Instruction::PopJumpIfFalse { .. } - | Instruction::PopJumpIfTrue { .. } - | Instruction::PopJumpIfNone { .. } - | Instruction::PopJumpIfNotNone { .. } + Opcode::PopJumpIfFalse + | Opcode::PopJumpIfTrue + | Opcode::PopJumpIfNone + | Opcode::PopJumpIfNotNone ) ) } /// Invert a conditional jump opcode. fn reversed_conditional(instr: &AnyInstruction) -> Option { - Some(match instr.real()? { - Instruction::PopJumpIfFalse { .. } => Instruction::PopJumpIfTrue { - delta: Arg::marker(), - } - .into(), - Instruction::PopJumpIfTrue { .. } => Instruction::PopJumpIfFalse { - delta: Arg::marker(), - } - .into(), - Instruction::PopJumpIfNone { .. } => Instruction::PopJumpIfNotNone { - delta: Arg::marker(), - } - .into(), - Instruction::PopJumpIfNotNone { .. } => Instruction::PopJumpIfNone { - delta: Arg::marker(), - } - .into(), + Some(match AnyOpcode::from(*instr).real()? { + Opcode::PopJumpIfFalse => Opcode::PopJumpIfTrue.into(), + Opcode::PopJumpIfTrue => Opcode::PopJumpIfFalse.into(), + Opcode::PopJumpIfNone => Opcode::PopJumpIfNotNone.into(), + Opcode::PopJumpIfNotNone => Opcode::PopJumpIfNone.into(), _ => return None, }) } @@ -2928,7 +2824,7 @@ fn normalize_jumps(blocks: &mut Vec) { if is_forward { // Insert NOT_TAKEN after forward conditional jump let not_taken = InstructionInfo { - instr: Instruction::NotTaken.into(), + instr: Opcode::NotTaken.into(), arg: OpArg::new(0), target: BlockIdx::NULL, location: last_ins.location, @@ -2957,7 +2853,7 @@ fn normalize_jumps(blocks: &mut Vec) { ..Block::default() }; new_block.instructions.push(InstructionInfo { - instr: Instruction::NotTaken.into(), + instr: Opcode::NotTaken.into(), arg: OpArg::new(0), target: BlockIdx::NULL, location: loc, @@ -2967,10 +2863,7 @@ fn normalize_jumps(blocks: &mut Vec) { cache_entries: 0, }); new_block.instructions.push(InstructionInfo { - instr: PseudoInstruction::Jump { - delta: Arg::marker(), - } - .into(), + instr: PseudoOpcode::Jump.into(), arg: OpArg::new(0), target, location: loc, @@ -3019,34 +2912,22 @@ fn normalize_jumps(blocks: &mut Vec) { continue; } let target_pos = block_order[target.idx()]; - info.instr = match info.instr { - AnyInstruction::Pseudo(PseudoInstruction::Jump { .. }) => { + info.instr = match info.instr.into() { + AnyOpcode::Pseudo(PseudoOpcode::Jump) => { if target_pos > source_pos { - Instruction::JumpForward { - delta: Arg::marker(), - } - .into() + Opcode::JumpForward.into() } else { - Instruction::JumpBackward { - delta: Arg::marker(), - } - .into() + Opcode::JumpBackward.into() } } - AnyInstruction::Pseudo(PseudoInstruction::JumpNoInterrupt { .. }) => { + AnyOpcode::Pseudo(PseudoOpcode::JumpNoInterrupt) => { if target_pos > source_pos { - Instruction::JumpForward { - delta: Arg::marker(), - } - .into() + Opcode::JumpForward.into() } else { - Instruction::JumpBackwardNoInterrupt { - delta: Arg::marker(), - } - .into() + Opcode::JumpBackwardNoInterrupt.into() } } - other => other, + _ => info.instr, }; } } @@ -4105,10 +3986,7 @@ pub(crate) fn convert_pseudo_ops(blocks: &mut [Block], cellfixedoffsets: &[u32]) let cell_relative = i.get(info.arg) as usize; let new_idx = cellfixedoffsets[cell_relative]; info.arg = OpArg::new(new_idx); - info.instr = Instruction::LoadFast { - var_num: Arg::marker(), - } - .into(); + info.instr = Opcode::LoadFast.into(); } // Jump pseudo ops are resolved during block linearization PseudoInstruction::Jump { .. } | PseudoInstruction::JumpNoInterrupt { .. } => {} @@ -4160,12 +4038,12 @@ pub(crate) fn fixup_deref_opargs(blocks: &mut [Block], cellfixedoffsets: &[u32]) continue; }; let needs_fixup = matches!( - instr, - Instruction::LoadDeref { .. } - | Instruction::StoreDeref { .. } - | Instruction::DeleteDeref { .. } - | Instruction::LoadFromDictOrDeref { .. } - | Instruction::MakeCell { .. } + instr.into(), + Opcode::LoadDeref + | Opcode::StoreDeref + | Opcode::DeleteDeref + | Opcode::LoadFromDictOrDeref + | Opcode::MakeCell ); if needs_fixup { let cell_relative = u32::from(info.arg) as usize; diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs index a17fe7945de..eeef411acbe 100644 --- a/crates/compiler-core/src/bytecode.rs +++ b/crates/compiler-core/src/bytecode.rs @@ -21,7 +21,8 @@ use rustpython_wtf8::{Wtf8, Wtf8Buf}; pub use crate::bytecode::{ instruction::{ - AnyInstruction, Arg, Instruction, InstructionMetadata, PseudoInstruction, StackEffect, + AnyInstruction, AnyOpcode, Arg, Instruction, InstructionMetadata, Opcode, + PseudoInstruction, PseudoOpcode, StackEffect, }, oparg::{ BinaryOperator, BuildSliceArgCount, CommonConstant, ComparisonOperator, ConvertValueOparg, diff --git a/crates/compiler-core/src/bytecode/instruction.rs b/crates/compiler-core/src/bytecode/instruction.rs index 6a5bd2a04f6..00608236f7a 100644 --- a/crates/compiler-core/src/bytecode/instruction.rs +++ b/crates/compiler-core/src/bytecode/instruction.rs @@ -1,4 +1,4 @@ -use core::{fmt, marker::PhantomData, mem}; +use core::{fmt, marker::PhantomData}; use crate::{ bytecode::{ @@ -13,452 +13,507 @@ use crate::{ marshal::MarshalError, }; -/// A Single bytecode instruction that are executed by the VM. -/// -/// Currently aligned with CPython 3.14. -/// -/// ## See also -/// - [CPython opcode IDs](https://github.com/python/cpython/blob/v3.14.2/Include/opcode_ids.h) -#[derive(Clone, Copy, Debug)] -#[repr(u8)] -pub enum Instruction { - // No-argument instructions (opcode < HAVE_ARGUMENT=44) - Cache = 0, - BinarySlice = 1, - BuildTemplate = 2, - BinaryOpInplaceAddUnicode = 3, - CallFunctionEx = 4, - CheckEgMatch = 5, - CheckExcMatch = 6, - CleanupThrow = 7, - DeleteSubscr = 8, - EndFor = 9, - EndSend = 10, - ExitInitCheck = 11, // Placeholder - FormatSimple = 12, - FormatWithSpec = 13, - GetAIter = 14, - GetANext = 15, - GetIter = 16, - Reserved = 17, - GetLen = 18, - GetYieldFromIter = 19, - InterpreterExit = 20, // Placeholder - LoadBuildClass = 21, - LoadLocals = 22, - MakeFunction = 23, - MatchKeys = 24, - MatchMapping = 25, - MatchSequence = 26, - Nop = 27, - NotTaken = 28, - PopExcept = 29, - PopIter = 30, - PopTop = 31, - PushExcInfo = 32, - PushNull = 33, - ReturnGenerator = 34, - ReturnValue = 35, - SetupAnnotations = 36, - StoreSlice = 37, - StoreSubscr = 38, - ToBool = 39, - UnaryInvert = 40, - UnaryNegative = 41, - UnaryNot = 42, - WithExceptStart = 43, - // CPython 3.14 opcodes with arguments (44-120) - BinaryOp { - op: Arg, - } = 44, - /// Build an Interpolation from value, expression string, and optional format_spec on stack. - /// - /// oparg encoding: (conversion << 2) | has_format_spec - /// - has_format_spec (bit 0): if 1, format_spec is on stack - /// - conversion (bits 2+): 0=None, 1=Str, 2=Repr, 3=Ascii - /// - /// Stack: [value, expression_str, format_spec?] -> [interpolation] - BuildInterpolation { - format: Arg, - } = 45, - BuildList { - count: Arg, - } = 46, - BuildMap { - count: Arg, - } = 47, - BuildSet { - count: Arg, - } = 48, - BuildSlice { - argc: Arg, - } = 49, - BuildString { - count: Arg, - } = 50, - BuildTuple { - count: Arg, - } = 51, - Call { - argc: Arg, - } = 52, - CallIntrinsic1 { - func: Arg, - } = 53, - CallIntrinsic2 { - func: Arg, - } = 54, - CallKw { - argc: Arg, - } = 55, - CompareOp { - opname: Arg, - } = 56, - ContainsOp { - invert: Arg, - } = 57, - ConvertValue { - oparg: Arg, - } = 58, - Copy { - i: Arg, - } = 59, - CopyFreeVars { - n: Arg, - } = 60, - DeleteAttr { - namei: Arg, - } = 61, - DeleteDeref { - i: Arg, - } = 62, - DeleteFast { - var_num: Arg, - } = 63, - DeleteGlobal { - namei: Arg, - } = 64, - DeleteName { - namei: Arg, - } = 65, - DictMerge { - i: Arg, - } = 66, - DictUpdate { - i: Arg, - } = 67, - EndAsyncFor = 68, - ExtendedArg = 69, - ForIter { - delta: Arg