diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 041b0e4e4..000000000 --- a/.gitattributes +++ /dev/null @@ -1,3 +0,0 @@ -data/*.tar.gz filter=lfs diff=lfs merge=lfs -text -data/*.tgz filter=lfs diff=lfs merge=lfs -text -data/*.zip filter=lfs diff=lfs merge=lfs -text diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c010ba8c1..a4042b8bc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @github/semantic-code +* @github/semantic-code @github/blackbird diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c561bd429..e8bb697e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,5 @@ name: Continuous integration + on: push: branches: [main] @@ -6,6 +7,9 @@ on: schedule: - cron: "0 0 1,15 * *" +permissions: + contents: read + # In the event that there is a new push to the ref, cancel any running jobs because there are now obsolete, and wasting resources. concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -17,18 +21,23 @@ jobs: strategy: matrix: rust: [stable] + env: + # cargo hack does not use the default-members in Cargo.toml, so we restrict to those explicitly + CARGO_HACK: cargo hack -p lsp-positions -p stack-graphs -p tree-sitter-stack-graphs --feature-powerset --exclude-features copious-debugging steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 with: rust-version: ${{ matrix.rust }} + - name: Install cargo-hack + run: cargo install cargo-hack - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Check formatting run: cargo fmt --all -- --check - name: Cache dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo @@ -36,43 +45,56 @@ jobs: key: ${{ runner.OS }}-cargo-${{ hashFiles('**/Cargo.lock') }} restore-keys: | ${{ runner.OS }}-cargo- - - name: Install cargo-valgrind + - name: Build library (all feature combinations) + run: ${{ env.CARGO_HACK }} --no-dev-deps build + - name: Run test suite (all feature combinations) + run: ${{ env.CARGO_HACK }} test + - name: Run test suite with all optimizations (default features) + run: cargo test --release + - name: Install valgrind run: | sudo apt-get update sudo apt-get install -y valgrind - cargo install cargo-valgrind - - name: Build library - run: cargo build + - name: Run test suite under valgrind (default features) + id: valgrind + # We only need to use valgrind to test the crates that have C bindings. + run: script/ci-test-valgrind -p stack-graphs + - name: Upload valgrind log + if: ${{ failure() && steps.valgrind.outcome == 'failure' }} + uses: actions/upload-artifact@v4 + with: + name: valgrind logs + path: ${{ runner.temp }}/valgrind.log - name: Ensure C headers are up to date run: | script/cbindgen test -z "$(git status --porcelain)" - - name: Run test suite - run: cargo test - - name: Run test suite under valgrind - # We only need to use valgrind to test the crates that have C bindings. - run: cargo valgrind test -p stack-graphs - - name: Run lsp-positions tests without tree-sitter - run: cargo test -p lsp-positions --no-default-features - - name: Run test suite with all features enabled - run: cargo test --all-features - - name: Run test suite with all optimizations - run: cargo test --release - # Do the new project test last, because it adds the crate in the current source - # folder, and that shouldn't influence other tests. + + test-init: + needs: [test-rust] + runs-on: ubuntu-latest + strategy: + matrix: + rust: [stable] + + steps: + - name: Install Rust environment + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 + with: + rust-version: ${{ matrix.rust }} + - name: Checkout code + uses: actions/checkout@v4 + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cargo + target + key: ${{ runner.OS }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.OS }}-cargo- - name: Generate, build, and run new language project - run: | - cargo run --bin tree-sitter-stack-graphs --features cli -- init \ - --language-name InitTest \ - --language-id init_test \ - --language-file-extension it \ - --grammar-crate-name tree-sitter-python \ - --grammar-crate-version 0.20.0 \ - --internal \ - --non-interactive - cargo check -p tree-sitter-stack-graphs-init_test --all-features - cargo test -p tree-sitter-stack-graphs-init_test - cargo run -p tree-sitter-stack-graphs-init_test --features cli -- help + run: script/ci-test-init list-languages: runs-on: ubuntu-latest @@ -81,7 +103,7 @@ jobs: working-directory: languages steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: List languages id: language-list run: echo "languages=$(find -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT @@ -98,11 +120,13 @@ jobs: steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 with: rust-version: ${{ matrix.rust }} + - name: Install cargo-hack + run: cargo install cargo-hack - name: Cache dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo @@ -111,11 +135,11 @@ jobs: restore-keys: | ${{ runner.OS }}-cargo- - name: Checkout code - uses: actions/checkout@v3 - - name: Build - run: cargo build -p ${{ matrix.language }} - - name: Test - run: cargo test -p ${{ matrix.language }} + uses: actions/checkout@v4 + - name: Build (all feature combinations) + run: cargo hack -p ${{ matrix.language }} --feature-powerset build + - name: Test (all features) + run: cargo test -p ${{ matrix.language }} --all-features test-cli: runs-on: ubuntu-latest @@ -128,11 +152,11 @@ jobs: steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 with: rust-version: ${{ matrix.rust }} - name: Cache dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo @@ -141,7 +165,7 @@ jobs: restore-keys: | ${{ runner.OS }}-cargo- - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: lfs: true - name: Build diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 5c4288c84..d68ffa75d 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -1,9 +1,14 @@ name: Performance testing + on: pull_request: paths: - 'stack-graphs/**' +permissions: + contents: read + pull-requests: write + # In the event that there is a new push to the ref, cancel any running jobs because there are now obsolete, and wasting resources. concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -37,7 +42,7 @@ jobs: done: ${{ steps.done.outputs.cache-hit }} steps: - name: "Checkout base code" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ env.BASE_REPO }} ref: ${{ env.BASE_SHA }} @@ -49,7 +54,7 @@ jobs: printf 'BASE_SHA=%s\n' "$(git rev-list -1 ${{ env.BASE_SHA }} -- stack-graphs)" >> $GITHUB_ENV working-directory: ${{ env.BASE_DIR }} - name: "Checkout head code" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ env.HEAD_REPO }} ref: ${{ env.HEAD_SHA }} @@ -62,7 +67,7 @@ jobs: working-directory: ${{ env.HEAD_DIR }} - name: "Check cached status" id: done - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: done key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }} @@ -79,11 +84,11 @@ jobs: BASE_SHA: ${{ needs.changes.outputs.base-sha }} steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 with: rust-version: stable - name: Cache Rust dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo @@ -94,7 +99,7 @@ jobs: sudo apt-get install -y valgrind - name: "Cache base result" id: cache-base-result - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ${{ env.MASSIF_OUT }} @@ -102,7 +107,7 @@ jobs: key: ${{ runner.os }}-perf-result-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.TEST_NAME }} - name: "Checkout base code" if: steps.cache-base-result.outputs.cache-hit != 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ env.BASE_REPO }} ref: ${{ env.BASE_SHA }} @@ -130,7 +135,7 @@ jobs: ${{ env.BASE_DIR }}/data/${{ env.TEST_NAME }} ms_print ${{ env.MASSIF_OUT }} > ${{ env.MASSIF_REPORT }} - name: Upload results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ env.BASE_ARTIFACT }} path: | @@ -148,11 +153,11 @@ jobs: HEAD_SHA: ${{ needs.changes.outputs.head-sha }} steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 with: rust-version: stable - name: Cache Rust dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo @@ -163,7 +168,7 @@ jobs: sudo apt-get install -y valgrind - name: "Cache head result" id: cache-head-result - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ${{ env.MASSIF_OUT }} @@ -171,7 +176,7 @@ jobs: key: ${{ runner.os }}-perf-result-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }} - name: "Checkout head code" if: steps.cache-head-result.outputs.cache-hit != 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ env.HEAD_REPO }} ref: ${{ env.HEAD_SHA }} @@ -199,7 +204,7 @@ jobs: ${{ env.HEAD_DIR }}/data/${{ env.TEST_NAME }} ms_print ${{ env.MASSIF_OUT }} > ${{ env.MASSIF_REPORT }} - name: Upload results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ env.HEAD_ARTIFACT }} path: | @@ -233,12 +238,12 @@ jobs: # Download results # - name: Download base results - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ env.BASE_ARTIFACT }} path: ${{ env.BASE_ARTIFACT }} - name: Download head results - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ env.HEAD_ARTIFACT }} path: ${{ env.HEAD_ARTIFACT }} @@ -246,7 +251,7 @@ jobs: # Create report # - name: "Checkout code" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: ${{ env.SRC_DIR }} - name: Generate summary @@ -268,7 +273,7 @@ jobs: - name: Create status marker run: touch done - name: "Cache status" - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 with: path: done key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }} diff --git a/.github/workflows/publish-lsp-positions.yml b/.github/workflows/publish-lsp-positions.yml index 210662a00..2ca05a133 100644 --- a/.github/workflows/publish-lsp-positions.yml +++ b/.github/workflows/publish-lsp-positions.yml @@ -5,6 +5,9 @@ on: tags: - lsp-positions-v* +permissions: + contents: write + jobs: publish-crate: runs-on: ubuntu-latest @@ -14,9 +17,9 @@ jobs: CRATE_DIR: './lsp-positions' steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # TODO Verify the crate version matches the tag - name: Test crate run: cargo test --all-features @@ -36,9 +39,9 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 with: body: | Find more info on all releases at https://crates.io/crates/lsp-positions. diff --git a/.github/workflows/publish-stack-graphs.yml b/.github/workflows/publish-stack-graphs.yml index 641a93488..29b21020b 100644 --- a/.github/workflows/publish-stack-graphs.yml +++ b/.github/workflows/publish-stack-graphs.yml @@ -5,6 +5,9 @@ on: tags: - stack-graphs-v* +permissions: + contents: write + jobs: publish-crate: runs-on: ubuntu-latest @@ -14,9 +17,9 @@ jobs: CRATE_DIR: './stack-graphs' steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # TODO Verify the crate version matches the tag - name: Test crate run: cargo test --all-features @@ -36,9 +39,9 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 with: body: | Find more info on all releases at https://crates.io/crates/stack-graphs. diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-java.yml b/.github/workflows/publish-tree-sitter-stack-graphs-java.yml index 6d7afb769..40878de95 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs-java.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs-java.yml @@ -5,6 +5,9 @@ on: tags: - tree-sitter-stack-graphs-java-v* +permissions: + contents: write + jobs: publish-crate: runs-on: ubuntu-latest @@ -14,9 +17,9 @@ jobs: CRATE_DIR: './languages/tree-sitter-stack-graphs-java' steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # TODO Verify the crate version matches the tag - name: Test crate run: cargo test --all-features @@ -36,9 +39,9 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 with: body: | Find more info on all releases at https://crates.io/crates/tree-sitter-stack-graphs-java. diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml b/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml index ff6826770..b9fbceec3 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml @@ -5,6 +5,9 @@ on: tags: - tree-sitter-stack-graphs-javascript-v* +permissions: + contents: write + jobs: publish-crate: runs-on: ubuntu-latest @@ -14,9 +17,9 @@ jobs: CRATE_DIR: './languages/tree-sitter-stack-graphs-javascript' steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # TODO Verify the crate version matches the tag - name: Test crate run: cargo test --all-features @@ -36,9 +39,9 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 with: body: | Find more info on all releases at https://crates.io/crates/tree-sitter-stack-graphs-javascript. diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-python.yml b/.github/workflows/publish-tree-sitter-stack-graphs-python.yml new file mode 100644 index 000000000..dc90a6974 --- /dev/null +++ b/.github/workflows/publish-tree-sitter-stack-graphs-python.yml @@ -0,0 +1,48 @@ +name: Publish tree-sitter-stack-graphs-python release + +on: + push: + tags: + - tree-sitter-stack-graphs-python-v* + +permissions: + contents: write + +jobs: + publish-crate: + runs-on: ubuntu-latest + env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + CRATE_DIR: './languages/tree-sitter-stack-graphs-python' + steps: + - name: Install Rust environment + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 + - name: Checkout repository + uses: actions/checkout@v4 + # TODO Verify the crate version matches the tag + - name: Test crate + run: cargo test --all-features + working-directory: ${{ env.CRATE_DIR }} + - name: Verify publish crate + run: cargo publish --dry-run + working-directory: ${{ env.CRATE_DIR }} + - name: Publish crate + run: cargo publish + working-directory: ${{ env.CRATE_DIR }} + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + create-release: + needs: publish-crate + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Create GitHub release + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 + with: + body: | + Find more info on all releases at https://crates.io/crates/tree-sitter-stack-graphs-python. + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml b/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml index a3409f397..9eb696e2a 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml @@ -5,6 +5,9 @@ on: tags: - tree-sitter-stack-graphs-typescript-v* +permissions: + contents: write + jobs: publish-crate: runs-on: ubuntu-latest @@ -14,9 +17,9 @@ jobs: CRATE_DIR: './languages/tree-sitter-stack-graphs-typescript' steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # TODO Verify the crate version matches the tag - name: Test crate run: cargo test --all-features @@ -36,9 +39,9 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 with: body: | Find more info on all releases at https://crates.io/crates/tree-sitter-stack-graphs-typescript. diff --git a/.github/workflows/publish-tree-sitter-stack-graphs.yml b/.github/workflows/publish-tree-sitter-stack-graphs.yml index e9395ab1f..0bf56c143 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs.yml @@ -5,6 +5,9 @@ on: tags: - tree-sitter-stack-graphs-v* +permissions: + contents: write + jobs: publish-crate: runs-on: ubuntu-latest @@ -14,9 +17,9 @@ jobs: CRATE_DIR: './tree-sitter-stack-graphs' steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # TODO Verify the crate version matches the tag - name: Test crate run: cargo test --all-features @@ -29,31 +32,6 @@ jobs: working-directory: ${{ env.CRATE_DIR }} env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - publish-npm: - needs: publish-crate - runs-on: ubuntu-latest - env: - PACKAGE_DIR: './tree-sitter-stack-graphs/npm' - steps: - - name: Install Node environment - uses: actions/setup-node@v3 - with: - node-version: 16.x - registry-url: 'https://registry.npmjs.org' - - name: Checkout repository - uses: actions/checkout@v3 - # TODO Verify the package version matches the tag - - name: Install dependencies - run: npm install - working-directory: ${{ env.PACKAGE_DIR }} - - name: Verify package - run: npm publish --dry-run - working-directory: ${{ env.PACKAGE_DIR }} - - name: Publish package - run: npm publish - working-directory: ${{ env.PACKAGE_DIR }} - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} create-release: needs: publish-crate runs-on: ubuntu-latest @@ -61,9 +39,9 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 with: body: | Find more info on all releases at https://crates.io/crates/tree-sitter-stack-graphs. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1992fceca..3827e3473 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,8 +32,8 @@ Here are a few things you can do that will increase the likelihood of your pull If you are one of the maintainers of this package, bump the version numbers in [`Cargo.toml`](Cargo.toml) and [`README.md`](README.md), then follow the typical instructions to publish a new version to [crates.io][]: ``` -$ cargo package -$ cargo publish +cargo package +cargo publish ``` [crates.io]: https://crates.io/stack-graphs/ @@ -43,4 +43,3 @@ $ cargo publish - [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) - [Using Pull Requests](https://help.github.com/articles/about-pull-requests/) - [GitHub Help](https://help.github.com) - diff --git a/README.md b/README.md index df51b81c6..6ae144656 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +**NOTE:** _This repository is no longer supported or updated by GitHub. If you wish to continue to develop this code yourself, we recommend you fork it._ + # Stack graphs The crates in this repository provide a Rust implementation of _stack graphs_, diff --git a/data/typescript_benchmark.zip b/data/typescript_benchmark.zip index 97f24b96d..9cf6105ad 100644 Binary files a/data/typescript_benchmark.zip and b/data/typescript_benchmark.zip differ diff --git a/data/typescript_minimal_project.zip b/data/typescript_minimal_project.zip index ada17bfe7..924d57bff 100644 Binary files a/data/typescript_minimal_project.zip and b/data/typescript_minimal_project.zip differ diff --git a/languages/tree-sitter-stack-graphs-java/CHANGELOG.md b/languages/tree-sitter-stack-graphs-java/CHANGELOG.md index f7483cabc..d295d8ce0 100644 --- a/languages/tree-sitter-stack-graphs-java/CHANGELOG.md +++ b/languages/tree-sitter-stack-graphs-java/CHANGELOG.md @@ -5,9 +5,31 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## v0.5.0 -- 2024-12-12 -## [0.2.0] - 2023-03-21 +- The `tree-sitter-stack-graphs` dependency is updated to version 0.10. + +- The `tree-sitter-java` dependency is updated to version 0.23.4. + +## v0.4.0 -- 2024-07-09 + +### Added + +- Add rules for the `condition` node that was missing. + +### Removed + +- The `FILE_PATH_VAR` constant has been replaced in favor of `tree_sitter_stack_graphs::FILE_PATH_VAR`. + +## v0.3.0 -- 2024-03-06 + +The `tree-sitter-stack-graphs` is updated to `v0.8`. + +### Changed + +- The `cli` feature is now required to install the CLI. + +## v0.2.0 -- 2023-03-21 ### Added diff --git a/languages/tree-sitter-stack-graphs-java/Cargo.toml b/languages/tree-sitter-stack-graphs-java/Cargo.toml index 8bd3e5f93..0ad8cb687 100644 --- a/languages/tree-sitter-stack-graphs-java/Cargo.toml +++ b/languages/tree-sitter-stack-graphs-java/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tree-sitter-stack-graphs-java" -version = "0.2.0" +version = "0.5.0" description = "Stack graphs for the Java programming language" homepage = "https://github.com/github/stack-graphs/tree/main/languages/tree-sitter-stack-graphs-java" @@ -23,6 +23,7 @@ keywords = ["tree-sitter", "stack-graphs", "java"] [[bin]] name = "tree-sitter-stack-graphs-java" path = "rust/bin.rs" +required-features = ["cli"] [lib] path = "rust/lib.rs" @@ -33,8 +34,15 @@ name = "test" path = "rust/test.rs" harness = false # need to provide own main function to handle running tests +[features] +cli = ["anyhow", "clap", "tree-sitter-stack-graphs/cli"] + [dependencies] -anyhow = "1.0" -clap = { version = "4", features = ["derive"] } -tree-sitter-stack-graphs = { version = "0.7", path = "../../tree-sitter-stack-graphs", features=["cli"] } -tree-sitter-java = { version = "=0.20.0" } +anyhow = { version = "1.0", optional = true } +clap = { version = "4", features = ["derive"], optional = true } +tree-sitter-java = { version = "=0.23.4" } +tree-sitter-stack-graphs = { version = "0.10", path = "../../tree-sitter-stack-graphs" } # explicit version is required to be able to publish crate + +[dev-dependencies] +anyhow = { version = "1.0" } +tree-sitter-stack-graphs = { path = "../../tree-sitter-stack-graphs", features = ["cli"] } diff --git a/languages/tree-sitter-stack-graphs-java/README.md b/languages/tree-sitter-stack-graphs-java/README.md index 821796b67..97820ed3f 100644 --- a/languages/tree-sitter-stack-graphs-java/README.md +++ b/languages/tree-sitter-stack-graphs-java/README.md @@ -1,30 +1,138 @@ # tree-sitter-stack-graphs definition for Java -This project defines tree-sitter-stack-graphs rules for Java using the [tree-sitter-java](https://www.npmjs.com/package/tree-sitter-java) grammar. +This project defines tree-sitter-stack-graphs rules for Java using the [tree-sitter-java][] grammar. -## Local Development +[tree-sitter-java]: https://crates.io/crates/tree-sitter-java + +- [API documentation](https://docs.rs/tree-sitter-stack-graphs-java/) +- [Release notes](https://github.com/github/stack-graphs/blob/main/languages/tree-sitter-stack-graphs-java/CHANGELOG.md) + +## Using the API + +To use this library, add the following to your `Cargo.toml`: + +```toml +[dependencies] +tree-sitter-stack-graphs-java = "0.5" +``` + +Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs-java/*/) for more details on how to use this library. + +## Using the Command-line Program + +The command-line program for `tree-sitter-stack-graphs-java` lets you do stack graph based analysis and lookup from the command line. + +The CLI can be run as follows: + +1. _(Installed)_ Install the CLI using Cargo as follows: + + ```sh + cargo install --features cli tree-sitter-stack-graphs-java + ``` + + After this, the CLI should be available as `tree-sitter-stack-graphs-java`. + +2. _(From source)_ Instead of installing the CLI, it can also be run directly from the crate directory, as a replacement for a `tree-sitter-stack-graphs-java` invocation, as follows: + + ```sh + cargo run --features cli -- + ``` + +The basic CLI workflow for the command-line program is to index source code and issue queries against the resulting database: + +1. Index a source folder as follows: + + ```sh + tree-sitter-stack-graphs-java index SOURCE_DIR + ``` + + _Indexing will skip any files that have already be indexed. To force a re-index, add the `-f` flag._ + + To check the status if a source folder, run: + + ```sh + tree-sitter-stack-graphs-java status SOURCE_DIR + ``` + + To clean the database and start with a clean slate, run: + + ```sh + tree-sitter-stack-graphs-java clean + ``` + + _Pass the `--delete` flag to not just empty the database, but also delete it. This is useful to resolve `unsupported database version` errors that may occur after a version update._ + +2. Run a query to find the definition(s) for a reference on a given line and column, run: + + ```sh + tree-sitter-stack-graphs-java query definition SOURCE_PATH:LINE:COLUMN + ``` + + Resulting definitions are printed, including a source line if the source file is available. + +Discover all available commands and flags by passing the `-h` flag to the CLI directly, or to any of the subcommands. + +## Development + +The project is written in Rust, and requires a recent version installed. Rust can be installed and updated using [rustup][]. + +[rustup]: https://rustup.rs/ The project is organized as follows: - The stack graph rules are defined in `src/stack-graphs.tsg`. +- Builtins sources and configuration are defined in `src/builtins.it` and `builtins.cfg` respectively. - Tests are put into the `test` directory. -The following commands are intended to be run from the repo root. +### Running Tests + +Run the tests as follows: + +```sh +cargo test +``` + +The project consists of a library and a CLI. By default, running `cargo` only applies to the library. To run `cargo` commands on the CLI as well, add `--features cli` or `--all-features`. + +Run the CLI from source as follows: + +```sh +cargo run --features cli -- ARGS +``` + +Sources are formatted using the standard Rust formatted, which is applied by running: + +```sh +cargo fmt +``` + +### Writing TSG + +The stack graph rules are written in [tree-sitter-graph][]. Checkout the [examples][], +which contain self-contained TSG rules for specific language features. A VSCode +[extension][] is available that provides syntax highlighting for TSG files. -Run all tests in the project by executing the following: +[tree-sitter-graph]: https://github.com/tree-sitter/tree-sitter-graph +[examples]: https://github.com/github/stack-graphs/blob/main/tree-sitter-stack-graphs/examples/ +[extension]: https://marketplace.visualstudio.com/items?itemName=tree-sitter.tree-sitter-graph - cargo test -p tree-sitter-stack-graphs-java +Parse and test a single file by executing the following commands: -Parse a single test file: - `cargo run -p tree-sitter-stack-graphs-java -- parse OPTIONS FILENAME` +```sh +cargo run --features cli -- parse FILES... +cargo run --features cli -- test TESTFILES... +``` -Test a single file: - `cargo run -p tree-sitter-stack-graphs-java -- test OPTIONS FILENAME` +Generate a visualization to debug failing tests by passing the `-V` flag: -For debugging purposes, it is useful to run the visualization tool to generate graphs for the tests being run. +```sh +cargo run --features cli -- test -V TESTFILES... +``` -To run a test and generate the visualization: +To generate the visualization regardless of test outcome, execute: -`cargo run -p tree-sitter-stack-graphs-java -- test --output-mode=always -V=%r/%d/%n.html FILENAME` +```sh +cargo run --features cli -- test -V --output-mode=always TESTFILES... +``` -Go to https://crates.io/crates/tree-sitter-stack-graphs for links to examples and documentation. +Go to for links to examples and documentation. diff --git a/languages/tree-sitter-stack-graphs-java/rust/lib.rs b/languages/tree-sitter-stack-graphs-java/rust/lib.rs index 09cb043a5..9b2cb39c8 100644 --- a/languages/tree-sitter-stack-graphs-java/rust/lib.rs +++ b/languages/tree-sitter-stack-graphs-java/rust/lib.rs @@ -1,4 +1,3 @@ -use tree_sitter_stack_graphs::loader::FileAnalyzers; use tree_sitter_stack_graphs::loader::LanguageConfiguration; use tree_sitter_stack_graphs::loader::LoadError; use tree_sitter_stack_graphs::CancellationFlag; @@ -15,8 +14,6 @@ pub const STACK_GRAPHS_BUILTINS_PATH: &str = "src/builtins.java"; /// The stack graphs builtins source for this language. pub const STACK_GRAPHS_BUILTINS_SOURCE: &str = include_str!("../src/builtins.java"); -/// The name of the file path global variable -pub const FILE_PATH_VAR: &str = "FILE_PATH"; /// The name of the project name global variable pub const PROJECT_NAME_VAR: &str = "PROJECT_NAME"; @@ -28,7 +25,7 @@ pub fn try_language_configuration( cancellation_flag: &dyn CancellationFlag, ) -> Result { LanguageConfiguration::from_sources( - tree_sitter_java::language(), + tree_sitter_java::LANGUAGE.into(), Some(String::from("source.java")), None, vec![String::from("java")], @@ -39,7 +36,6 @@ pub fn try_language_configuration( STACK_GRAPHS_BUILTINS_SOURCE, )), Some(STACK_GRAPHS_BUILTINS_CONFIG), - FileAnalyzers::new(), cancellation_flag, ) } diff --git a/languages/tree-sitter-stack-graphs-javascript/CHANGELOG.md b/languages/tree-sitter-stack-graphs-javascript/CHANGELOG.md index 3b21ea383..ae90a2f01 100644 --- a/languages/tree-sitter-stack-graphs-javascript/CHANGELOG.md +++ b/languages/tree-sitter-stack-graphs-javascript/CHANGELOG.md @@ -4,3 +4,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## v0.3.0 -- 2024-12-12 + +- The `tree-sitter-stack-graphs` dependency is updated to version 0.10. + +- The `tree-sitter-javascript` dependency is updated to version 0.23.1. + +## v0.2.0 -- 2024-07-09 + +### Removed + +- The `FILE_PATH_VAR` constant has been replaced in favor of `tree_sitter_stack_graphs::FILE_PATH_VAR`. + +## v0.1.0 -- 2024-03-06 + +Initial release. diff --git a/languages/tree-sitter-stack-graphs-javascript/Cargo.toml b/languages/tree-sitter-stack-graphs-javascript/Cargo.toml index 624edbab2..d1206c6e5 100644 --- a/languages/tree-sitter-stack-graphs-javascript/Cargo.toml +++ b/languages/tree-sitter-stack-graphs-javascript/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tree-sitter-stack-graphs-javascript" -version = "0.1.0" +version = "0.3.0" description = "Stack graphs definition for JavaScript using tree-sitter-javascript" readme = "README.md" keywords = ["tree-sitter", "stack-graphs", "javascript"] @@ -30,10 +30,11 @@ anyhow = { version = "1.0", optional = true } clap = { version = "4", optional = true } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -stack-graphs = { version = "0.12", path = "../../stack-graphs" } -tree-sitter-stack-graphs = { version = "0.7", path = "../../tree-sitter-stack-graphs" } -tree-sitter-javascript = { git = "https://github.com/tree-sitter/tree-sitter-javascript", rev = "5720b249490b3c17245ba772f6be4a43edb4e3b7" } +stack-graphs = { version = "0.14", path = "../../stack-graphs" } # explicit version is required to be able to publish crate +tree-sitter-graph = "0.12" +tree-sitter-javascript = "=0.23.1" +tree-sitter-stack-graphs = { version = "0.10", path = "../../tree-sitter-stack-graphs" } # explicit version is required to be able to publish crate [dev-dependencies] anyhow = "1.0" -tree-sitter-stack-graphs = { version = "0.7", path = "../../tree-sitter-stack-graphs", features = ["cli"] } +tree-sitter-stack-graphs = { path = "../../tree-sitter-stack-graphs", features = ["cli"] } diff --git a/languages/tree-sitter-stack-graphs-javascript/README.md b/languages/tree-sitter-stack-graphs-javascript/README.md index 3b53f03de..c7eefa955 100644 --- a/languages/tree-sitter-stack-graphs-javascript/README.md +++ b/languages/tree-sitter-stack-graphs-javascript/README.md @@ -4,27 +4,73 @@ This project defines tree-sitter-stack-graphs rules for JavaScript using the [tr [tree-sitter-javascript]: https://crates.io/crates/tree-sitter-javascript -## Usage +- [API documentation](https://docs.rs/tree-sitter-stack-graphs-javascript/) +- [Release notes](https://github.com/github/stack-graphs/blob/main/languages/tree-sitter-stack-graphs-javascript/CHANGELOG.md) + +## Using the API To use this library, add the following to your `Cargo.toml`: -``` toml +```toml [dependencies] -tree-sitter-stack-graphs-javascript = "0.0.1" +tree-sitter-stack-graphs-javascript = "0.3" ``` Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs-javascript/*/) for more details on how to use this library. -## Command-line Program +## Using the Command-line Program The command-line program for `tree-sitter-stack-graphs-javascript` lets you do stack graph based analysis and lookup from the command line. -Install the program using `cargo install` as follows: +The CLI can be run as follows: -``` sh -$ cargo install --features cli tree-sitter-stack-graphs-javascript -$ tree-sitter-stack-graphs-javascript --help -``` +1. _(Installed)_ Install the CLI using Cargo as follows: + + ```sh + cargo install --features cli tree-sitter-stack-graphs-javascript + ``` + + After this, the CLI should be available as `tree-sitter-stack-graphs-javascript`. + +2. _(From source)_ Instead of installing the CLI, it can also be run directly from the crate directory, as a replacement for a `tree-sitter-stack-graphs-javascript` invocation, as follows: + + ```sh + cargo run --features cli -- + ``` + +The basic CLI workflow for the command-line program is to index source code and issue queries against the resulting database: + +1. Index a source folder as follows: + + ```sh + tree-sitter-stack-graphs-javascript index SOURCE_DIR + ``` + + _Indexing will skip any files that have already be indexed. To force a re-index, add the `-f` flag._ + + To check the status if a source folder, run: + + ```sh + tree-sitter-stack-graphs-javascript status SOURCE_DIR + ``` + + To clean the database and start with a clean slate, run: + + ```sh + tree-sitter-stack-graphs-javascript clean + ``` + + _Pass the `--delete` flag to not just empty the database, but also delete it. This is useful to resolve `unsupported database version` errors that may occur after a version update._ + +2. Run a query to find the definition(s) for a reference on a given line and column, run: + + ```sh + tree-sitter-stack-graphs-javascript query definition SOURCE_PATH:LINE:COLUMN + ``` + + Resulting definitions are printed, including a source line if the source file is available. + +Discover all available commands and flags by passing the `-h` flag to the CLI directly, or to any of the subcommands. ## Development @@ -35,35 +81,29 @@ The project is written in Rust, and requires a recent version installed. Rust c The project is organized as follows: - The stack graph rules are defined in `src/stack-graphs.tsg`. -- Builtins sources and configuration are defined in `src/builtins.js` and `builtins.cfg` respectively. +- Builtins sources and configuration are defined in `src/builtins.it` and `builtins.cfg` respectively. - Tests are put into the `test` directory. -### Building and Running Tests - -Build the project by running: - -``` sh -$ cargo build -``` +### Running Tests Run the tests as follows: -``` sh -$ cargo test +```sh +cargo test ``` The project consists of a library and a CLI. By default, running `cargo` only applies to the library. To run `cargo` commands on the CLI as well, add `--features cli` or `--all-features`. Run the CLI from source as follows: -``` sh -$ cargo run --features cli -- ARGS +```sh +cargo run --features cli -- ARGS ``` Sources are formatted using the standard Rust formatted, which is applied by running: -``` sh -$ cargo fmt +```sh +cargo fmt ``` ### Writing TSG @@ -78,21 +118,21 @@ which contain self-contained TSG rules for specific language features. A VSCode Parse and test a single file by executing the following commands: -``` sh -$ cargo run --features cli -- parse FILES... -$ cargo run --features cli -- test TESTFILES... +```sh +cargo run --features cli -- parse FILES... +cargo run --features cli -- test TESTFILES... ``` Generate a visualization to debug failing tests by passing the `-V` flag: -``` sh -$ cargo run --features cli -- test -V TESTFILES... +```sh +cargo run --features cli -- test -V TESTFILES... ``` To generate the visualization regardless of test outcome, execute: -``` sh -$ cargo run --features cli -- test -V --output-mode=always TESTFILES... +```sh +cargo run --features cli -- test -V --output-mode=always TESTFILES... ``` -Go to https://crates.io/crates/tree-sitter-stack-graphs for links to examples and documentation. +Go to for links to examples and documentation. diff --git a/languages/tree-sitter-stack-graphs-javascript/docs/package-structure.md b/languages/tree-sitter-stack-graphs-javascript/docs/package-structure.md new file mode 100644 index 000000000..30b96945a --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/docs/package-structure.md @@ -0,0 +1,11 @@ +# How package structure is modeled + +_This is WIP and only contains links to the example, no explanation yet._ + +An [example](package-structure/example.js) test shows a basic setup with a package `foo` exporting a function `foo`, and a package `bar` depending on `foo` and importing the function. + +The stack graph for this example looks as follows: + +![stack graph for example](package-structure/example.png) + +_Generate this yourself by running `cargo run --features cli -- test --save-graph='%d/%n.html' --output-mode=always docs/package-structure/example.js`._ diff --git a/languages/tree-sitter-stack-graphs-javascript/docs/package-structure/example.js b/languages/tree-sitter-stack-graphs-javascript/docs/package-structure/example.js new file mode 100644 index 000000000..ba8165fa7 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/docs/package-structure/example.js @@ -0,0 +1,31 @@ +/* --- path: foo/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=54EA007B --- */ + +{ + "name": "foo" +} + +/* --- path: foo/index.js --- */ +/* --- global: FILE_PATH=index.js --- */ +/* --- global: PROJECT_NAME=54EA007B --- */ + +export function foo() {} + +/* --- path: bar/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=202D9AA4 --- */ + +{ + "name": "bar", + "dependencies": { + "foo": "" + } +} + +/* --- path: bar/index.js --- */ +/* --- global: FILE_PATH=index.js --- */ +/* --- global: PROJECT_NAME=202D9AA4 --- */ + +import { foo } from "foo" +// ^ defined: 13 diff --git a/languages/tree-sitter-stack-graphs-javascript/docs/package-structure/example.png b/languages/tree-sitter-stack-graphs-javascript/docs/package-structure/example.png new file mode 100644 index 000000000..83889ee42 Binary files /dev/null and b/languages/tree-sitter-stack-graphs-javascript/docs/package-structure/example.png differ diff --git a/languages/tree-sitter-stack-graphs-javascript/rust/lib.rs b/languages/tree-sitter-stack-graphs-javascript/rust/lib.rs index 508ba58b9..cbf80e5cd 100644 --- a/languages/tree-sitter-stack-graphs-javascript/rust/lib.rs +++ b/languages/tree-sitter-stack-graphs-javascript/rust/lib.rs @@ -5,7 +5,6 @@ // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ -use tree_sitter_stack_graphs::loader::FileAnalyzers; use tree_sitter_stack_graphs::loader::LanguageConfiguration; use tree_sitter_stack_graphs::loader::LoadError; use tree_sitter_stack_graphs::CancellationFlag; @@ -27,8 +26,7 @@ pub const STACK_GRAPHS_BUILTINS_PATH: &str = "src/builtins.js"; /// The stack graphs builtins source for this language. pub const STACK_GRAPHS_BUILTINS_SOURCE: &str = include_str!("../src/builtins.js"); -/// The name of the file path global variable. -pub const FILE_PATH_VAR: &str = "FILE_PATH"; +/// The name of the project name global variable pub const PROJECT_NAME_VAR: &str = "PROJECT_NAME"; pub fn language_configuration(cancellation_flag: &dyn CancellationFlag) -> LanguageConfiguration { @@ -38,8 +36,8 @@ pub fn language_configuration(cancellation_flag: &dyn CancellationFlag) -> Langu pub fn try_language_configuration( cancellation_flag: &dyn CancellationFlag, ) -> Result { - LanguageConfiguration::from_sources( - tree_sitter_javascript::language(), + let mut lc = LanguageConfiguration::from_sources( + tree_sitter_javascript::LANGUAGE.into(), Some(String::from("source.js")), None, vec![String::from("js")], @@ -50,7 +48,9 @@ pub fn try_language_configuration( STACK_GRAPHS_BUILTINS_SOURCE, )), Some(STACK_GRAPHS_BUILTINS_CONFIG), - FileAnalyzers::new().add("package.json".to_string(), NpmPackageAnalyzer {}), cancellation_flag, - ) + )?; + lc.special_files + .add("package.json".to_string(), NpmPackageAnalyzer {}); + Ok(lc) } diff --git a/languages/tree-sitter-stack-graphs-javascript/rust/npm_package.rs b/languages/tree-sitter-stack-graphs-javascript/rust/npm_package.rs index 5a46d6ee1..13c15c7ea 100644 --- a/languages/tree-sitter-stack-graphs-javascript/rust/npm_package.rs +++ b/languages/tree-sitter-stack-graphs-javascript/rust/npm_package.rs @@ -61,7 +61,6 @@ impl FileAnalyzer for NpmPackageAnalyzer { pkg_internal_name, "pkg_internal_name_pop", ); - // reach package internals via root // // [push pkg_internal_name] -> [push "GUARD:PKG_INTERNAL"] -> [root] @@ -83,13 +82,21 @@ impl FileAnalyzer for NpmPackageAnalyzer { // reach exports via package name // - // [root] -> [pop "GUARD:PKG"] -> [pop PKG_NAME]* -> [push PKG_INTERNAL_NAME] -> [push "GUARD:PKG_INTERNAL"] -> [root] + // [root] -> [pop "GUARD:PKG"] -> [pop PKG_NAME]* -> [push "GUARD:MODULE"] -> [push PKG_INTERNAL_NAME] -> [push "GUARD:PKG_INTERNAL"] -> [root] // if !npm_pkg.name.is_empty() { // NOTE Because all modules expose their exports at the top-level, both paths created below are equivalent for // exports of the main module. This means multiple equivalent paths to those exports, which is bad for // performance. At the moment, we have no mechanism to prevent this from happening. + let module_guard = add_push( + graph, + file, + pkg_internal_name_push, + MODULE_GUARD, + "main_guard", + ); + // reach package internals via package name // // [root] -> [pop "GUARD:PKG"] -> [pop pkg_name]* -> [push pkg_internal_name] @@ -102,31 +109,23 @@ impl FileAnalyzer for NpmPackageAnalyzer { pkg_guard_pop, "pkg_name_pop", ); - add_edge(graph, pkg_name_pop, pkg_internal_name_push, 0); + add_edge(graph, pkg_name_pop, module_guard, 0); - // reach main exports directly via package name (with precedence) - // - // [pop pkg_name] -1-> [pop "GUARD:EXPORTS"] -> [push "GUARD:EXPORTS"] -> [push main]* -> [push pkg_internal_name] - // - let exports_guard_pop = add_pop( - graph, - file, - pkg_name_pop, - EXPORTS_GUARD, - "exports_guard_pop", - ); - replace_edge(graph, pkg_name_pop, exports_guard_pop, 1); + // Common main let main = Some(npm_pkg.main) .filter(|main| !main.is_empty()) .and_then(|main| NormalizedRelativePath::from_str(&main)) .map(|p| p.into_path_buf()) .unwrap_or(PathBuf::from("index")) .with_extension(""); - let main_push = - add_module_pushes(graph, file, &main, pkg_internal_name_push, "main_push"); - let exports_guard_push = - add_push(graph, file, main_push, EXPORTS_GUARD, "exports_guard_push"); - add_edge(graph, exports_guard_pop, exports_guard_push, 0); + + let main_push = add_module_pushes(graph, file, &main, module_guard, "main_push"); + + // reach main directly via package name (with precedence) + // + // [pop pkg_name] -1-> [push main]* -> [push pkg_internal_name] + // + add_edge(graph, pkg_name_pop, main_push, 0); } // reach dependencies via package internal name diff --git a/languages/tree-sitter-stack-graphs-javascript/rust/util.rs b/languages/tree-sitter-stack-graphs-javascript/rust/util.rs index 1bed520cd..f4942df3b 100644 --- a/languages/tree-sitter-stack-graphs-javascript/rust/util.rs +++ b/languages/tree-sitter-stack-graphs-javascript/rust/util.rs @@ -14,11 +14,14 @@ use stack_graphs::arena::Handle; use stack_graphs::graph::File; use stack_graphs::graph::StackGraph; -pub const EXPORTS_GUARD: &str = "GUARD:EXPORTS"; +pub const MODULE_GUARD: &str = "GUARD:MODULE"; pub const PKG_GUARD: &str = "GUARD:PKG"; pub const PKG_INTERNAL_GUARD: &str = "GUARD:PKG_INTERNAL"; pub fn add_debug_name(graph: &mut StackGraph, node: Handle, name: &str) { + if name.is_empty() { + return; + } let key = graph.add_string("name"); let value = graph.add_string(name); graph.node_debug_info_mut(node).add(key, value); @@ -61,14 +64,6 @@ pub fn add_edge(graph: &mut StackGraph, from: Handle, to: Handle, pr graph.add_edge(from, to, precedence); } -pub fn replace_edge(graph: &mut StackGraph, from: Handle, to: Handle, precedence: i32) { - if from == to { - return; - } - graph.remove_edge(from, to); - graph.add_edge(from, to, precedence); -} - pub fn add_module_pops( graph: &mut StackGraph, file: Handle, diff --git a/languages/tree-sitter-stack-graphs-javascript/src/stack-graphs.tsg b/languages/tree-sitter-stack-graphs-javascript/src/stack-graphs.tsg index 254445369..7ca4eee88 100644 --- a/languages/tree-sitter-stack-graphs-javascript/src/stack-graphs.tsg +++ b/languages/tree-sitter-stack-graphs-javascript/src/stack-graphs.tsg @@ -230,7 +230,7 @@ attribute node_symbol = node => symbol = (source-text node), source_n ;; a connected subgraph that is never the less never traversed. ;; - `GUARD:LABEL` - used for the names of labels ;; - `GUARD:MEMBER` - used for the members/fields/properties of objects -;; - `GUARD:MODULE` - used for the final scope of the module +;; - `GUARD:MODULE` - used for module names ;; - `GUARD:RETURN` - used for the AST nodes for values returned by a function ;; in the function body ;; - `GUARD:THIS` - used for the implicit `this` argument of a function inside @@ -252,10 +252,12 @@ inherit .constructor inherit .export_statement inherit .exports inherit .hoist_point +inherit .closure_point inherit .import_statement inherit .pkg_pop inherit .pkg_push inherit .return_or_yield +inherit .containing_class_value @@ -280,6 +282,10 @@ inherit .return_or_yield (program)@prog { node @prog.after_scope node @prog.before_scope + let @prog.closure_point = @prog.after_scope + + ; apparently it's perfectly cromulent to `return` from the top level scope (because control flow happens there too), so we make a top-level return node. + node @prog.return_or_yield } ;; ### Program Queries @@ -287,7 +293,6 @@ inherit .return_or_yield (program)@prog { node prog_module_pop - node prog_module_scope node prog_exports_pop node prog_pkg_pop_guard node prog_pkg_push_guard @@ -307,7 +312,12 @@ inherit .return_or_yield edge @prog.pkg_push -> prog_pkg_push_guard edge prog_pkg_push_guard -> ROOT_NODE + node module_guard_pop + attr (module_guard_pop) pop_symbol = "GUARD:MODULE" + edge @prog.pkg_pop -> module_guard_pop + node prog_module_pop_start + edge module_guard_pop -> prog_module_pop_start var module_pop_end = prog_module_pop_start let module_name = (replace FILE_PATH "\.js$" "") scan module_name { @@ -320,23 +330,19 @@ inherit .return_or_yield "([^/]+)$" { attr (module_pop_end) symbol_definition = $1, source_node = @prog attr (module_pop_end) empty_source_span + attr (module_pop_end) definiens_node = @prog } } edge @prog.before_scope -> @prog.pkg_push edge @prog.before_scope -> @prog.hoist_point - attr (prog_module_scope) pop_symbol = "GUARD:MODULE" - edge @prog.pkg_pop -> prog_module_pop_start - edge module_pop_end -> prog_module_scope - edge prog_module_scope -> @prog.after_scope attr (prog_legacy_qname_guard) push_symbol = "GUARD:LEGACY_QNAME" edge @prog.before_scope -> prog_legacy_qname_guard edge prog_legacy_qname_guard -> ROOT_NODE attr (prog_module_pop) empty_source_span - attr (prog_module_scope) empty_source_span attr (@prog.exports) empty_source_span attr (prog_exports_pop) empty_source_span attr (@prog.before_scope) empty_source_span @@ -346,10 +352,6 @@ inherit .return_or_yield edge module_pop_end -> prog_exports_pop edge prog_exports_pop -> @prog.exports - ; allow direct access from the package to the modules exports - ; this is used from package.json - edge @prog.pkg_pop -> prog_exports_pop - ;; builtin types node @prog.builtins_number node @prog.builtins_string @@ -359,6 +361,10 @@ inherit .return_or_yield node @prog.builtins_Regex_prototype node @prog.builtins_arguments_prototype node @prog.builtins_empty_object + ; !!!! HACK + ; stack graphs currently make it impossible to test if an inherited variable + ; like this is defined or not + let @prog.containing_class_value = @prog.builtins_null } @@ -400,7 +406,19 @@ inherit .return_or_yield node @comment.after_scope node @comment.before_scope node @comment.value + node @comment.covalue + node @comment.new_bindings ; for object patterns edge @comment.after_scope -> @comment.before_scope + node @comment.source ; for export clauses with multiple exports +} + +(identifier) @identifier { + node @identifier.before_scope + node @identifier.after_scope + node @identifier.value + node @identifier.covalue + node @identifier.new_bindings + edge @identifier.after_scope -> @identifier.before_scope } @@ -505,7 +523,7 @@ inherit .return_or_yield (declaration)@decl)@export_stmt { node pop_guard_default - attr (pop_guard_default) symbol_definition = "GUARD:DEFAULT", source_node = @decl, definiens_node = @export_stmt.export_statement + attr (pop_guard_default) symbol_definition = "GUARD:DEFAULT", source_node = @decl, definiens_node = @export_stmt edge @export_stmt.exports -> pop_guard_default ;; edge export_stmt_pop_guard_default -> @decl.value ;; FIXME declarations have no .value @@ -542,13 +560,61 @@ inherit .return_or_yield node default_detour_pop attr (default_detour_push) push_symbol = "default" - attr (default_detour_pop) symbol_definition = "default", source_node = @decl, definiens_node = @decl + attr (default_detour_pop) symbol_definition = "default", source_node = @decl, definiens_node = @export_stmt edge pop_guard_default -> default_detour_push edge default_detour_push -> default_detour_pop ; edge default_detour_pop -> @decl.value ;; FIXME declarations have no .value } +(export_statement "default" + value:(_)@value)@export_stmt { + + node @export_stmt.pop_guard_default + attr (@export_stmt.pop_guard_default) symbol_definition = "GUARD:DEFAULT", source_node = @value, definiens_node = @export_stmt + edge @export_stmt.exports -> @export_stmt.pop_guard_default + edge @export_stmt.pop_guard_default -> @value.value + + ; !!!! HACK These detour nodes are a massive hack to allow find all refs land on defs + ; for the default values of modules that have useful names like the module name or + ; package name + + node detour_push + node @export_stmt.detour_pop + + scan FILE_PATH { + + "^(.+/)?([^/]+)/index\.js$" { + let module_name = $2 + attr (detour_push) push_symbol = module_name + attr (@export_stmt.detour_pop) symbol_definition = module_name, source_node = @value, definiens_node = @export_stmt + edge @export_stmt.pop_guard_default -> detour_push + edge detour_push -> @export_stmt.detour_pop + edge @export_stmt.detour_pop -> @value.value + } + + "^(.+/)?([^/]+)\.js$" { + let module_name = $2 + attr (detour_push) push_symbol = module_name + attr (@export_stmt.detour_pop) symbol_definition = module_name, source_node = @value, definiens_node = @export_stmt + edge @export_stmt.pop_guard_default -> detour_push + edge detour_push -> @export_stmt.detour_pop + edge @export_stmt.detour_pop -> @value.value + } + + } + + node default_detour_push + node @export_stmt.default_detour_pop + + attr (default_detour_push) push_symbol = "default" + attr (@export_stmt.default_detour_pop) symbol_definition = "default", source_node = @value, definiens_node = @export_stmt + edge @export_stmt.pop_guard_default -> default_detour_push + edge default_detour_push -> @export_stmt.default_detour_pop + edge @export_stmt.default_detour_pop -> @value.value + +} + (export_statement declaration: [ (function_declaration name:(identifier)@name) @@ -734,13 +800,8 @@ inherit .return_or_yield (export_statement value:(_)@default_expr)@export_stmt { - node pop_default_guard - - attr (pop_default_guard) symbol_definition = "GUARD:DEFAULT", source_node = @export_stmt, definiens_node = @export_stmt.export_statement edge @default_expr.before_scope -> @export_stmt.before_scope edge @export_stmt.after_scope -> @default_expr.after_scope - edge @export_stmt.exports -> pop_default_guard - edge pop_default_guard -> @default_expr.value } @@ -834,7 +895,12 @@ inherit .return_or_yield ; relative import let name = (replace (path-normalize (path-join (path-dir FILE_PATH) $1)) "\.js$" "") + node module_guard_push + attr (module_guard_push) push_symbol = "GUARD:MODULE" + edge module_guard_push -> @source.pkg_push + node source_push_end + edge source_push_end -> module_guard_push var push_start = source_push_end scan name { "([^/]+)/" { @@ -858,7 +924,6 @@ inherit .return_or_yield attr (push_start) is_reference, source_node = @source edge source_push_guard_exports -> push_start - edge source_push_end -> @source.pkg_push } "^[\"']([^\./].*)[\"']$" { ; package import @@ -1267,6 +1332,7 @@ inherit .return_or_yield node fun_decl_value_return node fun_decl_value_this node fun_decl_value_this_guard + let @body.closure_point = @body.after_scope attr (@name.pop) syntax_type = "function" @@ -1281,7 +1347,7 @@ inherit .return_or_yield ; function values have drop nodes that handle closures, that points to the ; before scope for the function attr (fun_decl_value_drop) type = "drop_scopes" - edge fun_decl_value_drop -> @fun_decl.before_scope + edge fun_decl_value_drop -> @fun_decl.closure_point ; the call sig's before scope comes from the drop node then flows into the body edge @call_sig.before_scope -> fun_decl_value_drop @@ -1351,6 +1417,7 @@ inherit .return_or_yield node fun_decl_value_return node fun_decl_value_this node fun_decl_value_this_guard + let @body.closure_point = @body.after_scope attr (@name.pop) syntax_type = "function" @@ -1365,7 +1432,7 @@ inherit .return_or_yield ; function values have drop nodes that handle closures, that points to the ; before scope of the declaration attr (fun_decl_value_drop) type = "drop_scopes" - edge fun_decl_value_drop -> @fun_decl.before_scope + edge fun_decl_value_drop -> @fun_decl.closure_point ; the call sig's before scope comes from the drop node then flows into the body @@ -1424,6 +1491,7 @@ inherit .return_or_yield node @name.pop node @class_decl.class_value + let @class_decl.containing_class_value = @class_decl.class_value node guard_prototype node @class_decl.prototype node @class_decl.constructor @@ -1437,7 +1505,7 @@ inherit .return_or_yield edge @class_decl.class_value -> guard_prototype edge guard_prototype -> @class_decl.prototype - edge @body.before_scope -> @class_decl.before_scope + edge @body.before_scope -> @class_decl.closure_point edge @class_decl.prototype -> @body.after_scope edge @class_decl.after_scope -> @class_decl.before_scope } @@ -1496,7 +1564,7 @@ inherit .return_or_yield node @method_def.after_scope node @method_def.before_scope node @method_def.method_value - + } ( @@ -1504,7 +1572,7 @@ inherit .return_or_yield name:(_)@name)@method_def (#eq? @name "constructor") ) { - + ; augmentation for the constructor attr (@name.pop) symbol_definition = "GUARD:CONSTRUCTOR", source_node = @name edge @method_def.class_value -> @name.pop @@ -1594,14 +1662,17 @@ inherit .return_or_yield -(field_definition - property:(property_identifier)@property)@field_def { - +(field_definition)@field_def { node @field_def.after_scope node @field_def.before_scope +} + +(field_definition + property:(_)@property)@field_def { + node @property.pop node property_pop_dot - + attr (@property.pop) node_definition = @property attr (property_pop_dot) pop_symbol = "GUARD:MEMBER" edge @field_def.after_scope -> property_pop_dot @@ -1623,6 +1694,15 @@ inherit .return_or_yield edge @property.pop -> @value.value } +(class_static_block body: (_) @body) @static_block { + node @static_block.before_scope + node @static_block.after_scope + + edge @body.before_scope -> @static_block.before_scope + + edge @static_block.after_scope -> @static_block.before_scope +} + ;; #### Statement Block @@ -1703,16 +1783,19 @@ inherit .return_or_yield edge @if_stmt.after_scope -> @alternative.after_scope } +(else_clause)@else_clause { + node @else_clause.after_scope + node @else_clause.before_scope +} + (else_clause - (_)@inner)@else_clause { + . (_)@inner)@else_clause { let @else_clause.hoist_point = @inner.before_scope } (else_clause (_)@inner)@else_clause { - node @else_clause.after_scope - node @else_clause.before_scope ; scopes flow in and right back out edge @inner.before_scope -> @else_clause.before_scope edge @else_clause.after_scope -> @inner.after_scope @@ -2208,12 +2291,14 @@ inherit .return_or_yield (member_expression) (parenthesized_expression) (undefined) - (primary_expression/identifier) (this) (super) (number) (string) (template_string) + (template_substitution) + (string_fragment) + (escape_sequence) (regex) (true) (false) @@ -2222,7 +2307,7 @@ inherit .return_or_yield (import) (object) (array) - (function) + (function_expression) (arrow_function) (generator_function) (class) @@ -2239,13 +2324,23 @@ inherit .return_or_yield (yield_expression) (spread_element) ]@expr { - + node @expr.after_scope node @expr.before_scope node @expr.value } +[ + (function_expression body:(_)@body) + (arrow_function body:(_)@body) + (generator_function body:(_)@body) +] { + + let @body.closure_point = @body.after_scope + +} + ;; ### Expression Queries @@ -2275,8 +2370,8 @@ inherit .return_or_yield ;; #### Template Strings ; template_strings w/ no substitutions -(template_string (_)* @substs)@template_string { - if (is-empty @substs) { +(template_string (_)* @parts)@template_string { + if (is-empty @parts) { edge @template_string.after_scope -> @template_string.before_scope } } @@ -2284,33 +2379,45 @@ inherit .return_or_yield ; nonempty template string, value ; LATER-TODO this isn't really right, but it gets flows through the template string ; which may be useful -(template_string (template_substitution (_)@inner_expr))@template_string { +(template_string (_)@part)@template_string { ; the value of a template string is a template string value built from the values of its substitutions ; attr (@template_string.value) "template_string_value" - edge @template_string.value -> @inner_expr.value + edge @template_string.value -> @part.value } ; nonempty template string, first substitution -(template_string . (template_substitution (_)@first_inner_expr))@template_string { +(template_string . (_)@first)@template_string { ; scopes propagate into the first subtitution of the template string - edge @first_inner_expr.before_scope -> @template_string.before_scope + edge @first.before_scope -> @template_string.before_scope } ; nonempty template string, between substitutions (template_string - (template_substitution (_)@left_inner_expr) + (_) @left . - (template_substitution (_)@right_inner_expr))@_template_string { + (_) @right) { ; scopes propagate from left substitutions to right substitutions - edge @right_inner_expr.before_scope -> @left_inner_expr.after_scope + edge @right.before_scope -> @left.after_scope } ; nonempty template string, last substitution -(template_string . (template_substitution (_)@last_inner_expr))@template_string { +(template_string (_) @last .)@template_string { ; scopes propagate out of the last substitution to the template string - edge @template_string.after_scope -> @last_inner_expr.after_scope + edge @template_string.after_scope -> @last.after_scope } +[ + (string_fragment) + (escape_sequence) +]@part { + edge @part.after_scope -> @part.before_scope +} + +(template_substitution (_)@expr)@subst { + edge @expr.before_scope -> @subst.before_scope + edge @subst.after_scope -> @expr.after_scope + edge @subst.value -> @expr.value +} ;; #### Numbers @@ -2326,10 +2433,10 @@ inherit .return_or_yield ;; #### Variables -(primary_expression/identifier)@variable { - ; scopes don't change - edge @variable.after_scope -> @variable.before_scope - +[ + (primary_expression/identifier)@variable + (member_expression object:(identifier)@variable) +] { ; value is a lookup, ie a push attr (@variable.value) node_reference = @variable edge @variable.value -> @variable.before_scope @@ -2364,6 +2471,14 @@ inherit .return_or_yield ; this is a lookup, ie a push attr (@this.value) symbol_reference = "this", source_node = @this edge @this.value -> @this.before_scope + + node pop_this + attr (pop_this) symbol_definition = "this" + node guard_prototype + attr (guard_prototype) push_symbol = "GUARD:PROTOTYPE" + edge @this.value -> pop_this + edge pop_this -> guard_prototype + edge guard_prototype -> @this.containing_class_value } @@ -2394,6 +2509,9 @@ inherit .return_or_yield ; the value of undefined is the undefined primitive edge @undefined.value -> @undefined.builtins_undefined + + ; !!!! HACK: `undefined` is a perfectly cromulent name for a parameter, but the parser thinks it means this node instead. For the moment, work around the problem this causes by giving all `undefined` nodes covalues like parameters enjoy. + node @undefined.covalue } @@ -2425,6 +2543,9 @@ inherit .return_or_yield attr (@object.member_pop) pop_symbol = "GUARD:MEMBER" edge @object.value -> @object.member_pop + node @object.class_value + node @object.constructor + } ; empty objects @@ -2462,7 +2583,7 @@ inherit .return_or_yield ; shorthand property identifier (shorthand_property_identifier)@shorthand_property_identifier { - + node @shorthand_property_identifier.after_scope node @shorthand_property_identifier.before_scope @@ -2492,10 +2613,13 @@ inherit .return_or_yield ; pairs -(computed_property_name (_)@expr)@computed_property_name { - +(computed_property_name)@computed_property_name { node @computed_property_name.after_scope node @computed_property_name.before_scope +} + +(computed_property_name (_)@expr)@computed_property_name { + edge @expr.before_scope -> @computed_property_name.before_scope edge @computed_property_name.after_scope -> @expr.after_scope @@ -2510,7 +2634,7 @@ inherit .return_or_yield ; This is done differently depending on what the key is. See next rules. ; attr @key.pop "pop" = @key, "definition" - + node @pair.key_pop edge @pair.key_pop -> @value.value edge @object.member_pop -> @pair.key_pop @@ -2676,7 +2800,7 @@ inherit .return_or_yield ;; #### Function Literals ; functions with names -(function +(function_expression name:(_)@name parameters:(_)@call_sig)@fun { @@ -2691,7 +2815,7 @@ inherit .return_or_yield ; function -(function +(function_expression parameters:(_)@call_sig body:(_)@body)@fun { @@ -2713,7 +2837,7 @@ inherit .return_or_yield ; function values have drop nodes that handle closures, that points to the ; before scope from the function attr (fun_value_drop) type = "drop_scopes" - edge fun_value_drop -> @fun.before_scope + edge fun_value_drop -> @fun.closure_point ; the call sig's before scope comes from the drop node, ; then flows into the body, and includes a variable binding for "this" @@ -2750,7 +2874,7 @@ inherit .return_or_yield edge @fun.value_arg_scope -> JUMP_TO_SCOPE_NODE } -(function +(function_expression parameters: (formal_parameters (_)@param))@fun { @@ -2791,7 +2915,7 @@ inherit .return_or_yield ; function values have drop nodes that handle closures, that points to the ; before scope from the function attr (fun_value_drop) type = "drop_scopes" - edge fun_value_drop -> @fun.before_scope + edge fun_value_drop -> @fun.closure_point ; the call sig's before scope comes from the drop node then flows into the body edge @call_sig.before_scope -> fun_value_drop @@ -2804,7 +2928,7 @@ inherit .return_or_yield ; function values have return nodes which need to be visible for returns attr (fun_value_return) pop_symbol = "GUARD:RETURN" edge fun_value_call -> fun_value_return - let @body.return_or_yield = fun_value_return + edge fun_value_return -> @fun.return_or_yield ; function values have this nodes which need to be visible for method calls attr (fun_value_this) push_symbol = "this" @@ -2825,16 +2949,9 @@ inherit .return_or_yield (arrow_function parameter:(_)@param)@fun { - - node @param.after_scope node param_arg_index - node @param.before_scope - node @param.covalue node param_pop - ; scope flows from the param right back out - edge @param.after_scope -> @param.before_scope - ; but augmented with a pop, b/c it's not a pattern attr (param_pop) node_definition = @param edge param_pop -> @param.covalue @@ -2900,7 +3017,7 @@ inherit .return_or_yield ; function values have drop nodes that handle closures, that points to the ; before scope from the function attr (fun_value_drop) type = "drop_scopes" - edge fun_value_drop -> @fun.before_scope + edge fun_value_drop -> @fun.closure_point ; the call sig's before scope comes from the drop node, ; then flows into the body, and includes a variable binding for "this" @@ -3072,8 +3189,7 @@ inherit .return_or_yield ;; ##### Member Expressions (member_expression - object: (_)@object - property: (_)@property)@member_expr + object:(_)@object property:(_)@property)@member_expr { node member_push @@ -3090,6 +3206,9 @@ inherit .return_or_yield edge @member_expr.value -> property_push edge member_push -> @object.value + ; (member_expression) nodes can occur in patterns + node @member_expr.covalue + node @member_expr.new_bindings } ;; ##### Subscript Expressions @@ -3114,6 +3233,9 @@ inherit .return_or_yield edge @subscript_expr.value -> @subscript_expr.index_push edge @subscript_expr.index_push -> subscript_expr_push_dot + ; subscript expressions can appear in array patterns, on the left, and thus require a covalue & bindings + node @subscript_expr.covalue + node @subscript_expr.new_bindings } (subscript_expression @@ -3170,9 +3292,6 @@ inherit .return_or_yield edge @new_expr.value -> new_expr_guard_this edge new_expr_guard_this -> new_expr_call - ; value coming from the field decls in the class - edge @new_expr.value -> @constructor.value - ; value also coming from the prototype node guard_prototype attr (guard_prototype) push_symbol = "GUARD:PROTOTYPE" @@ -3381,20 +3500,27 @@ inherit .return_or_yield ;; #### Comma Operator / Sequence Expressions -(sequence_expression - left: (_)@left - right: (_)@right)@sequence_expr { +(sequence_expression (_)* @elems)@sequence_expr { + if (is-empty @elems) { + edge @sequence_expr.after_scope -> @sequence_expr.before_scope + } +} - ; scopes propagate left to right - edge @left.before_scope -> @sequence_expr.before_scope +(sequence_expression . (_)@first)@sequence_expr { + edge @first.before_scope -> @sequence_expr.before_scope +} + +(sequence_expression (_)@left . (_)@right) { edge @right.before_scope -> @left.after_scope - edge @sequence_expr.after_scope -> @right.after_scope +} - ; the value is just the value of the right - edge @sequence_expr.value -> @right.value +(sequence_expression (_)@last .)@sequence_expr { + edge @sequence_expr.after_scope -> @last.after_scope + edge @sequence_expr.value -> @last.value } + ;; #### Ternary Expression (ternary_expression @@ -3435,12 +3561,13 @@ inherit .return_or_yield body:(_)@body)@class { node @class.class_value + let @class.containing_class_value = @class.class_value node guard_prototype node @class.prototype node @class.constructor attr (guard_prototype) pop_symbol = "GUARD:PROTOTYPE" - edge @body.before_scope -> @class.before_scope + edge @body.before_scope -> @class.closure_point edge @class.value -> @class.class_value edge @class.class_value -> guard_prototype edge guard_prototype -> @class.prototype @@ -3452,7 +3579,7 @@ inherit .return_or_yield (class name:(_)@name body:(_)@body)@class { - + node @name.pop attr (@name.pop) syntax_type = "class" @@ -3476,11 +3603,283 @@ inherit .return_or_yield edge @class.after_scope -> @name.after_scope } +(jsx_element + open_tag:(_)@open_tag + close_tag:(_)@close_tag + (_)*@children)@jsx_element { + + node @jsx_element.before_scope + node @jsx_element.after_scope + node @jsx_element.value + + edge @open_tag.before_scope -> @jsx_element.before_scope + edge @jsx_element.after_scope -> @close_tag.after_scope + + if (is-empty @children) { + edge @close_tag.before_scope -> @open_tag.after_scope + } + +} + +(jsx_element + open_tag:(_)@open_tag + . + [ + (jsx_text) + (jsx_element) + (jsx_self_closing_element) + (jsx_expression) + ]@first_child +) { + edge @first_child.before_scope -> @open_tag.after_scope +} + +(jsx_element + [ + (jsx_text) + (jsx_element) + (jsx_self_closing_element) + (jsx_expression) + ]@left_child + . + [ + (jsx_text) + (jsx_element) + (jsx_self_closing_element) + (jsx_expression) + ]@right_child +) { + edge @right_child.before_scope -> @left_child.after_scope +} + +(jsx_element + [ + (jsx_text) + (jsx_element) + (jsx_self_closing_element) + (jsx_expression) + ]@last_child + . + close_tag:(_)@close_tag +) { + edge @close_tag.before_scope -> @last_child.after_scope +} + +(jsx_text)@jsx_text { + node @jsx_text.before_scope + node @jsx_text.after_scope + + edge @jsx_text.after_scope -> @jsx_text.before_scope +} + +(jsx_opening_element)@jsx_opening_element { + + node @jsx_opening_element.before_scope + node @jsx_opening_element.after_scope + +} + +(jsx_opening_element + name:(_)@element_name)@jsx_opening_element { + + edge @element_name.before_scope -> @jsx_opening_element.before_scope + +} + +(jsx_opening_element + !name)@jsx_opening_element +{ + + edge @jsx_opening_element.after_scope -> @jsx_opening_element.before_scope + +} + +(jsx_opening_element + name:(_)@element_name + !attribute)@jsx_opening_element +{ + + edge @jsx_opening_element.after_scope -> @element_name.after_scope + +} + +(jsx_opening_element + name:(_)@element_name + . + attribute:(_)@first_attr +) { + + edge @first_attr.before_scope -> @element_name.after_scope + +} + +(jsx_opening_element + attribute:(_)@left_attr + . + attribute:(_)@right_attr +) { + + edge @right_attr.before_scope -> @left_attr.after_scope + +} + +(jsx_opening_element + attribute:(_)@last_attr + .)@jsx_opening_element +{ + edge @jsx_opening_element.after_scope -> @last_attr.after_scope +} +(jsx_attribute (_) . (_)?@attr_value)@jsx_attribute { + node @jsx_attribute.before_scope + node @jsx_attribute.after_scope + if none @attr_value { + edge @jsx_attribute.after_scope -> @jsx_attribute.before_scope + } else { + edge @attr_value.before_scope -> @jsx_attribute.before_scope + edge @jsx_attribute.after_scope -> @attr_value.after_scope + } + +} + +(jsx_namespace_name (_) @lhs (_) @rhs)@name { + node @name.before_scope + node @name.after_scope + + edge @lhs.before_scope -> @name.before_scope + edge @rhs.before_scope -> @lhs.after_scope + edge @name.after_scope -> @name.before_scope +} + +(jsx_self_closing_element + name:(_)@element_name)@jsx_self_closing_element { + + node @jsx_self_closing_element.before_scope + node @jsx_self_closing_element.after_scope + node @jsx_self_closing_element.value + + edge @element_name.before_scope -> @jsx_self_closing_element.before_scope + +} + +(jsx_self_closing_element + !name + !attribute)@jsx_self_closing_element +{ + + edge @jsx_self_closing_element.after_scope -> @jsx_self_closing_element.before_scope + +} + +(jsx_self_closing_element + name:(_)@element_name + !attribute)@jsx_self_closing_element +{ + + edge @jsx_self_closing_element.after_scope -> @element_name.after_scope + +} + +(jsx_self_closing_element + name:(_)@element_name + . + attribute:(_)@first_attr +) { + + edge @first_attr.before_scope -> @element_name.after_scope + +} + +(jsx_self_closing_element + attribute:(_)@left_attr + . + attribute:(_)@right_attr +) { + + edge @right_attr.before_scope -> @left_attr.after_scope + +} + +(jsx_self_closing_element + attribute:(_)@last_attr + .)@jsx_self_closing_element +{ + + edge @jsx_self_closing_element.after_scope -> @last_attr.after_scope + +} + +(jsx_expression)@jsx_expression { + + node @jsx_expression.before_scope + node @jsx_expression.after_scope + +} + +(jsx_expression (_)?@child)@jsx_expression { + + if none @child { + edge @jsx_expression.after_scope -> @jsx_expression.before_scope + } else { + edge @child.before_scope -> @jsx_expression.before_scope + edge @jsx_expression.after_scope -> @child.after_scope + } + +} + +(jsx_closing_element)@jsx_closing_element +{ + + node @jsx_closing_element.before_scope + node @jsx_closing_element.after_scope + +} + +(jsx_closing_element + !name)@jsx_closing_element +{ + + edge @jsx_closing_element.after_scope -> @jsx_closing_element.before_scope + +} + +(jsx_closing_element + name:(_)@element_name)@jsx_closing_element +{ + + edge @element_name.before_scope -> @jsx_closing_element.before_scope + edge @jsx_closing_element.after_scope -> @element_name.after_scope + +} + +[ + (jsx_opening_element + name:(identifier)@element_name) + (jsx_self_closing_element + name:(identifier)@element_name) + (jsx_closing_element + name:(identifier)@element_name) +] +{ + scan (source-text @element_name) { + ; standard HTML elements + "^(a|abbr|acronym|address|applet|area|article|aside|audio|b|base|basefont|bdi|bdo|big|blockquote|body|br|button|canvas|caption|center|cite|code|col|colgroup|data|datalist|dd|del|details|dfn|dialog|dir|div|dl|dt|em|embed|fieldset|figcaption|figure|font|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|header|hgroup|hr|html|i|iframe|input|ins|kbd|label|legend|li|link|main|map|mark|menu|meta|meter|nav|noframes|noscript|object|ol|optgroup|option|output|p|param|picture|pre|progress|q|rp|rt|ruby|s|samp|script|search|section|select|small|source|span|strike|strong|style|sub|summary|sup|svg|table|tbody|td|template|textarea|tfoot|th|thead|time|title|tr|track|tt|u|ul|var|video|wbr)$" { + ; do nothing! + } + + ; everything else + "^.+$" { + node element_name_pop + attr (element_name_pop) node_reference = @element_name + edge element_name_pop -> @element_name.before_scope + } + } + +} @@ -3724,13 +4123,11 @@ inherit .return_or_yield ;; ### Attributes Defined on Patterns ;; TODO [ - (assignment_expression left:(identifier)@pattern) (assignment_pattern)@pattern (object_pattern)@pattern (array_pattern)@pattern (rest_pattern)@pattern (pair_pattern)@pattern - (pattern/identifier)@pattern (pattern/property_identifier)@pattern (object_assignment_pattern)@pattern (shorthand_property_identifier_pattern)@pattern @@ -3755,7 +4152,6 @@ inherit .return_or_yield ; scope flows through, binding via a pop edge that goes to an unknown value attr (ident_pat_pop) node_definition = @ident_pat edge ident_pat_pop -> @ident_pat.covalue - edge @ident_pat.after_scope -> @ident_pat.before_scope edge @ident_pat.after_scope -> ident_pat_pop edge @ident_pat.new_bindings -> ident_pat_pop @@ -3764,9 +4160,6 @@ inherit .return_or_yield ;; #### Object Patterns -; LATER-TODO scope propagation through empty object patterns -; currently unsupported by tree sitter queries -; THIS IS A HUGE HACK AND MUST BE FIXED (object_pattern (_)* @entries)@object_pat { if (is-empty @entries) { edge @object_pat.after_scope -> @object_pat.before_scope @@ -4079,7 +4472,7 @@ inherit .return_or_yield node pop_default_guard node pop_dot - node pop_name + node @assignment_expr.pop_name attr (pop_default_guard) symbol_definition = "GUARD:DEFAULT", source_node = @exports edge @assignment_expr.exports -> pop_default_guard @@ -4087,47 +4480,47 @@ inherit .return_or_yield attr (pop_dot) pop_symbol = "GUARD:MEMBER" edge pop_default_guard -> pop_dot - attr (pop_name) node_definition = @property - attr (pop_name) definiens_node = @assignment_expr - edge pop_dot -> pop_name - edge pop_name -> @right.value + attr (@assignment_expr.pop_name) node_definition = @property + attr (@assignment_expr.pop_name) definiens_node = @assignment_expr + edge pop_dot -> @assignment_expr.pop_name + edge @assignment_expr.pop_name -> @right.value ;; For ES6 interoperability, expose members as named exports - edge @assignment_expr.exports -> pop_name + edge @assignment_expr.exports -> @assignment_expr.pop_name node detour_push - node detour_pop + node @assignment_expr.detour_pop scan FILE_PATH { "^(.+/)?([^/]+)/index\.js$" { let module_name = $2 attr (detour_push) push_symbol = module_name - attr (detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr + attr (@assignment_expr.detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr edge pop_default_guard -> detour_push - edge detour_push -> detour_pop - edge detour_pop -> @right.value + edge detour_push -> @assignment_expr.detour_pop + edge @assignment_expr.detour_pop -> @right.value } "^(.+/)?([^/]+)\.js$" { let module_name = $2 attr (detour_push) push_symbol = module_name - attr (detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr + attr (@assignment_expr.detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr edge pop_default_guard -> detour_push - edge detour_push -> detour_pop - edge detour_pop -> @right.value + edge detour_push -> @assignment_expr.detour_pop + edge @assignment_expr.detour_pop -> @right.value } } node default_detour_push - node default_detour_pop + node @assignment_expr.default_detour_pop attr (default_detour_push) push_symbol = "default" - attr (default_detour_pop) symbol_definition = "default", source_node = @assignment_expr, definiens_node = @assignment_expr + attr (@assignment_expr.default_detour_pop) symbol_definition = "default", source_node = @assignment_expr, definiens_node = @assignment_expr edge pop_default_guard -> default_detour_push - edge default_detour_push -> default_detour_pop - edge default_detour_pop -> @right.value + edge default_detour_push -> @assignment_expr.default_detour_pop + edge @assignment_expr.default_detour_pop -> @right.value } @@ -4141,52 +4534,52 @@ inherit .return_or_yield (#eq? @exports "exports") ) { - node pop_default_guard + node @assignment_expr.pop_default_guard node pop_dot - attr (pop_default_guard) symbol_definition = "GUARD:DEFAULT", source_node = @exports - attr (pop_default_guard) definiens_node = @assignment_expr - edge @assignment_expr.exports -> pop_default_guard - edge pop_default_guard -> @right.value + attr (@assignment_expr.pop_default_guard) symbol_definition = "GUARD:DEFAULT", source_node = @exports + attr (@assignment_expr.pop_default_guard) definiens_node = @assignment_expr + edge @assignment_expr.exports -> @assignment_expr.pop_default_guard + edge @assignment_expr.pop_default_guard -> @right.value ;; For ES6 interoperability, expose members as named exports - attr (pop_dot) pop_symbol = "GAURD:MEMBER" + attr (pop_dot) pop_symbol = "GUARD:MEMBER" edge @assignment_expr.exports -> pop_dot edge pop_dot -> @right.value node detour_push - node detour_pop + node @assignment_expr.detour_pop scan FILE_PATH { "^(.+/)?([^/]+)/index\.js$" { let module_name = $2 attr (detour_push) push_symbol = module_name - attr (detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr - edge pop_default_guard -> detour_push - edge detour_push -> detour_pop - edge detour_pop -> @right.value + attr (@assignment_expr.detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr + edge @assignment_expr.pop_default_guard -> detour_push + edge detour_push -> @assignment_expr.detour_pop + edge @assignment_expr.detour_pop -> @right.value } "^(.+/)?([^/]+)\.js$" { let module_name = $2 attr (detour_push) push_symbol = module_name - attr (detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr - edge pop_default_guard -> detour_push - edge detour_push -> detour_pop - edge detour_pop -> @right.value + attr (@assignment_expr.detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr + edge @assignment_expr.pop_default_guard -> detour_push + edge detour_push -> @assignment_expr.detour_pop + edge @assignment_expr.detour_pop -> @right.value } } node default_detour_push - node default_detour_pop + node @assignment_expr.default_detour_pop attr (default_detour_push) push_symbol = "default" - attr (default_detour_pop) symbol_definition = "default", source_node = @assignment_expr, definiens_node = @assignment_expr - edge pop_default_guard -> default_detour_push - edge default_detour_push -> default_detour_pop - edge default_detour_pop -> @right.value + attr (@assignment_expr.default_detour_pop) symbol_definition = "default", source_node = @assignment_expr, definiens_node = @assignment_expr + edge @assignment_expr.pop_default_guard -> default_detour_push + edge default_detour_push -> @assignment_expr.default_detour_pop + edge @assignment_expr.default_detour_pop -> @right.value } @@ -4302,11 +4695,19 @@ inherit .return_or_yield ;; These rules are all about declarations and terms that have the names ;; directly in them. -(class_declaration - name:(_)@name - body:(_)@_body)@class_decl { +( + (class_declaration + name:(_)@name + body:(class_body + member:(method_definition + name:(_)@_method_name)@constructor + ) + ) - attr (@name.pop) definiens_node = @class_decl + (#eq? @_method_name "constructor") +) { + + attr (@name.pop) definiens_node = @constructor } @@ -4337,7 +4738,7 @@ inherit .return_or_yield } -(function +(function_expression name:(_)@name parameters:(_)@_call_sig)@fun { @@ -4353,11 +4754,19 @@ inherit .return_or_yield } -(class - name:(_)@name - body:(_)@_body)@class { - - attr (@name.pop) definiens_node = @class +( + (class + name:(_)@name + body:(class_body + member:(method_definition + name:(_)@_method_name)@constructor + ) + ) + + (#eq? @_method_name "constructor") +) { + + attr (@name.pop) definiens_node = @constructor } @@ -4373,12 +4782,7 @@ inherit .return_or_yield (assignment_expression left: (identifier)@left - right: [ - (function) - (generator_function) - (arrow_function) - (class) - ])@assignment_expr { + right: (_))@assignment_expr { attr (@left.pop) definiens_node = @assignment_expr @@ -4390,9 +4794,9 @@ inherit .return_or_yield object:(identifier)@_object property:(_)@left ) - right: (_)@right)@assignment_expr - (#not-eq @_object "module") - (#not-eq @left "exports") + right: (_))@assignment_expr + (#not-eq? @_object "module") + (#not-eq? @left "exports") ) { node left_definiens_hook @@ -4400,7 +4804,7 @@ inherit .return_or_yield attr (left_ignore_guard) pop_symbol = "GUARD:GANDALF" attr (left_definiens_hook) node_definition = @left - attr (left_definiens_hook) definiens_node = @right + attr (left_definiens_hook) definiens_node = @assignment_expr edge @assignment_expr.pkg_pop -> left_ignore_guard edge left_ignore_guard -> left_definiens_hook } @@ -4411,9 +4815,9 @@ inherit .return_or_yield object:(identifier)@_object property:(_)@left ) - right: (_)@right)@assignment_expr - (#eq @_object "module") - (#eq @left "exports") + right: (_))@assignment_expr + (#eq? @_object "module") + (#eq? @left "exports") ) { node left_definiens_hook @@ -4421,7 +4825,7 @@ inherit .return_or_yield attr (left_ignore_guard) pop_symbol = "GUARD:GANDALF" attr (left_definiens_hook) node_definition = @left - attr (left_definiens_hook) definiens_node = @right + attr (left_definiens_hook) definiens_node = @assignment_expr edge @assignment_expr.pkg_pop -> left_ignore_guard edge left_ignore_guard -> left_definiens_hook @@ -4448,7 +4852,7 @@ inherit .return_or_yield (variable_declarator name:(identifier)@name value: [ - (function) + (function_expression) (generator_function) (arrow_function) ])) @@ -4456,7 +4860,7 @@ inherit .return_or_yield (variable_declarator name:(identifier)@name value: [ - (function) + (function_expression) (generator_function) (arrow_function) ])) @@ -4466,7 +4870,7 @@ inherit .return_or_yield ; (member_expression property:(_)@name) ; FIXME member expressions are references and have no .pop ] right: [ - (function) + (function_expression) (generator_function) (arrow_function) ]) @@ -4476,10 +4880,76 @@ inherit .return_or_yield } +( + (assignment_expression + left: [ + ( ; exports.foo = ... + (member_expression + object:(_)@_exports + property:(_)@_property) + (#eq? @_exports "exports") + ) + ( ; module.exports.foo = ... + (member_expression + object:(member_expression + object:(_)@_module + property:(_)@_exports) + property:(_)@_property) + (#eq? @_module "module") + (#eq? @_exports "exports") + ) + ] + right: [ + (function_expression) + (generator_function) + (arrow_function) + ])@assignment_expr + +) { + + attr (@assignment_expr.pop_name) syntax_type = "function" + attr (@assignment_expr.detour_pop) syntax_type = "function" + attr (@assignment_expr.default_detour_pop) syntax_type = "function" + +} + +( + (assignment_expression + left: (member_expression + object:(_)@_module + property:(_)@_exports) + right: [ + (function_expression) + (generator_function) + (arrow_function) + ])@assignment_expr + (#eq? @_module "module") + (#eq? @_exports "exports") +) { + + attr (@assignment_expr.pop_default_guard) syntax_type = "function" + attr (@assignment_expr.detour_pop) syntax_type = "function" + attr (@assignment_expr.default_detour_pop) syntax_type = "function" + +} + +(export_statement "default" + value:[ + (function_expression) + (generator_function) + (arrow_function) + ])@export_stmt { + + attr (@export_stmt.pop_guard_default) syntax_type = "function" + attr (@export_stmt.detour_pop) syntax_type = "function" + attr (@export_stmt.default_detour_pop) syntax_type = "function" + +} + (pair key: (_)@name value: [ - (function) + (function_expression) (generator_function) (arrow_function) ]) { @@ -4510,6 +4980,60 @@ inherit .return_or_yield } +( + (assignment_expression + left: [ + ( ; exports.foo = ... + (member_expression + object:(_)@_exports + property:(_)@_property) + (#eq? @_exports "exports") + ) + ( ; module.exports.foo = ... + (member_expression + object:(member_expression + object:(_)@_module + property:(_)@_exports) + property:(_)@_property) + (#eq? @_module "module") + (#eq? @_exports "exports") + ) + ] + right: (class))@assignment_expr + +) { + + attr (@assignment_expr.pop_name) syntax_type = "class" + attr (@assignment_expr.detour_pop) syntax_type = "class" + attr (@assignment_expr.default_detour_pop) syntax_type = "class" + +} + +( + (assignment_expression + left: (member_expression + object:(_)@_module + property:(_)@_exports) + right: (class))@assignment_expr + (#eq? @_module "module") + (#eq? @_exports "exports") +) { + + attr (@assignment_expr.pop_default_guard) syntax_type = "class" + attr (@assignment_expr.detour_pop) syntax_type = "class" + attr (@assignment_expr.default_detour_pop) syntax_type = "class" + +} + +(export_statement "default" + value:(class))@export_stmt { + + attr (@export_stmt.pop_guard_default) syntax_type = "class" + attr (@export_stmt.detour_pop) syntax_type = "class" + attr (@export_stmt.default_detour_pop) syntax_type = "class" + +} + (pair key: (_)@name value: (class)) { @@ -4520,12 +5044,7 @@ inherit .return_or_yield (pair key: (_)@name - value: [ - (function) - (generator_function) - (arrow_function) - (class) - ])@pair_expr { + value: (_))@pair_expr { node @name.definiens_hook node name_ignore_guard diff --git a/languages/tree-sitter-stack-graphs-javascript/test/base_syntax.js b/languages/tree-sitter-stack-graphs-javascript/test/base_syntax.js index 858f854cd..4cf3170d0 100644 --- a/languages/tree-sitter-stack-graphs-javascript/test/base_syntax.js +++ b/languages/tree-sitter-stack-graphs-javascript/test/base_syntax.js @@ -3,15 +3,37 @@ // foo export let x = 1; +export { + // x + A +}; import "foo"; debugger; var x; let x; +let x = { + get constructor() {} +}; function foo() { } +function foo(a) { } +function foo(undefined) { } function* foo() { } -class Foo { } +class Foo { + #x = null; + get [/**/ foo]() {} + static {} +} +@Foo class Bar { } +@Foo.Quux class Bar { } +@Foo() class Bar { } +@Foo.Quux() class Bar { } { } if (true) { } +if (true) { } else { } +if (/**/ true) /**/ { } else /**/ { } +if (true) return; +if (true) return; else return; +if (/**/ true) /**/ return; else /**/ return; switch (x) { } for (x; y; z) { } for (x in xs) { } @@ -22,6 +44,7 @@ with (x) { } break; continue; return; +return x; ; foo: x; @@ -45,8 +68,13 @@ undefined; []; [1, 2, 3]; function () { return; }; +function () { return function () { }; }; () => { }; +() => () => { }; function* () { yield 1; }; +function (/**/) { }; +function (x /**/) { }; +function (/**/ x) { }; foo(); foo(bar); foo.bar; @@ -59,7 +87,21 @@ x++; 1 + 1; -2; (x = 1); +(x[i] = 1); +([x] = 1); +({ x: y } = 1); +({/**/x: y } = 1); +({x:y.z} = 1); x += 1; (1, 2); 1 ? 2 : 3; -class { }; \ No newline at end of file +class { }; + + + <>doo + {garply} + { } + {/**/x} +; +; +; \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-javascript/test/computation_flow/class_declaration_fields_and_methods_visible_in_other_methods.js b/languages/tree-sitter-stack-graphs-javascript/test/computation_flow/class_declaration_fields_and_methods_visible_in_other_methods.js new file mode 100644 index 000000000..3bed1c415 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/test/computation_flow/class_declaration_fields_and_methods_visible_in_other_methods.js @@ -0,0 +1,23 @@ +class Foo { + bar = 1; + baz() { } + quux() { + this.bar; + // ^ defined: 2 + + this.baz(); + // ^ defined: 3 + } +} + +(class { + bar = 1; + baz() { } + quux() { + this.bar; + // ^ defined: 14 + + this.baz(); + // ^ defined: 15 + } +}); \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-javascript/test/computation_flow/function_closure_mutation.js b/languages/tree-sitter-stack-graphs-javascript/test/computation_flow/function_closure_mutation.js new file mode 100644 index 000000000..4b8c1a200 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/test/computation_flow/function_closure_mutation.js @@ -0,0 +1,43 @@ +let x = 1; + +function foo() { + /**/ x; + // ^ defined: 1, 38 +} + +function* foo() { + /**/ x; + // ^ defined: 1, 38 +} + +function () { + /**/ x; + // ^ defined: 1, 38 +}; + +function* () { + /**/ x; + // ^ defined: 1, 38 +}; + +() => x; +// ^ defined: 1, 38 + +() => { + /**/ x; + // ^ defined: 1, 38 +}; + +class C { + foo() { + /**/ x; + // ^ defined: 1, 38 + } +} + +x = 2; + +(function bar() { + /**/ bar; + // ^ defined: 40 +}); \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-javascript/test/expressions/member_expression.js b/languages/tree-sitter-stack-graphs-javascript/test/expressions/member_expression.js index b397d747a..438cf9dd6 100644 --- a/languages/tree-sitter-stack-graphs-javascript/test/expressions/member_expression.js +++ b/languages/tree-sitter-stack-graphs-javascript/test/expressions/member_expression.js @@ -15,4 +15,8 @@ let x = 1; // Flow around /**/ x; +// ^ defined: 1 + +// Optional chain +/**/ x?.foo // ^ defined: 1 \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_core.js b/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_core.js new file mode 100644 index 000000000..fc0674833 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_core.js @@ -0,0 +1,48 @@ +// The core of JSX tests here verify the behavior of the following node types: +// jsx_element +// jsx_identifier +// jsx_attribute +// jsx_expression +// jsx_opening_element +// jsx_closing_element +// There is no real way to avoid testing all of these at once, +// and so we don't even try to. + +let x = 1; + +// Flow In + +const el = {x}; +// ^ defined: 11 +// ^ defined: 11 + +const el2 = +// ^ defined: 11 +// ^ defined: 11 + +// Flow Out + +const el = + {z = 3} +; + +/**/ y; +// ^ defined: 25 + +/**/ z; +// ^ defined: 26 + +// Flow Across + +const el = + {w}; +// ^ defined: 37 + +const el = ; +// ^ defined: 41 + +// Flow Around + +/**/ x; +// ^ defined: 11 \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_fragment.js b/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_fragment.js new file mode 100644 index 000000000..02d5f5d68 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_fragment.js @@ -0,0 +1,8 @@ +let x = 1; + +// Flow Around + +const el = <>; + +/**/ x; +// ^ defined: 1 \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_namespace_name.js b/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_namespace_name.js new file mode 100644 index 000000000..ec59c3dca --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_namespace_name.js @@ -0,0 +1,21 @@ +let x = 1; + +// Flow Around + +const el = ; + +/**/ x; +// ^ defined: 1 + +// Flow In + +let foo = { + bar: { + baz: 1 + } +}; + +const el2 = ; +// ^ defined: 12 +// ^ defined: 13 +// ^ defined: 14 \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_self_closing_element.js b/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_self_closing_element.js new file mode 100644 index 000000000..937ca0970 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_self_closing_element.js @@ -0,0 +1,37 @@ +// The core of JSX tests here verify the behavior of the following node types: +// jsx_element +// jsx_identifier +// jsx_attribute +// jsx_expression +// jsx_opening_element +// jsx_closing_element +// There is no real way to avoid testing all of these at once, +// and so we don't even try to. + +let x = 1; + +// Flow In + +const el = ; +// ^ defined: 11 + +const el2 = +// ^ defined: 11 + +// Flow Out + +const el = ; + +/**/ y; +// ^ defined: 23 + +// Flow Across + +const el = ; +// ^ defined: 30 + +// Flow Around + +/**/ x; +// ^ defined: 11 \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_text.js b/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_text.js new file mode 100644 index 000000000..0cc359765 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/test/jsx/jsx_text.js @@ -0,0 +1,8 @@ +let x = 1; + +// Flow Around + +const el = bar; + +/**/ x; +// ^ defined: 1 \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-javascript/test/old/bug_regressions/variable_resolves_to_module.js b/languages/tree-sitter-stack-graphs-javascript/test/old/bug_regressions/variable_resolves_to_module.js new file mode 100644 index 000000000..e829ab7ff --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/test/old/bug_regressions/variable_resolves_to_module.js @@ -0,0 +1,4 @@ +/* --- path: debug.js --- */ + +/**/ debug(42); +// ^ defined: diff --git a/languages/tree-sitter-stack-graphs-javascript/test/old/packages/package_dependencies.js b/languages/tree-sitter-stack-graphs-javascript/test/old/packages/can-access-package-non-main-explicilty.js similarity index 64% rename from languages/tree-sitter-stack-graphs-javascript/test/old/packages/package_dependencies.js rename to languages/tree-sitter-stack-graphs-javascript/test/old/packages/can-access-package-non-main-explicilty.js index 7a67a873a..820c0368e 100644 --- a/languages/tree-sitter-stack-graphs-javascript/test/old/packages/package_dependencies.js +++ b/languages/tree-sitter-stack-graphs-javascript/test/old/packages/can-access-package-non-main-explicilty.js @@ -1,18 +1,3 @@ -/* --- path: foo/package.json --- */ -/* --- global: FILE_PATH=package.json --- */ -/* --- global: PROJECT_NAME=foo --- */ - -{ - "name": "foo", - "version": "1.0" -} - -/* --- path: foo/src/index.js --- */ -/* --- global: FILE_PATH=src/index.js --- */ -/* --- global: PROJECT_NAME=foo --- */ - -export let x; - /* --- path: acme_foo/package.json --- */ /* --- global: FILE_PATH=package.json --- */ /* --- global: PROJECT_NAME=acme_foo --- */ @@ -23,14 +8,14 @@ export let x; "main": "./api" } -/* --- path: acme_foo/core.js --- */ -/* --- global: FILE_PATH=core.js --- */ +/* --- path: acme_foo/api.js --- */ +/* --- global: FILE_PATH=api.js --- */ /* --- global: PROJECT_NAME=acme_foo --- */ export let x; -/* --- path: acme_foo/api.js --- */ -/* --- global: FILE_PATH=api.js --- */ +/* --- path: acme_foo/core.js --- */ +/* --- global: FILE_PATH=core.js --- */ /* --- global: PROJECT_NAME=acme_foo --- */ export let x; @@ -42,7 +27,6 @@ export let x; { "name": "bar", "dependencies": { - "foo": "1", "@acme/foo": "1" } } @@ -51,11 +35,5 @@ export let x; /* --- global: FILE_PATH=app.js --- */ /* --- global: PROJECT_NAME=bar --- */ -import { x } from "@acme/foo" -// ^ defined: 36 - -import { x } from "foo" -// ^ defined: 14 - import { x } from "@acme/foo/core" -// ^ defined: 30 +// ^ defined: 21 diff --git a/languages/tree-sitter-stack-graphs-javascript/test/old/packages/package-does-not-export-non-main.js b/languages/tree-sitter-stack-graphs-javascript/test/old/packages/package-does-not-export-non-main.js new file mode 100644 index 000000000..cea744788 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/test/old/packages/package-does-not-export-non-main.js @@ -0,0 +1,32 @@ +/* --- path: foo/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=foo --- */ + +{ + "name": "foo", + "version": "1.0" +} + +/* --- path: foo/impl.js --- */ +/* --- global: FILE_PATH=impl.js --- */ +/* --- global: PROJECT_NAME=foo --- */ + +export let x; + +/* --- path: bar/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=bar --- */ + +{ + "name": "bar", + "dependencies": { + "foo": "1" + } +} + +/* --- path: bar/app.js --- */ +/* --- global: FILE_PATH=app.js --- */ +/* --- global: PROJECT_NAME=bar --- */ + +import { x } from "foo" +// ^ defined: diff --git a/languages/tree-sitter-stack-graphs-javascript/test/old/packages/package-exports-explicit-main.js b/languages/tree-sitter-stack-graphs-javascript/test/old/packages/package-exports-explicit-main.js new file mode 100644 index 000000000..14ec28812 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/test/old/packages/package-exports-explicit-main.js @@ -0,0 +1,39 @@ +/* --- path: acme_foo/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=acme_foo --- */ + +{ + "name": "@acme/foo", + "version": "1.0", + "main": "./api" +} + +/* --- path: acme_foo/api.js --- */ +/* --- global: FILE_PATH=api.js --- */ +/* --- global: PROJECT_NAME=acme_foo --- */ + +export let x; + +/* --- path: acme_foo/core.js --- */ +/* --- global: FILE_PATH=core.js --- */ +/* --- global: PROJECT_NAME=acme_foo --- */ + +export let x; + +/* --- path: bar/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=bar --- */ + +{ + "name": "bar", + "dependencies": { + "@acme/foo": "1" + } +} + +/* --- path: bar/app.js --- */ +/* --- global: FILE_PATH=app.js --- */ +/* --- global: PROJECT_NAME=bar --- */ + +import { x } from "@acme/foo" +// ^ defined: 15 diff --git a/languages/tree-sitter-stack-graphs-javascript/test/old/packages/package-main-defaults-to-index.js b/languages/tree-sitter-stack-graphs-javascript/test/old/packages/package-main-defaults-to-index.js new file mode 100644 index 000000000..c06d0d625 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/test/old/packages/package-main-defaults-to-index.js @@ -0,0 +1,38 @@ +/* --- path: foo/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=foo --- */ + +{ + "name": "foo", + "version": "1.0" +} + +/* --- path: foo/index.js --- */ +/* --- global: FILE_PATH=index.js --- */ +/* --- global: PROJECT_NAME=foo --- */ + +export let x; + +/* --- path: foo/impl.js --- */ +/* --- global: FILE_PATH=impl.js --- */ +/* --- global: PROJECT_NAME=foo --- */ + +export let x; + +/* --- path: bar/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=bar --- */ + +{ + "name": "bar", + "dependencies": { + "foo": "1" + } +} + +/* --- path: bar/app.js --- */ +/* --- global: FILE_PATH=app.js --- */ +/* --- global: PROJECT_NAME=bar --- */ + +import { x } from "foo" +// ^ defined: 14 diff --git a/languages/tree-sitter-stack-graphs-python/.gitignore b/languages/tree-sitter-stack-graphs-python/.gitignore new file mode 100644 index 000000000..faf6459fb --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/.gitignore @@ -0,0 +1,3 @@ +*.html +/Cargo.lock +/target diff --git a/languages/tree-sitter-stack-graphs-python/CHANGELOG.md b/languages/tree-sitter-stack-graphs-python/CHANGELOG.md new file mode 100644 index 000000000..f21aecc9f --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/CHANGELOG.md @@ -0,0 +1,31 @@ +# Changelog for tree-sitter-stack-graphs-python + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## v0.3.0 -- 2024-12-12 + +- The `tree-sitter-stack-graphs` dependency is updated to version 0.10. + +- The `tree-sitter-python` dependency is updated to version 0.23.5. + +## v0.2.0 -- 2024-07-09 + +### Added + +- Added support for root paths. This fixes import problems when indexing using absolute directory paths. + +### Fixed + +- Fixed crash for lambdas with parameters. +- Fixed crash for nested functions definitions. + +### Removed + +- The `FILE_PATH_VAR` constant has been replaced in favor of `tree_sitter_stack_graphs::FILE_PATH_VAR`. + +## v0.1.0 -- 2024-03-06 + +Initial release. diff --git a/languages/tree-sitter-stack-graphs-python/Cargo.toml b/languages/tree-sitter-stack-graphs-python/Cargo.toml new file mode 100644 index 000000000..afdfe6091 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "tree-sitter-stack-graphs-python" +version = "0.3.0" +description = "Stack graphs definition for Python using tree-sitter-python" +readme = "README.md" +keywords = ["tree-sitter", "stack-graphs", "python"] +authors = [ + "GitHub ", +] +license = "MIT OR Apache-2.0" +edition = "2018" + +[[bin]] +name = "tree-sitter-stack-graphs-python" +path = "rust/bin.rs" +required-features = ["cli"] + +[lib] +path = "rust/lib.rs" +test = false + +[[test]] +name = "test" +path = "rust/test.rs" +harness = false + +[features] +cli = ["anyhow", "clap", "tree-sitter-stack-graphs/cli"] + +[dependencies] +anyhow = { version = "1.0", optional = true } +clap = { version = "4", optional = true, features = ["derive"] } +tree-sitter-stack-graphs = { version = "0.10", path = "../../tree-sitter-stack-graphs" } # explicit version is required to be able to publish crate +tree-sitter-python = "=0.23.5" + +[dev-dependencies] +anyhow = "1.0" +tree-sitter-stack-graphs = { path = "../../tree-sitter-stack-graphs", features = ["cli"] } diff --git a/languages/tree-sitter-stack-graphs-python/README.md b/languages/tree-sitter-stack-graphs-python/README.md new file mode 100644 index 000000000..7ec9ded72 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/README.md @@ -0,0 +1,138 @@ +# tree-sitter-stack-graphs definition for Python + +This project defines tree-sitter-stack-graphs rules for Python using the [tree-sitter-python][] grammar. + +[tree-sitter-python]: https://crates.io/crates/tree-sitter-python + +- [API documentation](https://docs.rs/tree-sitter-stack-graphs-python/) +- [Release notes](https://github.com/github/stack-graphs/blob/main/languages/tree-sitter-stack-graphs-python/CHANGELOG.md) + +## Using the API + +To use this library, add the following to your `Cargo.toml`: + +```toml +[dependencies] +tree-sitter-stack-graphs-python = "0.3" +``` + +Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs-python/*/) for more details on how to use this library. + +## Using the Command-line Program + +The command-line program for `tree-sitter-stack-graphs-python` lets you do stack graph based analysis and lookup from the command line. + +The CLI can be run as follows: + +1. _(Installed)_ Install the CLI using Cargo as follows: + + ```sh + cargo install --features cli tree-sitter-stack-graphs-python + ``` + + After this, the CLI should be available as `tree-sitter-stack-graphs-python`. + +2. _(From source)_ Instead of installing the CLI, it can also be run directly from the crate directory, as a replacement for a `tree-sitter-stack-graphs-python` invocation, as follows: + + ```sh + cargo run --features cli -- + ``` + +The basic CLI workflow for the command-line program is to index source code and issue queries against the resulting database: + +1. Index a source folder as follows: + + ```sh + tree-sitter-stack-graphs-python index SOURCE_DIR + ``` + + _Indexing will skip any files that have already be indexed. To force a re-index, add the `-f` flag._ + + To check the status if a source folder, run: + + ```sh + tree-sitter-stack-graphs-python status SOURCE_DIR + ``` + + To clean the database and start with a clean slate, run: + + ```sh + tree-sitter-stack-graphs-python clean + ``` + + _Pass the `--delete` flag to not just empty the database, but also delete it. This is useful to resolve `unsupported database version` errors that may occur after a version update._ + +2. Run a query to find the definition(s) for a reference on a given line and column, run: + + ```sh + tree-sitter-stack-graphs-python query definition SOURCE_PATH:LINE:COLUMN + ``` + + Resulting definitions are printed, including a source line if the source file is available. + +Discover all available commands and flags by passing the `-h` flag to the CLI directly, or to any of the subcommands. + +## Development + +The project is written in Rust, and requires a recent version installed. Rust can be installed and updated using [rustup][]. + +[rustup]: https://rustup.rs/ + +The project is organized as follows: + +- The stack graph rules are defined in `src/stack-graphs.tsg`. +- Builtins sources and configuration are defined in `src/builtins.it` and `builtins.cfg` respectively. +- Tests are put into the `test` directory. + +### Running Tests + +Run the tests as follows: + +```sh +cargo test +``` + +The project consists of a library and a CLI. By default, running `cargo` only applies to the library. To run `cargo` commands on the CLI as well, add `--features cli` or `--all-features`. + +Run the CLI from source as follows: + +```sh +cargo run --features cli -- ARGS +``` + +Sources are formatted using the standard Rust formatted, which is applied by running: + +```sh +cargo fmt +``` + +### Writing TSG + +The stack graph rules are written in [tree-sitter-graph][]. Checkout the [examples][], +which contain self-contained TSG rules for specific language features. A VSCode +[extension][] is available that provides syntax highlighting for TSG files. + +[tree-sitter-graph]: https://github.com/tree-sitter/tree-sitter-graph +[examples]: https://github.com/github/stack-graphs/blob/main/tree-sitter-stack-graphs/examples/ +[extension]: https://marketplace.visualstudio.com/items?itemName=tree-sitter.tree-sitter-graph + +Parse and test a single file by executing the following commands: + +```sh +cargo run --features cli -- parse FILES... +cargo run --features cli -- test TESTFILES... +``` + +Generate a visualization to debug failing tests by passing the `-V` flag: + +```sh +cargo run --features cli -- test -V TESTFILES... +``` + +To generate the visualization regardless of test outcome, execute: + +```sh +cargo run --features cli -- test -V --output-mode=always TESTFILES... +``` + +Go to for links to examples and documentation. diff --git a/languages/tree-sitter-stack-graphs-python/rust/bin.rs b/languages/tree-sitter-stack-graphs-python/rust/bin.rs new file mode 100644 index 000000000..dc9a6772c --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/rust/bin.rs @@ -0,0 +1,32 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use anyhow::anyhow; +use clap::Parser; +use tree_sitter_stack_graphs::cli::database::default_user_database_path_for_crate; +use tree_sitter_stack_graphs::cli::provided_languages::Subcommands; +use tree_sitter_stack_graphs::NoCancellation; + +fn main() -> anyhow::Result<()> { + let lc = match tree_sitter_stack_graphs_python::try_language_configuration(&NoCancellation) { + Ok(lc) => lc, + Err(err) => { + eprintln!("{}", err.display_pretty()); + return Err(anyhow!("Language configuration error")); + } + }; + let cli = Cli::parse(); + let default_db_path = default_user_database_path_for_crate(env!("CARGO_PKG_NAME"))?; + cli.subcommand.run(default_db_path, vec![lc]) +} + +#[derive(Parser)] +#[clap(about, version)] +pub struct Cli { + #[clap(subcommand)] + subcommand: Subcommands, +} diff --git a/languages/tree-sitter-stack-graphs-python/rust/lib.rs b/languages/tree-sitter-stack-graphs-python/rust/lib.rs new file mode 100644 index 000000000..332250511 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/rust/lib.rs @@ -0,0 +1,45 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use tree_sitter_stack_graphs::loader::LanguageConfiguration; +use tree_sitter_stack_graphs::loader::LoadError; +use tree_sitter_stack_graphs::CancellationFlag; + +/// The stack graphs tsg source for this language. +pub const STACK_GRAPHS_TSG_PATH: &str = "src/stack-graphs.tsg"; +/// The stack graphs tsg source for this language. +pub const STACK_GRAPHS_TSG_SOURCE: &str = include_str!("../src/stack-graphs.tsg"); + +/// The stack graphs builtins configuration for this language. +pub const STACK_GRAPHS_BUILTINS_CONFIG: &str = include_str!("../src/builtins.cfg"); +/// The stack graphs builtins path for this language +pub const STACK_GRAPHS_BUILTINS_PATH: &str = "src/builtins.py"; +/// The stack graphs builtins source for this language. +pub const STACK_GRAPHS_BUILTINS_SOURCE: &str = include_str!("../src/builtins.py"); + +pub fn language_configuration(cancellation_flag: &dyn CancellationFlag) -> LanguageConfiguration { + try_language_configuration(cancellation_flag).unwrap_or_else(|err| panic!("{}", err)) +} + +pub fn try_language_configuration( + cancellation_flag: &dyn CancellationFlag, +) -> Result { + LanguageConfiguration::from_sources( + tree_sitter_python::LANGUAGE.into(), + Some(String::from("source.py")), + None, + vec![String::from("py")], + STACK_GRAPHS_TSG_PATH.into(), + STACK_GRAPHS_TSG_SOURCE, + Some(( + STACK_GRAPHS_BUILTINS_PATH.into(), + STACK_GRAPHS_BUILTINS_SOURCE, + )), + Some(STACK_GRAPHS_BUILTINS_CONFIG), + cancellation_flag, + ) +} diff --git a/languages/tree-sitter-stack-graphs-python/rust/test.rs b/languages/tree-sitter-stack-graphs-python/rust/test.rs new file mode 100644 index 000000000..c81edf0d4 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/rust/test.rs @@ -0,0 +1,23 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use anyhow::anyhow; +use std::path::PathBuf; +use tree_sitter_stack_graphs::ci::Tester; +use tree_sitter_stack_graphs::NoCancellation; + +fn main() -> anyhow::Result<()> { + let lc = match tree_sitter_stack_graphs_python::try_language_configuration(&NoCancellation) { + Ok(lc) => lc, + Err(err) => { + eprintln!("{}", err.display_pretty()); + return Err(anyhow!("Language configuration error")); + } + }; + let test_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test"); + Tester::new(vec![lc], vec![test_path]).run() +} diff --git a/languages/tree-sitter-stack-graphs-python/src/builtins.cfg b/languages/tree-sitter-stack-graphs-python/src/builtins.cfg new file mode 100644 index 000000000..d685061be --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/src/builtins.cfg @@ -0,0 +1 @@ +[globals] diff --git a/languages/tree-sitter-stack-graphs-python/src/builtins.py b/languages/tree-sitter-stack-graphs-python/src/builtins.py new file mode 100644 index 000000000..e69de29bb diff --git a/languages/tree-sitter-stack-graphs-python/src/stack-graphs.scm b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.scm new file mode 100644 index 000000000..380af7d95 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.scm @@ -0,0 +1,779 @@ +;; -*- coding: utf-8 -*- +;; ------------------------------------------------------------------------------------------------ +;; Copyright © 2023, stack-graphs authors. +;; Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +;; Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +;; ------------------------------------------------------------------------------------------------ + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; LEGACY DEFINITION! INCLUDED FOR REFERENCE ONLY! ;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; Modules and Imports +;--------------------- + +(module) @mod +{ + var module_def = @mod.file_def + attr module_def "no_span" + var module_ref = @mod.file_ref + attr module_ref "no_span" + var parent_module_def = module_def + var parent_module_ref = module_ref + var grandparent_module_ref = module_ref + + scan filepath { + "([^/]+)/" + { + edge module_def -> module_def.dot + edge module_def.dot -> module_def.next_def + edge module_ref.next_ref -> module_ref.dot + edge module_ref.dot -> module_ref + + attr module_def "pop" = $1 + attr module_def.dot "pop" = "." + attr module_ref "push" = $1 + attr module_ref.dot "push" = "." + + set grandparent_module_ref = parent_module_ref + set parent_module_def = module_def + set parent_module_ref = module_ref + set module_ref = module_ref.next_ref + set module_def = module_def.next_def + attr module_def "no_span" + attr module_ref "no_span" + } + + "__init__\\.py$" + { + attr parent_module_def "definition" + } + + "([^/]+)$" + { + edge module_def -> module_def.dot + edge module_def.dot -> module_def.next_def + + attr module_def "definition", "pop" = (replace $1 "\\.py" "") + attr module_def.dot "pop" = "." + + set module_def = module_def.next_def + attr module_def "no_span" + attr module_ref "no_span" + } + } + + edge root -> @mod.file_def + edge @mod.file_ref -> root + edge module_def -> @mod.after_scope + + edge @mod.before_scope -> @mod.global_dot + edge @mod.global -> root + attr @mod.global "push" = "" + + edge @mod.global_dot -> @mod.global + attr @mod.global_dot "push" = "." + + var @mod::parent_module = parent_module_ref + var @mod::grandparent_module = grandparent_module_ref + var @mod::bottom = @mod.after_scope + var @mod::global = @mod.global + var @mod::global_dot = @mod.global_dot +} + +(import_statement + name: (dotted_name + . (identifier) @root_name)) @stmt +{ + edge @stmt.after_scope -> @root_name.def, "precedence" = 1 + edge @root_name.ref -> root + attr @root_name.ref "push", "reference" + attr @root_name.def "pop", "definition" +} + +(import_statement + name: (aliased_import + (dotted_name . (identifier) @root_name))) @stmt +{ + edge @stmt.after_scope -> @root_name.def + edge @root_name.ref -> root + attr @root_name.ref "push", "reference" +} + +(import_from_statement + module_name: (dotted_name + . (identifier) @prefix_root_name)) @stmt +{ + edge @prefix_root_name.ref -> root + attr @prefix_root_name.ref "push", "reference" +} + +(import_from_statement + name: (dotted_name + . (identifier) @import_root_name)) @stmt +{ + edge @stmt.after_scope -> @import_root_name.def, "precedence" = 1 + edge @import_root_name.ref -> @import_root_name.ref_dot + attr @import_root_name.def "pop", "definition" + attr @import_root_name.ref "push", "reference" + attr @import_root_name.ref_dot "push" = "." +} + +(import_from_statement + name: (aliased_import + (dotted_name + . (identifier) @import_root_name))) @stmt +{ + edge @import_root_name.ref -> @import_root_name.ref_dot + attr @import_root_name.ref "push", "reference" + attr @import_root_name.ref_dot "push" = "." +} + +(import_from_statement + module_name: [ + (dotted_name (identifier) @prefix_leaf_name .) + (relative_import (dotted_name (identifier) @prefix_leaf_name .)) + (relative_import (import_prefix) @prefix_leaf_name .) + ] + name: [ + (dotted_name + . (identifier) @import_root_name) + (aliased_import + (dotted_name + . (identifier) @import_root_name)) + ]) +{ + edge @import_root_name.ref_dot -> @prefix_leaf_name.ref +} + +[ + (import_from_statement + (aliased_import + name: (dotted_name (identifier) @name .) + alias: (identifier) @alias)) + (import_statement + (aliased_import + name: (dotted_name (identifier) @name .) + alias: (identifier) @alias)) +] @stmt +{ + edge @stmt.after_scope -> @alias + edge @alias -> @name.ref + attr @alias "pop", "definition" +} + +[ + (import_statement + name: (dotted_name + (identifier) @leaf_name .)) + (import_from_statement + name: (dotted_name + (identifier) @leaf_name .)) +] +{ + attr @leaf_name.def "pop", "definition" + attr @leaf_name.ref "push", "reference" + edge @leaf_name.def -> @leaf_name.ref +} + +(relative_import + (import_prefix) @prefix + (#eq? @prefix ".")) @import +{ + edge @prefix.ref -> @import::parent_module +} + +(relative_import + (import_prefix) @prefix + (#eq? @prefix "..")) @import +{ + edge @prefix.ref -> @import::grandparent_module +} + +(relative_import + (import_prefix) @prefix + (dotted_name + . (identifier) @name)) +{ + attr @name.ref "push", "reference" + attr @name.ref_dot "push" = "." + edge @name.ref -> @name.ref_dot + edge @name.ref_dot -> @prefix.ref +} + +[ + (import_from_statement + module_name: (relative_import + (dotted_name + (identifier) @parent_name + . + (identifier) @child_name))) + (import_from_statement + module_name: (dotted_name + (identifier) @parent_name + . + (identifier) @child_name)) +] +{ + attr @child_name.ref "push", "reference" + attr @child_name.ref_dot "push" = "." + edge @child_name.ref -> @child_name.ref_dot + edge @child_name.ref_dot -> @parent_name.ref +} + +(import_from_statement + module_name: (dotted_name + . (identifier) @root_name)) +{ + attr @root_name.ref "push", "reference" + edge @root_name.ref -> root +} + +(import_from_statement + module_name: (dotted_name + (identifier) @leaf_name .) + (wildcard_import) @star) @stmt +{ + edge @stmt.after_scope -> @star.ref_dot, "precedence" = 1 + edge @star.ref_dot -> @leaf_name.ref + attr @star.ref_dot "push" = "." +} + +[ + (import_statement + name: (dotted_name + (identifier) @parent_name + . + (identifier) @child_name)) + (import_from_statement + name: (dotted_name + (identifier) @parent_name + . + (identifier) @child_name)) + (import_from_statement + name: (aliased_import + name: (dotted_name + (identifier) @parent_name + . + (identifier) @child_name))) +] +{ + edge @child_name.ref -> @child_name.ref_dot + edge @child_name.ref_dot -> @parent_name.ref + edge @parent_name.def -> @parent_name.def_dot + edge @parent_name.def_dot -> @child_name.def + attr @child_name.def "pop", "definition" + attr @child_name.ref "push","reference" + attr @parent_name.def_dot "pop" = "." + attr @child_name.ref_dot "push" = "." +} + +;-------- +; Scopes +;-------- + +[ + (module (_) @last_stmt .) + (block (_) @last_stmt .) +] @block +{ + edge @block.after_scope -> @last_stmt.after_scope +} + +[ + (module (_) @stmt1 . (_) @stmt2) + (block (_) @stmt1 . (_) @stmt2) +] +{ + edge @stmt2.before_scope -> @stmt1.after_scope +} + +[ + (module (_) @stmt) + (block (_) @stmt) +] +{ + edge @stmt.after_scope -> @stmt.before_scope + let @stmt::local_scope = @stmt.before_scope +} + +[ + (block . (_) @stmt) + (module . (_) @stmt) +] @block +{ + edge @stmt.before_scope -> @block.before_scope +} + +(block (_) @stmt . ) @block +{ + edge @block.after_scope -> @stmt.after_scope +} + +(function_definition (block) @block) +{ + edge @block.before_scope -> @block::local_scope +} + +[ + (while_statement (block) @block) + (if_statement (block) @block) + (with_statement (block) @block) + (try_statement (block) @block) + (for_statement (block) @block) + (_ [ + (else_clause (block) @block) + (elif_clause (block) @block) + (except_clause (block) @block) + (finally_clause (block) @block) + ]) +] @stmt +{ + edge @block.before_scope -> @block::local_scope + edge @stmt.after_scope -> @block.after_scope +} + +(match_statement (case_clause) @block) @stmt +{ + let @block::local_scope = @block.before_scope + edge @block.before_scope -> @stmt.before_scope + edge @stmt.after_scope -> @block.after_scope +} + +[ + (for_statement) + (while_statement) +] @stmt +{ + edge @stmt.before_scope -> @stmt.after_scope +} + +;------------- +; Definitions +;------------- + +[ + (assignment + left: (_) @pattern + right: (_) @value) + (with_item + value: + (as_pattern + (_) @value + alias: (as_pattern_target (_) @pattern))) +] +{ + edge @pattern.input -> @value.output +} + +(function_definition + name: (identifier) @name + parameters: (parameters) @params + body: (block) @body) @func +{ + attr @name "definiens" = @func + edge @func.after_scope -> @name + edge @name -> @func.call + edge @func.call -> @func.return_value + edge @body.before_scope -> @params.after_scope + edge @body.before_scope -> @func.drop_scope + edge @func.drop_scope -> @func::bottom + attr @func.drop_scope "drop" + attr @name "pop", "definition" + attr @func.call "pop" = "()", "pop-scope" + attr @params.before_scope "jump-to" + attr @func.return_value "endpoint" + let @func::function_returns = @func.return_value + + ; Prevent functions defined inside of method bodies from being treated like methods + let @body::class_self_scope = nil + let @body::class_member_attr_scope = nil +} + +;; +;; BEGIN BIG GNARLY DISJUNCTION +;; +;; The following pair of rules is intended to capture the following behavior: +;; +;; If a function definition is used to define a method, by being inside a class +;; definition, then we make its syntax type `method`. Otherwise, we make it's +;; syntax type `function`. Unfortunately, because of the limitations on negation +;; and binding in tree sitter queries, we cannot negate `class_definition` or +;; similar things directly. Instead, we have to manually push the negation down +;; to form the finite disjunction it corresponds to. +;; + +[ + (class_definition (block (decorated_definition (function_definition name: (_)@name)))) + (class_definition (block (function_definition name: (_)@name))) +] +{ + attr @name "syntax_type" = "method" +} + +[ + (module (decorated_definition (function_definition name: (_)@name))) + (module (function_definition name: (_)@name)) + + (if_statement (block (decorated_definition (function_definition name: (_)@name)))) + (if_statement (block (function_definition name: (_)@name))) + + (elif_clause (block (decorated_definition (function_definition name: (_)@name)))) + (elif_clause (block (function_definition name: (_)@name))) + + (else_clause (block (decorated_definition (function_definition name: (_)@name)))) + (else_clause (block (function_definition name: (_)@name))) + + (case_clause (block (decorated_definition (function_definition name: (_)@name)))) + (case_clause (block (function_definition name: (_)@name))) + + (for_statement (block (decorated_definition (function_definition name: (_)@name)))) + (for_statement (block (function_definition name: (_)@name))) + + (while_statement (block (decorated_definition (function_definition name: (_)@name)))) + (while_statement (block (function_definition name: (_)@name))) + + (try_statement (block (decorated_definition (function_definition name: (_)@name)))) + (try_statement (block (function_definition name: (_)@name))) + + (except_clause (block (decorated_definition (function_definition name: (_)@name)))) + (except_clause (block (function_definition name: (_)@name))) + + (finally_clause (block (decorated_definition (function_definition name: (_)@name)))) + (finally_clause (block (function_definition name: (_)@name))) + + (with_statement (block (decorated_definition (function_definition name: (_)@name)))) + (with_statement (block (function_definition name: (_)@name))) + + (function_definition (block (decorated_definition (function_definition name: (_)@name)))) + (function_definition (block (function_definition name: (_)@name))) +] +{ + attr @name "syntax_type" = "function" +} + +;; +;; END BIG GNARLY DISJUNCTION +;; + +(function_definition + parameters: (parameters + . (identifier) @param) + body: (block) @body) +{ + edge @param.input -> @param::class_self_scope + edge @param::class_member_attr_scope -> @param.output + edge @param.output -> @body.after_scope + attr @param.output "push" +} + +(parameter/identifier) @param +{ + attr @param.input "definition", "pop" + attr @param.param_name "push" + edge @param.input -> @param.param_index + edge @param.input -> @param.param_name +} + +[ + (parameter/default_parameter + name: (identifier) @name + value: (_) @value) @param + (parameter/typed_default_parameter + name: (_) @name + value: (_) @value) @param +] +{ + attr @name "definition", "pop" + attr @param.param_name "push" = @name + edge @name -> @param.param_name + edge @name -> @param.param_index + edge @param.input -> @name + edge @name -> @value.output +} + +[ + (parameter/typed_parameter + . (_) @name) @param + (parameter/list_splat_pattern + (_) @name) @param + (parameter/dictionary_splat_pattern + (_) @name) @param +] +{ + attr @name "definition", "pop" + attr @param.param_name "push" = @name + edge @name -> @param.param_name + edge @name -> @param.param_index + edge @param.input -> @name +} + +[ + (pattern_list (_) @pattern) + (tuple_pattern (_) @pattern) +] @list +{ + let statement_scope = @list::local_scope + let @pattern::local_scope = @pattern.pattern_before_scope + edge statement_scope -> @pattern::local_scope, "precedence" = (+ 1 (child-index @pattern)) + + edge @pattern.pattern_index -> @list.input + edge @pattern.input -> @pattern.pattern_index + attr @pattern.pattern_index "push" = (child-index @pattern) +} + +(parameters + (_) @param) @params +{ + attr @param.param_index "push" = (child-index @param) + edge @param.param_index -> @params.before_scope + edge @params.after_scope -> @param.input + edge @param.param_name -> @params.before_scope +} + +(return_statement (_) @expr) @stmt +{ + edge @stmt::function_returns -> @expr.output +} + +(class_definition + name: (identifier) @name) @class +{ + attr @name "definiens" = @class + attr @name "syntax_type" = "class" + edge @class.parent_scope -> @class::class_parent_scope + edge @class.parent_scope -> @class::local_scope + edge @class.after_scope -> @name + edge @name -> @class.call + edge @name -> @class.dot + edge @class.dot -> @class.members + edge @class.call -> @class.call_drop + edge @class.call_drop -> @class.self_scope + edge @class.self_scope -> @class.super_scope + edge @class.self_scope -> @class.self_dot + edge @class.self_dot -> @class.members + edge @class.members -> @class.member_attrs + attr @class.call "pop" = "()", "pop-scope" + attr @class.call_drop "drop" + attr @class.dot "pop" = "." + attr @class.self_dot "pop" = "." + attr @name "pop", "definition" + attr @class.member_attrs "push" = "." + attr @class.self_scope "endpoint" + let @class::super_scope = @class.super_scope + let @class::class_parent_scope = @class.parent_scope + let @class::class_self_scope = @class.call_drop + let @class::class_member_attr_scope = @class.member_attrs +} + +(class_definition + body: (block + (_) @last_stmt .) @body) @class +{ + edge @class.members -> @last_stmt.after_scope +} + +(class_definition + superclasses: (argument_list + (_) @superclass)) @class +{ + edge @class.super_scope -> @superclass.output +} + +(decorated_definition + definition: (_) @def) @stmt +{ + edge @def.before_scope -> @stmt.before_scope + edge @stmt.after_scope -> @def.after_scope +} + +(case_clause + pattern: (_) @pattern + consequence: (_) @consequence) @clause +{ + edge @consequence.before_scope -> @pattern.new_bindings + edge @consequence.before_scope -> @clause.before_scope + edge @clause.after_scope -> @consequence.after_scope +} + +;------------- +; Expressions +;------------- + +(call + function: (_) @fn + arguments: (argument_list) @args) @call +{ + edge @call.output -> @call.output_args + edge @call.output_args -> @fn.output + attr @call.output_args "push" = "()", "push-scope" = @args +} + +(call + function: (attribute + object: (_) @receiver) + arguments: (argument_list + (expression) @arg) @args) +{ + edge @args -> @arg.arg_index + edge @receiver -> @receiver.arg_index + + attr @receiver.arg_index "pop" = "0" + edge @receiver.arg_index -> @receiver.output + + attr @arg.arg_index "pop" = (+ 1 (child-index @arg)) + edge @arg.arg_index -> @arg.output +} + +(call + arguments: (argument_list + (keyword_argument + name: (identifier) @name + value: (_) @val) @arg) @args) @call +{ + edge @args -> @arg.arg_name + attr @arg.arg_name "pop" = @name + edge @arg.arg_name -> @val.output +} + +(argument_list + (expression) @arg) @args +{ + edge @args -> @arg.arg_index + attr @arg.arg_index "pop" = (child-index @arg) + edge @arg.arg_index -> @arg.output +} + +( + (call + function: (identifier) @fn-name) @call + (#eq? @fn-name "super") +) +{ + edge @call.output -> @call::super_scope +} + +[ + (tuple (_) @element) + (expression_list (_) @element) +] @tuple +{ + edge @tuple.output -> @element.el_index + attr @element.el_index "pop" = (child-index @element) + edge @element.el_index -> @element.output + + edge @tuple.new_bindings -> @element.new_bindings +} + +(attribute + object: (_) @object + attribute: (identifier) @name) @expr +{ + edge @expr.output -> @name.output + edge @name.output -> @expr.output_dot + edge @expr.output_dot -> @object.output + edge @object.input -> @expr.input_dot + edge @expr.input_dot -> @name.input + edge @name.input -> @expr.input + attr @expr.output_dot "push" = "." + attr @expr.input_dot "pop" = "." + attr @name.input "pop" + attr @name.output "push" +} + +(pattern/attribute + attribute: (identifier) @name) +{ + attr @name.input "definition" +} + +(primary_expression/attribute + attribute: (identifier) @name) +{ + attr @name.output "reference" +} + +(primary_expression/identifier) @id +{ + edge @id.output -> @id::local_scope + edge @id.output -> @id::class_parent_scope + edge @id::local_scope -> @id.input + attr @id.input "pop" + attr @id.output "push", "reference" + + attr @id.new_binding_pop "pop", "definition" + edge @id.new_bindings -> @id.new_binding_pop +} + +(pattern/identifier) @id +{ + edge @id.output -> @id::local_scope + edge @id.output -> @id::class_parent_scope + edge @id::local_scope -> @id.input, "precedence" = 1 + attr @id.input "pop", "definition" + attr @id.output "push" + + attr @id.new_binding_pop "pop", "definition" + edge @id.new_bindings -> @id.new_binding_pop +} + +(as_pattern + (expression) @value + alias: (as_pattern_target (primary_expression/identifier) @id)) @as_pattern +{ + edge @id.output -> @id::local_scope + edge @id.output -> @id::class_parent_scope + edge @id::local_scope -> @id.input, "precedence" = 1 + attr @id.input "pop", "definition" + attr @id.output "push" + + edge @as_pattern.new_bindings -> @value.new_bindings + edge @as_pattern.new_bindings -> @id.new_bindings +} + +(list) @list +{ + edge @list.output -> @list.called + edge @list.called -> @list::global_dot + attr @list.called "push" = "list" +} + +(list (_) @el) @list +{ + edge @list.new_bindings -> @el.new_bindings +} + +(dictionary (pair) @pair) @dict +{ + edge @dict.new_bindings -> @pair.new_bindings +} + +(pair + value: (_) @value) @pair +{ + edge @pair.new_bindings -> @value.new_bindings +} + +(set (_) @el) @set +{ + edge @set.new_bindings -> @el.new_bindings +} + +(list_splat (_) @splatted) @splat +{ +attr @splat.new_bindings_pop "pop" = @splatted, "definition" +edge @splat.new_bindings -> @splat.new_bindings_pop +} + +(binary_operator + (_) @left + (_) @right) @binop +{ + edge @binop.new_bindings -> @left.new_bindings + edge @binop.new_bindings -> @right.new_bindings +} + +(case_pattern (_) @expr) @pat +{ + edge @pat.new_bindings -> @expr.new_bindings +} diff --git a/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg new file mode 100644 index 000000000..5fb407ae8 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg @@ -0,0 +1,1377 @@ +;; -*- coding: utf-8 -*- +;; ------------------------------------------------------------------------------------------------ +;; Copyright © 2023, stack-graphs authors. +;; Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +;; Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +;; ------------------------------------------------------------------------------------------------ + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Stack graphs definition for Python +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Global Variables +;; ^^^^^^^^^^^^^^^^ + +global FILE_PATH +global ROOT_PATH = "" +global ROOT_NODE +global JUMP_TO_SCOPE_NODE + +;; Attribute Shorthands +;; ^^^^^^^^^^^^^^^^^^^^ + +attribute node_definition = node => type = "pop_symbol", node_symbol = node, is_definition +attribute node_reference = node => type = "push_symbol", node_symbol = node, is_reference +attribute pop_node = node => type = "pop_symbol", node_symbol = node +attribute pop_scoped_node = node => type = "pop_scoped_symbol", node_symbol = node +attribute pop_scoped_symbol = symbol => type = "pop_scoped_symbol", symbol = symbol +attribute pop_symbol = symbol => type = "pop_symbol", symbol = symbol +attribute push_node = node => type = "push_symbol", node_symbol = node +attribute push_scoped_node = node => type = "push_scoped_symbol", node_symbol = node +attribute push_scoped_symbol = symbol => type = "push_scoped_symbol", symbol = symbol +attribute push_symbol = symbol => type = "push_symbol", symbol = symbol +attribute scoped_node_definition = node => type = "pop_scoped_symbol", node_symbol = node, is_definition +attribute scoped_node_reference = node => type = "push_scoped_symbol", node_symbol = node, is_reference +attribute symbol_definition = symbol => type = "pop_symbol", symbol = symbol, is_definition +attribute symbol_reference = symbol => type = "push_symbol", symbol = symbol, is_reference + +attribute node_symbol = node => symbol = (source-text node), source_node = node + +;; Nodes +;; ^^^^^ + +(module) @node { + node @node.after_scope + node @node.before_scope +} + +[ + ; _statement + ; _simple_statement + (future_import_statement) + (import_statement) + (import_from_statement) + (print_statement) + (assert_statement) + (expression_statement) + (return_statement) + (delete_statement) + (raise_statement) + (pass_statement) + (break_statement) + (continue_statement) + (global_statement) + (nonlocal_statement) + (exec_statement) + (type_alias_statement) + ; _compound_statement + (if_statement) + (for_statement) + (while_statement) + (try_statement) + (with_statement) + (function_definition) + (class_definition) + (decorated_definition) + (match_statement) + ; block + (block) + ; statement clauses + (if_clause) + (elif_clause) + (else_clause) + (except_group_clause) + (except_clause) + (finally_clause) + (with_clause) + (case_clause) +] @node { + node @node.after_scope + node @node.before_scope +} + +[ + (parameters) + (lambda_parameters) +] @node { + node @node.after_scope + node @node.before_scope +} + +[ + (identifier) +] @node { + node @node.def + node @node.def_dot + node @node.ref + node @node.ref_dot +} + +[ + (dotted_name) + (aliased_import) + (relative_import) + (wildcard_import) + (import_prefix) +] @node { + node @node.after_scope + node @node.before_scope + node @node.def + node @node.ref +} + +[ + ; expressions + (comparison_operator) + (not_operator) + (boolean_operator) + (lambda) + ;(primary_expression) ; unfolded below + (conditional_expression) + (named_expression) + (as_pattern) + ; primary_expression + (await) + (binary_operator) + (identifier) + ;(keyword_identifier) ; invalid query pattern? + (string) + (concatenated_string) + (integer) + (float) + (true) + (false) + (none) + (unary_operator) + (attribute) + (subscript) + (call) + (list) + (list_comprehension) + (dictionary) + (dictionary_comprehension) + (set) + (set_comprehension) + (tuple) + (pair) + (parenthesized_expression) + (generator_expression) + (ellipsis) + (list_splat) + + ; expression list + (expression_list) + + ; pattern + (pattern/identifier) + ;(keyword_identifier) ; invalid query pattern? + ;(subscript) + ;(attribute) + (list_splat_pattern) + (tuple_pattern) + (list_pattern) + ; _simple_patterns + (class_pattern) + (splat_pattern) + (union_pattern) + ;(list_pattern) ; already in pattern + ;(tuple_pattern) ; already in pattern + (dict_pattern) + ;(string) ; already in primary_expression + ;(concatenated_string) ; already in primary_expression + ;(true) ; already in primary_expression + ;(false) ; already in primary_expression + ;(none) ; already in primary_expression + ;(integer) ; already in primary_expression + ;(float) ; already in primary_expression + (complex_pattern) + (dotted_name) + ; _as_attern + (as_pattern) + ; keyword pattern + (keyword_pattern) + ; case pattern + (case_pattern) + ; with item + (with_item) + + ; pattern list + (pattern_list) + + ; parameter + ;(identifier) ; already in expressions + (typed_parameter) + (default_parameter) + (typed_default_parameter) + ;(list_splat_pattern) ; already in patterns + ;(tuple_pattern) ; already in patterns + (keyword_separator) + (positional_separator) + (dictionary_splat_pattern) + + ; parameters + (parameters) +] @node { + node @node.input + node @node.new_bindings + node @node.output +} + +(comment) @node { + node @node.after_scope + node @node.before_scope + node @node.def + node @node.def_dot + node @node.input + node @node.new_bindings + node @node.output + node @node.ref + node @node.ref_dot +} + +;; Inherited Variables +;; ^^^^^^^^^^^^^^^^^^^ + +inherit .bottom +inherit .class_member_attr_scope +inherit .class_parent_scope +inherit .class_self_scope +inherit .class_super_scope +inherit .function_returns +inherit .global +inherit .global_dot +inherit .grandparent_module +inherit .local_scope +inherit .parent_module + +;; +;; # # +;; ## ## #### ##### # # # ###### #### +;; # # # # # # # # # # # # # +;; # # # # # # # # # # ##### #### +;; # # # # # # # # # # # +;; # # # # # # # # # # # # +;; # # #### ##### #### ###### ###### #### +;; +;; Modules + +(module) @mod +{ + node mod_file_def + node mod_file_ref + + var module_def = mod_file_def + + node parent_module_def_node + var parent_module_def = parent_module_def_node + + var module_ref = mod_file_ref + + node parent_module_ref_node + var parent_module_ref = parent_module_ref_node + + node grandparent_module_ref_node + var grandparent_module_ref = grandparent_module_ref_node + + ; get the file path relative to the root path + let rel_path = (replace FILE_PATH ROOT_PATH "") + scan rel_path { + "([^/]+)/" + { + node def_dot + attr (def_dot) pop_symbol = "." + node next_def + ; + edge module_def -> def_dot + edge def_dot -> next_def + ; + attr (module_def) pop_symbol = $1 + ; + set parent_module_def = module_def + set module_def = next_def + + node ref_dot + attr (ref_dot) push_symbol = "." + node next_ref + ; + edge next_ref -> ref_dot + edge ref_dot -> module_ref + ; + attr (module_ref) push_symbol = $1 + ; + set grandparent_module_ref = parent_module_ref + set parent_module_ref = module_ref + set module_ref = next_ref + } + + "__init__\.py$" + { + attr (parent_module_def) is_definition, source_node = @mod, empty_source_span + } + + "([^/]+)\.py$" + { + node def_dot + attr (def_dot) pop_symbol = "." + node next_def + ; + edge module_def -> def_dot + edge def_dot -> next_def + ; + attr (module_def) pop_symbol = $1, is_definition, source_node = @mod, empty_source_span + ; + set module_def = next_def + } + } + + edge ROOT_NODE -> mod_file_def + edge mod_file_ref -> ROOT_NODE + edge module_def -> @mod.after_scope + + node global + node global_dot + + edge @mod.before_scope -> global_dot + edge global -> ROOT_NODE + attr (global) push_symbol = "" + + edge global_dot -> global + attr (global_dot) push_symbol = "." + + let @mod.parent_module = parent_module_ref + let @mod.grandparent_module = grandparent_module_ref + let @mod.bottom = @mod.after_scope + let @mod.global = global + let @mod.global_dot = global_dot + + ;; add a dummy nodes for inherited variables + node @mod.class_member_attr_scope + node @mod.class_parent_scope + node @mod.class_self_scope + node @mod.class_super_scope + node @mod.function_returns + node @mod.local_scope +} + +;; +;; ### +;; # # # ##### #### ##### ##### #### +;; # ## ## # # # # # # # # +;; # # ## # # # # # # # # #### +;; # # # ##### # # ##### # # +;; # # # # # # # # # # # +;; ### # # # #### # # # #### +;; +;; Imports + +;; Import References +;; ^^^^^^^^^^^^^^^^^ + +;;;; Dotted Names +;; +;; (dotted_name).ref node to connect to to use the reference +;; (dotted_name).before_scope node to connect from to ensure the reference resolves + +;; all names are references +[ + (import_statement name: (dotted_name (identifier) @name)) + (future_import_statement name: (dotted_name (identifier) @name)) + (import_from_statement name: (dotted_name (identifier) @name)) + (import_statement name: (aliased_import name: (dotted_name (identifier) @name))) + (future_import_statement name: (aliased_import name: (dotted_name (identifier) @name))) + (import_from_statement name: (aliased_import name: (dotted_name (identifier) @name))) + (import_from_statement module_name: (dotted_name (identifier) @name)) + (import_from_statement module_name: (relative_import (dotted_name (identifier) @name))) +] { + attr (@name.ref) node_reference = @name +} + +;; references are chained +[ + (import_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (future_import_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (import_from_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (import_statement name: (aliased_import name: (dotted_name (identifier) @left . (identifier) @right))) + (future_import_statement name: (aliased_import name: (dotted_name (identifier) @left . (identifier) @right))) + (import_from_statement name: (aliased_import name: (dotted_name (identifier) @left . (identifier) @right))) + (import_from_statement module_name: (dotted_name (identifier) @left . (identifier) @right)) + (import_from_statement module_name: (relative_import (dotted_name (identifier) @left . (identifier) @right))) +] { + node push_dot + attr (push_dot) push_symbol = "." + + edge @right.ref -> push_dot + edge push_dot -> @left.ref +} + +;; lookup first reference +[ + (import_statement name: (dotted_name . (identifier) @first) @dotted) + (future_import_statement name: (dotted_name . (identifier) @first) @dotted) + (import_from_statement name: (dotted_name . (identifier) @first) @dotted) + (import_statement name: (aliased_import name: (dotted_name . (identifier) @first) @dotted)) + (future_import_statement name: (aliased_import name: (dotted_name . (identifier) @first) @dotted)) + (import_from_statement name: (aliased_import name: (dotted_name . (identifier) @first) @dotted)) + (import_from_statement module_name: (dotted_name . (identifier) @first) @dotted) + (import_from_statement module_name: (relative_import (dotted_name . (identifier) @first) @dotted)) +] { + edge @first.ref -> @dotted.before_scope +} + +;; expose last reference +[ + (import_statement name: (dotted_name (identifier) @last .) @dotted) + (future_import_statement name: (dotted_name (identifier) @last .) @dotted) + (import_from_statement name: (dotted_name (identifier) @last .) @dotted) + (import_statement name: (aliased_import name: (dotted_name (identifier) @last .) @dotted)) + (future_import_statement name: (aliased_import name: (dotted_name (identifier) @last .) @dotted)) + (import_from_statement name: (aliased_import name: (dotted_name (identifier) @last .) @dotted)) + (import_from_statement module_name: (dotted_name (identifier) @last .) @dotted) + (import_from_statement module_name: (relative_import (dotted_name (identifier) @last .) @dotted)) +] { + edge @dotted.ref -> @last.ref +} + +;;;; Aliased Import +;; +;; An aliased import behaves like its wrapped dotted_name as a reference +;; +;; (aliased_import).ref node to connect to, to use the reference +;; (aliased_import).before_scope node to connect from to ensure the reference resolves + +(aliased_import name: (dotted_name) @dotted) @aliased { + edge @aliased.ref -> @dotted.ref + edge @dotted.before_scope -> @aliased.before_scope +} + +;;;; Relative Import +;; +;; A relative import behaves like its wrapped dotted_name as a reference +;; +;; (relative_import).ref node to connect to, to use the reference +;; (relative_import).before_scope node to connect from to ensure the reference resolves + +(relative_import (import_prefix) . (dotted_name) @dotted) @relative { + edge @relative.ref -> @dotted.ref + + node push_dot + attr (push_dot) push_symbol = "." + + edge @dotted.before_scope -> push_dot + edge push_dot -> @relative.before_scope +} + +(relative_import (import_prefix) .) @relative { + edge @relative.ref -> @relative.before_scope +} + +;;;; Wildcard Import +;; +;; A wildcard import simply passes through +;; +;; (wildcard_import).ref node to connect to, to use the reference +;; (wildcard_import).before_scope node to connect from to ensure the reference resolves + +(wildcard_import) @wildcard { + edge @wildcard.ref -> @wildcard.before_scope +} + +;;;; Import from +;; +;; The imported references are resolved in the from module + +[ + (import_from_statement module_name: (_) @left name: (_) @right) + (import_from_statement module_name: (_) @left (wildcard_import) @right) +] { + node push_dot + attr (push_dot) push_symbol = "." + + edge @right.before_scope -> push_dot + edge push_dot -> @left.ref +} + +;;;; Non-relative Imports +;; +;; Non-relative imports are resolved in the root scope + +[ + (import_statement name: (_) @name) + (import_from_statement module_name: (dotted_name) @name) +] { + edge @name.before_scope -> ROOT_NODE +} + +;;;; Relative Imports +;; +;; Relative imports are resolved in scopes related to the current module + +;; . imports resolve in parent module scope +(import_from_statement module_name: (relative_import (import_prefix) @prefix (#eq? @prefix ".")) @relative) { + edge @relative.before_scope -> @prefix.parent_module +} + +;; .. imports resolve in grandparent module scope +(import_from_statement module_name: (relative_import (import_prefix) @prefix (#eq? @prefix "..")) @relative) { + edge @relative.before_scope -> @prefix.grandparent_module +} + +;;;; Future Imports +;; +;; We don't know the future, so we cannot connect references of future imports anywhere. Maybe one day? + +;; Import Definitions +;; ^^^^^^^^^^^^^^^^^^ + +;;;; Dotted Names & Aliased Imports +;; +;; (dotted_name).after_scope node to connect to, to expose the definition +;; (dotted_name).def node to connect from, to give definition content + +;; unaliased names and aliases are definitions +[ + (import_statement name: (dotted_name (identifier) @name)) + (future_import_statement name: (dotted_name (identifier) @name)) + (import_from_statement name: (dotted_name (identifier) @name)) + (import_statement name: (aliased_import alias: (identifier) @name)) + (future_import_statement name: (aliased_import alias: (identifier) @name)) + (import_from_statement name: (aliased_import alias: (identifier) @name)) +] { + attr (@name.def) node_definition = @name +} + +;; definitions are chained +[ + (import_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (future_import_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (import_from_statement name: (dotted_name (identifier) @left . (identifier) @right)) +] { + node pop_dot + attr (pop_dot) pop_symbol = "." + + edge @left.def -> pop_dot + edge pop_dot -> @right.def +} + +;; connect last definition +[ + (import_statement name: (dotted_name (identifier) @last .) @outer) + (future_import_statement name: (dotted_name (identifier) @last .) @outer) + (import_from_statement name: (dotted_name (identifier) @last .) @outer) + (import_statement name: (aliased_import alias: (identifier) @last ) @outer) + (future_import_statement name: (aliased_import alias: (identifier) @last ) @outer) + (import_from_statement name: (aliased_import alias: (identifier) @last ) @outer) +] { + edge @last.def -> @outer.def +} + +;; expose first definition +[ + (import_statement name: (dotted_name . (identifier) @first) @outer) + (future_import_statement name: (dotted_name . (identifier) @first) @outer) + (import_from_statement name: (dotted_name . (identifier) @first) @outer) + (import_statement name: (aliased_import alias: (identifier) @first ) @outer) + (future_import_statement name: (aliased_import alias: (identifier) @first ) @outer) + (import_from_statement name: (aliased_import alias: (identifier) @first ) @outer) +] { + edge @outer.after_scope -> @first.def +} + +;;;; Wildcard Import +;; +;; Wildcard imports simply pass through + +(wildcard_import) @wildcard { + edge @wildcard.after_scope -> @wildcard.def +} + +;;;; Import Definitions -> References +;; +;; The definitions introduced by imports are connected to the corresponding references + +[ + (import_statement name: (_) @name) + (future_import_statement name: (_) @name) + (import_from_statement name: (_) @name) + (import_from_statement (wildcard_import) @name) +] { + edge @name.def -> @name.ref +} + +;;;; Imports +;; +;; The definitions introduced by imports are visible after the import statement + +[ + (import_statement name: (_) @name) + (future_import_statement name: (_) @name) + (import_from_statement name: (_) @name) + (import_from_statement (wildcard_import) @name) +] @stmt { + edge @stmt.after_scope -> @name.after_scope + attr (@stmt.after_scope -> @name.after_scope) precedence = 1 +} + +;; +;; ###### +;; # # # #### #### # # #### +;; # # # # # # # # # # +;; ###### # # # # #### #### +;; # # # # # # # # # +;; # # # # # # # # # # # +;; ###### ###### #### #### # # #### +;; +;; Blocks + +[ + (module (_) @last_stmt .) + (block (_) @last_stmt .) +] @block +{ + edge @block.after_scope -> @last_stmt.after_scope +} + +[ + (module (_) @stmt1 . (_) @stmt2) + (block (_) @stmt1 . (_) @stmt2) +] +{ + edge @stmt2.before_scope -> @stmt1.after_scope +} + +[ + (module (_) @stmt) + (block (_) @stmt) +] +{ + edge @stmt.after_scope -> @stmt.before_scope + let @stmt.local_scope = @stmt.before_scope +} + +[ + (block . (_) @stmt) + (module . (_) @stmt) +] @block +{ + edge @stmt.before_scope -> @block.before_scope +} + +(function_definition (block) @block) +{ + edge @block.before_scope -> @block.local_scope +} + +[ + (while_statement (block) @block) + (if_statement (block) @block) + (with_statement (block) @block) + (try_statement (block) @block) + (for_statement (block) @block) + (_ [ + (else_clause (block) @block) + (elif_clause (block) @block) + (except_clause (block) @block) + (finally_clause (block) @block) + ]) +] @stmt +{ + edge @block.before_scope -> @block.local_scope + edge @stmt.after_scope -> @block.after_scope +} + +(match_statement body: (_) @block) @stmt +{ + let @block.local_scope = @block.before_scope + edge @block.before_scope -> @stmt.before_scope + edge @stmt.after_scope -> @block.after_scope +} + +[ + (for_statement) + (while_statement) +] @stmt +{ + edge @stmt.before_scope -> @stmt.after_scope +} + +;; +;; ##### +;; # # ##### ## ##### ###### # # ###### # # ##### #### +;; # # # # # # ## ## # ## # # # +;; ##### # # # # ##### # ## # ##### # # # # #### +;; # # ###### # # # # # # # # # # +;; # # # # # # # # # # # ## # # # +;; ##### # # # # ###### # # ###### # # # #### +;; +;; Statements + +;;;; Simple Statements + +(print_statement) {} + +(assert_statement) {} + +(expression_statement) {} + +(return_statement (_) @expr) @stmt +{ + edge @stmt.function_returns -> @expr.output +} + +(delete_statement) {} + +(raise_statement) {} + +(pass_statement) {} + +(break_statement) {} + +(continue_statement) {} + +(global_statement) {} + +(nonlocal_statement) {} + +(exec_statement) {} + +(type_alias_statement) {} + +;;;; Compound Statements + +(if_statement) {} + +(if_clause) {} + +(elif_clause) {} + +(else_clause) {} + +(for_statement) {} + +(while_statement) {} + +(try_statement) {} + +(except_group_clause) {} + +(except_clause) {} + +(finally_clause) {} + +(with_statement) {} + +(with_clause) {} + +[ + (function_definition + parameters: (_) @params + body: (_) @body + ) @func + (lambda + parameters: (_) @params + body: (_) @body + )@func +] { + node @func.call + node return_value + node drop_scope + + edge @func.call -> return_value + edge @body.before_scope -> @params.after_scope + edge @body.before_scope -> drop_scope + edge drop_scope -> @func.bottom + attr (drop_scope) type = "drop_scopes" + attr (@func.call) pop_scoped_symbol = "()" + edge @params.before_scope -> JUMP_TO_SCOPE_NODE + attr (return_value) is_exported + let @func.function_returns = return_value +} + +(function_definition + name: (identifier) @name + body: (_) @body +) @func { + attr (@name.def) node_definition = @name + attr (@name.def) definiens_node = @func + edge @func.after_scope -> @name.def + edge @name.def -> @func.call + + ; Prevent functions defined inside of method bodies from being treated like methods + let @body.class_self_scope = #null + let @body.class_member_attr_scope = #null +} + +; method definition +(class_definition + body: (block + (function_definition + parameters: + (parameters . (identifier) @first_param) + body: (block) @method_body + ) + ) +) +{ + edge @first_param.def -> @first_param.class_self_scope + edge @first_param.class_member_attr_scope -> @first_param.output + edge @first_param.output -> @method_body.after_scope + attr (@first_param.output) push_node = @first_param +} + +[ + (parameters (_) @param) @params + (lambda_parameters (_) @param) @params +] +{ + node @param.param_index + node @param.param_name + + attr (@param.param_index) push_symbol = (named-child-index @param) + edge @param.param_index -> @params.before_scope + edge @params.after_scope -> @param.input + edge @param.param_name -> @params.before_scope +} + + +(parameter/identifier) @param @name +{ + attr (@name.def) node_definition = @name + attr (@param.param_name) push_node = @param + edge @name.def -> @param.param_name + edge @name.def -> @param.param_index + edge @param.input -> @name.def +} + +[ + (parameter/default_parameter + name: (_) @name + value: (_) @value) @param + (parameter/typed_default_parameter + name: (_) @name + value: (_) @value) @param +] { + attr (@name.def) node_definition = @name + attr (@param.param_name) push_node = @name + edge @name.def -> @param.param_name + edge @name.def -> @param.param_index + edge @param.input -> @name.def + edge @name.def -> @value.output +} + +[ + (parameter/typed_parameter + . (_) @name) @param + (parameter/list_splat_pattern + (_) @name) @param + (parameter/dictionary_splat_pattern + (_) @name) @param +] { + attr (@name.def) node_definition = @name + attr (@param.param_name) push_node = @name + edge @name.def -> @param.param_name + edge @name.def -> @param.param_index + edge @param.input -> @name.def +} + +(class_definition) @class { + node @class.call_drop + node @class.member_attrs + node @class.members + node @class.super_scope +} + +(class_definition + name: (identifier) @name) { + attr (@name.def) node_definition = @name +} + +(class_definition + name: (identifier) @name) @class +{ + node call + node self_dot + node self_scope + node members_dot + + attr (@name.def) definiens_node = @class + attr (@name.def) syntax_type = "class" + edge @class.after_scope -> @name.def + edge @name.def -> call + edge @name.def -> members_dot + edge members_dot -> @class.members + edge call -> @class.call_drop + edge @class.call_drop -> self_scope + edge self_scope -> @class.super_scope + edge self_scope -> self_dot + edge self_dot -> @class.members + edge @class.members -> @class.member_attrs + attr (call) pop_scoped_symbol = "()" + attr (@class.call_drop) type = "drop_scopes" + attr (members_dot) pop_symbol = "." + attr (self_dot) pop_symbol = "." + attr (@class.member_attrs) push_symbol = "." + attr (self_scope) is_exported +} + +(class_definition + body: (_) @body) @class +{ + let @body.class_member_attr_scope = @class.member_attrs + node @body.class_parent_scope + edge @body.class_parent_scope -> @class.class_parent_scope + edge @body.class_parent_scope -> @class.local_scope + let @body.class_self_scope = @class.call_drop + let @body.class_super_scope = @class.super_scope +} + +(class_definition + body: (block + (_) @last_stmt .)) @class +{ + edge @class.members -> @last_stmt.after_scope +} + +(class_definition + superclasses: (argument_list + (_) @superclass)) @class +{ + edge @class.super_scope -> @superclass.output +} + +(decorated_definition + definition: (_) @def) @stmt +{ + edge @def.before_scope -> @stmt.before_scope + edge @stmt.after_scope -> @def.after_scope +} + +(match_statement) {} + +(case_clause + (case_pattern) @pattern + consequence: (_) @consequence) +{ + edge @consequence.before_scope -> @pattern.new_bindings +} + +(case_clause + consequence: (_) @consequence) @clause +{ + edge @consequence.before_scope -> @clause.before_scope + edge @clause.after_scope -> @consequence.after_scope +} + +;; +;; ####### +;; # # # ##### ##### ###### #### #### # #### # # #### +;; # # # # # # # # # # # # # ## # # +;; ##### ## # # # # ##### #### #### # # # # # # #### +;; # ## ##### ##### # # # # # # # # # # +;; # # # # # # # # # # # # # # # ## # # +;; ####### # # # # # ###### #### #### # #### # # #### +;; +;; Expressions + +(lambda + body: (_) @body +)@lam { + ;; TODO Unify .before_scope, .local_scope, and .input to simplify + ;; uniform treatment of lambdas and function definitions. + node @body.before_scope + let @body.local_scope = @body.before_scope + edge @body.input -> @body.before_scope + + edge @lam.output -> @lam.call +} + +(conditional_expression) {} + +(named_expression) {} + +(as_pattern) {} + +(await) {} + +(binary_operator + (_) @left + (_) @right) @binop +{ + edge @binop.new_bindings -> @left.new_bindings + edge @binop.new_bindings -> @right.new_bindings +} + +(primary_expression/identifier) @name { + attr (@name.ref) node_reference = @name +} + +(primary_expression/identifier) @name +{ + edge @name.output -> @name.local_scope + edge @name.output -> @name.class_parent_scope + edge @name.local_scope -> @name.input + attr (@name.input) pop_node = @name + attr (@name.output) node_reference = @name + + edge @name.new_bindings -> @name.def +} + +(string) {} + +(concatenated_string) {} + +(integer) {} + +(float) {} + +(true) {} + +(false) {} + +(none) {} + +(unary_operator) {} + +(attribute + object: (_) @object + attribute: (identifier) @name) @expr +{ + node input_dot + node output_dot + + edge @expr.output -> @name.output + edge @name.output -> output_dot + edge output_dot -> @object.output + edge @object.input -> input_dot + edge input_dot -> @name.input + edge @name.input -> @expr.input + attr (output_dot) push_symbol = "." + attr (input_dot) pop_symbol = "." + attr (@name.input) pop_node = @name + attr (@name.output) push_node = @name +} + +(primary_expression/attribute + attribute: (identifier) @name) +{ + attr (@name.output) is_reference +} + +(subscript) {} + +(call + function: (_) @fn + arguments: (argument_list) @args) @call +{ + node output_args + + edge @call.output -> output_args + edge output_args -> @fn.output + attr (output_args) push_scoped_symbol = "()", scope = @args.args +} + +(call + function: (attribute + object: (_) @receiver) + arguments: (argument_list + (expression) @arg) @args) +{ + node receiver_arg_index + attr (receiver_arg_index) pop_symbol = "0" + edge @args.args -> receiver_arg_index + edge receiver_arg_index -> @receiver.output + + ;; FIXME the arguments will also exist with their unshifted indices because of the general + ;; rule below! + node arg_index + attr (arg_index) pop_symbol = (plus 1 (named-child-index @arg)) + edge arg_index -> @arg.output +} + +(call + arguments: (argument_list + (keyword_argument + name: (identifier) @name + value: (_) @val)) @args) +{ + node arg_name + edge @args.args -> arg_name + attr (arg_name) pop_node = @name + edge arg_name -> @val.output +} + +(argument_list) @args { + node @args.args + attr (@args.args) is_exported +} + +(argument_list + (expression) @arg) @args +{ + node arg_index + edge @args.args -> arg_index + attr (arg_index) pop_symbol = (named-child-index @arg) + edge arg_index -> @arg.output +} + +( + (call + function: (identifier) @_fn_name) @call + (#eq? @_fn_name "super") +) +{ + edge @call.output -> @call.class_super_scope +} + +(list) @list +{ + node called + + edge @list.output -> called + edge called -> @list.global_dot + attr (called) push_symbol = "list" +} + +(list (_) @el) @list +{ + edge @list.new_bindings -> @el.new_bindings +} + +(list_comprehension) {} + +(dictionary (pair) @pair) @dict +{ + edge @dict.new_bindings -> @pair.new_bindings +} + +(pair + value: (_) @value) @pair +{ + edge @pair.new_bindings -> @value.new_bindings +} + +(dictionary_comprehension) {} + +(set (_) @el) @set +{ + edge @set.new_bindings -> @el.new_bindings +} + +(set_comprehension) {} + +[ + (tuple (_) @element) + (expression_list (_) @element) +] @tuple +{ + node el_index + + edge @tuple.output -> el_index + attr (el_index) pop_symbol = (named-child-index @element) + edge el_index -> @element.output + + edge @tuple.new_bindings -> @element.new_bindings +} + +(parenthesized_expression) {} + +(generator_expression) {} + +(ellipsis) {} + +(list_splat (_) @splatted) @splat +{ + edge @splat.new_bindings -> @splatted.new_bindings +} + +;; +;; ###### +;; # # ## ##### ##### ###### ##### # # #### +;; # # # # # # # # # ## # # +;; ###### # # # # ##### # # # # # #### +;; # ###### # # # ##### # # # # +;; # # # # # # # # # ## # # +;; # # # # # ###### # # # # #### +;; +;; Patterns + + +[ + (pattern/identifier) @name @pattern + ; identifiers in patterns + (as_pattern (identifier) @name .) @pattern + (keyword_pattern (identifier) @name) @pattern + (splat_pattern (identifier) @name) @pattern + ; every context where _simple_pattern is used, because matching + ; pattern/dotted_name does not work + (case_pattern (dotted_name . (_) @name .) @pattern) + (keyword_pattern (dotted_name . (_) @name .) @pattern) + (union_pattern (dotted_name . (_) @name .) @pattern) +] { + attr (@name.def) node_definition = @name + edge @pattern.output -> @pattern.local_scope + edge @pattern.output -> @pattern.class_parent_scope + edge @pattern.local_scope -> @pattern.input + attr (@pattern.local_scope -> @pattern.input) precedence = 1 + attr (@pattern.input) node_definition = @name + attr (@pattern.output) push_node = @name + + edge @pattern.new_bindings -> @name.def +} + +(pattern/subscript) {} + +(pattern/attribute + attribute: (identifier) @name) +{ + attr (@name.input) is_definition +} + + +(list_splat_pattern (_) @pattern) @list { + edge @list.new_bindings -> @pattern.new_bindings +} + +[ + (pattern_list (_) @pattern) + (tuple_pattern (_) @pattern) + (list_pattern (_) @pattern) +] @list +{ + node pattern_index + + let statement_scope = @list.local_scope + node @pattern.local_scope + edge statement_scope -> @pattern.local_scope + attr (statement_scope -> @pattern.local_scope) precedence = (plus 1 (named-child-index @pattern)) + + edge pattern_index -> @list.input + edge @pattern.input -> pattern_index + attr (pattern_index) push_symbol = (named-child-index @pattern) +} + +(class_pattern (case_pattern) @pattern) @class { + edge @class.new_bindings -> @pattern.new_bindings +} + +(splat_pattern (_) @pattern) @splat { + edge @splat.new_bindings -> @pattern.new_bindings +} + +(union_pattern (_) @pattern) @union { + edge @union.new_bindings -> @pattern.new_bindings +} + +(dict_pattern (_) @pattern) @dict { + edge @dict.new_bindings -> @pattern.new_bindings +} + +(complex_pattern) {} + +(as_pattern + (expression) @value + alias: (as_pattern_target (identifier) @name)) @as_pattern +{ + attr (@name.local_scope -> @name.input) precedence = 1 + attr (@name.input) is_definition + + edge @as_pattern.new_bindings -> @value.new_bindings + edge @as_pattern.new_bindings -> @name.new_bindings +} + +(keyword_pattern) {} + +(case_pattern (_) @pattern) @case +{ + edge @case.new_bindings -> @pattern.new_bindings +} + +[ + (assignment + left: (_) @pattern + right: (_) @value) + (with_item + value: + (as_pattern + (_) @value + alias: (as_pattern_target (_) @pattern))) +] +{ + edge @pattern.input -> @value.output +} + +;; +;; ##### ####### +;; # # # # # # ##### ## # # # # # ##### ###### #### +;; # # # ## # # # # # # # # # # # # # +;; ##### # # # # # # # ## # # # # ##### #### +;; # # # # # # ###### ## # # ##### # # +;; # # # # ## # # # # # # # # # # # +;; ##### # # # # # # # # # # # ###### #### +;; +;; Syntax Types + +;; +;; BEGIN BIG GNARLY DISJUNCTION +;; +;; The following pair of rules is intended to capture the following behavior: +;; +;; If a function definition is used to define a method, by being inside a class +;; definition, then we make its syntax type `method`. Otherwise, we make it's +;; syntax type `function`. Unfortunately, because of the limitations on negation +;; and binding in tree sitter queries, we cannot negate `class_definition` or +;; similar things directly. Instead, we have to manually push the negation down +;; to form the finite disjunction it corresponds to. +;; + +[ + (class_definition (block (decorated_definition (function_definition name: (_)@name)))) + (class_definition (block (function_definition name: (_)@name))) +] +{ + attr (@name.def) syntax_type = "method" +} + +[ + (module (decorated_definition (function_definition name: (_)@name))) + (module (function_definition name: (_)@name)) + + (if_statement (block (decorated_definition (function_definition name: (_)@name)))) + (if_statement (block (function_definition name: (_)@name))) + + (elif_clause (block (decorated_definition (function_definition name: (_)@name)))) + (elif_clause (block (function_definition name: (_)@name))) + + (else_clause (block (decorated_definition (function_definition name: (_)@name)))) + (else_clause (block (function_definition name: (_)@name))) + + (case_clause (block (decorated_definition (function_definition name: (_)@name)))) + (case_clause (block (function_definition name: (_)@name))) + + (for_statement (block (decorated_definition (function_definition name: (_)@name)))) + (for_statement (block (function_definition name: (_)@name))) + + (while_statement (block (decorated_definition (function_definition name: (_)@name)))) + (while_statement (block (function_definition name: (_)@name))) + + (try_statement (block (decorated_definition (function_definition name: (_)@name)))) + (try_statement (block (function_definition name: (_)@name))) + + (except_clause (block (decorated_definition (function_definition name: (_)@name)))) + (except_clause (block (function_definition name: (_)@name))) + + (finally_clause (block (decorated_definition (function_definition name: (_)@name)))) + (finally_clause (block (function_definition name: (_)@name))) + + (with_statement (block (decorated_definition (function_definition name: (_)@name)))) + (with_statement (block (function_definition name: (_)@name))) + + (function_definition (block (decorated_definition (function_definition name: (_)@name)))) + (function_definition (block (function_definition name: (_)@name))) +] +{ + attr (@name.def) syntax_type = "function" +} + +;; +;; END BIG GNARLY DISJUNCTION +;; diff --git a/languages/tree-sitter-stack-graphs-python/test/aliased_imports.py b/languages/tree-sitter-stack-graphs-python/test/aliased_imports.py new file mode 100644 index 000000000..0373a33be --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/aliased_imports.py @@ -0,0 +1,31 @@ +#------ path: foo.py ------# + +# module +class A: + a = 1 + +class B: + class C: + class D: + d = 2 + +#------ path: main.py ---# + +from foo import A as X, B.C.D as Y +import foo as f + +print X.a, Y.d +# ^ defined: 5 +# ^ defined: 10 + +print A, B.C +# ^ defined: +# ^ defined: +# ^ defined: + +print f.B +# ^ defined: 3, 15 +# ^ defined: 7 + +print foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/attributes.py b/languages/tree-sitter-stack-graphs-python/test/attributes.py new file mode 100644 index 000000000..6dd3d0e48 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/attributes.py @@ -0,0 +1,14 @@ +a = 1 + +a.b = 2 + +a.c.d = 5 + +print a.b +# ^ defined: 1 +# ^ defined: 3 + +print a.c, a.c.d +# ^ defined: 1 +# ^ defined: +# ^ defined: 5 diff --git a/languages/tree-sitter-stack-graphs-python/test/blocks.py b/languages/tree-sitter-stack-graphs-python/test/blocks.py new file mode 100644 index 000000000..462890baf --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/blocks.py @@ -0,0 +1,21 @@ +def f(): + if a: + b = 1 + else: + c = 2 + + print b, c + # ^ defined: 3 + # ^ defined: 5 + +class G: + if d: + e = 1 + + print e + # ^ defined: 13 + +print b, c, e +# ^ defined: +# ^ defined: +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/chained_functions.py b/languages/tree-sitter-stack-graphs-python/test/chained_functions.py new file mode 100644 index 000000000..ef1dac21d --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/chained_functions.py @@ -0,0 +1,23 @@ +class A: + w = 1 + x = 2 + y = 3 + z = 4 + +def get_a(): + return A +def get_b(): + return get_a() +def get_c(): + return get_b() +def get_d(): + return get_c() +def get_e(): + return get_d() +def get_f(): + return get_e() + +g = get_f(A) +print g.x, g.y +# ^ defined: 3 +# ^ defined: 4 diff --git a/languages/tree-sitter-stack-graphs-python/test/chained_methods.py.skip b/languages/tree-sitter-stack-graphs-python/test/chained_methods.py.skip new file mode 100644 index 000000000..9c4f53115 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/chained_methods.py.skip @@ -0,0 +1,27 @@ +class Builder: + def set_a(self, a): + self.a = a + return self + + def set_b(self, b): + self.b = b + return self + + def set_c(self, c): + self.c = c + return self + + def set_d(self, d): + self.d = d + return self + + def set_e(self, e): + self.d = d + return self + +Builder().set_a('a1').set_b('b2').set_c('c3').set_d('d4').set_e('e4') +# ^ defined: 2 +# ^ defined: 6 +# ^ defined: 10 +# ^ defined: 14 +# ^ defined: 18 diff --git a/languages/tree-sitter-stack-graphs-python/test/class_members.py b/languages/tree-sitter-stack-graphs-python/test/class_members.py new file mode 100644 index 000000000..c67ace992 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/class_members.py @@ -0,0 +1,23 @@ +a = 1 + +class B: + c = a + # ^ defined: 1 + + def d(self): + return self.c + + class E: + f = a + # ^ defined: 1 + +print B.c +# ^ defined: 3 +# ^ defined: 1, 4 + +print B.d(1) +# ^ defined: 7 + +print B.a, E.a +# ^ defined: +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/decorators.py b/languages/tree-sitter-stack-graphs-python/test/decorators.py new file mode 100644 index 000000000..b55f44b2f --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/decorators.py @@ -0,0 +1,10 @@ +from a import deprecated +from b import ignore_warnings + +class A: + @deprecated + # ^ defined: 1 + @ignore_warnings.all + # ^ defined: 2 + def b(self): + pass diff --git a/languages/tree-sitter-stack-graphs-python/test/exceptions.py b/languages/tree-sitter-stack-graphs-python/test/exceptions.py new file mode 100644 index 000000000..b6b2e57e1 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/exceptions.py @@ -0,0 +1,5 @@ +try: + print() +except Exception as e: + x = e + # ^ defined: 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/functions.py b/languages/tree-sitter-stack-graphs-python/test/functions.py new file mode 100644 index 000000000..31a1d6782 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/functions.py @@ -0,0 +1,31 @@ +import a.x.y +import b.x.y + +def get_x(value): + return value.x + # ^ defined: 4 + +print get_x(a).y +# ^ defined: 1 + +print get_x(b).y +# ^ defined: 2 + +def get_a(): + return a + +print get_a(b).x +# ^ defined: 1 + +print get_x(foo=1, value=a).y +# ^ defined: 1 + +def foo(w: int, x, y=1, z: int=4, *args, **dict): + local = x +# ^ defined: 23 + print(args, w, z) +# ^ defined: 23 +# ^ defined: 23 +# ^ defined: 23 + return y +# ^ defined: 23 diff --git a/languages/tree-sitter-stack-graphs-python/test/imported_functions.py b/languages/tree-sitter-stack-graphs-python/test/imported_functions.py new file mode 100644 index 000000000..495697740 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imported_functions.py @@ -0,0 +1,17 @@ +#------ path: a.py ------# + +def foo(x): + return x + +#------ path: b.py ------# + +class A: + bar = 1 + +#------ path: main.py ---------# + +from a import * +from b import * + +foo(A).bar +# ^ defined: 9 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports.py b/languages/tree-sitter-stack-graphs-python/test/imports.py new file mode 100644 index 000000000..96d561813 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports.py @@ -0,0 +1,31 @@ +#------ path: one/two.py -----------# + +# module +import a.b.c + +d = 1 + +e = a.b + +#------ path: three/__init__.py ---# + +# module +f = 3 + +#------ path: main.py -------------# + +from one.two import d, e.c +# ^ defined: 3 +# ^ defined: 6 +# ^ defined: 4, 8 + +import three +# ^ defined: 12 + +print(d, e.c) +# ^ defined: 6, 17 +# ^ defined: 4, 17 + +print three.f +# ^ defined: 12, 22 +# ^ defined: 13 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_submodule.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_submodule.py new file mode 100644 index 000000000..ea51f9450 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_submodule.py @@ -0,0 +1,14 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +from foo import bar + +bar.BAR +# ^ defined: 7, 3 +# ^ defined: 3 + +foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value.py new file mode 100644 index 000000000..1eb97ffe2 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value.py @@ -0,0 +1,13 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +from foo import FOO + +FOO +# ^ defined: 7, 3 + +foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value_aliased.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value_aliased.py new file mode 100644 index 000000000..088ca9544 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value_aliased.py @@ -0,0 +1,13 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +from foo import FOO as QUX + +QUX +# ^ defined: 7, 3 + +FOO +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_wildcard.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_wildcard.py new file mode 100644 index 000000000..3a1de4ffe --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_wildcard.py @@ -0,0 +1,13 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +from foo import * + +FOO +# ^ defined: 3 + +foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_submodule_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_submodule_a_value.py new file mode 100644 index 000000000..8d89d8538 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_submodule_a_value.py @@ -0,0 +1,16 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +from foo.bar import BAR + +BAR +# ^ defined: 7, 3 + +foo +# ^ defined: + +bar +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_a_value.py new file mode 100644 index 000000000..961cd1622 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_a_value.py @@ -0,0 +1,10 @@ +# --- path: foo/__init__.py --- + +FOO = 42 + +# --- path: foo/bar/test.py --- + +from .. import FOO + +FOO +# ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_submodule_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_submodule_a_value.py new file mode 100644 index 000000000..75652f7cd --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_submodule_a_value.py @@ -0,0 +1,10 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: foo/baz/test.py --- + +from ..bar import BAR + +BAR +# ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_a_value.py new file mode 100644 index 000000000..f8de8042c --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_a_value.py @@ -0,0 +1,10 @@ +# --- path: foo/__init__.py --- + +FOO = 42 + +# --- path: foo/test.py --- + +from . import FOO + +FOO +# ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_submodule_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_submodule_a_value.py new file mode 100644 index 000000000..dc64ce387 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_submodule_a_value.py @@ -0,0 +1,10 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: foo/test.py --- + +from .bar import BAR + +BAR +# ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_module.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_module.py new file mode 100644 index 000000000..897acdf63 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_module.py @@ -0,0 +1,11 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +import foo + +foo.FOO +# ^ defined: 7, 3 +# ^ defined: 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_module_aliased.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_module_aliased.py new file mode 100644 index 000000000..220a6b72d --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_module_aliased.py @@ -0,0 +1,14 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +import foo as qux + +qux.FOO +# ^ defined: 7, 3 +# ^ defined: 3 + +foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule.py new file mode 100644 index 000000000..312cd66a1 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule.py @@ -0,0 +1,15 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +import foo.bar + +foo.bar.BAR +# ^ defined: 7 +# ^ defined: 7, 3 +# ^ defined: 3 + +bar +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule_aliased.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule_aliased.py new file mode 100644 index 000000000..b6bfdf7b7 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule_aliased.py @@ -0,0 +1,17 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +import foo.bar as qux + +qux.BAR +# ^ defined: 7, 3 +# ^ defined: 3 + +foo +# ^ defined: + +bar +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/relative_import_resolves_to_itself.py.skip b/languages/tree-sitter-stack-graphs-python/test/imports/relative_import_resolves_to_itself.py.skip new file mode 100644 index 000000000..2cdb2cf66 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/relative_import_resolves_to_itself.py.skip @@ -0,0 +1,14 @@ +# --- path: foo/__init__.py --- +from . import bar +# ^ defined: 6 + +# --- path: foo/bar/__init__.py --- +BAR = 'b' + +# --- path: main.py --- +from foo import bar +# ^ defined: 6 + +bar.BAR +# ^ defined: 9, 6 +# ^ defined: 6 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/require_explicit_submodule_import.py.skip b/languages/tree-sitter-stack-graphs-python/test/imports/require_explicit_submodule_import.py.skip new file mode 100644 index 000000000..a9d12848c --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/require_explicit_submodule_import.py.skip @@ -0,0 +1,18 @@ +# --- path: foo/__init__.py --- + +FOO = 42 + +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +import foo + +foo.FOO +# ^ defined: 11, 3 +# ^ defined: 3 + +foo.bar +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/instance_members.py b/languages/tree-sitter-stack-graphs-python/test/instance_members.py new file mode 100644 index 000000000..38a6b6d75 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/instance_members.py @@ -0,0 +1,38 @@ +#---- path: a.py ------- + +class A: + def __init__(self, b, c): + self.b = b + self.c = A(c, 1) + + def get_b(self): + return self.b + # ^ defined: 4, 5 + + def get_c(self): + return self.c + # ^ defined: 6 + + def get_all(self): + return [self.get_b(), self.get_c()] + # ^ defined: 8 + # ^ defined: 12 + +a = A(1, 2) +a.get_all() +# ^ defined: 16 + +a.b +# ^ defined: 4, 5 + +a.c.b +# ^ defined: 4, 5 +# ^ defined: 6 + +#----- path: main.py --------- + +import a + +print a.A, a.a +# ^ defined: 3 +# ^ defined: 21 diff --git a/languages/tree-sitter-stack-graphs-python/test/lambdas.py b/languages/tree-sitter-stack-graphs-python/test/lambdas.py new file mode 100644 index 000000000..9812df4de --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/lambdas.py @@ -0,0 +1,2 @@ +sorted([1, 2, 3], key=lambda x: x) +# ^ defined: 1 diff --git a/languages/tree-sitter-stack-graphs-python/test/loops.py b/languages/tree-sitter-stack-graphs-python/test/loops.py new file mode 100644 index 000000000..2f0663295 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/loops.py @@ -0,0 +1,19 @@ +class Node3: + value = 3 +class Node2: + value = 2 + next = Node3 +class Node1: + value = 1 + next = Node2 + +def linked_list_search(l, item): + node = l + while node: + if node.value == item: + return node + # ^ defined: 10, 11, 16 + node = node.next + +linked_list_search(Node1, 5).value +# ^ defined: 6 diff --git a/languages/tree-sitter-stack-graphs-python/test/many_definitions.py b/languages/tree-sitter-stack-graphs-python/test/many_definitions.py new file mode 100644 index 000000000..f85f3a8ec --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/many_definitions.py @@ -0,0 +1,76 @@ +#--- path: a.py ---# + +def f0(a): return X.a + X.b +def f1(a): return f0(1) +def f2(a): return f1(2) +def f3(a): return f2(3) +def f4(a): return f3(4) +def f5(a): return f4(5) +def f6(a): return f5(6) +def f7(a): return f6(7) +def f8(a): return f7(8) +def f9(a): return f8(9) + +class C1: + def m0(self, b): return f9(0) + def m1(self, b): return self.m0(1) + def m2(self, b): return self.m1(2) + def m3(self, b): return self.m2(3) + def m4(self, b): return self.m3(4) + def m5(self, b): return self.m4(5) + def m6(self, b): return self.m5(6) + def m7(self, b): return self.m6(7) + def m8(self, b): return self.m7(8) + +def f10(): return C1.m8(0) +def f11(): return f10(1) +def f12(): return X.c(2) +def f13(): return X.c(3) +def f14(): return x(4) +def f15(): return x(5) +def f16(): return x(6) +def f17(): return x(7) +def f18(): return x(8) + +class C2: + def m0(self): return X.d(0) + def m1(self): return X.d(1) + def m2(self): return X.d(2) + def m3(self): return X.d(3) + def m4(self): return X.d(4) + def m5(self): return X.d(5) + def m6(self): return X.d(6) + def m7(self): return X.d(7) + def m8(self): return X.d(8) + +#--- path: main.py ---# + +from a import * + +print f0(), f4(), f8() +# ^ defined: 3 +# ^ defined: 7 +# ^ defined: 11 + +print C1.m0, C1().m0(), C1.m4, C1().m4, C1.m8, C1.m8 +# ^ defined: 14 +# ^ defined: 15 +# ^ defined: 15 +# ^ defined: 19 +# ^ defined: 19 +# ^ defined: 23 +# ^ defined: 23 + +print f10(), f14(), f18() +# ^ defined: 25 +# ^ defined: 29 +# ^ defined: 33 + +print C2.m0, C2().m0(), C2.m4, C2().m4, C2.m8, C2.m8 +# ^ defined: 35 +# ^ defined: 36 +# ^ defined: 36 +# ^ defined: 40 +# ^ defined: 40 +# ^ defined: 44 +# ^ defined: 44 diff --git a/languages/tree-sitter-stack-graphs-python/test/nested_functions.py b/languages/tree-sitter-stack-graphs-python/test/nested_functions.py new file mode 100644 index 000000000..db655e3a3 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/nested_functions.py @@ -0,0 +1,6 @@ +def outer(a): + def inner(b): + pass + + inner(1) + # ^ defined: 2 diff --git a/languages/tree-sitter-stack-graphs-python/test/not_a_method.py b/languages/tree-sitter-stack-graphs-python/test/not_a_method.py new file mode 100644 index 000000000..f0a207eb0 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/not_a_method.py @@ -0,0 +1,20 @@ +class Foo: + a = 1 + + def first_method(self): + self.a + # ^ defined: 2 + + def second_method(self): + self.a + # ^ defined: 2 + + # First argument here is not self + def not_a_method(not_self): + return not_self.a + # ^ defined: + + +def function(self): + self.a + # ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/pattern_matching.py b/languages/tree-sitter-stack-graphs-python/test/pattern_matching.py new file mode 100644 index 000000000..d6b8980a3 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/pattern_matching.py @@ -0,0 +1,42 @@ +command = 1 +current_room = 2 +character = 3 + +match command.split(): + # ^ defined: 1 + case ["quit"]: + print("Goodbye!") + quit_game() + case ["look"]: + current_room.describe() + # ^ defined: 2 + case ["get", obj]: + character.get(obj, current_room) + # ^ defined: 3 + # ^ defined: 13 + # ^ defined: 2 + case ["go", direction]: + current_room = current_room.neighbor(direction) + # ^ defined: 18 + case { "foo": foo }: + print(foo) + # ^ defined: 21 + case {"bar": bar, "quux": quux}: + print(bar,quux) + # ^ defined: 24 + # ^ defined: 24 + case ["grab", { "key": {"garply": garply}}]: + print(garply) + # ^ defined: 28 + case ["drop", *objs]: + print(objs) + # ^ defined: 31 + case ["get", obj] | ["pick", "up", obj] | ["pick", obj, "up"]: + print(obj) + # ^ defined: 34, 34, 34 + case ["go", ("north" | "south" | "east" | "west") as direction2]: + current_room = current_room.neighbor(direction2) + # ^ defined: 37 + case (foo, "bar"): + print(foo) + # ^ defined: 40 diff --git a/languages/tree-sitter-stack-graphs-python/test/redundant_reexport.py b/languages/tree-sitter-stack-graphs-python/test/redundant_reexport.py new file mode 100644 index 000000000..53afca29e --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/redundant_reexport.py @@ -0,0 +1,15 @@ +#--- path: a/__init__.py ---# + +from . import child + +#--- path: a/child.py ---- + +def f(): + pass + +#--- path: main.py ---# + +import a + +print a.child.f() +# ^ defined: 7 diff --git a/languages/tree-sitter-stack-graphs-python/test/relative_imports.py b/languages/tree-sitter-stack-graphs-python/test/relative_imports.py new file mode 100644 index 000000000..f14571575 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/relative_imports.py @@ -0,0 +1,60 @@ +#------ path: foo/bar/main.py ------# + +from . import a +from .b import B +from ..c import see +from ..c.d import D + +print a.A +# ^ defined: 3, 25 +# ^ defined: 26 + +print B.bee +# ^ defined: 4, 31 +# ^ defined: 32 + +print see() +# ^ defined: 5, 37 + + +print D.d +# ^ defined: 44 + +#------ path: foo/bar/a.py --------# + +# module +A = "a" + +#------ path: foo/bar/b.py --------# + +# module +class B: + bee = 1 + +#------ path: foo/c.py ------------# + +# module +def see(): + pass + +#------ path: foo/c/d.py ---------# + +# module +class D: + d = "d" + +#------ path: foo/e/g.py ---# + +# module +G = 1 + +#------ path: foo/e/__init__.py ---# + +# module +from .g import G +# ^ defined: 49 + +from ..c import see +# ^ defined: 37 + +E = 1 diff --git a/languages/tree-sitter-stack-graphs-python/test/root_path.py b/languages/tree-sitter-stack-graphs-python/test/root_path.py new file mode 100644 index 000000000..23937db7c --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/root_path.py @@ -0,0 +1,21 @@ +# ------ path: foo/bar/module.py -----------# +# ------ global: ROOT_PATH=foo/bar -----------# + +foo = 42 + +# ------ path: foo/bar/baz/module.py -----------# +# ------ global: ROOT_PATH=foo/bar -----------# + +bar = "hello" + +# ------ path: foo/bar/main.py -------------# +# ------ global: ROOT_PATH=foo/bar -----------# + +from module import foo +from baz.module import bar + +print(foo) +# ^ defined: 4, 14 + +print(bar) +# ^ defined: 9, 15 diff --git a/languages/tree-sitter-stack-graphs-python/test/self.py b/languages/tree-sitter-stack-graphs-python/test/self.py new file mode 100644 index 000000000..763a52eb6 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/self.py @@ -0,0 +1,11 @@ +class Foo: + a = 1 + + def mathod_1(self): + return self.a + # ^ defined: 2 + + # Self can be named anything + def method_2(actually_self): + return actually_self.a + # ^ defined: 2 diff --git a/languages/tree-sitter-stack-graphs-python/test/statement_bindings.py b/languages/tree-sitter-stack-graphs-python/test/statement_bindings.py new file mode 100644 index 000000000..6e3a13abb --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/statement_bindings.py @@ -0,0 +1,9 @@ +import a.b +import c.d + +with a as x, c as y: + print x.b, y.d + # ^ defined: 1, 4 + # ^ defined: 1 + # ^ defined: 2, 4 + # ^ defined: 2 diff --git a/languages/tree-sitter-stack-graphs-python/test/superclasses.py b/languages/tree-sitter-stack-graphs-python/test/superclasses.py new file mode 100644 index 000000000..849a192a6 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/superclasses.py @@ -0,0 +1,19 @@ +class A: + def __init__(self): + self.some_attr = 2 + + def some_method(self): + print self + +class B(A): + def method2(self): + print self.some_attr, self.some_method() + # ^ defined: 3 + # ^ defined: 5, 14 + + def some_method(self): + pass + + def other(self): + super().some_method() + # ^ defined: 5 diff --git a/languages/tree-sitter-stack-graphs-python/test/test.py b/languages/tree-sitter-stack-graphs-python/test/test.py new file mode 100644 index 000000000..e69de29bb diff --git a/languages/tree-sitter-stack-graphs-python/test/tuples.py b/languages/tree-sitter-stack-graphs-python/test/tuples.py new file mode 100644 index 000000000..1ea6ee3ef --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/tuples.py @@ -0,0 +1,38 @@ +class a: + x = 1 +class b: + x = 1 +class c: + x = 1 + +(a1, b1) = (a, b) +a2, b2 = (a, b) +(a3, b3) = (a, b) +a4, b4 = a, b + +print a1.x, b1.x +# ^ defined: 2 +# ^ defined: 4 +print a2.x, b2.x +# ^ defined: 2 +# ^ defined: 4 +print a3.x, b3.x +# ^ defined: 2 +# ^ defined: 4 +print a4.x, b4.x +# ^ defined: 2 +# ^ defined: 4 + +t = (a, b), c +(a5, b5), c5 = t + +print a5.x, b5.x, c5.x +# ^ defined: 2 +# ^ defined: 4 +# ^ defined: 6 + +(a6, (b6, c6)) = (a, (b, c)) + +print a6.x, b6.x +# ^ defined: 2 +# ^ defined: 4 diff --git a/languages/tree-sitter-stack-graphs-python/test/wildcard_import.py b/languages/tree-sitter-stack-graphs-python/test/wildcard_import.py new file mode 100644 index 000000000..f2ac040f2 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/wildcard_import.py @@ -0,0 +1,21 @@ +#------ path: one/two.py ------# + +a = 1 +b = 2 + +#------ path: one/three.py ------# + +b = 3 +c = 4 + +#------ path: main.py ---------# + +from one.two import * +from one.three import * + +print a +# ^ defined: 3 +print b +# ^ defined: 8 +print c +# ^ defined: 9 diff --git a/languages/tree-sitter-stack-graphs-typescript/CHANGELOG.md b/languages/tree-sitter-stack-graphs-typescript/CHANGELOG.md index e5b478f09..5a3bfa3d4 100644 --- a/languages/tree-sitter-stack-graphs-typescript/CHANGELOG.md +++ b/languages/tree-sitter-stack-graphs-typescript/CHANGELOG.md @@ -5,7 +5,43 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## v0.4.0 -- 2024-12-13 + +- The `tree-sitter-stack-graphs` dependency is updated to version 0.10. + +- The `tree-sitter-typescript` dependency is updated to version 0.23.2. + +## v0.3.0 -- 2024-07-09 + +### Added + +- Support for TSX. A new language configuration for TSX is available with `{try_,}language_configuration_tsx`. TSX is enabled in the CLI next to TypeScript. + +### Fixed + +- Imports are more robust to the presence of file extensions in the import name. + +### Changed + +- The functions `{try_,}language_configuration` have been renamed to `{try_,}language_configuration_typescript`. + +### Removed + +- The `FILE_PATH_VAR` constant has been replaced in favor of `tree_sitter_stack_graphs::FILE_PATH_VAR`. + +## v0.2.0 -- 2024-03-06 + +The `tree-sitter-stack-graphs` is updated to `v0.8`. + +### Added + +- An experimental VSCode LSP plugin that supports code navigation based on the stack graph rules. _Purely an experiment, not ready for serious use!_ Requires the `lsp` feature to be enabled. + +### Changed + +- Various improvements to the rules for imports and packages. + +## v0.1.0 -- 2023-01-27 ### Added diff --git a/languages/tree-sitter-stack-graphs-typescript/Cargo.toml b/languages/tree-sitter-stack-graphs-typescript/Cargo.toml index da5df7ef9..ed14d9112 100644 --- a/languages/tree-sitter-stack-graphs-typescript/Cargo.toml +++ b/languages/tree-sitter-stack-graphs-typescript/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "tree-sitter-stack-graphs-typescript" -version = "0.1.0" -description = "Stack graphs definition for TypeScript using tree-sitter-typescript" +version = "0.4.0" +description = "Stack graphs definition for TypeScript & TSX using tree-sitter-typescript" readme = "README.md" -keywords = ["tree-sitter", "stack-graphs", "typescript"] +keywords = ["tree-sitter", "stack-graphs", "typescript", "tsx"] authors = ["Hendrik van Antwerpen "] license = "MIT OR Apache-2.0" edition = "2018" @@ -32,11 +32,15 @@ clap = { version = "4", optional = true } glob = "0.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -stack-graphs = { version = ">=0.11, <=0.12", path = "../../stack-graphs" } -tree-sitter-stack-graphs = { version = "0.7", path = "../../tree-sitter-stack-graphs" } -tree-sitter-typescript = "=0.20.2" +stack-graphs = { version = "0.14", path = "../../stack-graphs" } # explicit version is required to be able to publish crate +tree-sitter-stack-graphs = { version = "0.10", path = "../../tree-sitter-stack-graphs" } # explicit version is required to be able to publish crate +tree-sitter-typescript = "=0.23.2" tsconfig = "0.1.0" [dev-dependencies] anyhow = { version = "1.0" } -tree-sitter-stack-graphs = { version = "0.7", path = "../../tree-sitter-stack-graphs", features = ["cli"] } +tree-sitter-stack-graphs = { path = "../../tree-sitter-stack-graphs", features = ["cli"] } + +[build-dependencies] +anyhow = { version = "1.0" } +regex = "1.10.3" diff --git a/languages/tree-sitter-stack-graphs-typescript/README.md b/languages/tree-sitter-stack-graphs-typescript/README.md index 0c1f9d5ee..c83d5849d 100644 --- a/languages/tree-sitter-stack-graphs-typescript/README.md +++ b/languages/tree-sitter-stack-graphs-typescript/README.md @@ -4,72 +4,106 @@ This project defines tree-sitter-stack-graphs rules for TypeScript using the [tr [tree-sitter-typescript]: https://crates.io/crates/tree-sitter-typescript -## Usage +- [API documentation](https://docs.rs/tree-sitter-stack-graphs-typescript/) +- [Release notes](https://github.com/github/stack-graphs/blob/main/languages/tree-sitter-stack-graphs-typescript/CHANGELOG.md) + +## Using the API To use this library, add the following to your `Cargo.toml`: -``` toml +```toml [dependencies] -tree-sitter-stack-graphs-typescript = "0.1" +tree-sitter-stack-graphs-typescript = "0.4" ``` -Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs-typescript/*/) for -more details on how to use this library. +Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs-typescript/*/) for more details on how to use this library. -## Command-line Program +## Using the Command-line Program -The command-line program for `tree-sitter-stack-graphs-typescript` lets you do -stack graph based analysis and lookup from the command line. +The command-line program for `tree-sitter-stack-graphs-typescript` lets you do stack graph based analysis and lookup from the command line. -Install the program using `cargo install` as follows: +The CLI can be run as follows: -``` sh -$ cargo install --features cli tree-sitter-stack-graphs-typescript -$ tree-sitter-stack-graphs-typescript --help -``` +1. _(Installed)_ Install the CLI using Cargo as follows: + + ```sh + cargo install --features cli tree-sitter-stack-graphs-typescript + ``` + + After this, the CLI should be available as `tree-sitter-stack-graphs-typescript`. + +2. _(From source)_ Instead of installing the CLI, it can also be run directly from the crate directory, as a replacement for a `tree-sitter-stack-graphs-typescript` invocation, as follows: + + ```sh + cargo run --features cli -- + ``` + +The basic CLI workflow for the command-line program is to index source code and issue queries against the resulting database: + +1. Index a source folder as follows: + + ```sh + tree-sitter-stack-graphs-typescript index SOURCE_DIR + ``` + + _Indexing will skip any files that have already be indexed. To force a re-index, add the `-f` flag._ + + To check the status if a source folder, run: + + ```sh + tree-sitter-stack-graphs-typescript status SOURCE_DIR + ``` + + To clean the database and start with a clean slate, run: + + ```sh + tree-sitter-stack-graphs-typescript clean + ``` + + _Pass the `--delete` flag to not just empty the database, but also delete it. This is useful to resolve `unsupported database version` errors that may occur after a version update._ + +2. Run a query to find the definition(s) for a reference on a given line and column, run: + + ```sh + tree-sitter-stack-graphs-typescript query definition SOURCE_PATH:LINE:COLUMN + ``` + + Resulting definitions are printed, including a source line if the source file is available. + +Discover all available commands and flags by passing the `-h` flag to the CLI directly, or to any of the subcommands. ## Development -The project is written in Rust, and requires a recent version installed. -Rust can be installed and updated using [rustup][]. +The project is written in Rust, and requires a recent version installed. Rust can be installed and updated using [rustup][]. [rustup]: https://rustup.rs/ - The project is organized as follows: - The stack graph rules are defined in `src/stack-graphs.tsg`. -- Builtins sources and configuration are defined in `src/builtins.ts` and `builtins.cfg` respectively. +- Builtins sources and configuration are defined in `src/builtins.it` and `builtins.cfg` respectively. - Tests are put into the `test` directory. -### Building and Running Tests - -Build the project by running: - -``` sh -$ cargo build -``` +### Running Tests Run the tests as follows: -``` sh -$ cargo test +```sh +cargo test ``` -The project consists of a library and a CLI. -By default, running `cargo` only applies to the library. -To run `cargo` commands on the CLI as well, add `--features cli` or `--all-features`. +The project consists of a library and a CLI. By default, running `cargo` only applies to the library. To run `cargo` commands on the CLI as well, add `--features cli` or `--all-features`. Run the CLI from source as follows: -``` sh -$ cargo run --features cli -- ARGS +```sh +cargo run --features cli -- ARGS ``` Sources are formatted using the standard Rust formatted, which is applied by running: -``` sh -$ cargo fmt +```sh +cargo fmt ``` ### Writing TSG @@ -84,21 +118,21 @@ which contain self-contained TSG rules for specific language features. A VSCode Parse and test a single file by executing the following commands: -``` sh -$ cargo run --features cli -- parse FILES... -$ cargo run --features cli -- test TESTFILES... +```sh +cargo run --features cli -- parse FILES... +cargo run --features cli -- test TESTFILES... ``` Generate a visualization to debug failing tests by passing the `-V` flag: -``` sh -$ cargo run --features cli -- test -V TESTFILES... +```sh +cargo run --features cli -- test -V TESTFILES... ``` To generate the visualization regardless of test outcome, execute: -``` sh -$ cargo run --features cli -- test -V --output-mode=always TESTFILES... +```sh +cargo run --features cli -- test -V --output-mode=always TESTFILES... ``` -Go to https://crates.io/crates/tree-sitter-stack-graphs for links to examples and documentation. +Go to for links to examples and documentation. diff --git a/languages/tree-sitter-stack-graphs-typescript/build.rs b/languages/tree-sitter-stack-graphs-typescript/build.rs new file mode 100644 index 000000000..31e3dd9de --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/build.rs @@ -0,0 +1,92 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2024, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use std::path::Path; + +use anyhow::{bail, Result}; +use regex::Regex; + +const TSG_SOURCE: &str = "src/stack-graphs.tsg"; +const DIALECTS: [&str; 2] = ["typescript", "tsx"]; + +/// preprocess the input file, removing lines that are not for the selected dialect +fn preprocess( + input: impl std::io::Read, + mut output: impl std::io::Write, + dialect: &str, +) -> Result<()> { + // Matches: ; #dialect typescript + let directive_start = Regex::new(r";\s*#dialect\s+(\w+)").unwrap(); + + // Matches: ; #end + let directive_end = Regex::new(r";\s*#end").unwrap(); + + let no_code = Regex::new(r"^[\s;]+").unwrap(); + + let input = std::io::read_to_string(input)?; + + // If the filter is None or Some(true), the lines are written to the output + let mut filter: Option = None; + + for (mut line_no, line) in input.lines().enumerate() { + // Line numbers are one based + line_no += 1; + + if let Some(captures) = directive_start.captures(line) { + if !no_code.is_match(line) { + bail!("Line {line_no}: unexpected code before directive"); + } + + if filter.is_some() { + bail!("Line {line_no}: dialect directive cannot be nested"); + } + + let directive = captures.get(1).unwrap().as_str(); + if !DIALECTS.contains(&directive) { + bail!("Line {line_no}: unknown dialect: {directive}"); + } + + filter = Some(dialect == directive); + } else if directive_end.is_match(line) { + if !no_code.is_match(line) { + bail!("Line {line_no}: unexpected code before directive end"); + } + + if filter.is_none() { + bail!("Line {line_no}: unmatched directive end"); + } + + filter = None; + } else if filter.unwrap_or(true) { + output.write_all(line.as_bytes())?; + } + // a new line is always written so that removed lines are padded to preserve line numbers + output.write(b"\n")?; + } + + if filter.is_some() { + bail!("Unmatched directive end at the end of the file"); + } + + Ok(()) +} + +fn main() { + let out_dir = std::env::var_os("OUT_DIR").expect("OUT_DIR is not set"); + + for dialect in DIALECTS { + let input = std::fs::File::open(TSG_SOURCE).expect("Failed to open stack-graphs.tsg"); + + let out_filename = Path::new(&out_dir).join(format!("stack-graphs-{dialect}.tsg")); + let output = std::fs::File::create(out_filename).expect("Failed to create output file"); + + preprocess(input, output, dialect).expect("Failed to preprocess stack-graphs.tsg"); + } + + println!("cargo:rerun-if-changed={TSG_SOURCE}"); + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/languages/tree-sitter-stack-graphs-typescript/rust/bin.rs b/languages/tree-sitter-stack-graphs-typescript/rust/bin.rs index b1ee4f568..ea7713595 100644 --- a/languages/tree-sitter-stack-graphs-typescript/rust/bin.rs +++ b/languages/tree-sitter-stack-graphs-typescript/rust/bin.rs @@ -12,17 +12,23 @@ use tree_sitter_stack_graphs::cli::provided_languages::Subcommands; use tree_sitter_stack_graphs::NoCancellation; fn main() -> anyhow::Result<()> { - let lc = match tree_sitter_stack_graphs_typescript::try_language_configuration(&NoCancellation) - { - Ok(lc) => lc, - Err(err) => { - eprintln!("{}", err.display_pretty()); - return Err(anyhow!("Language configuration error")); - } - }; let cli = Cli::parse(); + let mut lcs = Vec::new(); + for r in [ + tree_sitter_stack_graphs_typescript::try_language_configuration_typescript(&NoCancellation), + tree_sitter_stack_graphs_typescript::try_language_configuration_tsx(&NoCancellation), + ] { + let lc = match r { + Ok(lc) => lc, + Err(err) => { + eprintln!("{}", err.display_pretty()); + return Err(anyhow!("Language configuration error")); + } + }; + lcs.push(lc); + } let default_db_path = default_user_database_path_for_crate(env!("CARGO_PKG_NAME"))?; - cli.subcommand.run(default_db_path, vec![lc]) + cli.subcommand.run(default_db_path, lcs) } #[derive(Parser)] diff --git a/languages/tree-sitter-stack-graphs-typescript/rust/lib.rs b/languages/tree-sitter-stack-graphs-typescript/rust/lib.rs index 45ce6d255..8fd63881d 100644 --- a/languages/tree-sitter-stack-graphs-typescript/rust/lib.rs +++ b/languages/tree-sitter-stack-graphs-typescript/rust/lib.rs @@ -5,7 +5,6 @@ // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ -use tree_sitter_stack_graphs::loader::FileAnalyzers; use tree_sitter_stack_graphs::loader::LanguageConfiguration; use tree_sitter_stack_graphs::loader::LoadError; use tree_sitter_stack_graphs::CancellationFlag; @@ -20,7 +19,10 @@ pub mod util; /// The stacks graphs tsg path for this language. pub const STACK_GRAPHS_TSG_PATH: &str = "src/stack-graphs.tsg"; /// The stack graphs tsg source for this language -pub const STACK_GRAPHS_TSG_SOURCE: &str = include_str!("../src/stack-graphs.tsg"); +pub const STACK_GRAPHS_TSG_TS_SOURCE: &str = + include_str!(concat!(env!("OUT_DIR"), "/stack-graphs-typescript.tsg")); +pub const STACK_GRAPHS_TSG_TSX_SOURCE: &str = + include_str!(concat!(env!("OUT_DIR"), "/stack-graphs-tsx.tsg")); /// The stack graphs builtins configuration for this language pub const STACK_GRAPHS_BUILTINS_CONFIG: &str = include_str!("../src/builtins.cfg"); @@ -29,33 +31,65 @@ pub const STACK_GRAPHS_BUILTINS_PATH: &str = "src/builtins.ts"; /// The stack graphs builtins source for this language pub const STACK_GRAPHS_BUILTINS_SOURCE: &str = include_str!("../src/builtins.ts"); -/// The name of the file path global variable -pub const FILE_PATH_VAR: &str = "FILE_PATH"; /// The name of the project name global variable pub const PROJECT_NAME_VAR: &str = "PROJECT_NAME"; -pub fn language_configuration(cancellation_flag: &dyn CancellationFlag) -> LanguageConfiguration { - try_language_configuration(cancellation_flag).unwrap_or_else(|err| panic!("{}", err)) +pub fn language_configuration_typescript( + cancellation_flag: &dyn CancellationFlag, +) -> LanguageConfiguration { + try_language_configuration_typescript(cancellation_flag).unwrap_or_else(|err| panic!("{}", err)) } -pub fn try_language_configuration( +pub fn try_language_configuration_typescript( cancellation_flag: &dyn CancellationFlag, ) -> Result { - LanguageConfiguration::from_sources( - tree_sitter_typescript::language_typescript(), + let mut lc = LanguageConfiguration::from_sources( + tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), Some(String::from("source.ts")), None, vec![String::from("ts")], STACK_GRAPHS_TSG_PATH.into(), - STACK_GRAPHS_TSG_SOURCE, + STACK_GRAPHS_TSG_TS_SOURCE, + Some(( + STACK_GRAPHS_BUILTINS_PATH.into(), + STACK_GRAPHS_BUILTINS_SOURCE, + )), + Some(STACK_GRAPHS_BUILTINS_CONFIG), + cancellation_flag, + )?; + lc.special_files + .add("tsconfig.json".to_string(), TsConfigAnalyzer {}) + .add("package.json".to_string(), NpmPackageAnalyzer {}); + lc.no_similar_paths_in_file = true; + Ok(lc) +} + +pub fn language_configuration_tsx( + cancellation_flag: &dyn CancellationFlag, +) -> LanguageConfiguration { + try_language_configuration_tsx(cancellation_flag).unwrap_or_else(|err| panic!("{}", err)) +} + +pub fn try_language_configuration_tsx( + cancellation_flag: &dyn CancellationFlag, +) -> Result { + let mut lc = LanguageConfiguration::from_sources( + tree_sitter_typescript::LANGUAGE_TSX.into(), + Some(String::from("source.tsx")), + None, + vec![String::from("tsx")], + STACK_GRAPHS_TSG_PATH.into(), + STACK_GRAPHS_TSG_TSX_SOURCE, Some(( STACK_GRAPHS_BUILTINS_PATH.into(), STACK_GRAPHS_BUILTINS_SOURCE, )), Some(STACK_GRAPHS_BUILTINS_CONFIG), - FileAnalyzers::new() - .add("tsconfig.json".to_string(), TsConfigAnalyzer {}) - .add("package.json".to_string(), NpmPackageAnalyzer {}), cancellation_flag, - ) + )?; + lc.special_files + .add("tsconfig.json".to_string(), TsConfigAnalyzer {}) + .add("package.json".to_string(), NpmPackageAnalyzer {}); + lc.no_similar_paths_in_file = true; + Ok(lc) } diff --git a/languages/tree-sitter-stack-graphs-typescript/rust/npm_package.rs b/languages/tree-sitter-stack-graphs-typescript/rust/npm_package.rs index 9b67db598..b7351454a 100644 --- a/languages/tree-sitter-stack-graphs-typescript/rust/npm_package.rs +++ b/languages/tree-sitter-stack-graphs-typescript/rust/npm_package.rs @@ -8,6 +8,7 @@ use serde::Deserialize; use std::collections::HashMap; use std::path::Path; +use std::path::PathBuf; use stack_graphs::arena::Handle; use stack_graphs::graph::File; @@ -15,6 +16,7 @@ use stack_graphs::graph::StackGraph; use tree_sitter_stack_graphs::BuildError; use tree_sitter_stack_graphs::FileAnalyzer; +use crate::tsconfig::NormalizedRelativePath; use crate::util::*; pub struct NpmPackageAnalyzer {} @@ -74,16 +76,28 @@ impl FileAnalyzer for NpmPackageAnalyzer { }; // package definition - let pkg_def = add_module_pops( - graph, - file, - NON_REL_M_NS, - Path::new(&npm_pkg.name), - root, - "npm_package.pkg_def", - ); - let pkg_ref = add_push(graph, file, proj_scope, PKG_M_NS, "npm_package.pkg_ref"); - add_edge(graph, pkg_def, pkg_ref, 0); + if !npm_pkg.name.is_empty() { + let pkg_def = add_module_pops( + graph, + file, + NON_REL_M_NS, + Path::new(&npm_pkg.name), + root, + "npm_package.pkg_def", + ); + let pkg_ref = add_push(graph, file, proj_scope, PKG_M_NS, "npm_package.pkg_ref"); + add_edge(graph, pkg_def, pkg_ref, 0); + + let main = Some(npm_pkg.main) + .filter(|main| !main.is_empty()) + .and_then(|main| NormalizedRelativePath::from_str(&main)) + .map(|p| p.into_path_buf()) + .unwrap_or(PathBuf::from("index")) + .with_extension(""); + let main_ref = + add_module_pushes(graph, file, M_NS, &main, proj_scope, "npm_package.main_ref"); + add_edge(graph, pkg_def, main_ref, 0); + } // dependencies (package references) for (i, (pkg_name, _)) in npm_pkg.dependencies.iter().enumerate() { @@ -113,7 +127,10 @@ impl FileAnalyzer for NpmPackageAnalyzer { #[derive(Default, Debug, Clone, PartialEq, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NpmPackage { + #[serde(default)] pub name: String, #[serde(default)] + pub main: String, + #[serde(default)] pub dependencies: HashMap, } diff --git a/languages/tree-sitter-stack-graphs-typescript/rust/test.rs b/languages/tree-sitter-stack-graphs-typescript/rust/test.rs index ca5b8494c..51ef9cdca 100644 --- a/languages/tree-sitter-stack-graphs-typescript/rust/test.rs +++ b/languages/tree-sitter-stack-graphs-typescript/rust/test.rs @@ -11,14 +11,18 @@ use tree_sitter_stack_graphs::ci::Tester; use tree_sitter_stack_graphs::NoCancellation; fn main() -> anyhow::Result<()> { - let lc = match tree_sitter_stack_graphs_typescript::try_language_configuration(&NoCancellation) - { - Ok(lc) => lc, - Err(err) => { - eprintln!("{}", err.display_pretty()); - return Err(anyhow!("Language configuration error")); - } - }; + let lc_factories = [ + tree_sitter_stack_graphs_typescript::try_language_configuration_typescript, + tree_sitter_stack_graphs_typescript::try_language_configuration_tsx, + ]; + + let lcs = lc_factories + .iter() + .map(|lc_factory| lc_factory(&NoCancellation)) + .collect::, _>>() + .inspect_err(|err| eprintln!("{}", err.display_pretty())) + .map_err(|_| anyhow!("Language configuration error"))?; + let test_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test"); - Tester::new(vec![lc], vec![test_path]).run() + Tester::new(lcs, vec![test_path]).run() } diff --git a/languages/tree-sitter-stack-graphs-typescript/rust/tsconfig.rs b/languages/tree-sitter-stack-graphs-typescript/rust/tsconfig.rs index c885c298f..5c9ecae28 100644 --- a/languages/tree-sitter-stack-graphs-typescript/rust/tsconfig.rs +++ b/languages/tree-sitter-stack-graphs-typescript/rust/tsconfig.rs @@ -498,15 +498,15 @@ fn longest_common_prefix(left: &Path, right: &Path) -> Option { Some(prefix) } -struct NormalizedRelativePath(PathBuf); +pub(crate) struct NormalizedRelativePath(PathBuf); impl NormalizedRelativePath { - pub(self) fn from_str(path: &str) -> Option { + pub(crate) fn from_str(path: &str) -> Option { Self::from_path(Path::new(path)) } /// Creates a new normalized, relative path from a path. - pub(self) fn from_path(path: &Path) -> Option { + pub(crate) fn from_path(path: &Path) -> Option { let mut np = PathBuf::new(); let mut normal_components = 0usize; for c in path.components() { @@ -538,18 +538,18 @@ impl NormalizedRelativePath { } /// Returns if the relative path escapes to the parent. - pub(self) fn escapes(&self) -> bool { + pub(crate) fn escapes(&self) -> bool { self.0 .components() .next() .map_or(false, |c| c == Component::ParentDir) } - pub(self) fn as_path(&self) -> &Path { + pub(crate) fn as_path(&self) -> &Path { &self.0 } - pub(self) fn into_path_buf(self) -> PathBuf { + pub(crate) fn into_path_buf(self) -> PathBuf { self.0 } } diff --git a/languages/tree-sitter-stack-graphs-typescript/src/stack-graphs.tsg b/languages/tree-sitter-stack-graphs-typescript/src/stack-graphs.tsg index 0a2a0d0b5..773abde7c 100644 --- a/languages/tree-sitter-stack-graphs-typescript/src/stack-graphs.tsg +++ b/languages/tree-sitter-stack-graphs-typescript/src/stack-graphs.tsg @@ -540,7 +540,8 @@ attribute node_symbol = node => symbol = (source-text node), source_n ; module reference var mod_scope = mod_ref__ns - scan (path-normalize mod_path) { + ; normalize path and remove the extension as we want to match 'foo', 'foo.js', 'foo.ts', etc. + scan (path-normalize (replace mod_path "\.(js|ts|jsx|tsx)$" "")) { "([^/]+)/?" { node mod_ref attr (mod_ref) push_symbol = $1 @@ -626,16 +627,6 @@ attribute node_symbol = node => symbol = (source-text node), source_n (import_statement "type"?@is_type (import_clause (named_imports (import_specifier name:(_)@name))@clause)) ] { if none @is_type { - node @name.expr_def - node @name.expr_def__ns - node @name.expr_ref - node @name.expr_ref__ns - - ; expr reference - attr (@name.expr_ref) node_reference = @name - edge @name.expr_ref -> @name.expr_ref__ns - ; - attr (@name.expr_ref__ns) push_symbol = "%E" edge @name.expr_ref__ns -> @clause.lexical_scope } } @@ -645,6 +636,9 @@ if none @is_type { (import_statement "type"?@is_type (import_clause (named_imports (import_specifier name:(_)@name !alias))@clause)) ] { if none @is_type { + node @name.expr_def + node @name.expr_def__ns + ; expr definition edge @clause.defs -> @name.expr_def__ns ; FIXME defs, lexical_defs? ; @@ -663,7 +657,6 @@ if none @is_type { if none @is_type { node @alias.expr_def node @alias.expr_def__ns - node @alias.expr_ref ; expr definition edge @clause.defs -> @alias.expr_def__ns @@ -687,8 +680,25 @@ if none @is_type { (export_statement "*" source:(_)@name )@export_stmt { - ; export everything from source module - edge @export_stmt.exports -> @name.mod_ref + node expr_pop + node expr_push + attr (expr_pop) pop_symbol = "%E" + attr (expr_push) push_symbol = "%E" + edge expr_pop -> expr_push + + node type_pop + node type_push + attr (type_pop) pop_symbol = "%E" + attr (type_push) push_symbol = "%E" + edge type_pop -> type_push + + ; export all expressions from source module + edge @export_stmt.exports -> expr_pop + edge expr_push -> @name.mod_ref + + ; export all types from source module + edge @export_stmt.exports -> type_pop + edge type_push -> @name.mod_ref } ; export CLAUSE from MODULE @@ -906,18 +916,38 @@ if none @is_default { ; ) (export_statement - (namespace_export (identifier)) - source:(_)@name + (namespace_export (identifier)@name) + source:(_)@from )@export_stmt { ; namespace definitions are specified together with (module) and (internal_module) - ; export definitions - edge @export_stmt.exports -> @name.expr_def__ns - edge @export_stmt.exports -> @name.type_def__ns + ; export expression definition + node expr_ns_pop + attr (expr_ns_pop) pop_symbol = "%E" + ; + edge @export_stmt.exports -> expr_ns_pop + edge expr_ns_pop -> @name.expr_def - ; connect definitions to exports - edge @export_stmt.expr_member__ns -> @name.mod_ref - edge @export_stmt.type_member__ns -> @name.mod_ref + ; export type definition + node type_ns_pop + attr (type_ns_pop) pop_symbol = "%T" + ; + edge @export_stmt.exports -> type_ns_pop + edge type_ns_pop -> @name.type_def + + ; connect expression definition to exports + node expr_ns_push + attr (expr_ns_push) push_symbol = "%E" + ; + edge @name.expr_def_member -> expr_ns_push + edge expr_ns_push -> @from.mod_ref + + ; connect type definition to exports + node type_ns_push + attr (type_ns_push) push_symbol = "%T" + ; + edge @name.type_def_member -> type_ns_push + edge type_ns_push -> @from.mod_ref } ; export as namespace NAME @@ -930,16 +960,36 @@ if none @is_default { (program (export_statement "as" "namespace" . (_)@name -)@export_stmt)@prog { +))@prog { ; namespace definitions are specified together with (module) and (internal_module) - ; make definitions global - edge @prog.globals -> @name.expr_def__ns - edge @prog.globals -> @name.type_def__ns + ; make expression definition global + node expr_ns_pop + attr (expr_ns_pop) pop_symbol = "%E" + ; + edge @prog.globals -> expr_ns_pop + edge expr_ns_pop -> @name.expr_def + + ; make type definition global + node type_ns_pop + attr (type_ns_pop) pop_symbol = "%T" + ; + edge @prog.globals -> type_ns_pop + edge type_ns_pop -> @name.type_def + + ; connect expression definition to exports + node expr_ns_push + attr (expr_ns_push) push_symbol = "%E" + ; + edge @name.expr_def_member -> expr_ns_push + edge expr_ns_push -> @prog.exports - ; connect definitions to exports - edge @export_stmt.expr_member__ns -> @prog.exports - edge @export_stmt.type_member__ns -> @prog.exports + ; connect type definition to exports + node type_ns_push + attr (type_ns_push) push_symbol = "%T" + ; + edge @name.type_def_member -> type_ns_push + edge type_ns_push -> @prog.exports } ;; Imports @@ -1068,14 +1118,38 @@ if none @is_type { (import_statement "type"? - (import_clause (namespace_import (identifier))) + (import_clause (namespace_import (identifier)@name)) source:(_)@from )@import_stmt { ; namespace definitions are specified together with (module) and (internal_module) - ; connect definitions to import - edge @import_stmt.expr_member__ns -> @from.mod_ref - edge @import_stmt.type_member__ns -> @from.mod_ref + ; expose expression definition + node expr_ns_pop + attr (expr_ns_pop) pop_symbol = "%E" + ; + edge @import_stmt.lexical_defs -> expr_ns_pop + edge expr_ns_pop -> @name.expr_def + + ; expose type definition + node type_ns_pop + attr (type_ns_pop) pop_symbol = "%T" + ; + edge @import_stmt.lexical_defs -> type_ns_pop + edge type_ns_pop -> @name.type_def + + ; connect expression definition to import + node expr_ns_push + attr (expr_ns_push) push_symbol = "%E" + ; + edge @name.expr_def_member -> expr_ns_push + edge expr_ns_push -> @from.mod_ref + + ; connect type definition to import + node type_ns_push + attr (type_ns_push) push_symbol = "%T" + ; + edge @name.type_def_member -> type_ns_push + edge type_ns_push -> @from.mod_ref } ;; Import Alias @@ -1259,7 +1333,7 @@ if none @is_type { attr (@name.expr_def__ns) pop_symbol = "%E" edge @name.expr_def__ns -> @name.expr_def ; - attr (@name.expr_def) node_definition = @name + attr (@name.expr_def) node_definition = @name, syntax_type = "function" ; function type edge @name.expr_def -> @name.expr_def__typeof @@ -1387,6 +1461,8 @@ if none @is_async { edge @dec.lexical_scope -> @class_decl.lexical_scope } + + ; definitions [ (abstract_class_declaration name:(_)@name) @@ -1413,10 +1489,17 @@ if none @is_async { edge @class_decl.type_export -> @class_decl.generic_type } [ - (abstract_class_declaration name:(_)) - (class_declaration name:(_)) + (interface_declaration name:(_)@name) +] { + attr (@name.type_def) syntax_type = "interface" +} +[ + (abstract_class_declaration name:(_)@name) + (class_declaration name:(_)@name) ; NOTE not for interface_declaration as body is already a type with an endpoint of needed ]@class_decl { + attr (@name.type_def) syntax_type = "class" + ; mark type scope as endpoint attr (@class_decl.generic_inner_type) is_endpoint } @@ -1432,7 +1515,7 @@ if none @is_async { attr (@name.expr_def__ns) pop_symbol = "%E" edge @name.expr_def__ns -> @name.expr_def ; - attr (@name.expr_def) node_definition = @name + attr (@name.expr_def) node_definition = @name, syntax_type = "class" ; type of expression is .static_type edge @name.expr_def -> @name.expr_def__typeof @@ -1537,6 +1620,7 @@ if none @is_async { [ (abstract_class_declaration body:(_)@body) (class_declaration body:(_)@body) + (interface_declaration body:(_)@body) ]@class_decl { ; propagate lexical scope ; FIXME the static members have access to type variables like this @@ -1550,16 +1634,6 @@ if none @is_async { edge @class_decl.static_type -> @body.static_members attr (@class_decl.static_type -> @body.static_members) precedence = 2 } -[ - (interface_declaration body:(_)@body) -]@class_decl { - ; propagate lexical scope - edge @body.lexical_scope -> @class_decl.generic_inner_lexical_scope - - ; interface type equals body type - edge @class_decl.generic_inner_type -> @body.type -} - ;; Class Body @@ -1573,23 +1647,35 @@ if none @is_async { ; (public_field_definition)] ; ) -(class_body)@class_body { - node @class_body.lexical_scope - node @class_body.type_members - node @class_body.static_members +[ + (class_body) + (interface_body) +]@body { + node @body.lexical_scope + node @body.type_members + node @body.static_members +} + +; decorators +[ + (class_body decorator:(_)@dec) +]@class_body { + ; connect lexical scope + edge @dec.lexical_scope -> @class_body.lexical_scope } -(class_body - (_)@mem -)@class_body { +[ + (class_body (_)@mem) + (interface_body (_)@mem) +]@body { ; propagate lexical scope - edge @mem.lexical_scope -> @class_body.lexical_scope + edge @mem.lexical_scope -> @body.lexical_scope ; body members are member definitions - edge @class_body.type_members -> @mem.type_members + edge @body.type_members -> @mem.type_members ; body static members are static member definitions - edge @class_body.static_members -> @mem.static_members + edge @body.static_members -> @mem.static_members } @@ -2238,147 +2324,104 @@ if none @is_async { [ ; X - (module name:(identifier)@name @mod) - (internal_module name:(identifier)@name @mod) - (export_statement "as" "namespace" . (identifier)@name) @mod - (export_statement (namespace_export (identifier)@name))@mod - (import_statement (import_clause (namespace_import (identifier)@name)))@mod - (ambient_declaration (module name:(string)@name @mod)) + (module name:(identifier)@name) + (internal_module name:(identifier)@name) + (export_statement "as" "namespace" . (identifier)@name) + (export_statement (namespace_export (identifier)@name)) + (import_statement (import_clause (namespace_import (identifier)@name))) + (ambient_declaration (module name:(string) @name)) ; X._ - (module name:(nested_identifier . (identifier)@name @mod)) - (internal_module name:(nested_identifier . (identifier)@name @mod)) - ; X._._ - (module name:(nested_identifier . (nested_identifier . (identifier)@name @mod))) - (internal_module name:(nested_identifier . (nested_identifier . (identifier)@name @mod))) - ; X._._._ - (module name:(nested_identifier . (nested_identifier . (nested_identifier . (identifier)@name @mod)))) - (internal_module name:(nested_identifier . (nested_identifier . (nested_identifier . (identifier)@name @mod)))) - ; X._._._._ - (module name:(nested_identifier . (nested_identifier . (nested_identifier . (nested_identifier . (identifier)@name @mod))))) - (internal_module name:(nested_identifier . (nested_identifier . (nested_identifier . (nested_identifier . (identifier)@name @mod))))) -]@mod_decl { + (nested_identifier object:(identifier)@name) + ; _.X._ + (member_expression object:(identifier)@name) + ; _.X + (nested_identifier property:(_)@name) + ; _._.X + (member_expression property:(_)@name) +] { node @name.expr_def - node @name.expr_def__ns - node @name.expr_def__typeof - node @mod.expr_member - node @mod.expr_member__ns + node expr_def_typeof + node @name.expr_def_member node @name.type_def - node @name.type_def__ns - node @mod.type_member - node @mod.type_member__ns - - ; expose expression definition - edge @mod_decl.lexical_defs -> @name.expr_def__ns + node @name.type_def_member ; expression definition - attr (@name.expr_def__ns) pop_symbol = "%E" - edge @name.expr_def__ns -> @name.expr_def + attr (@name.expr_def) node_definition = @name, syntax_type = "module" + attr (expr_def_typeof) pop_symbol = ":" + attr (@name.expr_def_member) pop_symbol = "." ; - attr (@name.expr_def) node_definition = @name - - ; namespaces mimic objects with members to access exported expressions - edge @name.expr_def -> @name.expr_def__typeof - ; - attr (@name.expr_def__typeof) pop_symbol = ":" - edge @name.expr_def__typeof -> @mod.expr_member - ; - attr (@mod.expr_member) pop_symbol = "." - edge @mod.expr_member -> @mod.expr_member__ns - ; - attr (@mod.expr_member__ns) push_symbol = "%E" - - ; expose type definition - edge @mod_decl.lexical_defs -> @name.type_def__ns + edge @name.expr_def -> expr_def_typeof + edge expr_def_typeof -> @name.expr_def_member ; type definition - attr (@name.type_def__ns) pop_symbol = "%T" - edge @name.type_def__ns -> @name.type_def - ; - attr (@name.type_def) node_definition = @name - - ; namespaces mimic types with member types to access exported expressions - edge @name.type_def -> @mod.type_member + attr (@name.type_def) node_definition = @name, syntax_type = "module" + attr (@name.type_def_member) pop_symbol = "." ; - attr (@mod.type_member) pop_symbol = "." - edge @mod.type_member -> @mod.type_member__ns - ; - attr (@mod.type_member__ns) push_symbol = "%T" + edge @name.type_def -> @name.type_def_member } [ - ; _.X - (module name:(nested_identifier . (_)@basemod . (identifier)@name)@mod) - (internal_module name:(nested_identifier . (_)@basemod . (identifier)@name)@mod) - ; _._.X - (module name:(nested_identifier . (nested_identifier . (_)@basemod . (identifier)@name)@mod)) - (internal_module name:(nested_identifier . (nested_identifier . (_)@basemod . (identifier)@name)@mod)) - ; _._._.X - (module name:(nested_identifier . (nested_identifier . (nested_identifier . (_)@basemod . (identifier)@name)@mod))) - (internal_module name:(nested_identifier . (nested_identifier . (nested_identifier . (_)@basemod . (identifier)@name)@mod))) - ; _._._._.X - (module name:(nested_identifier . (nested_identifier . (nested_identifier . (nested_identifier . (_)@basemod . (identifier)@name)@mod)))) - (internal_module name:(nested_identifier . (nested_identifier . (nested_identifier . (nested_identifier . (_)@basemod . (identifier)@name)@mod)))) -] { - node @name.expr_def - node @name.expr_def__ns - node @name.expr_def__typeof - node @mod.expr_member - node @mod.expr_member__ns - node @name.type_def - node @name.type_def__ns - node @mod.type_member - node @mod.type_member__ns - - ; expose expression definition - edge @basemod.expr_member__ns -> @name.expr_def__ns - - ; expression definition - attr (@name.expr_def__ns) pop_symbol = "%E" - edge @name.expr_def__ns -> @name.expr_def - ; - attr (@name.expr_def) node_definition = @name + (nested_identifier object:(_)@mod) + (member_expression object:[(member_expression) (identifier)]@mod) +]@nested { + node @nested.expr_def + node @nested.type_def - ; namespaces mimic objects with members to access exported expressions - edge @name.expr_def -> @name.expr_def__typeof - ; - attr (@name.expr_def__typeof) pop_symbol = ":" - edge @name.expr_def__typeof -> @mod.expr_member - ; - attr (@mod.expr_member) pop_symbol = "." - edge @mod.expr_member -> @mod.expr_member__ns - ; - attr (@mod.expr_member__ns) push_symbol = "%E" + edge @nested.expr_def -> @mod.expr_def + edge @nested.type_def -> @mod.type_def +} - ; expose type definition - edge @basemod.type_member__ns -> @name.type_def__ns +[ + (nested_identifier object:(_)@mod property:(_)@name) + (member_expression object:[(member_expression) (identifier)]@mod property:(_)@name) +] { + edge @mod.expr_def_member -> @name.expr_def + edge @mod.type_def_member -> @name.type_def +} - ; type definition - attr (@name.type_def__ns) pop_symbol = "%T" - edge @name.type_def__ns -> @name.type_def - ; - attr (@name.type_def) node_definition = @name +[ + (nested_identifier property:(_)@name) + (member_expression property:(_)@name) +]@nested { + node @nested.expr_def_member + node @nested.type_def_member - ; namespaces mimic types with member types to access exported expressions - edge @name.type_def -> @mod.type_member - ; - attr (@mod.type_member) pop_symbol = "." - edge @mod.type_member -> @mod.type_member__ns - ; - attr (@mod.type_member__ns) push_symbol = "%T" + edge @name.expr_def_member -> @nested.expr_def_member + edge @name.type_def_member -> @nested.type_def_member } [ - (module name:(_)@mod body:(_)@body) - (internal_module name:(_)@mod body:(_)@body) + (module name:(_)@mod body:(_)@body) + (internal_module name:(_)@mod body:(_)@body) ]@mod_decl { ; propagate lexical scope edge @body.lexical_scope -> @mod_decl.lexical_scope - ; connect object to exports - edge @mod.expr_member__ns -> @body.exports + ; expose expression definition + node expr_ns_pop + attr (expr_ns_pop) pop_symbol = "%E" + ; + edge @mod_decl.lexical_defs -> expr_ns_pop + edge expr_ns_pop -> @mod.expr_def + + ; expose type definition + node type_ns_pop + attr (type_ns_pop) pop_symbol = "%T" + ; + edge @mod_decl.lexical_defs -> type_ns_pop + edge type_ns_pop -> @mod.type_def + + ; connect expression definition to exports + node expr_ns_push + attr (expr_ns_push) push_symbol = "%E" + edge @mod.expr_def_member -> expr_ns_push + edge expr_ns_push -> @body.exports - ; connect type to exports - edge @mod.type_member__ns -> @body.exports + ; connect type definition to exports + node type_ns_push + attr (type_ns_push) push_symbol = "%T" + edge @mod.type_def_member -> type_ns_push + edge type_ns_push -> @body.exports } ; NOTE internal_module is also an expression in the grammar, not only a statement. @@ -2455,7 +2498,7 @@ if none @is_async { attr (@name.expr_def__ns) pop_symbol = "%E" edge @name.expr_def__ns -> @name.expr_def ; - attr (@name.expr_def) node_definition = @name + attr (@name.expr_def) node_definition = @name, syntax_type = "type" ; type of enum expression is members edge @name.expr_def -> @name.expr_def__typeof @@ -2477,7 +2520,7 @@ if none @is_async { attr (@name.type_def__ns) pop_symbol = "%T" edge @name.type_def__ns -> @name.type_def ; - attr (@name.type_def) node_definition = @name + attr (@name.type_def) node_definition = @name, syntax_type = "type" edge @name.type_def -> @enum_decl.type ; edge @enum_decl.type -> @body.type_members @@ -2567,7 +2610,7 @@ if none @is_async { attr (@name.type_def__ns) pop_symbol = "%T" edge @name.type_def__ns -> @name.type_def ; - attr (@name.type_def) node_definition = @name + attr (@name.type_def) node_definition = @name, syntax_type = "type" edge @name.type_def -> @decl.generic_type ; ; type parameters are handled by generics rules below @@ -2613,7 +2656,7 @@ if none @is_async { (call_expression) (class) (false) - (function) + (function_expression) (generator_function) (import) (member_expression) @@ -2625,6 +2668,7 @@ if none @is_async { (object) (parenthesized_expression) (regex) + (satisfies_expression) (sequence_expression) (spread_element) (string) @@ -2634,7 +2678,16 @@ if none @is_async { (ternary_expression) (this) (true) +; #dialect typescript (type_assertion) +; #end +; #dialect tsx + (jsx_element) + (jsx_self_closing_element) + (jsx_opening_element) + (jsx_closing_element) + (jsx_expression) +; #end (unary_expression) (undefined) (update_expression) @@ -2736,91 +2789,13 @@ if none @is_async { ; x; -[ - (primary_expression/identifier)@name - ; FIXME expansion of _lhs_expression/identifier and _augmented_assignment_lhs - (for_in_statement ["var" "let" "const"]?@is_def left:(identifier)@name) - (assignment_expression left:(identifier)@name) - (augmented_assignment_expression left:(identifier)@name) - (asserts (identifier)@name) - (type_predicate name:(identifier)@name) - ; FIXME type_query has its own restricted expression production - ; we need to do this for every (identifier) inside a type query - ; this cannot be expressed, so we manually unroll three levels here - (type_query (identifier)@name) - (type_query (member_expression object:(identifier)@name)) - (type_query (member_expression object:(member_expression object:(identifier)@name))) - (type_query (member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) - (type_query (subscript_expression object:(identifier)@name)) - (type_query (subscript_expression object:(member_expression object:(identifier)@name))) - (type_query (subscript_expression object:(member_expression object:(member_expression object:(identifier)@name)))) - (type_query (call_expression function:(identifier)@name)) - (type_query (call_expression function:(member_expression object:(identifier)@name))) - (type_query (call_expression function:(member_expression object:(member_expression object:(identifier)@name)))) - (type_query (call_expression function:(member_expression object:(member_expression object:(member_expression object:(identifier)@name))))) - (type_query (call_expression function:(subscript_expression object:(identifier)@name))) - (type_query (call_expression function:(subscript_expression object:(member_expression object:(identifier)@name)))) - (type_query (call_expression function:(subscript_expression object:(member_expression object:(member_expression object:(identifier)@name))))) - ; FIXME decorator has its own restricted expression production - ; we need to do this for every (identifier) inside a decorator - ; this cannot be expressed, so we manually unroll three levels here - (decorator (identifier)@name) - (decorator (member_expression object:(identifier)@name)) - (decorator (member_expression object:(member_expression object:(identifier)@name))) - (decorator (member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) - (decorator (call_expression function:(identifier)@name)) - (decorator (call_expression function:(member_expression object:(identifier)@name))) - (decorator (call_expression function:(member_expression object:(member_expression object:(identifier)@name)))) - (decorator (call_expression function:(member_expression object:(member_expression object:(member_expression object:(identifier)@name))))) -] { -if none @is_def { +(identifier)@name { node @name.cotype node @name.lexical_defs node @name.lexical_scope node @name.type node @name.var_defs -} -} -[ - (primary_expression/identifier)@name - (decorator (identifier)@name) - ; FIXME expansion of _lhs_expression/identifier and _augmented_assignment_lhs - ; we need to do this for every (identifier) inside a type query - ; this cannot be expressed, so we manually unroll three levels here - (for_in_statement ["var" "let" "const"]?@is_def left:(identifier)@name) - (assignment_expression left:(identifier)@name) - (augmented_assignment_expression left:(identifier)@name) - (asserts (identifier)@name) - (type_predicate name:(identifier)@name) - ; FIXME type_query has its own restricted expression production - (type_query (identifier)@name) - (type_query (member_expression object:(identifier)@name)) - (type_query (member_expression object:(member_expression object:(identifier)@name))) - (type_query (member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) - (type_query (subscript_expression object:(identifier)@name)) - (type_query (subscript_expression object:(member_expression object:(identifier)@name))) - (type_query (subscript_expression object:(member_expression object:(member_expression object:(identifier)@name)))) - (type_query (call_expression function:(identifier)@name)) - (type_query (call_expression function:(member_expression object:(identifier)@name))) - (type_query (call_expression function:(member_expression object:(member_expression object:(identifier)@name)))) - (type_query (call_expression function:(member_expression object:(member_expression object:(member_expression object:(identifier)@name))))) - (type_query (call_expression function:(subscript_expression object:(identifier)@name))) - (type_query (call_expression function:(subscript_expression object:(member_expression object:(identifier)@name)))) - (type_query (call_expression function:(subscript_expression object:(member_expression object:(member_expression object:(identifier)@name))))) - ; FIXME decorator has its own restricted expression production - ; we need to do this for every (identifier) inside a decorator - ; this cannot be expressed, so we manually unroll three levels here - (decorator (identifier)@name) - (decorator (member_expression object:(identifier)@name)) - (decorator (member_expression object:(member_expression object:(identifier)@name))) - (decorator (member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) - (decorator (call_expression function:(identifier)@name)) - (decorator (call_expression function:(member_expression object:(identifier)@name))) - (decorator (call_expression function:(member_expression object:(member_expression object:(identifier)@name)))) - (decorator (call_expression function:(member_expression object:(member_expression object:(member_expression object:(identifier)@name))))) -] { -if none @is_def { node @name.expr_ref node @name.expr_ref__ns node @name.expr_ref__typeof @@ -2838,7 +2813,67 @@ if none @is_def { attr (@name.expr_ref__typeof) push_symbol = ":" edge @name.expr_ref__typeof -> @name.expr_ref } -} + +; [ +; (primary_expression/identifier)@name +; ; FIXME expansion of _lhs_expression/identifier and _augmented_assignment_lhs +; ; we need to do this for every (identifier) inside a type query +; ; this cannot be expressed, so we manually unroll three levels here +; (for_in_statement ["var" "let" "const"]?@is_def left:(identifier)@name) +; (assignment_expression left:(identifier)@name) +; (augmented_assignment_expression left:(identifier)@name) +; (asserts (identifier)@name) +; (type_predicate name:(identifier)@name) +; ; FIXME type_query has its own restricted expression production +; ; we need to do this for every (identifier) inside a type query +; ; this cannot be expressed, so we manually unroll three levels here +; (type_query (identifier)@name) +; (type_query (member_expression object:(identifier)@name)) +; (type_query (member_expression object:(member_expression object:(identifier)@name))) +; (type_query (member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) +; (type_query (subscript_expression object:(identifier)@name)) +; (type_query (subscript_expression object:(member_expression object:(identifier)@name))) +; (type_query (subscript_expression object:(member_expression object:(member_expression object:(identifier)@name)))) +; (type_query (call_expression function:(identifier)@name)) +; (type_query (call_expression function:(member_expression object:(identifier)@name))) +; (type_query (call_expression function:(member_expression object:(member_expression object:(identifier)@name)))) +; (type_query (call_expression function:(member_expression object:(member_expression object:(member_expression object:(identifier)@name))))) +; (type_query (call_expression function:(subscript_expression object:(identifier)@name))) +; (type_query (call_expression function:(subscript_expression object:(member_expression object:(identifier)@name)))) +; (type_query (call_expression function:(subscript_expression object:(member_expression object:(member_expression object:(identifier)@name))))) +; ; FIXME decorator has its own restricted expression production +; ; we need to do this for every (identifier) inside a decorator +; ; this cannot be expressed, so we manually unroll three levels here +; (decorator (identifier)@name) +; (decorator (member_expression object:(identifier)@name)) +; (decorator (member_expression object:(member_expression object:(identifier)@name))) +; (decorator (member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) +; (decorator (call_expression function:(identifier)@name)) +; (decorator (call_expression function:(member_expression object:(identifier)@name))) +; (decorator (call_expression function:(member_expression object:(member_expression object:(identifier)@name)))) +; (decorator (call_expression function:(member_expression object:(member_expression object:(member_expression object:(identifier)@name))))) +; ; FIXME nested_identifier has its own restricted expression production +; ; we need to do this for every (identifier) inside a decorator +; ; this cannot be expressed, so we manually unroll three levels here +; (nested_identifier object:(identifier)@name) +; (nested_identifier object:(member_expression object:(identifier)@name)) +; (nested_identifier object:(member_expression object:(member_expression object:(identifier)@name))) +; (nested_identifier object:(member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) +; ; #dialect tsx +; (jsx_opening_element name: (identifier)@name) +; (jsx_closing_element name: (identifier)@name) +; (jsx_self_closing_element name: (identifier)@name) +; ; #end +; ] { +; if none @is_def { +; ; node @name.cotype +; ; node @name.lexical_defs +; ; node @name.lexical_scope +; ; node @name.type +; ; node @name.var_defs + +; } +; } @@ -2937,6 +2972,21 @@ if none @is_def { +;; Satisfies Expressions + +; { foo: 42 } satisfies { foo: number } + +(satisfies_expression (_)@expr . (_)@type)@satisfies { + ; propagate lexical scope + edge @expr.lexical_scope -> @satisfies.lexical_scope + edge @type.lexical_scope -> @satisfies.lexical_scope + + ; type is union of both + edge @satisfies.type -> @expr.type + edge @satisfies.type -> @type.type +} + + ;; Spread Element (spread_element (_)@expr)@spread_elem { @@ -3188,20 +3238,20 @@ if none @is_def { ; function (x) {}; -; (function +; (function_expression ; (formal_parameters (identifier)) ; (statement_block)) ; this captures the parameters -; (function +; (function_expression ; parameters: (_)@params) ; this captures the body -; (function +; (function_expression ; body:(_)@body)@function ; functions with names -; (function +; (function_expression ; name:(_)@name ; parameters:(_)@call_sig)@fun { ; } @@ -3236,7 +3286,7 @@ if none @is_def { ; } [ - (function) + (function_expression) (arrow_function) (generator_function) ]@fun { @@ -3263,9 +3313,9 @@ if none @is_def { } [ - (function parameters:(_)@call_sig) - (arrow_function parameters:(_)@call_sig) - (generator_function parameters:(_)@call_sig) + (function_expression parameters:(_)@call_sig) + (arrow_function parameters:(_)@call_sig) + (generator_function parameters:(_)@call_sig) ]@fun { ; propagate lexical scope edge @call_sig.lexical_scope -> @fun.lexical_scope @@ -3314,9 +3364,9 @@ if none @is_def { } [ - (function body:(_)@body) - (arrow_function body:(_)@body) - (generator_function body:(_)@body) + (function_expression body:(_)@body) + (arrow_function body:(_)@body) + (generator_function body:(_)@body) ]@fun { ; propagate lexical scope edge @body.lexical_scope -> @fun.lexical_scope @@ -3325,8 +3375,8 @@ if none @is_def { ;;;; specified return type [ - (function return_type:(_)@return_type) - (arrow_function return_type:(_)@return_type) + (function_expression return_type:(_)@return_type) + (arrow_function return_type:(_)@return_type) ]@fun { ; propagate lexical scope edge @return_type.lexical_scope -> @fun.lexical_scope @@ -3338,8 +3388,8 @@ if none @is_def { ;;;; inferred return type [ - (function "async"?@is_async !return_type body:(_)@body) - (arrow_function "async"?@is_async !return_type body:(statement_block)@body) + (function_expression "async"?@is_async !return_type body:(_)@body) + (arrow_function "async"?@is_async !return_type body:(statement_block)@body) ]@fun { if none @is_async { ; callable is type of return statement @@ -3357,8 +3407,8 @@ if none @is_async { ;;;; inferred async return type [ - (function "async" !return_type body:(_)@body) - (arrow_function "async" body:(statement_block)@body) + (function_expression "async" !return_type body:(_)@body) + (arrow_function "async" body:(statement_block)@body) ]@fun_decl { ; function returns body return type edge @fun_decl.callable__return -> @fun_decl.async_type @@ -3477,8 +3527,8 @@ if none @is_async { ; (subscript_expression (identifier) (string)) (member_expression - object: (_)@object - property: (_)@prop + object:(_)@object + property:(_)@prop )@member_expr { node @member_expr.member node @prop.expr_ref @@ -3501,7 +3551,6 @@ if none @is_async { edge @prop.expr_ref__typeof -> @prop.expr_ref } - (subscript_expression object: (_)@object index: (_)@index @@ -3807,15 +3856,13 @@ if none @is_async { ; (sequence_expression (number) (number)) -(sequence_expression - left: (_)@left - right: (_)@right -)@sequence_expr { +(sequence_expression (_)@expr)@sequence_expr { ; propagate lexical scope - edge @left.lexical_scope -> @sequence_expr.lexical_scope - edge @right.lexical_scope -> @sequence_expr.lexical_scope + edge @expr.lexical_scope -> @sequence_expr.lexical_scope +} - ; FIXME @sequence_expr.type is type of last, but cannot express because of nesting +(sequence_expression (_)@last .)@sequence_expr { + edge @sequence_expr.type -> @last.type } @@ -3957,6 +4004,12 @@ if none @is_async { edge @class_expr.this__type_def -> @class_expr.generic_inner_type } +; decorators +(class decorator:(_)@dec)@class_expr { + ; connect lexical scope + edge @dec.lexical_scope -> @class_expr.lexical_scope +} + ; default constructor ; FIXME only if no other constructor is defined (class)@class_expr { @@ -3994,7 +4047,7 @@ if none @is_async { } - +; #dialect typescript ;; Type Assertion ; (type_assertion @@ -4009,7 +4062,7 @@ if none @is_async { ; propagate lexical scope edge @expr.lexical_scope -> @type_assert.lexical_scope } - +; #end ;; As Expression @@ -4072,25 +4125,16 @@ if none @is_async { (this) (member_expression) (subscript_expression) + (undefined) ]@pat { node @pat.defs } -[ ; NOTE these are the ones not also variables - (for_in_statement ["var" "let" "const"] left:(identifier)@name) - (variable_declarator name:(identifier)@name) - (pattern/identifier)@name - (rest_pattern (_)@name) -] { - node @name.cotype - node @name.lexical_scope -} - [ (for_in_statement ["var" "let" "const"] left:(identifier)@name) (variable_declarator name:(identifier)@name) (pattern/identifier)@name - (rest_pattern (_)@name) + (rest_pattern (identifier)@name) ] { node @name.defs node @name.expr_def @@ -4301,6 +4345,7 @@ if none @is_async { [ (abstract_method_signature) (call_signature) + (class_static_block) (construct_signature) (decorator) ; not strictly a member, but appears in same positions (index_signature) @@ -4324,6 +4369,14 @@ if none @is_async { +;; Static Block + +(class_static_block body:(_)@body)@static { + edge @body.lexical_scope -> @static.lexical_scope +} + + + ;; Construct Signature ; (construct_signature @@ -4451,7 +4504,7 @@ if none @is_async { attr (@sig.member) pop_symbol = "." edge @sig.member -> @name.expr_def ; - attr (@name.expr_def) node_definition = @name + attr (@name.expr_def) node_definition = @name, syntax_type = "field" } (property_signature @@ -4478,6 +4531,12 @@ if none @is_async { ; value:(_)@value ; opt ; )@def +; decorators +(public_field_definition decorator:(_)@dec)@def { + ; connect lexical scope + edge @dec.lexical_scope -> @def.lexical_scope +} + (public_field_definition name:(_)@name )@def { @@ -4489,7 +4548,7 @@ if none @is_async { attr (@def.member) pop_symbol = "." edge @def.member -> @name.expr_def ; - attr (@name.expr_def) node_definition = @name + attr (@name.expr_def) node_definition = @name, syntax_type = "field" } (public_field_definition @@ -4573,7 +4632,7 @@ if none @is_acc { attr (@mem.member) pop_symbol = "." edge @mem.member -> @name.expr_def ; - attr (@name.expr_def) node_definition = @name + attr (@name.expr_def) node_definition = @name, syntax_type = "method" ; type of method is callable generic type edge @name.expr_def -> @name.expr_def__typeof @@ -4658,7 +4717,7 @@ if none @is_acc { attr (@mem.member) pop_symbol = "." edge @mem.member -> @name.expr_def ; - attr (@name.expr_def) node_definition = @name + attr (@name.expr_def) node_definition = @name, syntax_type = "field" } [ @@ -4863,6 +4922,7 @@ if none @is_acc { [ (array_type) (asserts) + (asserts_annotation) (conditional_type) (constraint) (constructor_type) @@ -4978,6 +5038,9 @@ if none @is_acc { edge @type.lexical_scope -> @asserts.lexical_scope } +(asserts_annotation (_)@asserts)@asserts_annotation { + edge @asserts.lexical_scope -> @asserts_annotation.lexical_scope +} ;; Optional Type @@ -5501,70 +5564,59 @@ if none @is_acc { [ ; X - (nested_type_identifier module:(identifier)@name @mod) + (nested_type_identifier module:(identifier)@name) ; X._ - (nested_type_identifier module:(nested_identifier . (identifier)@name @mod)) - ; X._._ - (nested_type_identifier module:(nested_identifier . (nested_identifier . (identifier)@name @mod))) - ; X._._._ - (nested_type_identifier module:(nested_identifier . (nested_identifier . (nested_identifier . (identifier)@name @mod)))) - ; X._._._._ - (nested_type_identifier module:(nested_identifier . (nested_identifier . (nested_identifier . (nested_identifier . (identifier)@name @mod))))) -]@nested_type_id { + (nested_identifier object:(identifier)@name) + ; _.X._ + (member_expression object:(identifier)@name) + ; _.X + (nested_identifier property:(_)@name) + ; _._.X + (member_expression property:(_)@name) +] { node @name.type_ref - node @name.type_ref__ns - node @mod.type - - ; type reference to namespace name - edge @mod.type -> @name.type_ref - ; attr (@name.type_ref) node_reference = @name - edge @name.type_ref -> @name.type_ref__ns - ; - attr (@name.type_ref__ns) push_symbol = "%T" - edge @name.type_ref__ns -> @nested_type_id.lexical_scope + + node @name.type_ref_member + attr (@name.type_ref_member) push_symbol = "." + + edge @name.type_ref_member -> @name.type_ref } [ - ; _.X - (nested_type_identifier module:(nested_identifier . (_)@basemod . (identifier)@name .)@mod) - ; _._.X - (nested_type_identifier module:(nested_identifier . (nested_identifier . (_)@basemod . (identifier)@name .)@mod)) - ; _._._.X - (nested_type_identifier module:(nested_identifier . (nested_identifier . (nested_identifier . (_)@basemod . (identifier)@name .)@mod))) - ; _._._._.X - (nested_type_identifier module:(nested_identifier . (nested_identifier . (nested_identifier . (nested_identifier . (_)@basemod . (identifier)@name .)@mod)))) + (nested_identifier object:(_)@mod) + (member_expression object:[(member_expression) (identifier)]@mod) +]@nested { + node @nested.type_ref + edge @mod.type_ref -> @nested.type_ref +} + +[ + (nested_identifier object:(_)@mod property:(_)@name) + (member_expression object:[(member_expression) (identifier)]@mod property:(_)@name) ] { - node @name.type_ref - node @name.type_ref__ns - node @mod.member - node @mod.type + edge @name.type_ref -> @mod.type_ref_member +} - ; type reference to namespace name - edge @mod.type -> @name.type_ref - ; - attr (@name.type_ref) node_reference = @name - edge @name.type_ref -> @mod.member - ; - attr (@mod.member) push_symbol = "." - edge @mod.member -> @basemod.type +[ + (nested_identifier property:(_)@name) + (member_expression property:(_)@name) +]@nested { + node @nested.type_ref_member + edge @nested.type_ref_member -> @name.type_ref_member } -(nested_type_identifier - module:(_)@mod - name:(_)@name -)@nested_type_id { - node @name.nested_type_ref ; FIXME non-standard name because the general (type_identifier) rule is also applied to @name - node @nested_type_id.member +(nested_type_identifier module:(_)@mod)@nested_type_id { + node type_ns + attr (type_ns) push_symbol = "%T" - ; type reference into namespace type - edge @nested_type_id.type -> @name.nested_type_ref - ; - attr (@name.nested_type_ref) node_reference = @name - edge @name.nested_type_ref -> @nested_type_id.member - ; - attr (@nested_type_id.member) push_symbol = "." - edge @nested_type_id.member -> @mod.type + edge @mod.type_ref -> type_ns + edge type_ns -> @nested_type_id.lexical_scope +} + +(nested_type_identifier module:(_)@mod name:(_)@name)@nested_type_id { + edge @nested_type_id.type -> @name.type_ref + edge @name.type_ref -> @mod.type_ref_member } @@ -6092,7 +6144,7 @@ if none @is_acc { [ (arrow_function "async") - (function "async") + (function_expression "async") (function_declaration "async") (generator_function "async") (generator_function_declaration "async") @@ -6122,9 +6174,124 @@ if none @is_acc { attr (@async.async_type__type_app) push_scoped_symbol = "<>", scope = @async.async_type__type_args edge @async.async_type__type_app -> @async.async_type__promise_ref ; - attr (@async.async_type__promise_ref) symbol_reference = "Promise", source_node = @async + attr (@async.async_type__promise_ref) symbol_reference = "Promise", source_node = @async, empty_source_span edge @async.async_type__promise_ref -> @async.async_type__promise_ref__ns ; attr (@async.async_type__promise_ref__ns) push_symbol = "%T" edge @async.async_type__promise_ref__ns -> @async.lexical_scope } + +; +; # ##### # # +; # # # # # +; # # # # +; # ##### # +; # # # # # +; # # # # # # +; ##### ##### # # +; +; ;;;;;;;;;;;;;;;;;;;; +; #dialect tsx +; ;;;;;;;;;;;;;;;;;;;; + +(jsx_element + open_tag:(_)@open_tag + close_tag:(_)@close_tag)@jsx_element { + + edge @open_tag.lexical_scope -> @jsx_element.lexical_scope + edge @close_tag.lexical_scope -> @jsx_element.lexical_scope +} + +(jsx_element + [ + (jsx_text) + (jsx_element) + (jsx_self_closing_element) + (jsx_expression) + ]@child +)@parent { + edge @child.lexical_scope -> @parent.lexical_scope +} + +(jsx_text)@jsx_text { + node @jsx_text.lexical_scope +} + +[ + (jsx_opening_element name: (_)@name)@element + (jsx_closing_element name: (_)@name)@element + (jsx_self_closing_element name: (_)@name)@element +] { + edge @name.lexical_scope -> @element.lexical_scope +} + +(jsx_opening_element + name:(_)@element_name + attribute:(_)@attr +) { + edge @attr.lexical_scope -> @element_name.lexical_scope +} + +(jsx_attribute (_) . (_)?@attr_value)@jsx_attribute { + node @jsx_attribute.lexical_scope + + if some @attr_value { + edge @attr_value.lexical_scope -> @jsx_attribute.lexical_scope + } +} + +(jsx_namespace_name (_) @lhs (_) @rhs)@name { + node @name.lexical_scope + + edge @lhs.lexical_scope -> @name.lexical_scope + edge @rhs.lexical_scope -> @name.lexical_scope +} + +(jsx_self_closing_element + name:(_)@element_name)@element { + edge @element_name.lexical_scope -> @element.lexical_scope +} + +(jsx_self_closing_element + name:(_)@element_name + attribute:(_)@attr +) { + edge @attr.lexical_scope -> @element_name.lexical_scope +} + +(jsx_expression (_)@child)@expr { + edge @child.lexical_scope -> @expr.lexical_scope +} + +(jsx_closing_element + name:(_)@element_name)@element +{ + edge @element_name.lexical_scope -> @element.lexical_scope +} + +[ + (jsx_opening_element + name:(identifier)@element_name) + (jsx_self_closing_element + name:(identifier)@element_name) + (jsx_closing_element + name:(identifier)@element_name) +] { + scan (source-text @element_name) { + ; standard HTML elements + "^(a|abbr|acronym|address|applet|area|article|aside|audio|b|base|basefont|bdi|bdo|big|blockquote|body|br|button|canvas|caption|center|cite|code|col|colgroup|data|datalist|dd|del|details|dfn|dialog|dir|div|dl|dt|em|embed|fieldset|figcaption|figure|font|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|header|hgroup|hr|html|i|iframe|input|ins|kbd|label|legend|li|link|main|map|mark|menu|meta|meter|nav|noframes|noscript|object|ol|optgroup|option|output|p|param|picture|pre|progress|q|rp|rt|ruby|s|samp|script|search|section|select|small|source|span|strike|strong|style|sub|summary|sup|svg|table|tbody|td|template|textarea|tfoot|th|thead|time|title|tr|track|tt|u|ul|var|video|wbr)$" { + ; do nothing! + } + + ; everything else + "^.+$" { + node element_name_pop + attr (element_name_pop) node_reference = @element_name + edge element_name_pop -> @element_name.lexical_scope + } + } +} + +; ;;;;;;;;;;;;;;;;;;;; +; #end +; ;;;;;;;;;;;;;;;;;;;; diff --git a/languages/tree-sitter-stack-graphs-typescript/test/expressions/rest-array-attern.ts b/languages/tree-sitter-stack-graphs-typescript/test/expressions/rest-array-attern.ts new file mode 100644 index 000000000..7bc38c4a0 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/expressions/rest-array-attern.ts @@ -0,0 +1,6 @@ +function foo(...[bar]) { + bar; +// ^ defined: 1 +} + +export {} \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-typescript/test/expressions/satisfies-type.ts b/languages/tree-sitter-stack-graphs-typescript/test/expressions/satisfies-type.ts new file mode 100644 index 000000000..48b3a86f5 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/expressions/satisfies-type.ts @@ -0,0 +1,8 @@ +type Foo = { foo: number }; + +let x = { foo: 42 } satisfies Foo; + + x.foo +// ^ defined: 3, 1 + +export {} diff --git a/languages/tree-sitter-stack-graphs-typescript/test/expressions/undefined-pattern.ts b/languages/tree-sitter-stack-graphs-typescript/test/expressions/undefined-pattern.ts new file mode 100644 index 000000000..652adf3c6 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/expressions/undefined-pattern.ts @@ -0,0 +1,5 @@ +interface Foo { + bar(undefined?: any): any; +} + +export {} \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_core.tsx b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_core.tsx new file mode 100644 index 000000000..93285bc93 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_core.tsx @@ -0,0 +1,39 @@ +// The core of JSX tests here verify the behavior of the following node types: +// jsx_element +// jsx_identifier +// jsx_attribute +// jsx_expression +// jsx_opening_element +// jsx_closing_element +// There is no real way to avoid testing all of these at once, +// and so we don't even try to. + +let x = 1; + +// Flow In + +const el = {x}; +// ^ defined: 11 +// ^ defined: 11 + +const el2 = +// ^ defined: 11 +// ^ defined: 11 + +let y = 0; +let z = 2; + +const el = +// ^ defined: 23 + {z = 3} +// ^ defined: 24 +; + +/**/ y; +// ^ defined: 23 + +/**/ z; +// ^ defined: 24 + +/**/ x; +// ^ defined: 11 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_fragment.tsx b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_fragment.tsx new file mode 100644 index 000000000..1713694f7 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_fragment.tsx @@ -0,0 +1,12 @@ +let x = 1; + +// Flow Around + +const el = <>; + +/**/ x; +// ^ defined: 1 + +// Children +({x}); +// ^ defined: 1 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_namespace_name.tsx b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_namespace_name.tsx new file mode 100644 index 000000000..74fe0ff9b --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_namespace_name.tsx @@ -0,0 +1,30 @@ +let x = 1; + +// Flow Around +namespace noo.bar { + export let baz = 1; +} + +const el = ; +// ^ defined: 4 +// ^ defined: 4 +// ^ defined: 5 +// ^ defined: 4 +// ^ defined: 4 +// ^ defined: 5 + +/**/ x; +// ^ defined: 1 + +// Flow In + +let foo = { + bar: { + baz: 1 + } +}; + +const el2 = ; +// ^ defined: 21 +// ^ defined: 22 +// ^ defined: 23 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_self_closing_element.tsx b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_self_closing_element.tsx new file mode 100644 index 000000000..d26746137 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_self_closing_element.tsx @@ -0,0 +1,37 @@ +// The core of JSX tests here verify the behavior of the following node types: +// jsx_element +// jsx_identifier +// jsx_attribute +// jsx_expression +// jsx_opening_element +// jsx_closing_element +// There is no real way to avoid testing all of these at once, +// and so we don't even try to. + +let x = 1; + +// Flow In + +const el = ; +// ^ defined: 11 + +const el2 = +// ^ defined: 11 + +// Flow Out + +let y = 2; + +const el = ; +// ^ defined: 23 + +// Flow Across + +const el = ; +// ^ defined: 23 + +// Flow Around + +/**/ x; +// ^ defined: 11 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_text.tsx b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_text.tsx new file mode 100644 index 000000000..0cc359765 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_text.tsx @@ -0,0 +1,8 @@ +let x = 1; + +// Flow Around + +const el = bar; + +/**/ x; +// ^ defined: 1 \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-directly-exported-variable.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-directly-exported-variable.ts index bd6a1126b..91e5da6c7 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-directly-exported-variable.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-directly-exported-variable.ts @@ -1,10 +1,10 @@ -/*--- path: ./ModA.ts ---*/ +/*--- path: ModA.ts ---*/ export let a = { v: 42 }; -/*--- path: ./ModB.ts ---*/ +/*--- path: ModB.ts ---*/ import { a } from "./ModA"; // ^ defined: 3 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-js.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-js.ts new file mode 100644 index 000000000..92817fbaf --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-js.ts @@ -0,0 +1,6 @@ +/* --- path: src/foo.ts --- */ +export const bar = 42; + +/* --- path: src/index.ts --- */ +import { bar } from "./foo.js"; +// ^ defined: 2 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-none.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-none.ts new file mode 100644 index 000000000..f221684b4 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-none.ts @@ -0,0 +1,6 @@ +/* --- path: src/foo.ts --- */ +export const bar = 42; + +/* --- path: src/index.ts --- */ +import { bar } from "./foo"; +// ^ defined: 2 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-ts.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-ts.ts new file mode 100644 index 000000000..1e46349c9 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-ts.ts @@ -0,0 +1,6 @@ +/* --- path: src/foo.ts --- */ +export const bar = 42; + +/* --- path: src/index.ts --- */ +import { bar } from "./foo.ts"; +// ^ defined: 2 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-ambient-module-declaration.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-ambient-module-declaration.ts index 75aff4238..abef61e5e 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-ambient-module-declaration.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-ambient-module-declaration.ts @@ -1,8 +1,8 @@ -/* --- path: ./index.ts --- */ +/* --- path: index.ts --- */ import { foo } from "@my/lib"; // ^ defined: 8 -/* --- path: ./mod.ts --- */ +/* --- path: mod.ts --- */ declare module "@my/lib" { export const foo = 42; diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-own-direcotry-in-index.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-own-direcotry-in-index.ts new file mode 100644 index 000000000..b62a2fe45 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-own-direcotry-in-index.ts @@ -0,0 +1,8 @@ +/*--- path: foo/index.ts ---*/ + +import { FOO } from "./bar"; +// ^ defined: 8 + +/*--- path: foo/bar.ts ---*/ + +export const FOO = 42; diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-own-project-subdirectory.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-own-project-subdirectory.ts index 2d399830c..743efa90a 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-own-project-subdirectory.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-own-project-subdirectory.ts @@ -1,11 +1,11 @@ -/*--- path: ./A/ModA.ts ---*/ +/*--- path: A/ModA.ts ---*/ /*--- global: PROJECT_NAME=foo/bar ---*/ export let a = { v: 42 }; -/*--- path: ./ModB.ts ---*/ +/*--- path: ModB.ts ---*/ /*--- global: PROJECT_NAME=foo/bar ---*/ import { a } from "./A/ModA"; diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-subdirectory.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-subdirectory.ts index 19e6c03b2..11292dff4 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-subdirectory.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-subdirectory.ts @@ -1,10 +1,10 @@ -/*--- path: ./A/ModA.ts ---*/ +/*--- path: A/ModA.ts ---*/ export let a = { v: 42 }; -/*--- path: ./ModB.ts ---*/ +/*--- path: ModB.ts ---*/ import { a } from "./A/ModA"; // ^ defined: 3 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-superdirectory.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-superdirectory.ts index bf7865e08..8f728d91c 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-superdirectory.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-from-superdirectory.ts @@ -1,10 +1,10 @@ -/*--- path: ./ModA.ts ---*/ +/*--- path: ModA.ts ---*/ export let a = { v: 42 }; -/*--- path: ./B/ModB.ts ---*/ +/*--- path: B/ModB.ts ---*/ import { a } from "../ModA"; // ^ defined: 3 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-indirectly-exported-variable.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-indirectly-exported-variable.ts index 3676bf85c..e2906595e 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-indirectly-exported-variable.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-indirectly-exported-variable.ts @@ -1,4 +1,4 @@ -/*--- path: ./ModA.ts ---*/ +/*--- path: ModA.ts ---*/ let a = { v: 42 @@ -7,7 +7,7 @@ let a = { export { a }; // ^ defined: 3 -/*--- path: ./ModB.ts ---*/ +/*--- path: ModB.ts ---*/ import { a } from "./ModA"; // ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-reexports-from-subdirectory-index.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-reexports-from-subdirectory-index.ts new file mode 100644 index 000000000..e1871de89 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-reexports-from-subdirectory-index.ts @@ -0,0 +1,12 @@ +/* --- path: foo/index.ts --- */ + +export * from './bar' + +/* --- path: foo/bar.ts --- */ + +export let baz = 42 + +/* --- path: qux.ts --- */ + +import { baz } from './foo' +// ^ defined: 7 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-reexports-from-superdirectory-index.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-reexports-from-superdirectory-index.ts new file mode 100644 index 000000000..c37ae3cf7 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-reexports-from-superdirectory-index.ts @@ -0,0 +1,12 @@ +/* --- path: index.ts --- */ + +export * from './bar' + +/* --- path: bar.ts --- */ + +export let baz = 42 + +/* --- path: foo/qux.ts --- */ + +import { baz } from '..' +// ^ defined: 7 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-relative-to-project-root.ts.bug b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-relative-to-project-root.ts.skip similarity index 55% rename from languages/tree-sitter-stack-graphs-typescript/test/modules/import-relative-to-project-root.ts.bug rename to languages/tree-sitter-stack-graphs-typescript/test/modules/import-relative-to-project-root.ts.skip index c3a2589da..eed68b4ee 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-relative-to-project-root.ts.bug +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-relative-to-project-root.ts.skip @@ -1,8 +1,8 @@ -/*--- path: ./ModA.ts ---*/ +/*--- path: ModA.ts ---*/ export const a = 42; -/*--- path: ./B/ModB.ts ---*/ +/*--- path: B/ModB.ts ---*/ import { a } from "./ModA"; // ^ defined: \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-via-index.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-via-index.ts index ef0504741..3730b1b73 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-via-index.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-via-index.ts @@ -1,10 +1,10 @@ -/*--- path: ./A/index.ts ---*/ +/*--- path: A/index.ts ---*/ export let a = { v: 42 }; -/*--- path: ./B/ModB.ts ---*/ +/*--- path: B/ModB.ts ---*/ import { a } from "../A"; // ^ defined: 3 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-via-superdirectory.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-via-superdirectory.ts index 332d6474d..a67e79311 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-via-superdirectory.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-via-superdirectory.ts @@ -1,10 +1,10 @@ -/*--- path: ./A/ModA.ts ---*/ +/*--- path: A/ModA.ts ---*/ export let a = { v: 42 }; -/*--- path: ./B/ModB.ts ---*/ +/*--- path: B/ModB.ts ---*/ import { a } from "../A/ModA"; // ^ defined: 3 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-via-two-reexports.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-via-two-reexports.ts new file mode 100644 index 000000000..981e3d15d --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-via-two-reexports.ts @@ -0,0 +1,15 @@ +/* --- path: src/foo/index.ts --- */ + +export * from "../bar"; + +/* --- path: src/bar/index.ts --- */ + +export * from "./quz"; + +/* --- path: src/bar/quz.ts --- */ + +export const QUZ = 42; + +/* --- path: src/test.ts --- */ + +import { QUZ } from "./foo"; diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/reexport-as-object.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/reexport-as-object.ts new file mode 100644 index 000000000..54e5494b9 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/reexport-as-object.ts @@ -0,0 +1,15 @@ +/* --- path: foo.ts --- */ + +export const FOO = 42; + +/* --- path: bar.ts --- */ + +export * as quz from "./foo"; + +/* --- path: test.ts --- */ + +import { quz } from "./bar"; + + quz.FOO +// ^ defined: 11, 7 +// ^ defined: 3 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/rename-imported-directly-exported-variable.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/rename-imported-directly-exported-variable.ts index f3b403494..ecfdc2d2c 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/modules/rename-imported-directly-exported-variable.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/rename-imported-directly-exported-variable.ts @@ -1,10 +1,10 @@ -/*--- path: ./ModA.ts ---*/ +/*--- path: ModA.ts ---*/ export let a = { v: 42 }; -/*--- path: ./ModB.ts ---*/ +/*--- path: ModB.ts ---*/ import { a as b } from "./ModA"; // ^ defined: 3 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/rename-imported-indirectly-exported-variable.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/rename-imported-indirectly-exported-variable.ts index 48f62099e..e1e99db34 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/modules/rename-imported-indirectly-exported-variable.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/rename-imported-indirectly-exported-variable.ts @@ -1,4 +1,4 @@ -/*--- path: ./ModA.ts ---*/ +/*--- path: ModA.ts ---*/ let a = { v: 42 @@ -7,7 +7,7 @@ let a = { export { a }; // ^ defined: 3 -/*--- path: ./ModB.ts ---*/ +/*--- path: ModB.ts ---*/ import { a as b } from "./ModA"; // ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/very-deep-namespace.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/very-deep-namespace.ts new file mode 100644 index 000000000..71af87eee --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/very-deep-namespace.ts @@ -0,0 +1,6 @@ +namespace a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y { + export const Z = 42; +} + +a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.Z +// ^ defined: 2 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/packages/can-access-package-non-main-explicilty.ts b/languages/tree-sitter-stack-graphs-typescript/test/packages/can-access-package-non-main-explicilty.ts new file mode 100644 index 000000000..dda91f57b --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/packages/can-access-package-non-main-explicilty.ts @@ -0,0 +1,51 @@ +/* --- path: acme_foo/tsconfig.json --- */ +/* --- global: FILE_PATH=tsconfig.json --- */ +/* --- global: PROJECT_NAME=acme_foo --- */ + +{} + +/* --- path: acme_foo/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=acme_foo --- */ + +{ + "name": "@acme/foo", + "version": "1.0", + "main": "./api" +} + +/* --- path: acme_foo/api.ts --- */ +/* --- global: FILE_PATH=api.ts --- */ +/* --- global: PROJECT_NAME=acme_foo --- */ + +export let x; + +/* --- path: acme_foo/core.ts --- */ +/* --- global: FILE_PATH=core.ts --- */ +/* --- global: PROJECT_NAME=acme_foo --- */ + +export let x; + +/* --- path: bar/tsconfig.json --- */ +/* --- global: FILE_PATH=tsconfig.json --- */ +/* --- global: PROJECT_NAME=bar --- */ + +{} + +/* --- path: bar/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=bar --- */ + +{ + "name": "bar", + "dependencies": { + "@acme/foo": "1" + } +} + +/* --- path: bar/app.ts --- */ +/* --- global: FILE_PATH=app.ts --- */ +/* --- global: PROJECT_NAME=bar --- */ + +import { x } from "@acme/foo/core" +// ^ defined: 27 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/packages/package-does-not-export-non-main.ts b/languages/tree-sitter-stack-graphs-typescript/test/packages/package-does-not-export-non-main.ts new file mode 100644 index 000000000..d39b0d79f --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/packages/package-does-not-export-non-main.ts @@ -0,0 +1,44 @@ +/* --- path: foo/tsconfig.json --- */ +/* --- global: FILE_PATH=tsconfig.json --- */ +/* --- global: PROJECT_NAME=foo --- */ + +{} + +/* --- path: foo/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=foo --- */ + +{ + "name": "foo", + "version": "1.0" +} + +/* --- path: foo/impl.ts --- */ +/* --- global: FILE_PATH=impl.ts --- */ +/* --- global: PROJECT_NAME=foo --- */ + +export let x; + +/* --- path: bar/tsconfig.json --- */ +/* --- global: FILE_PATH=tsconfig.json --- */ +/* --- global: PROJECT_NAME=bar --- */ + +{} + +/* --- path: bar/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=bar --- */ + +{ + "name": "bar", + "dependencies": { + "foo": "1" + } +} + +/* --- path: bar/app.ts --- */ +/* --- global: FILE_PATH=app.ts --- */ +/* --- global: PROJECT_NAME=bar --- */ + +import { x } from "foo" +// ^ defined: diff --git a/languages/tree-sitter-stack-graphs-typescript/test/packages/package-exports-explicit-main.ts b/languages/tree-sitter-stack-graphs-typescript/test/packages/package-exports-explicit-main.ts new file mode 100644 index 000000000..f6a4f5e45 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/packages/package-exports-explicit-main.ts @@ -0,0 +1,51 @@ +/* --- path: acme_foo/tsconfig.json --- */ +/* --- global: FILE_PATH=tsconfig.json --- */ +/* --- global: PROJECT_NAME=acme_foo --- */ + +{} + +/* --- path: acme_foo/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=acme_foo --- */ + +{ + "name": "@acme/foo", + "version": "1.0", + "main": "./api" +} + +/* --- path: acme_foo/api.ts --- */ +/* --- global: FILE_PATH=api.ts --- */ +/* --- global: PROJECT_NAME=acme_foo --- */ + +export let x; + +/* --- path: acme_foo/core.ts --- */ +/* --- global: FILE_PATH=core.ts --- */ +/* --- global: PROJECT_NAME=acme_foo --- */ + +export let x; + +/* --- path: bar/tsconfig.json --- */ +/* --- global: FILE_PATH=tsconfig.json --- */ +/* --- global: PROJECT_NAME=bar --- */ + +{} + +/* --- path: bar/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=bar --- */ + +{ + "name": "bar", + "dependencies": { + "@acme/foo": "1" + } +} + +/* --- path: bar/app.ts --- */ +/* --- global: FILE_PATH=app.ts --- */ +/* --- global: PROJECT_NAME=bar --- */ + +import { x } from "@acme/foo" +// ^ defined: 21 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/package-invisible-without-dependency.ts b/languages/tree-sitter-stack-graphs-typescript/test/packages/package-invisible-without-dependency.ts similarity index 71% rename from languages/tree-sitter-stack-graphs-typescript/test/projects/package-invisible-without-dependency.ts rename to languages/tree-sitter-stack-graphs-typescript/test/packages/package-invisible-without-dependency.ts index ca9852a68..87248abe2 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/package-invisible-without-dependency.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/packages/package-invisible-without-dependency.ts @@ -1,36 +1,40 @@ -/* --- path: ./my_lib/package.json --- */ +/* --- path: my_lib/package.json --- */ /* --- global: FILE_PATH=package.json --- */ /* --- global: PROJECT_NAME=my_lib --- */ + { "name": "@my/lib" } -/* --- path: ./my_lib/tsconfig.json --- */ +/* --- path: my_lib/tsconfig.json --- */ /* --- global: FILE_PATH=tsconfig.json --- */ /* --- global: PROJECT_NAME=my_lib --- */ -{ -} -/* --- path: ./my_lib/src/foo.ts --- */ +{} + +/* --- path: my_lib/src/foo.ts --- */ /* --- global: FILE_PATH=src/foo.ts --- */ /* --- global: PROJECT_NAME=my_lib --- */ + export const bar = 42; -/* --- path: ./my_app/package.json --- */ +/* --- path: my_app/package.json --- */ /* --- global: FILE_PATH=package.json --- */ /* --- global: PROJECT_NAME=my_app --- */ + { "name": "@my/app" } -/* --- path: ./my_app/tsconfig.json --- */ +/* --- path: my_app/tsconfig.json --- */ /* --- global: FILE_PATH=tsconfig.json --- */ /* --- global: PROJECT_NAME=my_app --- */ -{ -} -/* --- path: ./my_app/src/index.ts --- */ +{} + +/* --- path: my_app/src/index.ts --- */ /* --- global: FILE_PATH=src/index.ts --- */ /* --- global: PROJECT_NAME=my_app --- */ + import { bar } from "@my/lib/foo"; // ^ defined: diff --git a/languages/tree-sitter-stack-graphs-typescript/test/packages/package-main-defaults-to-index.ts b/languages/tree-sitter-stack-graphs-typescript/test/packages/package-main-defaults-to-index.ts new file mode 100644 index 000000000..b7983551d --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/packages/package-main-defaults-to-index.ts @@ -0,0 +1,50 @@ +/* --- path: foo/tsconfig.json --- */ +/* --- global: FILE_PATH=tsconfig.json --- */ +/* --- global: PROJECT_NAME=foo --- */ + +{} + +/* --- path: foo/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=foo --- */ + +{ + "name": "foo", + "version": "1.0" +} + +/* --- path: foo/index.ts --- */ +/* --- global: FILE_PATH=index.ts --- */ +/* --- global: PROJECT_NAME=foo --- */ + +export let x; + +/* --- path: foo/impl.ts --- */ +/* --- global: FILE_PATH=impl.ts --- */ +/* --- global: PROJECT_NAME=foo --- */ + +export let x; + +/* --- path: bar/tsconfig.json --- */ +/* --- global: FILE_PATH=tsconfig.json --- */ +/* --- global: PROJECT_NAME=bar --- */ + +{} + +/* --- path: bar/package.json --- */ +/* --- global: FILE_PATH=package.json --- */ +/* --- global: PROJECT_NAME=bar --- */ + +{ + "name": "bar", + "dependencies": { + "foo": "1" + } +} + +/* --- path: bar/app.ts --- */ +/* --- global: FILE_PATH=app.ts --- */ +/* --- global: PROJECT_NAME=bar --- */ + +import { x } from "foo" +// ^ defined: 20 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/baseurl-to-subdir.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/baseurl-to-subdir.ts index a867d24b1..80f5a00d4 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/baseurl-to-subdir.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/baseurl-to-subdir.ts @@ -1,4 +1,4 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "composite": true, @@ -6,9 +6,9 @@ } } -/* --- path: ./src/foo.ts --- */ +/* --- path: src/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "foo"; // ^ defined: 10 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/composite-project.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/composite-project.ts index 732536dbf..e7fd1b57d 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/composite-project.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/composite-project.ts @@ -1,13 +1,13 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "composite": true, } } -/* --- path: ./src/foo.ts --- */ +/* --- path: src/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "./foo"; // ^ defined: 9 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/declarations-are-ignored.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/declarations-are-ignored.ts index 27cf54427..8c3d2a6b5 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/declarations-are-ignored.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/declarations-are-ignored.ts @@ -1,12 +1,12 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { } -/* --- path: ./src/foo.ts --- */ +/* --- path: src/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "./foo"; // ^ defined: 6 -/* --- path: ./types/index.d.ts --- */ +/* --- path: types/index.d.ts --- */ diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/exclude-second-subdir.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/exclude-second-subdir.ts index 083f0293b..326557cc7 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/exclude-second-subdir.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/exclude-second-subdir.ts @@ -1,13 +1,13 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "exclude": ["test/**/*"] } -/* --- path: ./src/foo.ts --- */ +/* --- path: src/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "./foo"; // ^ defined: 7 -/* --- path: ./test/index.ts --- */ +/* --- path: test/index.ts --- */ diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/explicit-root-dir.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/explicit-root-dir.ts index c51108621..37dbade29 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/explicit-root-dir.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/explicit-root-dir.ts @@ -1,13 +1,13 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "rootDir": "." } } -/* --- path: ./core/foo.ts --- */ +/* --- path: core/foo.ts --- */ export const bar = 42; -/* --- path: ./core/index.ts --- */ +/* --- path: core/index.ts --- */ import { bar } from "./foo"; // ^ defined: 9 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/import-from-rootdirs-subdir.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/import-from-rootdirs-subdir.ts index 6c8715cec..5c9d2c896 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/import-from-rootdirs-subdir.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/import-from-rootdirs-subdir.ts @@ -1,4 +1,4 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "rootDirs": [ @@ -8,13 +8,13 @@ } } -/* --- path: ./src/core/index.ts --- */ +/* --- path: src/core/index.ts --- */ import { bar } from "./foo/baz"; // ^ defined: 16 -/* --- path: ./src/util/foo/baz.ts --- */ +/* --- path: src/util/foo/baz.ts --- */ export const bar = 42; -/* --- path: ./src/util/index.ts --- */ +/* --- path: src/util/index.ts --- */ -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/import-from-rootdirs.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/import-from-rootdirs.ts index d7720e510..7603a8f91 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/import-from-rootdirs.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/import-from-rootdirs.ts @@ -1,4 +1,4 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "rootDirs": [ @@ -8,13 +8,13 @@ } } -/* --- path: ./src/core/index.ts --- */ +/* --- path: src/core/index.ts --- */ import { bar } from "./foo"; // ^ defined: 16 -/* --- path: ./src/util/foo.ts --- */ +/* --- path: src/util/foo.ts --- */ export const bar = 42; -/* --- path: ./src/util/index.ts --- */ +/* --- path: src/util/index.ts --- */ -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/import-with-own-project-baseurl.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/import-with-own-project-baseurl.ts index 29c2892d8..1e40e23db 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/import-with-own-project-baseurl.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/import-with-own-project-baseurl.ts @@ -1,4 +1,4 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ /* --- global: PROJECT_NAME=a --- */ { "compilerOptions": { @@ -7,11 +7,11 @@ } } -/* --- path: ./src/foo.ts --- */ +/* --- path: src/foo.ts --- */ /* --- global: PROJECT_NAME=a --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ /* --- global: PROJECT_NAME=a --- */ import { bar } from "foo"; // ^ defined: 12 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/include-one-subdir.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/include-one-subdir.ts index 42f991b95..8aed79105 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/include-one-subdir.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/include-one-subdir.ts @@ -1,13 +1,13 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "include": ["src/**/*"] } -/* --- path: ./src/foo.ts --- */ +/* --- path: src/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "./foo"; // ^ defined: 7 -/* --- path: ./test/index.ts --- */ +/* --- path: test/index.ts --- */ diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/invalid-paths-mappings.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/invalid-paths-mappings.ts index 6e422f928..926f28e97 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/invalid-paths-mappings.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/invalid-paths-mappings.ts @@ -1,4 +1,4 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "paths": { diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/js-sources-are-ignored.ts.skip b/languages/tree-sitter-stack-graphs-typescript/test/projects/js-sources-are-ignored.ts.skip index 3ad81a358..21f8fae85 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/js-sources-are-ignored.ts.skip +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/js-sources-are-ignored.ts.skip @@ -1,12 +1,12 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { } -/* --- path: ./lib/index.js --- */ +/* --- path: lib/index.js --- */ -/* --- path: ./src/foo.ts --- */ +/* --- path: src/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "./foo"; // ^ defined: 8 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/js-sources-can-be-allowed.ts.skip b/languages/tree-sitter-stack-graphs-typescript/test/projects/js-sources-can-be-allowed.ts.skip index 1b4bd4619..ed304a1b9 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/js-sources-can-be-allowed.ts.skip +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/js-sources-can-be-allowed.ts.skip @@ -1,15 +1,15 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "allowJs": true } } -/* --- path: ./lib/index.js --- */ +/* --- path: lib/index.js --- */ -/* --- path: ./src/foo.ts --- */ +/* --- path: src/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "./foo"; // ^ defined: 11 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/non-relative-single-module-remap.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/non-relative-single-module-remap.ts index c948f40ee..64036fc0d 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/non-relative-single-module-remap.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/non-relative-single-module-remap.ts @@ -1,4 +1,4 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "composite": true, @@ -9,9 +9,9 @@ } } -/* --- path: ./lib/the_foo.ts --- */ +/* --- path: lib/the_foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "foo"; // ^ defined: 13 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/non-relative-star-remap.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/non-relative-star-remap.ts index 2d0c07608..5961eeffa 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/non-relative-star-remap.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/non-relative-star-remap.ts @@ -1,4 +1,4 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "composite": true, @@ -9,9 +9,9 @@ } } -/* --- path: ./lib/foo.ts --- */ +/* --- path: lib/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "foo"; // ^ defined: 13 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/non-relative-star-remaps.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/non-relative-star-remaps.ts index f8146cf96..6d3e26955 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/non-relative-star-remaps.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/non-relative-star-remaps.ts @@ -1,4 +1,4 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "composite": true, @@ -9,9 +9,9 @@ } } -/* --- path: ./ext/foo.ts --- */ +/* --- path: ext/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "foo"; // ^ defined: 13 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/one-file-in-subdir.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/one-file-in-subdir.ts index 5b399e022..9aed45a39 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/one-file-in-subdir.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/one-file-in-subdir.ts @@ -1,13 +1,13 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "files": ["src/index.ts"] } -/* --- path: ./src/foo.ts --- */ +/* --- path: src/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "./foo"; // ^ defined: 7 -/* --- path: ./test/index.ts --- */ +/* --- path: test/index.ts --- */ diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/package-dependency-with-nested-source-root.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/package-dependency-with-nested-source-root.ts index f8b2a69fb..16215e31c 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/package-dependency-with-nested-source-root.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/package-dependency-with-nested-source-root.ts @@ -1,22 +1,22 @@ -/* --- path: ./my_lib/package.json --- */ +/* --- path: my_lib/package.json --- */ /* --- global: FILE_PATH=package.json --- */ /* --- global: PROJECT_NAME=my_lib --- */ { "name": "@my/lib" } -/* --- path: ./my_lib/tsconfig.json --- */ +/* --- path: my_lib/tsconfig.json --- */ /* --- global: FILE_PATH=tsconfig.json --- */ /* --- global: PROJECT_NAME=my_lib --- */ { } -/* --- path: ./my_lib/src/foo.ts --- */ +/* --- path: my_lib/src/foo.ts --- */ /* --- global: FILE_PATH=src/foo.ts --- */ /* --- global: PROJECT_NAME=my_lib --- */ export const bar = 42; -/* --- path: ./my_app/package.json --- */ +/* --- path: my_app/package.json --- */ /* --- global: FILE_PATH=package.json --- */ /* --- global: PROJECT_NAME=my_app --- */ { @@ -26,13 +26,13 @@ export const bar = 42; } } -/* --- path: ./my_app/tsconfig.json --- */ +/* --- path: my_app/tsconfig.json --- */ /* --- global: FILE_PATH=tsconfig.json --- */ /* --- global: PROJECT_NAME=my_app --- */ { } -/* --- path: ./my_app/src/index.ts --- */ +/* --- path: my_app/src/index.ts --- */ /* --- global: FILE_PATH=src/index.ts --- */ /* --- global: PROJECT_NAME=my_app --- */ import { bar } from "@my/lib/foo"; diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/relative-import.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/relative-import.ts index 1c2670335..e1951f3af 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/relative-import.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/relative-import.ts @@ -1,13 +1,13 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "composite": true } } -/* --- path: ./src/foo.ts --- */ +/* --- path: src/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "./foo"; // ^ defined: 9 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/relative-single-module-remap.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/relative-single-module-remap.ts index 03ee84488..71b6ce834 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/relative-single-module-remap.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/relative-single-module-remap.ts @@ -1,4 +1,4 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "composite": true, @@ -8,9 +8,9 @@ } } -/* --- path: ./lib/the_foo.ts --- */ +/* --- path: lib/the_foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "foo"; // ^ defined: 12 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/relative-star-remap.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/relative-star-remap.ts index a116cfbbd..4a169c524 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/relative-star-remap.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/relative-star-remap.ts @@ -1,4 +1,4 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "composite": true, @@ -8,9 +8,9 @@ } } -/* --- path: ./lib/foo.ts --- */ +/* --- path: lib/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "foo"; // ^ defined: 12 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/remap-with-baseurl.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/remap-with-baseurl.ts index 53d47534d..cf2788ba0 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/remap-with-baseurl.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/remap-with-baseurl.ts @@ -1,4 +1,4 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { "compilerOptions": { "composite": true, @@ -9,9 +9,9 @@ } } -/* --- path: ./src/util_impl/foo.ts --- */ +/* --- path: src/util_impl/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "util/foo"; // ^ defined: 13 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/sources-in-multiple-subdirs.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/sources-in-multiple-subdirs.ts index e4248c32f..76d763b33 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/sources-in-multiple-subdirs.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/sources-in-multiple-subdirs.ts @@ -1,12 +1,12 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { } -/* --- path: ./src/foo.ts --- */ +/* --- path: src/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "./foo"; // ^ defined: 6 -/* --- path: ./test/index.ts --- */ +/* --- path: test/index.ts --- */ diff --git a/languages/tree-sitter-stack-graphs-typescript/test/projects/sources-in-one-subdir.ts b/languages/tree-sitter-stack-graphs-typescript/test/projects/sources-in-one-subdir.ts index e97ea71ed..24a9a62ca 100644 --- a/languages/tree-sitter-stack-graphs-typescript/test/projects/sources-in-one-subdir.ts +++ b/languages/tree-sitter-stack-graphs-typescript/test/projects/sources-in-one-subdir.ts @@ -1,10 +1,10 @@ -/* --- path: ./tsconfig.json --- */ +/* --- path: tsconfig.json --- */ { } -/* --- path: ./src/foo.ts --- */ +/* --- path: src/foo.ts --- */ export const bar = 42; -/* --- path: ./src/index.ts --- */ +/* --- path: src/index.ts --- */ import { bar } from "./foo"; // ^ defined: 6 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/statements/class-static-block.ts b/languages/tree-sitter-stack-graphs-typescript/test/statements/class-static-block.ts new file mode 100644 index 000000000..6ccdadea4 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/statements/class-static-block.ts @@ -0,0 +1,12 @@ +let bar = 42; + +class Foo { + static foo: number; + static { + this.foo = bar; + // ^ defined: 4 + // ^ defined: 1 + } +} + +export {} diff --git a/languages/tree-sitter-stack-graphs-typescript/vscode/package-lock.json b/languages/tree-sitter-stack-graphs-typescript/vscode/package-lock.json index 9cc00e7ae..4d95b10b3 100644 --- a/languages/tree-sitter-stack-graphs-typescript/vscode/package-lock.json +++ b/languages/tree-sitter-stack-graphs-typescript/vscode/package-lock.json @@ -38,9 +38,9 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dependencies": { "balanced-match": "^1.0.0" } diff --git a/lsp-positions/CHANGELOG.md b/lsp-positions/CHANGELOG.md index 36a0d4ebd..f80bc5c5d 100644 --- a/lsp-positions/CHANGELOG.md +++ b/lsp-positions/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v0.3.4 -- 2024-12-12 + +Upgraded the `tree-sitter` dependency to version 0.24. + +## v0.3.3 -- 2024-03-05 + +The `tree-sitter` dependency version was updated to fix install problems. + ## v0.3.2 -- 2023-06-08 ### Added diff --git a/lsp-positions/Cargo.toml b/lsp-positions/Cargo.toml index 2b2d496b3..a9f7acb20 100644 --- a/lsp-positions/Cargo.toml +++ b/lsp-positions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lsp-positions" -version = "0.3.2" +version = "0.3.4" description = "LSP-compatible character positions" homepage = "https://github.com/github/stack-graphs/tree/main/lsp-positions" repository = "https://github.com/github/stack-graphs/" @@ -17,12 +17,14 @@ edition = "2018" test = false [features] -default = ["tree-sitter"] bincode = ["dep:bincode"] +tree-sitter = ["dep:tree-sitter"] [dependencies] memchr = "2.4" -tree-sitter = { version=">= 0.19", optional=true } -unicode-segmentation = { version="1.8" } -serde = { version="1", optional=true, features=["derive"] } -bincode = { version="2.0.0-rc.3", optional=true } +tree-sitter = { version = "0.24", optional = true } # keep the same minor version as the tree-sitter + # dependency of tree-sitter-stack-graphs to prevent + # install problems +unicode-segmentation = { version = "1.8" } +serde = { version = "1", features = ["derive"], optional = true } +bincode = { version = "2.0.0-rc.3", optional = true } diff --git a/script/ci-test-init b/script/ci-test-init new file mode 100755 index 000000000..0ecc60788 --- /dev/null +++ b/script/ci-test-init @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -eu + +cargo run --bin tree-sitter-stack-graphs --features cli -- init \ + --language-name InitTest \ + --language-id init_test \ + --language-file-extension it \ + --grammar-crate-name tree-sitter-python \ + --grammar-crate-version 0.23.2 \ + --internal \ + --non-interactive + +cargo check -p tree-sitter-stack-graphs-init_test --all-features + +cargo test -p tree-sitter-stack-graphs-init_test + +cargo run -p tree-sitter-stack-graphs-init_test --features cli -- -V diff --git a/script/ci-test-valgrind b/script/ci-test-valgrind new file mode 100755 index 000000000..3f5e3dae6 --- /dev/null +++ b/script/ci-test-valgrind @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -eu + +# Start by building the test binary, before we try to get its filename +cargo test "$@" --no-run + +# Cargo does not have a clean way to get the test binary, so we do some text processing here to get it +test="$(cargo test "$@" --no-run 2>&1 | grep 'Executable' | head -1 | sed -e 's/[^(]*(\(.*\)).*$/\1/')" +log="${RUNNER_TEMP-.}/valgrind.log" + +# Run the test binary under valgrind +if ! valgrind \ + --leak-check=full \ + --show-leak-kinds=all \ + --error-exitcode=1 \ + --suppressions=valgrind.supp \ + --gen-suppressions=all \ + --log-file="$log" \ + "$test"; then + echo "Valgrind detected errors! See logs: $log" + exit 1 +fi diff --git a/stack-graphs/CHANGELOG.md b/stack-graphs/CHANGELOG.md index c61221e8e..802275fc2 100644 --- a/stack-graphs/CHANGELOG.md +++ b/stack-graphs/CHANGELOG.md @@ -5,6 +5,44 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v0.14.1 -- 2024-12-12 + +### Fixed + +- A panic when using `StackGraph::incoming_edge_degree` on some nodes without incoming edges. + +## v0.14.0 -- 2024-07-09 + +### Changed + +- The method `StackGraph::add_from_graph` returns the handles of the newly added files. + +## v0.13.0 -- 2024-03-06 + +### Added + +- New type `StitcherConfig` to specify configuration flags that control stitching. +- Path stiching statistics can now be collected during stitching for debugging purposes. Disabled by default and can be enabled with `StitcherConfig`. +- A method `StackGraph::set_edge_precedence` that allows changing the precendence of an existing edge in a stack graph. +- A method `StackGraph::incoming_edge_degree` that returns the number of edges ending in a given node. +- A method `Database::get_incoming_path_degree` that returns the number of partial paths in the database ending in a given node. +- Cycle detection improved by using incoming path or edge degrees to reduce the amount of data to keep in memory. +- Visualization uses different colors for nodes of different files, which makes understanding multi-file graphs easier. + +### Changed + +- Methods and types that do stitching in their implementation now require a `StitcherConfig` value. These include `Assertion::run`, `ForwardPartialPathStitcher::find_*`, and `Test::run`. +- The SQLite storage data encoding and queries changed to improve performance. + +### Removed + +- Method `StackGraph::remove_edge` has been removed. + +### Fixed + +- A panic when using an arena after `Arena::clear` or `SupplementalArena::clear` was called. +- Missing candidates when looking up root paths in a `Database`. + ## v0.12.0 -- 2023-07-27 ### Added diff --git a/stack-graphs/Cargo.toml b/stack-graphs/Cargo.toml index 2e55e1768..e542f4163 100644 --- a/stack-graphs/Cargo.toml +++ b/stack-graphs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "stack-graphs" -version = "0.12.0" +version = "0.14.1" description = "Name binding for arbitrary programming languages" homepage = "https://github.com/github/stack-graphs/tree/main/stack-graphs" repository = "https://github.com/github/stack-graphs/" @@ -25,14 +25,14 @@ test = false [dependencies] bincode = { version = "2.0.0-rc.3", optional = true } -bitvec = "1.0" -controlled-option = "0.4" +bitvec = "1.0.1" +controlled-option = "0.4.1" either = "1.6" enumset = "1.1" fxhash = "0.2" -itertools = "0.10" +itertools = "0.10.2" libc = "0.2" -lsp-positions = { version = "0.3", path = "../lsp-positions" } +lsp-positions = { version = "0.3", path = "../lsp-positions" } # explicit version is required to be able to publish crate rusqlite = { version = "0.28", optional = true, features = ["bundled", "functions"] } serde = { version = "1.0", optional = true, features = ["derive"] } serde_json = { version = "1.0", optional = true } @@ -42,7 +42,6 @@ thiserror = { version = "1.0" } [dev-dependencies] assert-json-diff = "2" -itertools = "0.10" maplit = "1.0" pretty_assertions = "0.7" serde_json = { version = "1.0" } diff --git a/stack-graphs/README.md b/stack-graphs/README.md index 7805d19ef..69efa5bd8 100644 --- a/stack-graphs/README.md +++ b/stack-graphs/README.md @@ -9,7 +9,7 @@ To use this library, add the following to your `Cargo.toml`: ``` toml [dependencies] -stack-graphs = "0.12" +stack-graphs = "0.14" ``` Check out our [documentation](https://docs.rs/stack-graphs/) for more details on diff --git a/stack-graphs/include/stack-graphs.h b/stack-graphs/include/stack-graphs.h index 50b7062ed..aeac9e636 100644 --- a/stack-graphs/include/stack-graphs.h +++ b/stack-graphs/include/stack-graphs.h @@ -437,6 +437,11 @@ struct sg_partial_path { struct sg_partial_path_edge_list edges; }; +struct sg_stitcher_config { + // Enables similar path detection during stiching. + bool detect_similar_paths; +}; + // An array of all of the partial paths in a partial path database. Partial path handles are // indices into this array. There will never be a valid partial path at index 0; a handle with // the value 0 represents a missing partial path. @@ -733,6 +738,7 @@ enum sg_result sg_partial_path_arena_find_partial_paths_in_file(const struct sg_ struct sg_partial_path_arena *partials, sg_file_handle file, struct sg_partial_path_list *partial_path_list, + const struct sg_stitcher_config *stitcher_config, const size_t *cancellation_flag); // Finds all complete paths reachable from a set of starting nodes, placing the result into the @@ -748,6 +754,7 @@ enum sg_result sg_partial_path_arena_find_all_complete_paths(const struct sg_sta size_t starting_node_count, const sg_node_handle *starting_nodes, struct sg_partial_path_list *path_list, + const struct sg_stitcher_config *stitcher_config, const size_t *cancellation_flag); // Returns a reference to the array of partial path data in this partial path database. The diff --git a/stack-graphs/src/arena.rs b/stack-graphs/src/arena.rs index 6e4747267..134a90d14 100644 --- a/stack-graphs/src/arena.rs +++ b/stack-graphs/src/arena.rs @@ -183,10 +183,9 @@ impl Arena { /// Clear the arena, keeping underlying allocated capacity. After this, all previous handles into /// the arena are invalid. - #[cfg_attr(not(feature = "storage"), allow(dead_code))] #[inline(always)] - pub(crate) fn clear(&mut self) { - self.items.clear(); + pub fn clear(&mut self) { + self.items.truncate(1); } /// Adds a new instance to this arena, returning a stable handle to it. @@ -290,10 +289,9 @@ impl SupplementalArena { /// Clear the supplemantal arena, keeping underlying allocated capacity. After this, /// all previous handles into the arena are invalid. - #[cfg_attr(not(feature = "storage"), allow(dead_code))] #[inline(always)] - pub(crate) fn clear(&mut self) { - self.items.clear(); + pub fn clear(&mut self) { + self.items.truncate(1); } /// Creates a new, empty supplemental arena, preallocating enough space to store supplemental diff --git a/stack-graphs/src/assert.rs b/stack-graphs/src/assert.rs index 70237614c..6e1190673 100644 --- a/stack-graphs/src/assert.rs +++ b/stack-graphs/src/assert.rs @@ -20,6 +20,7 @@ use crate::partial::PartialPaths; use crate::stitching::Database; use crate::stitching::DatabaseCandidates; use crate::stitching::ForwardPartialPathStitcher; +use crate::stitching::StitcherConfig; use crate::CancellationError; use crate::CancellationFlag; @@ -149,12 +150,19 @@ impl Assertion { graph: &StackGraph, partials: &mut PartialPaths, db: &mut Database, + stitcher_config: StitcherConfig, cancellation_flag: &dyn CancellationFlag, ) -> Result<(), AssertionError> { match self { - Self::Defined { source, targets } => { - self.run_defined(graph, partials, db, source, targets, cancellation_flag) - } + Self::Defined { source, targets } => self.run_defined( + graph, + partials, + db, + source, + targets, + stitcher_config, + cancellation_flag, + ), Self::Defines { source, symbols } => self.run_defines(graph, source, symbols), Self::Refers { source, symbols } => self.run_refers(graph, source, symbols), } @@ -167,6 +175,7 @@ impl Assertion { db: &mut Database, source: &AssertionSource, expected_targets: &Vec, + stitcher_config: StitcherConfig, cancellation_flag: &dyn CancellationFlag, ) -> Result<(), AssertionError> { let references = source.iter_references(graph).collect::>(); @@ -182,6 +191,7 @@ impl Assertion { ForwardPartialPathStitcher::find_all_complete_partial_paths( &mut DatabaseCandidates::new(graph, partials, db), vec![*reference], + stitcher_config, cancellation_flag, |_, _, p| { reference_paths.push(p.clone()); diff --git a/stack-graphs/src/c.rs b/stack-graphs/src/c.rs index f4857c791..9b50349ec 100644 --- a/stack-graphs/src/c.rs +++ b/stack-graphs/src/c.rs @@ -32,6 +32,7 @@ use crate::stitching::Database; use crate::stitching::DatabaseCandidates; use crate::stitching::ForwardPartialPathStitcher; use crate::stitching::GraphEdgeCandidates; +use crate::stitching::StitcherConfig; use crate::CancellationError; use crate::CancellationFlag; @@ -1173,18 +1174,21 @@ pub extern "C" fn sg_partial_path_arena_find_partial_paths_in_file( partials: *mut sg_partial_path_arena, file: sg_file_handle, partial_path_list: *mut sg_partial_path_list, + stitcher_config: *const sg_stitcher_config, cancellation_flag: *const usize, ) -> sg_result { let graph = unsafe { &(*graph).inner }; let partials = unsafe { &mut (*partials).inner }; let file = file.into(); let partial_path_list = unsafe { &mut *partial_path_list }; + let stitcher_config = unsafe { *stitcher_config }; let cancellation_flag: Option<&AtomicUsize> = unsafe { std::mem::transmute(cancellation_flag.as_ref()) }; ForwardPartialPathStitcher::find_minimal_partial_path_set_in_file( graph, partials, file, + stitcher_config.into(), &AtomicUsizeCancellationFlag(cancellation_flag), |_graph, partials, path| { let mut path = path.clone(); @@ -1210,17 +1214,20 @@ pub extern "C" fn sg_partial_path_arena_find_all_complete_paths( starting_node_count: usize, starting_nodes: *const sg_node_handle, path_list: *mut sg_partial_path_list, + stitcher_config: *const sg_stitcher_config, cancellation_flag: *const usize, ) -> sg_result { let graph = unsafe { &(*graph).inner }; let partials = unsafe { &mut (*partials).inner }; let starting_nodes = unsafe { std::slice::from_raw_parts(starting_nodes, starting_node_count) }; + let stitcher_config = unsafe { *stitcher_config }; let path_list = unsafe { &mut *path_list }; let cancellation_flag: Option<&AtomicUsize> = unsafe { std::mem::transmute(cancellation_flag.as_ref()) }; ForwardPartialPathStitcher::find_all_complete_partial_paths( &mut GraphEdgeCandidates::new(graph, partials, None), starting_nodes.iter().copied().map(sg_node_handle::into), + stitcher_config.into(), &AtomicUsizeCancellationFlag(cancellation_flag), |graph, _partials, path| { if path.is_complete(graph) { @@ -1391,6 +1398,20 @@ pub struct sg_forward_partial_path_stitcher { pub is_complete: bool, } +// Configuration for partial path stitchers. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct sg_stitcher_config { + /// Enables similar path detection during stiching. + pub detect_similar_paths: bool, +} + +impl Into for sg_stitcher_config { + fn into(self) -> StitcherConfig { + StitcherConfig::default().with_detect_similar_paths(self.detect_similar_paths) + } +} + // This is the Rust equivalent of a common C trick, where you have two versions of a struct — a // publicly visible one and a private one containing internal implementation details. In our case, // `sg_forward_partial_path_stitcher` is the public struct, and diff --git a/stack-graphs/src/cycles.rs b/stack-graphs/src/cycles.rs index eda31e171..212d14085 100644 --- a/stack-graphs/src/cycles.rs +++ b/stack-graphs/src/cycles.rs @@ -44,12 +44,14 @@ use crate::partial::Cyclicity; use crate::partial::PartialPath; use crate::partial::PartialPaths; use crate::paths::PathResolutionError; +use crate::stats::FrequencyDistribution; use crate::stitching::Appendable; use crate::stitching::ToAppendable; /// Helps detect similar paths in the path-finding algorithm. pub struct SimilarPathDetector

{ - paths: HashMap>, + paths: HashMap>, + counts: Option>>, } #[doc(hidden)] @@ -92,13 +94,23 @@ where pub fn new() -> SimilarPathDetector

{ SimilarPathDetector { paths: HashMap::new(), + counts: None, } } - /// Determines whether we should process this path during the path-finding algorithm. If we have seen - /// a path with the same start and end node, and the same pre- and postcondition, then we return false. - /// Otherwise, we return true. - pub fn has_similar_path( + /// Set whether to collect statistics for this similar path detector. + pub fn set_collect_stats(&mut self, collect_stats: bool) { + if !collect_stats { + self.counts = None; + } else if self.counts.is_none() { + self.counts = Some(HashMap::new()); + } + } + + /// Add a path, and determine whether we should process this path during the path-finding algorithm. + /// If we have seen a path with the same start and end node, and the same pre- and postcondition, then + /// we return false. Otherwise, we return true. + pub fn add_path( &mut self, _graph: &StackGraph, arena: &mut P::Arena, @@ -110,15 +122,47 @@ where { let key = path.key(); - let possibly_similar_paths = self.paths.entry(key).or_default(); - for other_path in possibly_similar_paths.iter() { + // Iterate through the bucket to determine if this paths is better than any already known + // path. Note that the bucket might be modified during the loop if a path is removed which + // is shadowed by the new path! + let possibly_similar_paths = self.paths.entry(key.clone()).or_default(); + let mut possible_similar_counts = self + .counts + .as_mut() + .map(move |cs| cs.entry(key).or_default()); + let mut idx = 0; + let mut count = 0; + while idx < possibly_similar_paths.len() { + let other_path = &mut possibly_similar_paths[idx]; match cmp(arena, path, other_path) { - Some(ord) if ord != Ordering::Less => return true, - _ => continue, + Some(Ordering::Less) => { + // the new path is better, remove the old one + possibly_similar_paths.remove(idx); + if let Some(possible_similar_counts) = possible_similar_counts.as_mut() { + count += possible_similar_counts[idx]; + possible_similar_counts.remove(idx); + } + // keep `idx` which now points to the next element + continue; + } + Some(_) => { + // the new path is equal or worse, and ignored + if let Some(possible_similar_counts) = possible_similar_counts { + possible_similar_counts[idx] += 1; + } + return true; + } + None => { + idx += 1; + } } } + // this path is either new or better, keep it possibly_similar_paths.push(path.clone()); + if let Some(possible_similar_counts) = possible_similar_counts { + possible_similar_counts.push(count); + } false } @@ -126,6 +170,42 @@ where pub fn max_bucket_size(&self) -> usize { self.paths.iter().map(|b| b.1.len()).max().unwrap_or(0) } + + // Returns the distribution of similar path counts. + pub fn stats(&self) -> SimilarPathStats { + let mut stats = SimilarPathStats::default(); + if let Some(counts) = &self.counts { + for bucket in counts.values() { + stats.similar_path_bucket_size.record(bucket.len()); + for count in bucket.iter() { + stats.similar_path_count.record(*count); + } + } + } + stats + } +} + +#[derive(Clone, Debug, Default)] +pub struct SimilarPathStats { + // The distribution of the number of similar paths detected + pub similar_path_count: FrequencyDistribution, + // The distribution of the internal bucket sizes in the similar path detector + pub similar_path_bucket_size: FrequencyDistribution, +} + +impl std::ops::AddAssign for SimilarPathStats { + fn add_assign(&mut self, rhs: Self) { + self.similar_path_bucket_size += rhs.similar_path_bucket_size; + self.similar_path_count += rhs.similar_path_count; + } +} + +impl std::ops::AddAssign<&Self> for SimilarPathStats { + fn add_assign(&mut self, rhs: &Self) { + self.similar_path_bucket_size += &rhs.similar_path_bucket_size; + self.similar_path_count += &rhs.similar_path_count; + } } // ---------------------------------------------------------------------------- diff --git a/stack-graphs/src/graph.rs b/stack-graphs/src/graph.rs index ae2b9aa62..d55dfc48a 100644 --- a/stack-graphs/src/graph.rs +++ b/stack-graphs/src/graph.rs @@ -425,7 +425,7 @@ impl Handle { /// Each node (except for the _root node_ and _jump to scope_ node) lives in a file, and has a /// _local ID_ that must be unique within its file. #[repr(C)] -#[derive(Clone, Copy, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct NodeID { file: ControlledOption>, local_id: u32, @@ -1303,14 +1303,20 @@ impl StackGraph { let edges = &mut self.outgoing_edges[source]; if let Err(index) = edges.binary_search_by_key(&sink, |o| o.sink) { edges.insert(index, OutgoingEdge { sink, precedence }); + self.incoming_edges[sink] += Degree::One; } } - /// Removes an edge from the stack graph. - pub fn remove_edge(&mut self, source: Handle, sink: Handle) { + /// Sets edge precedence of the given edge. + pub fn set_edge_precedence( + &mut self, + source: Handle, + sink: Handle, + precedence: i32, + ) { let edges = &mut self.outgoing_edges[source]; if let Ok(index) = edges.binary_search_by_key(&sink, |o| o.sink) { - edges.remove(index); + edges[index].precedence = precedence; } } @@ -1325,6 +1331,14 @@ impl StackGraph { None => Either::Left(std::iter::empty()), } } + + /// Returns the number of edges that end at a particular sink node. + pub fn incoming_edge_degree(&self, sink: Handle) -> Degree { + self.incoming_edges + .get(sink) + .cloned() + .unwrap_or(Degree::Zero) + } } //------------------------------------------------------------------------------------------------- @@ -1430,6 +1444,36 @@ impl StackGraph { //------------------------------------------------------------------------------------------------- // Stack graphs +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u8)] +pub enum Degree { + Zero, + One, + Multiple, +} + +impl Default for Degree { + fn default() -> Self { + Self::Zero + } +} + +impl std::ops::Add for Degree { + type Output = Self; + fn add(self, rhs: Self) -> Self::Output { + match (self, rhs) { + (Self::Zero, result) | (result, Self::Zero) => result, + _ => Self::Multiple, + } + } +} + +impl std::ops::AddAssign for Degree { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; + } +} + /// Contains all of the nodes and edges that make up a stack graph. pub struct StackGraph { interned_strings: InternedStringArena, @@ -1442,9 +1486,10 @@ pub struct StackGraph { pub(crate) nodes: Arena, pub(crate) source_info: SupplementalArena, node_id_handles: NodeIDHandles, - outgoing_edges: SupplementalArena>, + outgoing_edges: SupplementalArena>, + incoming_edges: SupplementalArena, pub(crate) node_debug_info: SupplementalArena, - pub(crate) edge_debug_info: SupplementalArena, DebugInfo); 8]>>, + pub(crate) edge_debug_info: SupplementalArena, DebugInfo); 4]>>, } impl StackGraph { @@ -1455,12 +1500,16 @@ impl StackGraph { /// Copies the given stack graph into this stack graph. Panics if any of the files /// in the other stack graph are already defined in the current one. - pub fn add_from_graph(&mut self, other: &StackGraph) -> Result<(), Handle> { + pub fn add_from_graph( + &mut self, + other: &StackGraph, + ) -> Result>, Handle> { let mut files = HashMap::new(); for other_file in other.iter_files() { let file = self.add_file(other[other_file].name())?; files.insert(other_file, file); } + let files = files; let node_id = |other_node_id: NodeID| { if other_node_id.is_root() { NodeID::root() @@ -1603,7 +1652,7 @@ impl StackGraph { } } } - Ok(()) + Ok(files.into_values().collect()) } } @@ -1625,6 +1674,7 @@ impl Default for StackGraph { source_info: SupplementalArena::new(), node_id_handles: NodeIDHandles::new(), outgoing_edges: SupplementalArena::new(), + incoming_edges: SupplementalArena::new(), node_debug_info: SupplementalArena::new(), edge_debug_info: SupplementalArena::new(), } diff --git a/stack-graphs/src/lib.rs b/stack-graphs/src/lib.rs index 009076af9..a8573c5e5 100644 --- a/stack-graphs/src/lib.rs +++ b/stack-graphs/src/lib.rs @@ -69,6 +69,7 @@ pub mod graph; pub mod partial; pub mod paths; pub mod serde; +pub mod stats; pub mod stitching; #[cfg(feature = "storage")] pub mod storage; diff --git a/stack-graphs/src/stats.rs b/stack-graphs/src/stats.rs new file mode 100644 index 000000000..6b9f6ead8 --- /dev/null +++ b/stack-graphs/src/stats.rs @@ -0,0 +1,106 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use std::collections::HashMap; +use std::hash::Hash; + +use itertools::Itertools; + +/// Frequency distribution maintains the frequency of T values. +#[derive(Clone, Debug, Default)] +pub struct FrequencyDistribution +where + T: Eq + Hash, +{ + values: HashMap, + total: usize, +} + +impl FrequencyDistribution { + pub fn record(&mut self, value: T) { + *self.values.entry(value).or_default() += 1; + self.total += 1; + } + + // The number of recorded values. + pub fn count(&self) -> usize { + return self.total; + } + + // The number of unique recorded values. + pub fn unique(&self) -> usize { + return self.values.len(); + } + + pub fn frequencies(&self) -> FrequencyDistribution { + let mut fs = FrequencyDistribution::default(); + for count in self.values.values() { + fs.record(*count); + } + fs + } +} + +impl FrequencyDistribution { + pub fn quantiles(&self, q: usize) -> Vec<&T> { + if q == 0 || self.total == 0 { + return vec![]; + } + + let mut it = self.values.iter().sorted_by_key(|e| e.0); + let mut total_count = 0; + let mut last_value; + let mut result = Vec::new(); + + if let Some((value, count)) = it.next() { + total_count += count; + last_value = value; + } else { + return vec![]; + } + result.push(last_value); + + for k in 1..=q { + let limit = ((self.total as f64 * k as f64) / q as f64).round() as usize; + while total_count < limit { + if let Some((value, count)) = it.next() { + total_count += count; + last_value = value; + } else { + break; + } + } + result.push(last_value); + } + + result + } +} + +impl std::ops::AddAssign for FrequencyDistribution +where + T: Eq + Hash, +{ + fn add_assign(&mut self, rhs: Self) { + for (value, count) in rhs.values { + *self.values.entry(value).or_default() += count; + } + self.total += rhs.total; + } +} + +impl std::ops::AddAssign<&Self> for FrequencyDistribution +where + T: Eq + Hash + Clone, +{ + fn add_assign(&mut self, rhs: &Self) { + for (value, count) in &rhs.values { + *self.values.entry(value.clone()).or_default() += count; + } + self.total += rhs.total; + } +} diff --git a/stack-graphs/src/stitching.rs b/stack-graphs/src/stitching.rs index 4872afe91..99aaa8df1 100644 --- a/stack-graphs/src/stitching.rs +++ b/stack-graphs/src/stitching.rs @@ -41,6 +41,9 @@ use std::collections::VecDeque; #[cfg(feature = "copious-debugging")] use std::fmt::Display; +use itertools::izip; +use itertools::Itertools; + use crate::arena::Arena; use crate::arena::Handle; use crate::arena::HandleSet; @@ -51,6 +54,8 @@ use crate::arena::SupplementalArena; use crate::cycles::Appendables; use crate::cycles::AppendingCycleDetector; use crate::cycles::SimilarPathDetector; +use crate::cycles::SimilarPathStats; +use crate::graph::Degree; use crate::graph::Edge; use crate::graph::File; use crate::graph::Node; @@ -60,7 +65,9 @@ use crate::partial::Cyclicity; use crate::partial::PartialPath; use crate::partial::PartialPaths; use crate::partial::PartialSymbolStack; +use crate::paths::Extend; use crate::paths::PathResolutionError; +use crate::stats::FrequencyDistribution; use crate::CancellationError; use crate::CancellationFlag; @@ -198,6 +205,9 @@ where where R: std::iter::Extend; + /// Get the number of available candidates that share the given path's end node. + fn get_joining_candidate_degree(&self, path: &PartialPath) -> Degree; + /// Get the graph, partial path arena, and database backing this candidates instance. fn get_graph_partials_and_db(&mut self) -> (&StackGraph, &mut PartialPaths, &Db); } @@ -239,6 +249,10 @@ impl ForwardCandidates for GraphEdgeC })); } + fn get_joining_candidate_degree(&self, path: &PartialPath) -> Degree { + self.graph.incoming_edge_degree(path.end_node) + } + fn get_graph_partials_and_db(&mut self) -> (&StackGraph, &mut PartialPaths, &GraphEdges) { (self.graph, self.partials, &self.edges) } @@ -272,7 +286,13 @@ pub struct Database { symbol_stack_keys: ListArena>, symbol_stack_key_cache: HashMap, paths_by_start_node: SupplementalArena>>, - root_paths_by_precondition: SupplementalArena>>, + root_paths_by_precondition_prefix: + SupplementalArena>>, + root_paths_by_precondition_with_variable: + SupplementalArena>>, + root_paths_by_precondition_without_variable: + SupplementalArena>>, + incoming_paths: SupplementalArena, } impl Database { @@ -284,7 +304,10 @@ impl Database { symbol_stack_keys: List::new_arena(), symbol_stack_key_cache: HashMap::new(), paths_by_start_node: SupplementalArena::new(), - root_paths_by_precondition: SupplementalArena::new(), + root_paths_by_precondition_prefix: SupplementalArena::new(), + root_paths_by_precondition_with_variable: SupplementalArena::new(), + root_paths_by_precondition_without_variable: SupplementalArena::new(), + incoming_paths: SupplementalArena::new(), } } @@ -297,7 +320,10 @@ impl Database { self.symbol_stack_keys.clear(); self.symbol_stack_key_cache.clear(); self.paths_by_start_node.clear(); - self.root_paths_by_precondition.clear(); + self.root_paths_by_precondition_prefix.clear(); + self.root_paths_by_precondition_with_variable.clear(); + self.root_paths_by_precondition_without_variable.clear(); + self.incoming_paths.clear(); } /// Adds a partial path to this database. We do not deduplicate partial paths in any way; it's @@ -309,6 +335,7 @@ impl Database { path: PartialPath, ) -> Handle { let start_node = path.start_node; + let end_node = path.end_node; copious_debugging!( " Add {} path to database {}", if graph[start_node].is_root() { @@ -325,20 +352,28 @@ impl Database { if graph[start_node].is_root() { // The join node is root, so there's no need to use half-open symbol stacks here, as we // do for [`PartialPath::concatenate`][]. - let key = SymbolStackKey::from_partial_symbol_stack( + let mut key = SymbolStackKey::from_partial_symbol_stack( partials, self, symbol_stack_precondition, ); if !key.is_empty() { - let key_handle = key.back_handle(); - self.root_paths_by_precondition[key_handle].push(handle); + match symbol_stack_precondition.has_variable() { + true => self.root_paths_by_precondition_with_variable[key.back_handle()] + .push(handle), + false => self.root_paths_by_precondition_without_variable[key.back_handle()] + .push(handle), + } + } + while key.pop_back(self).is_some() && !key.is_empty() { + self.root_paths_by_precondition_prefix[key.back_handle()].push(handle); } } else { // Otherwise index it by its source node. self.paths_by_start_node[start_node].push(handle); } + self.incoming_paths[end_node] += Degree::One; handle } @@ -357,12 +392,12 @@ impl Database { if graph[path.end_node].is_root() { // The join node is root, so there's no need to use half-open symbol stacks here, as we // do for [`PartialPath::concatenate`][]. - let key = SymbolStackKey::from_partial_symbol_stack( + self.find_candidate_partial_paths_from_root( + graph, partials, - self, - path.symbol_stack_postcondition, + Some(path.symbol_stack_postcondition), + result, ); - self.find_candidate_partial_paths_from_root(graph, partials, Some(key), result); } else { self.find_candidate_partial_paths_from_node(graph, partials, path.end_node, result); } @@ -375,7 +410,7 @@ impl Database { &mut self, graph: &StackGraph, partials: &mut PartialPaths, - symbol_stack: Option, + symbol_stack: Option, result: &mut R, ) where R: std::iter::Extend>, @@ -383,31 +418,76 @@ impl Database { // If the path currently ends at the root node, then we need to look up partial paths whose // symbol stack precondition is compatible with the path. match symbol_stack { - Some(mut symbol_stack) => loop { + Some(symbol_stack) => { + let mut key = + SymbolStackKey::from_partial_symbol_stack(partials, self, symbol_stack); copious_debugging!( " Search for symbol stack <{}>", - symbol_stack.display(graph, self) + key.display(graph, self) ); - let key_handle = symbol_stack.back_handle(); - if let Some(paths) = self.root_paths_by_precondition.get(key_handle) { + // paths that have exactly this symbol stack + if let Some(paths) = self + .root_paths_by_precondition_without_variable + .get(key.back_handle()) + { #[cfg(feature = "copious-debugging")] { for path in paths { copious_debugging!( - " Found path {}", + " Found path with exact stack {}", self[*path].display(graph, partials) ); } } result.extend(paths.iter().copied()); } - if symbol_stack.pop_back(self).is_none() { - break; + // paths that have an extension of this symbol stack + if symbol_stack.has_variable() { + if let Some(paths) = self + .root_paths_by_precondition_prefix + .get(key.back_handle()) + { + #[cfg(feature = "copious-debugging")] + { + for path in paths { + copious_debugging!( + " Found path with smaller stack {}", + self[*path].display(graph, partials) + ); + } + } + result.extend(paths.iter().copied()); + } } - }, + loop { + // paths that have a prefix of this symbol stack + if let Some(paths) = self + .root_paths_by_precondition_with_variable + .get(key.back_handle()) + { + #[cfg(feature = "copious-debugging")] + { + for path in paths { + copious_debugging!( + " Found path with smaller stack {}", + self[*path].display(graph, partials) + ); + } + } + result.extend(paths.iter().copied()); + } + if key.pop_back(self).is_none() { + break; + } + } + } None => { copious_debugging!(" Search for all root paths"); - for (_, paths) in self.root_paths_by_precondition.iter() { + for (_, paths) in self + .root_paths_by_precondition_with_variable + .iter() + .chain(self.root_paths_by_precondition_without_variable.iter()) + { #[cfg(feature = "copious-debugging")] { for path in paths { @@ -453,6 +533,11 @@ impl Database { } } + /// Returns the number of paths in this database that share the given end node. + pub fn get_incoming_path_degree(&self, end_node: Handle) -> Degree { + self.incoming_paths[end_node] + } + /// Determines which nodes in the stack graph are “local”, taking into account the partial /// paths in this database. /// @@ -596,6 +681,10 @@ impl ForwardCandidates, PartialPath, Database, CancellationE .find_candidate_partial_paths(self.graph, self.partials, path, result); } + fn get_joining_candidate_degree(&self, path: &PartialPath) -> Degree { + self.database.get_incoming_path_degree(path.end_node) + } + fn get_graph_partials_and_db(&mut self) -> (&StackGraph, &mut PartialPaths, &Database) { (self.graph, self.partials, self.database) } @@ -736,16 +825,24 @@ impl<'a> Display for DisplaySymbolStackKey<'a> { /// [`find_all_complete_partial_paths`]: #method.find_all_complete_partial_paths pub struct ForwardPartialPathStitcher { candidates: Vec, - queue: VecDeque<(PartialPath, AppendingCycleDetector)>, + extensions: Vec<(PartialPath, AppendingCycleDetector)>, + queue: VecDeque<(PartialPath, AppendingCycleDetector, bool)>, // tracks the number of initial paths in the queue because we do not want call // extend_until on those - initial_paths: usize, + initial_paths_in_queue: usize, // next_iteration is a tuple of queues instead of an queue of tuples so that the path queue // can be cheaply exposed through the C API as a continuous memory block - next_iteration: (VecDeque, VecDeque>), + next_iteration: ( + VecDeque, + VecDeque>, + VecDeque, + ), appended_paths: Appendables, similar_path_detector: Option>, + check_only_join_nodes: bool, max_work_per_phase: usize, + initial_paths: usize, + stats: Option, #[cfg(feature = "copious-debugging")] phase_number: usize, } @@ -763,26 +860,87 @@ impl ForwardPartialPathStitcher { I: IntoIterator, { let mut appended_paths = Appendables::new(); - let next_iteration: (VecDeque<_>, VecDeque<_>) = initial_partial_paths + let next_iteration: (VecDeque<_>, VecDeque<_>, VecDeque<_>) = initial_partial_paths .into_iter() .map(|p| { let c = AppendingCycleDetector::from(&mut appended_paths, p.clone().into()); - (p, c) + (p, c, false) }) - .unzip(); + .multiunzip(); + let initial_paths = next_iteration.0.len(); Self { candidates: Vec::new(), + extensions: Vec::new(), queue: VecDeque::new(), - initial_paths: next_iteration.0.len(), + initial_paths_in_queue: initial_paths, next_iteration, appended_paths, + // By default, all paths are checked for similarity similar_path_detector: Some(SimilarPathDetector::new()), + // By default, all nodes are checked for cycles and (if enabled) similarity + check_only_join_nodes: false, // By default, there's no artificial bound on the amount of work done per phase max_work_per_phase: usize::MAX, + initial_paths, + stats: None, #[cfg(feature = "copious-debugging")] phase_number: 1, } } + + /// Sets whether similar path detection should be enabled during path stitching. Paths are similar + /// if start and end node, and pre- and postconditions are the same. The presence of similar paths + /// can lead to exponential blow up during path stitching. Similar path detection is enabled by + /// default. + pub fn set_similar_path_detection(&mut self, detect_similar_paths: bool) { + if !detect_similar_paths { + self.similar_path_detector = None; + } else if self.similar_path_detector.is_none() { + let mut similar_path_detector = SimilarPathDetector::new(); + similar_path_detector.set_collect_stats(self.stats.is_some()); + self.similar_path_detector = Some(similar_path_detector); + } + } + + /// Sets whether all nodes are checked for cycles and (if enabled) similar paths, or only nodes with multiple + /// incoming candidates. Checking only join nodes is **unsafe** unless the database of candidates is stable + /// between all stitching phases. If paths are added to the database from one phase to another, for example if + /// paths are dynamically loaded from storage, setting this to true is incorrect and might lead to non-termination! + pub fn set_check_only_join_nodes(&mut self, check_only_join_nodes: bool) { + self.check_only_join_nodes = check_only_join_nodes; + } + + /// Sets the maximum amount of work that can be performed during each phase of the algorithm. + /// By bounding our work this way, you can ensure that it's not possible for our CPU-bound + /// algorithm to starve any worker threads or processes that you might be using. If you don't + /// call this method, then we allow ourselves to process all of the extensions of all of the + /// paths found in the previous phase, with no additional bound. + pub fn set_max_work_per_phase(&mut self, max_work_per_phase: usize) { + self.max_work_per_phase = max_work_per_phase; + } + + /// Sets whether to collect statistics during stitching. + pub fn set_collect_stats(&mut self, collect_stats: bool) { + if !collect_stats { + self.stats = None; + } else if self.stats.is_none() { + let mut stats = Stats::default(); + stats.initial_paths.record(self.initial_paths); + self.stats = Some(stats); + } + if let Some(similar_path_detector) = &mut self.similar_path_detector { + similar_path_detector.set_collect_stats(collect_stats); + } + } + + pub fn into_stats(mut self) -> Stats { + if let (Some(stats), Some(similar_path_detector)) = + (&mut self.stats, self.similar_path_detector) + { + stats.similar_paths_stats = similar_path_detector.stats(); + } + self.stats.unwrap_or_default() + } } impl ForwardPartialPathStitcher { @@ -806,27 +964,6 @@ impl ForwardPartialPathStitcher { self.next_iteration.0.as_mut_slices().0 } - /// Sets whether similar path detection should be enabled during path stitching. Paths are similar - /// if start and end node, and pre- and postconditions are the same. The presence of similar paths - /// can lead to exponential blow up during path stitching. Similar path detection is enabled by - /// default. - pub fn set_similar_path_detection(&mut self, detect_similar_paths: bool) { - if !detect_similar_paths { - self.similar_path_detector = None; - } else if self.similar_path_detector.is_none() { - self.similar_path_detector = Some(SimilarPathDetector::new()); - } - } - - /// Sets the maximum amount of work that can be performed during each phase of the algorithm. - /// By bounding our work this way, you can ensure that it's not possible for our CPU-bound - /// algorithm to starve any worker threads or processes that you might be using. If you don't - /// call this method, then we allow ourselves to process all of the extensions of all of the - /// paths found in the previous phase, with no additional bound. - pub fn set_max_work_per_phase(&mut self, max_work_per_phase: usize) { - self.max_work_per_phase = max_work_per_phase; - } - /// Attempts to extend one partial path as part of the algorithm. When calling this function, /// you are responsible for ensuring that `db` already contains all of the possible appendables /// that we might want to extend `partial_path` with. @@ -835,51 +972,59 @@ impl ForwardPartialPathStitcher { candidates: &mut C, partial_path: &PartialPath, cycle_detector: AppendingCycleDetector, + has_split: bool, ) -> usize where A: Appendable, Db: ToAppendable, C: ForwardCandidates, { + let check_cycle = !self.check_only_join_nodes + || partial_path.start_node == partial_path.end_node + || candidates.get_joining_candidate_degree(partial_path) == Degree::Multiple; + let (graph, partials, db) = candidates.get_graph_partials_and_db(); copious_debugging!(" Extend {}", partial_path.display(graph, partials)); - // check is path is cyclic, in which case we do not extend it - let has_precondition_variables = partial_path.symbol_stack_precondition.has_variable() - || partial_path.scope_stack_precondition.has_variable(); - let cycles = cycle_detector - .is_cyclic(graph, partials, db, &mut self.appended_paths) - .expect("cyclic test failed when stitching partial paths"); - let cyclic = match has_precondition_variables { - // If the precondition has no variables, we allow cycles that strengthen the - // precondition, because we know they cannot strengthen the precondition of - // the overall path. - false => !cycles - .into_iter() - .all(|c| c == Cyclicity::StrengthensPrecondition), - // If the precondition has variables, do not allow any cycles, not even those - // that strengthen the precondition. This is more strict than necessary. Better - // might be to disallow precondition strengthening cycles only if they would - // strengthen the overall path precondition. - true => !cycles.is_empty(), - }; - if cyclic { - copious_debugging!(" is discontinued: cyclic"); - return 0; + if check_cycle { + // Check is path is cyclic, in which case we do not extend it. We only do this if the start and end nodes are the same, + // or the current end node has multiple incoming edges. If neither of these hold, the path cannot end in a cycle. + let has_precondition_variables = partial_path.symbol_stack_precondition.has_variable() + || partial_path.scope_stack_precondition.has_variable(); + let cycles = cycle_detector + .is_cyclic(graph, partials, db, &mut self.appended_paths) + .expect("cyclic test failed when stitching partial paths"); + let cyclic = match has_precondition_variables { + // If the precondition has no variables, we allow cycles that strengthen the + // precondition, because we know they cannot strengthen the precondition of + // the overall path. + false => !cycles + .into_iter() + .all(|c| c == Cyclicity::StrengthensPrecondition), + // If the precondition has variables, do not allow any cycles, not even those + // that strengthen the precondition. This is more strict than necessary. Better + // might be to disallow precondition strengthening cycles only if they would + // strengthen the overall path precondition. + true => !cycles.is_empty(), + }; + if cyclic { + copious_debugging!(" is discontinued: cyclic"); + return 0; + } } // find candidates to append self.candidates.clear(); candidates.get_forward_candidates(partial_path, &mut self.candidates); + let (graph, partials, db) = candidates.get_graph_partials_and_db(); // try to extend path with candidates - let extension_count = self.candidates.len(); - self.next_iteration.0.reserve(extension_count); - self.next_iteration.1.reserve(extension_count); - for extension in &self.candidates { - let (graph, partials, db) = candidates.get_graph_partials_and_db(); - let extension_path = db.get_appendable(extension); - copious_debugging!(" with {}", extension_path.display(graph, partials)); + let candidate_count = self.candidates.len(); + self.extensions.clear(); + self.extensions.reserve(candidate_count); + for candidate in &self.candidates { + let appendable = db.get_appendable(candidate); + copious_debugging!(" with {}", appendable.display(graph, partials)); let mut new_partial_path = partial_path.clone(); let mut new_cycle_detector = cycle_detector.clone(); @@ -887,14 +1032,30 @@ impl ForwardPartialPathStitcher { // partial path, just skip the extension — it's not a fatal error. #[cfg_attr(not(feature = "copious-debugging"), allow(unused_variables))] { - if let Err(err) = extension_path.append_to(graph, partials, &mut new_partial_path) { + if let Err(err) = appendable.append_to(graph, partials, &mut new_partial_path) { copious_debugging!(" is invalid: {:?}", err); continue; } - copious_debugging!(" is {}", new_partial_path.display(graph, partials)); - new_cycle_detector.append(&mut self.appended_paths, extension.clone()); + } + new_cycle_detector.append(&mut self.appended_paths, candidate.clone()); + copious_debugging!(" is {}", new_partial_path.display(graph, partials)); + self.extensions.push((new_partial_path, new_cycle_detector)); + } + + let extension_count = self.extensions.len(); + let new_has_split = has_split || self.extensions.len() > 1; + self.next_iteration.0.reserve(extension_count); + self.next_iteration.1.reserve(extension_count); + self.next_iteration.2.reserve(extension_count); + for (new_partial_path, new_cycle_detector) in self.extensions.drain(..) { + let check_similar_path = new_has_split + && (!self.check_only_join_nodes + || candidates.get_joining_candidate_degree(&new_partial_path) + == Degree::Multiple); + let (graph, partials, _) = candidates.get_graph_partials_and_db(); + if check_similar_path { if let Some(similar_path_detector) = &mut self.similar_path_detector { - if similar_path_detector.has_similar_path( + if similar_path_detector.add_path( graph, partials, &new_partial_path, @@ -912,16 +1073,38 @@ impl ForwardPartialPathStitcher { } }, ) { + copious_debugging!( + " extension {}", + new_partial_path.display(graph, partials) + ); copious_debugging!(" is rejected: too many similar"); continue; } } } - self.next_iteration.0.push_back(new_partial_path); - self.next_iteration.1.push_back(new_cycle_detector); + + self.next_iteration.0.push(new_partial_path); + self.next_iteration.1.push(new_cycle_detector); + self.next_iteration.2.push(new_has_split); } - extension_count + if let Some(stats) = &mut self.stats { + let (graph, _, _) = candidates.get_graph_partials_and_db(); + let end_node = &graph[partial_path.end_node]; + if end_node.is_root() { + stats.candidates_per_root_path.record(candidate_count); + stats.extensions_per_root_path.record(extension_count); + stats.root_visits += 1; + } else { + stats.candidates_per_node_path.record(candidate_count); + stats.extensions_per_node_path.record(extension_count); + stats.node_visits.record(end_node.id()); + } + if extension_count == 0 { + stats.terminal_path_lengh.record(partial_path.edges.len()); + } + } + candidate_count } /// Returns whether the algorithm has completed. @@ -949,21 +1132,23 @@ impl ForwardPartialPathStitcher { E: Fn(&StackGraph, &mut PartialPaths, &PartialPath) -> bool, { copious_debugging!("==> Start phase {}", self.phase_number); - self.queue.extend( - self.next_iteration - .0 - .drain(..) - .zip(self.next_iteration.1.drain(..)), - ); + self.queue.extend(izip!( + self.next_iteration.0.drain(..), + self.next_iteration.1.drain(..), + self.next_iteration.2.drain(..), + )); + if let Some(stats) = &mut self.stats { + stats.queued_paths_per_phase.record(self.queue.len()); + } let mut work_performed = 0; - while let Some((partial_path, cycle_detector)) = self.queue.pop_front() { + while let Some((partial_path, cycle_detector, has_split)) = self.queue.pop_front() { let (graph, partials, _) = candidates.get_graph_partials_and_db(); copious_debugging!( "--> Candidate partial path {}", partial_path.display(graph, partials) ); - if self.initial_paths > 0 { - self.initial_paths -= 1; + if self.initial_paths_in_queue > 0 { + self.initial_paths_in_queue -= 1; } else if !extend_while(graph, partials, &partial_path) { copious_debugging!( " Do not extend {}", @@ -971,11 +1156,14 @@ impl ForwardPartialPathStitcher { ); continue; } - work_performed += self.extend(candidates, &partial_path, cycle_detector); + work_performed += self.extend(candidates, &partial_path, cycle_detector, has_split); if work_performed >= self.max_work_per_phase { break; } } + if let Some(stats) = &mut self.stats { + stats.processed_paths_per_phase.record(work_performed); + } #[cfg(feature = "copious-debugging")] { @@ -1012,15 +1200,16 @@ impl ForwardPartialPathStitcher { graph: &StackGraph, partials: &mut PartialPaths, file: Handle, + config: StitcherConfig, cancellation_flag: &dyn CancellationFlag, mut visit: F, - ) -> Result<(), CancellationError> + ) -> Result where F: FnMut(&StackGraph, &mut PartialPaths, &PartialPath), { fn as_complete_as_necessary(graph: &StackGraph, path: &PartialPath) -> bool { path.starts_at_endpoint(graph) - && (path.ends_at_endpoint(graph) || graph[path.end_node].is_jump_to()) + && (path.ends_at_endpoint(graph) || path.ends_in_jump(graph)) } let initial_paths = graph @@ -1031,6 +1220,10 @@ impl ForwardPartialPathStitcher { .collect::>(); let mut stitcher = ForwardPartialPathStitcher::from_partial_paths(graph, partials, initial_paths); + config.apply(&mut stitcher); + stitcher.set_check_only_join_nodes(true); + + let mut accepted_path_length = FrequencyDistribution::default(); while !stitcher.is_complete() { cancellation_flag.check("finding complete partial paths")?; stitcher.process_next_phase( @@ -1039,11 +1232,16 @@ impl ForwardPartialPathStitcher { ); for path in stitcher.previous_phase_partial_paths() { if as_complete_as_necessary(graph, path) { + accepted_path_length.record(path.edges.len()); visit(graph, partials, path); } } } - Ok(()) + + Ok(Stats { + accepted_path_length, + ..stitcher.into_stats() + }) } } @@ -1062,9 +1260,10 @@ impl ForwardPartialPathStitcher { pub fn find_all_complete_partial_paths( candidates: &mut C, starting_nodes: I, + config: StitcherConfig, cancellation_flag: &dyn CancellationFlag, mut visit: F, - ) -> Result<(), Err> + ) -> Result where I: IntoIterator>, A: Appendable, @@ -1073,19 +1272,22 @@ impl ForwardPartialPathStitcher { F: FnMut(&StackGraph, &mut PartialPaths, &PartialPath), Err: std::convert::From, { - let mut stitcher = { - let (graph, partials, _) = candidates.get_graph_partials_and_db(); - let initial_paths = starting_nodes - .into_iter() - .filter(|n| graph[*n].is_reference()) - .map(|n| { - let mut p = PartialPath::from_node(graph, partials, n); - p.eliminate_precondition_stack_variables(partials); - p - }) - .collect::>(); - ForwardPartialPathStitcher::from_partial_paths(graph, partials, initial_paths) - }; + let (graph, partials, _) = candidates.get_graph_partials_and_db(); + let initial_paths = starting_nodes + .into_iter() + .filter(|n| graph[*n].is_reference()) + .map(|n| { + let mut p = PartialPath::from_node(graph, partials, n); + p.eliminate_precondition_stack_variables(partials); + p + }) + .collect::>(); + let mut stitcher = + ForwardPartialPathStitcher::from_partial_paths(graph, partials, initial_paths); + config.apply(&mut stitcher); + stitcher.set_check_only_join_nodes(true); + + let mut accepted_path_length = FrequencyDistribution::default(); while !stitcher.is_complete() { cancellation_flag.check("finding complete partial paths")?; for path in stitcher.previous_phase_partial_paths() { @@ -1095,10 +1297,121 @@ impl ForwardPartialPathStitcher { let (graph, partials, _) = candidates.get_graph_partials_and_db(); for path in stitcher.previous_phase_partial_paths() { if path.is_complete(graph) { + accepted_path_length.record(path.edges.len()); visit(graph, partials, path); } } } - Ok(()) + + Ok(Stats { + accepted_path_length, + ..stitcher.into_stats() + }) + } +} + +#[derive(Clone, Debug, Default)] +pub struct Stats { + /// The distribution of the number of initial paths + pub initial_paths: FrequencyDistribution, + /// The distribution of the number of queued paths per stitching phase + pub queued_paths_per_phase: FrequencyDistribution, + /// The distribution of the number of processed paths per stitching phase + pub processed_paths_per_phase: FrequencyDistribution, + /// The distribution of the length of accepted paths + pub accepted_path_length: FrequencyDistribution, + /// The distribution of the maximal length of paths (when they cannot be extended more) + pub terminal_path_lengh: FrequencyDistribution, + /// The distribution of the number of candidates for paths ending in a regular node + pub candidates_per_node_path: FrequencyDistribution, + /// The distribution of the number of candidates for paths ending in the root node + pub candidates_per_root_path: FrequencyDistribution, + /// The distribution of the number of extensions (accepted candidates) for paths ending in a regular node + pub extensions_per_node_path: FrequencyDistribution, + /// The distribution of the number of extensions (accepted candidates) for paths ending in the root node + pub extensions_per_root_path: FrequencyDistribution, + /// The number of times the root node is visited + pub root_visits: usize, + /// The distribution of the number of times a regular node is visited + pub node_visits: FrequencyDistribution, + /// The distribution of the number of similar paths between node pairs. + pub similar_paths_stats: SimilarPathStats, +} + +impl std::ops::AddAssign for Stats { + fn add_assign(&mut self, rhs: Self) { + self.initial_paths += rhs.initial_paths; + self.queued_paths_per_phase += rhs.queued_paths_per_phase; + self.processed_paths_per_phase += rhs.processed_paths_per_phase; + self.accepted_path_length += rhs.accepted_path_length; + self.terminal_path_lengh += rhs.terminal_path_lengh; + self.candidates_per_node_path += rhs.candidates_per_node_path; + self.candidates_per_root_path += rhs.candidates_per_root_path; + self.extensions_per_node_path += rhs.extensions_per_node_path; + self.extensions_per_root_path += rhs.extensions_per_root_path; + self.root_visits += rhs.root_visits; + self.node_visits += rhs.node_visits; + self.similar_paths_stats += rhs.similar_paths_stats; + } +} + +impl std::ops::AddAssign<&Self> for Stats { + fn add_assign(&mut self, rhs: &Self) { + self.initial_paths += &rhs.initial_paths; + self.processed_paths_per_phase += &rhs.processed_paths_per_phase; + self.accepted_path_length += &rhs.accepted_path_length; + self.terminal_path_lengh += &rhs.terminal_path_lengh; + self.candidates_per_node_path += &rhs.candidates_per_node_path; + self.candidates_per_root_path += &rhs.candidates_per_root_path; + self.extensions_per_node_path += &rhs.extensions_per_node_path; + self.extensions_per_root_path += &rhs.extensions_per_root_path; + self.root_visits += rhs.root_visits; + self.node_visits += &rhs.node_visits; + self.similar_paths_stats += &rhs.similar_paths_stats; + } +} + +/// Configuration for partial path stitchers. +#[derive(Clone, Copy, Debug)] +pub struct StitcherConfig { + /// Enables similar path detection during path stitching. + detect_similar_paths: bool, + /// Collect statistics about path stitching. + collect_stats: bool, +} + +impl StitcherConfig { + pub fn detect_similar_paths(&self) -> bool { + self.detect_similar_paths + } + + pub fn with_detect_similar_paths(mut self, detect_similar_paths: bool) -> Self { + self.detect_similar_paths = detect_similar_paths; + self + } + + pub fn collect_stats(&self) -> bool { + self.collect_stats + } + + pub fn with_collect_stats(mut self, collect_stats: bool) -> Self { + self.collect_stats = collect_stats; + self + } +} + +impl StitcherConfig { + fn apply(&self, stitcher: &mut ForwardPartialPathStitcher) { + stitcher.set_similar_path_detection(self.detect_similar_paths); + stitcher.set_collect_stats(self.collect_stats); + } +} + +impl Default for StitcherConfig { + fn default() -> Self { + Self { + detect_similar_paths: true, + collect_stats: false, + } } } diff --git a/stack-graphs/src/storage.rs b/stack-graphs/src/storage.rs index 3cd89d797..c3b3e8ebf 100644 --- a/stack-graphs/src/storage.rs +++ b/stack-graphs/src/storage.rs @@ -7,6 +7,7 @@ use bincode::error::DecodeError; use bincode::error::EncodeError; +use itertools::Itertools; use rusqlite::functions::FunctionFlags; use rusqlite::types::ValueRef; use rusqlite::Connection; @@ -19,6 +20,7 @@ use std::path::PathBuf; use thiserror::Error; use crate::arena::Handle; +use crate::graph::Degree; use crate::graph::File; use crate::graph::Node; use crate::graph::StackGraph; @@ -32,7 +34,7 @@ use crate::stitching::ForwardCandidates; use crate::CancellationError; use crate::CancellationFlag; -const VERSION: usize = 5; +const VERSION: usize = 6; const SCHEMA: &str = r#" CREATE TABLE metadata ( @@ -417,6 +419,7 @@ impl SQLiteWriter { graph: StackGraph::new(), partials: PartialPaths::new(), db: Database::new(), + stats: Stats::default(), } } } @@ -430,6 +433,7 @@ pub struct SQLiteReader { graph: StackGraph, partials: PartialPaths, db: Database, + stats: Stats, } impl SQLiteReader { @@ -452,6 +456,7 @@ impl SQLiteReader { graph: StackGraph::new(), partials: PartialPaths::new(), db: Database::new(), + stats: Stats::default(), }) } @@ -465,6 +470,8 @@ impl SQLiteReader { self.loaded_root_paths.clear(); self.partials.clear(); self.db.clear(); + + self.stats.clear(); } /// Clear path data that has been loaded into this reader instance. @@ -475,6 +482,8 @@ impl SQLiteReader { self.loaded_root_paths.clear(); self.partials.clear(); self.db.clear(); + + self.stats.clear_paths(); } /// Get the file's status in the database. If a tag is provided, it must match or the file @@ -516,7 +525,13 @@ impl SQLiteReader { /// Ensure the graph for the given file is loaded. pub fn load_graph_for_file(&mut self, file: &str) -> Result> { - Self::load_graph_for_file_inner(file, &mut self.graph, &mut self.loaded_graphs, &self.conn) + Self::load_graph_for_file_inner( + file, + &mut self.graph, + &mut self.loaded_graphs, + &self.conn, + &mut self.stats, + ) } fn load_graph_for_file_inner( @@ -524,13 +539,16 @@ impl SQLiteReader { graph: &mut StackGraph, loaded_graphs: &mut HashSet, conn: &Connection, + stats: &mut Stats, ) -> Result> { copious_debugging!("--> Load graph for {}", file); if !loaded_graphs.insert(file.to_string()) { copious_debugging!(" * Already loaded"); + stats.file_cached += 1; return Ok(graph.get_file(file).expect("loaded file to exist")); } copious_debugging!(" * Load from database"); + stats.file_loads += 1; let mut stmt = conn.prepare_cached("SELECT value FROM graphs WHERE file = ?")?; let value = stmt.query_row([file], |row| row.get::<_, Vec>(0))?; let (file_graph, _): (serde::StackGraph, usize) = @@ -552,6 +570,7 @@ impl SQLiteReader { &mut self.graph, &mut self.loaded_graphs, &self.conn, + &mut self.stats, )?; } Ok(()) @@ -566,8 +585,10 @@ impl SQLiteReader { copious_debugging!(" * Load extensions from node {}", node.display(&self.graph)); if !self.loaded_node_paths.insert(node) { copious_debugging!(" > Already loaded"); + self.stats.node_path_cached += 1; return Ok(()); } + self.stats.node_path_loads += 1; let id = self.graph[node].id(); let file = id.file().expect("file node required"); let file = self.graph[file].name(); @@ -589,6 +610,7 @@ impl SQLiteReader { &mut self.graph, &mut self.loaded_graphs, &self.conn, + &mut self.stats, )?; let (path, _): (serde::PartialPath, usize) = bincode::decode_from_slice(&value, BINCODE_CONFIG)?; @@ -615,21 +637,23 @@ impl SQLiteReader { " * Load extensions from root with symbol stack {}", symbol_stack.display(&self.graph, &mut self.partials) ); - let symbol_stack_prefixes = - symbol_stack.storage_key_prefixes(&self.graph, &mut self.partials); - for symbol_stack in symbol_stack_prefixes { + let mut stmt = self.conn.prepare_cached( + "SELECT file,value from root_paths WHERE symbol_stack LIKE ? ESCAPE ?", + )?; + let (symbol_stack_patterns, escape) = + symbol_stack.storage_key_patterns(&self.graph, &mut self.partials); + for symbol_stack in symbol_stack_patterns { copious_debugging!( " * Load extensions from root with prefix symbol stack {}", symbol_stack ); if !self.loaded_root_paths.insert(symbol_stack.clone()) { copious_debugging!(" > Already loaded"); + self.stats.root_path_cached += 1; continue; } - let mut stmt = self - .conn - .prepare_cached("SELECT file,value from root_paths WHERE symbol_stack = ?")?; - let paths = stmt.query_map([symbol_stack], |row| { + self.stats.root_path_loads += 1; + let paths = stmt.query_map([symbol_stack, escape.clone()], |row| { let file = row.get::<_, String>(0)?; let value = row.get::<_, Vec>(1)?; Ok((file, value)) @@ -644,6 +668,7 @@ impl SQLiteReader { &mut self.graph, &mut self.loaded_graphs, &self.conn, + &mut self.stats, )?; let (path, _): (serde::PartialPath, usize) = bincode::decode_from_slice(&value, BINCODE_CONFIG)?; @@ -681,41 +706,64 @@ impl SQLiteReader { } /// Get the stack graph, partial paths arena, and path database for the currently loaded data. - pub fn get(&mut self) -> (&StackGraph, &mut PartialPaths, &mut Database) { - (&self.graph, &mut self.partials, &mut self.db) + pub fn get(&mut self) -> (&mut StackGraph, &mut PartialPaths, &mut Database) { + (&mut self.graph, &mut self.partials, &mut self.db) + } + + /// Return stats about this database reader. + pub fn stats(&self) -> Stats { + self.stats.clone() } } +// Methods for computing keys and patterns for a symbol stack. The format of a storage key is: +// +// has-var GS ( symbol (US symbol)* )? +// +// where has-var is "V" if the symbol stack has a variable, "X" otherwise. impl PartialSymbolStack { /// Returns a string representation of this symbol stack for indexing in the database. - fn storage_key(mut self, graph: &StackGraph, partials: &mut PartialPaths) -> String { + fn storage_key(self, graph: &StackGraph, partials: &mut PartialPaths) -> String { let mut key = String::new(); - while let Some(symbol) = self.pop_front(partials) { - if !key.is_empty() { - key += "\u{241F}"; - } - key += &graph[symbol.symbol]; + match self.has_variable() { + true => key += "V\u{241E}", + false => key += "X\u{241E}", } + key += &self + .iter(partials) + .map(|s| &graph[s.symbol]) + .join("\u{241F}"); key } /// Returns string representations for all prefixes of this symbol stack for querying the /// index in the database. - fn storage_key_prefixes( + fn storage_key_patterns( mut self, graph: &StackGraph, partials: &mut PartialPaths, - ) -> Vec { - let mut key_prefixes = vec![String::new()]; + ) -> (Vec, String) { + let mut key_patterns = Vec::new(); + let mut symbols = String::new(); while let Some(symbol) = self.pop_front(partials) { - let mut key = key_prefixes.last().unwrap().to_string(); - if !key.is_empty() { - key += "\u{241F}"; + if !symbols.is_empty() { + symbols += "\u{241F}"; } - key += &graph[symbol.symbol]; - key_prefixes.push(key); + let symbol = graph[symbol.symbol] + .replace("%", "\\%") + .replace("_", "\\_") + .to_string(); + symbols += &symbol; + // patterns for paths matching a prefix of this stack + key_patterns.push("V\u{241E}".to_string() + &symbols); } - key_prefixes + // pattern for paths matching exactly this stack + key_patterns.push("X\u{241E}".to_string() + &symbols); + if self.has_variable() { + // patterns for paths for which this stack is a prefix + key_patterns.push("_\u{241E}".to_string() + &symbols + "\u{241F}%"); + } + (key_patterns, "\\".to_string()) } } @@ -736,11 +784,39 @@ impl ForwardCandidates, PartialPath, Database, StorageError> .find_candidate_partial_paths(&self.graph, &mut self.partials, path, result); } + fn get_joining_candidate_degree(&self, path: &PartialPath) -> Degree { + self.db.get_incoming_path_degree(path.end_node) + } + fn get_graph_partials_and_db(&mut self) -> (&StackGraph, &mut PartialPaths, &Database) { (&self.graph, &mut self.partials, &self.db) } } +#[derive(Clone, Debug, Default)] +pub struct Stats { + pub file_loads: usize, + pub file_cached: usize, + pub root_path_loads: usize, + pub root_path_cached: usize, + pub node_path_loads: usize, + pub node_path_cached: usize, +} + +impl Stats { + fn clear(&mut self) { + *self = Stats::default(); + } + + fn clear_paths(&mut self) { + *self = Stats { + file_loads: self.file_loads, + file_cached: self.file_cached, + ..Stats::default() + } + } +} + /// Check if the database has the version supported by this library version. fn check_version(conn: &Connection) -> Result<()> { let version = conn.query_row("SELECT version FROM metadata", [], |r| r.get::<_, usize>(0))?; diff --git a/stack-graphs/src/visualization/visualization.css b/stack-graphs/src/visualization/visualization.css index 360594182..c3dfc091b 100644 --- a/stack-graphs/src/visualization/visualization.css +++ b/stack-graphs/src/visualization/visualization.css @@ -9,15 +9,33 @@ /* Paul Tol's Colorblind Friendly Color Scheme (vibrant) * Source: https://personal.sron.nl/~pault/ * + * In default order: + * + * orange #ee7733 * blue #0077bb * cyan #33bbee - * teal #009988 - * orange #ee7733 - * red #cc3311 * magenta #ee3377 + * red #cc3311 + * teal #009988 * grey #bbbbbb */ +/* Paul Tol's Colorblind Friendly Color Scheme (light) + * Source: https://personal.sron.nl/~pault/ + * + * In default order: + * + * light blue #77aadd + * orange #ee8866 + * light yellow #eedd88 + * pink #ffaabb + * light cyan #99ddff + * mint #44bb99 + * pear #bbcc33 + * olive #aaaa00 + * pale grey #dddddd + */ + .sg { width: 100%; height: 100%; @@ -50,30 +68,17 @@ /* --- drop scopes --- */ .sg .node.drop_scopes .background { - fill: #cc3311; r: 6px; } /* --- jump to scope --- */ .sg .node.jump_to_scope .background { - fill: #ee7733; r: 6px; - stroke: black; } /* --- pop symbol --- */ -.sg .node.pop_symbol .background, -.sg .node.pop_scoped_symbol .background { - fill: #009988; -} - -.sg .node.pop_symbol .arrow, -.sg .node.pop_scoped_symbol .arrow { - fill: #005b51; -} - .sg .node.pop_scoped_symbol .pop_scope { fill: #ee7733; r: 6px; @@ -81,21 +86,11 @@ } .sg .node.definition .background { - stroke: black; + stroke-width: 2px; } /* --- push symbol --- */ -.sg .node.push_symbol .background, -.sg .node.push_scoped_symbol .background { - fill: #33bbee; -} - -.sg .node.push_symbol .arrow, -.sg .node.push_scoped_symbol .arrow { - fill: #006e96; -} - .sg .node.push_scoped_symbol .push_scope { fill: #bbbbbb; r: 6px; @@ -112,15 +107,13 @@ } .sg .node.reference .background { - stroke: black; + stroke-width: 2px; } /* --- root --- */ .sg .node.root .background { - fill: #0077bb; r: 6px; - stroke: black; } /* --- scope --- */ @@ -131,14 +124,9 @@ } .sg .node.scope .background { - fill: #bbbbbb; r: 6px; } -.sg .node.scope.exported .background { - stroke: black; -} - .sg .node.scope .focus-point { r: 3px; fill: none; @@ -148,6 +136,10 @@ fill: black; } +.sg .node.scope.exported .background { + stroke-width: 2px; +} + /* --- plain labeled node --- */ .sg .node.scope.plain_labeled_node .border { @@ -156,18 +148,12 @@ } .sg .node.scope.plain_labeled_node .background { - fill: #bbbbbb; rx: 6px; } -.sg .node.scope.plain_labeled_node.exported .background { - stroke: black; -} - /* --- path highlight --- */ .sg .node.path-node .border { - stroke: #ee3377; stroke-width: 4px; stroke-dasharray: 5, 5; } @@ -185,30 +171,21 @@ } .sg .edge path { - stroke: black; stroke-width: 1px; fill: none; } .sg .edge text { font-size: 11pt; - stroke: black; stroke-width: 1px; dominant-baseline: central; } .sg .edge.path-edge path { - stroke: #ee3377; stroke-width: 3px; } -.sg .edge.path-edge text -{ - stroke: #ee3377; - fill: #ee3377; -} - /* ------------------------------------------------------------------------------------------------ * Jumps */ @@ -345,6 +322,40 @@ margin: 0px 1px; } +/* ------------------------------------------------------------------------------------------------ + * Legend + */ + +#sg-legend { + position: absolute; + left: 10px; + top: 10px; + background-color: #bbbbbb; + padding: 6px; + border-radius: 6px; + z-index: 1; +} + +#sg-legend h1 { + font-variant: small-caps; + font-weight: bold; + font-size: inherit; + border-bottom: solid 1px #777777; + margin: 0px; +} + +#sg-legend ul { + list-style: none; + padding: 0px; + margin: 0px; +} + +#sg-legend li { + padding: 3px 6px; + margin: 3px 0px; + border: 1px solid white; +} + /* ------------------------------------------------------------------------------------------------ * Help */ @@ -442,3 +453,189 @@ margin: 0px 6px; cursor: pointer; } + +/* ------------------------------------------------------------------------------------------------ + * Colors + */ + +.sg .node.global .background { + fill: #0077bb; /* blue */ +} +#sg-legend .global { + background-color: #0077bb; /* blue */ +} +.sg .edge.global path, +.sg .edge.global text { + stroke: #0077bb; /* blue */ +} + +.sg .node.file-0 .background { + fill: #77aadd; /* light blue */ +} +#sg-legend .file-0 { + background-color: #77aadd; /* light blue */ +} +.sg .node.file-0 .arrow, +.sg .edge.file-0 text +{ + fill: #0e2236; /* light blue (darkened) */ +} +.sg .node.file-0.reference .background, +.sg .node.file-0.definition .background, +.sg .node.file-0.scope.exported .background, +.sg .edge.file-0 path, +.sg .edge.file-0 text +{ + stroke: #0e2236; /* light blue (darkened) */ +} + +.sg .node.file-1 .background, +.sg .edge.file-1 text +{ + fill: #ee8866; /* orange */ +} +#sg-legend .file-1 { + background-color: #ee8866; /* orange */ +} +.sg .node.file-1 .arrow { + fill: #3d1407; /* orange (darkened) */ +} +.sg .node.file-1.reference .background, +.sg .node.file-1.definition .background, +.sg .node.file-1.scope.exported .background, +.sg .edge.file-1 path, +.sg .edge.file-1 text +{ + stroke: #3d1407; /* orange (darkened) */ +} + +.sg .node.file-2 .background, +.sg .edge.file-2 text +{ + fill: #eedd88; /* light yellow */ +} +#sg-legend .file-2 { + background-color: #eedd88; /* light yellow */ +} +.sg .node.file-2 .arrow { + fill: #413809; /* light yellow (darkened) */ +} +.sg .node.file-2.reference .background, +.sg .node.file-2.definition .background, +.sg .node.file-2.scope.exported .background, +.sg .edge.file-2 path, +.sg .edge.file-2 text +{ + stroke: #413809; /* light yellow (darkened) */ +} + +.sg .node.file-3 .background, +.sg .edge.file-3 text +{ + fill: #ffaabb; /* pink */ +} +#sg-legend .file-3 { + background-color: #ffaabb; /* pink */ +} +.sg .node.file-3 .arrow { + fill: #550011; /* pink (darkened) */ +} +.sg .node.file-3.reference .background, +.sg .node.file-3.definition .background, +.sg .node.file-3.scope.exported .background, +.sg .edge.file-3 path, +.sg .edge.file-3 text +{ + stroke: #550011; /* pink (darkened) */ +} + +.sg .node.file-4 .background, +.sg .edge.file-4 text +{ + fill: #99ddff; /* light cyan */ +} +#sg-legend .file-4 { + background-color: #99ddff; /* light cyan */ +} +.sg .node.file-4 .arrow { + fill: #003652; /* light cyan (darkened) */ +} +.sg .node.file-4.reference .background, +.sg .node.file-4.definition .background, +.sg .node.file-4.scope.exported .background, +.sg .edge.file-4 path, +.sg .edge.file-4 text +{ + stroke: #003652; /* light cyan (darkened) */ +} + +.sg .node.file-5 .background, +.sg .edge.file-5 text +{ + fill: #44bb99; /* mint */ +} +#sg-legend .file-5 { + background-color: #44bb99; /* mint */ +} +.sg .node.file-5 .arrow { + fill: #0e251f; /* mint (darkened) */ +} +.sg .node.file-5.reference .background, +.sg .node.file-5.definition .background, +.sg .node.file-5.scope.exported .background, +.sg .edge.file-5 path, +.sg .edge.file-5 text +{ + stroke: #0e251f; /* mint (darkened) */ +} + +.sg .node.file-6 .background, +.sg .edge.file-6 text +{ + fill: #bbcc33; /* pear */ +} +#sg-legend .file-6 { + background-color: #bbcc33; /* pear */ +} +.sg .node.file-6 .arrow { + fill: #25290a; /* pear (darkened) */ +} +.sg .node.file-6.reference .background, +.sg .node.file-6.definition .background, +.sg .node.file-6.scope.exported .background, +.sg .edge.file-6 path, +.sg .edge.file-6 text +{ + stroke: #25290a; /* pear (darkened) */ +} + +.sg .node.file-7 .background, +.sg .edge.file-7 text +{ + fill: #aaaa00; /* olive */ +} +#sg-legend .file-7 { + background-color: #aaaa00; /* olive */ +} +.sg .node.file-7 .arrow { + fill: #222200; /* olive (darkened) */ +} +.sg .node.file-7.reference .background, +.sg .node.file-7.definition .background, +.sg .node.file-7.scope.exported .background, +.sg .edge.file-7 path, +.sg .edge.file-7 text +{ + stroke: #222200; /* olive (darkened) */ +} + +.sg .node.path-node .border, +.sg .edge.path-edge path, +.sg .edge.path-edge text +{ + stroke: #ee7733; +} + +.sg .edge.path-edge text { + fill: #ee7733; +} diff --git a/stack-graphs/src/visualization/visualization.js b/stack-graphs/src/visualization/visualization.js index a1d9bbaac..6fdab1ef6 100644 --- a/stack-graphs/src/visualization/visualization.js +++ b/stack-graphs/src/visualization/visualization.js @@ -17,11 +17,14 @@ class StackGraph { static arrow_head_w = 16; static arrow_head_h = 8; + static number_of_file_colors = 8; + constructor(container, graph, paths, metadata) { this.metadata = metadata; this.graph = graph; this.paths = paths; + this.cleanup_data(); this.compute_data(); this.current_node = null; @@ -31,13 +34,35 @@ class StackGraph { this.render(); } + cleanup_data() { + let idx = 0; + while (idx < this.graph.edges.length) { + let edge = this.graph.edges[idx]; + if (edge.source.file === edge.sink.file && edge.source.local_id === edge.sink.local_id) { + console.log("ignoring self loop", edge); + this.graph.edges.splice(idx, 1); + } else { + idx += 1; + } + } + } + compute_data() { + this.F = {}; this.ID = {}; this.N = []; + this.compute_file_data(); this.compute_node_data(); this.compute_path_data(); } + compute_file_data() { + for (let i in graph.files) { + const file = graph.files[i]; + this.F[file] = i; + } + } + compute_node_data() { for (let i in graph.nodes) { const node = graph.nodes[i]; @@ -151,7 +176,8 @@ class StackGraph { // render UI this.render_help(); this.render_tooltip(); - this.render_graph() + this.render_legend(); + this.render_graph(); // pan & zoom let zoom = d3.zoom() @@ -225,7 +251,7 @@ class StackGraph { .data(dag.links()) .enter() .append("g") - .attr("class", (d) => d.data.is_jump ? "jump" : "edge") + .attr("class", (d) => `${d.data.is_jump ? "jump" : "edge"} ${this.edge_to_file_class(d.data)}`) .attr("id", (d) => this.edge_to_id_str(d.data)); edges.append("path") .attr("id", (d) => this.edge_to_id_str(d.data) + ":path") @@ -286,14 +312,14 @@ class StackGraph { render_node(node, g) { g.attr('id', this.node_to_id_str(node)); - g.attr('class', `node ${node.type}`); + g.attr('class', `node ${node.type} ${this.node_to_file_class(node)}`); switch (node.type) { case "drop_scopes": - this.render_scope(g); + this.render_symbol_node(g, "[drop]", null, ""); break; case "jump_to_scope": - this.render_scope(g); + this.render_symbol_node(g, "[jump]", null, ""); break; case "pop_symbol": this.render_symbol_node(g, node.symbol, null, "pop"); @@ -322,7 +348,7 @@ class StackGraph { } break; case "root": - this.render_scope(g); + this.render_symbol_node(g, "[root]", null, ""); break; case "scope": if (this.show_all_node_labels()) { @@ -347,6 +373,7 @@ class StackGraph { break; } } + render_symbol_node(g, text, scope, shape) { let content = g.append("g"); content.append('text').text(text); @@ -458,9 +485,9 @@ class StackGraph { } } - /* ------------------------------------------------------------------------------------------------ - * Path Highlighting - */ + // ------------------------------------------------------------------------------------------------ + // Path Highlighting + // paths_mouseover(e, node) { if (this.paths_lock !== null) { @@ -768,6 +795,30 @@ class StackGraph { || (this.current_edge !== null && path.derived.edges.hasOwnProperty(this.edge_to_id_str(this.current_edge))); } + // ------------------------------------------------------------------------------------------------ + // Legend + // + + render_legend() { + const legend = d3.select('body').append('div') + .attr('id', 'sg-legend') + legend.append("h1").text("Files"); + const items = legend.append("ul"); + items.append("li") + .classed("global", true) + .text("[global]"); + for (const file in this.F) { + items.append("li") + .classed('file-' + this.F[file], true) + .text(file); + } + } + + legend_update() { + const legend = d3.select('#sg-legend'); + legend.style('visibility', this.show_file_legend() ? null : 'hidden'); + } + // ------------------------------------------------------------------------------------------------ // Help // @@ -789,6 +840,10 @@ class StackGraph { Pan by dragging the background with the mouse. Zoom using the scroll wheel. `); + this.show_files_legend_toggle = this.new_setting(help_content, "sg-files-legend", "Show files legend (f)", true); + this.show_files_legend_toggle.on("change", (e => { + this.legend_update(); + })); this.show_all_node_labels_toggle = this.new_setting(help_content, "sg-scope-labels", "Show all node labels (l)", false); this.show_all_node_labels_toggle.on("change", (e => { this.render_graph(); @@ -825,6 +880,10 @@ class StackGraph { help_keypress(e) { switch (e.keyCode) { + case 70: // f + this.show_files_legend_toggle.property("checked", !this.show_files_legend_toggle.property("checked")); + this.legend_update(); + break; case 72: // h this.help_toggle.property("checked", !this.help_toggle.property("checked")); break; @@ -847,6 +906,10 @@ class StackGraph { return this.show_all_node_labels_toggle.property("checked"); } + show_file_legend() { + return this.show_files_legend_toggle.property("checked"); + } + new_setting(element, id, html, initial) { const toggle = element.append("div"); const toggle_input = toggle.append('input') @@ -869,10 +932,24 @@ class StackGraph { return this.node_id_to_str(node.id); } + node_to_file_class(node) { + return this.node_id_to_file_class(node.id); + } + edge_to_id_str(edge) { return this.node_id_to_str(edge.source) + "->" + this.node_id_to_str(edge.sink); } + edge_to_file_class(edge) { + if (edge.source.hasOwnProperty('file')) { + return "file-" + (this.F[edge.source.file] % StackGraph.number_of_file_colors); + } else if (edge.sink.hasOwnProperty('file')) { + return "file-" + (this.F[edge.sink.file] % StackGraph.number_of_file_colors); + } else { + return "global"; + } + } + node_id_to_str(id) { if (id.hasOwnProperty('file')) { return id.file + "#" + id.local_id; @@ -881,6 +958,14 @@ class StackGraph { } } + node_id_to_file_class(id) { + if (id.hasOwnProperty('file')) { + return "file-" + (this.F[id.file] % StackGraph.number_of_file_colors); + } else { + return "global"; + } + } + id_selector(id) { const sel = "#" + id.replaceAll(/[^a-zA-Z0-9]/g, '\\$&'); return sel; diff --git a/stack-graphs/tests/it/arena.rs b/stack-graphs/tests/it/arena.rs index 5af25d66e..6963dae29 100644 --- a/stack-graphs/tests/it/arena.rs +++ b/stack-graphs/tests/it/arena.rs @@ -229,3 +229,28 @@ fn can_compare_deques() { deque10.ensure_backwards(&mut arena); assert_eq!(deque1.cmp(&mut arena, deque10), Ordering::Less); } + +#[test] +fn can_use_arena_after_clear() { + let mut a = Arena::new(); + let h = a.add(12 as u8); + assert_eq!(12, *a.get(h)); + + a.clear(); + let h = a.add(7); + assert_eq!(7, *a.get(h)); +} + +#[test] +fn can_use_supplemental_arena_after_clear() { + let mut a = Arena::new(); + let h = a.add(()); + + let mut x = SupplementalArena::new(); + x[h] = 12; + assert_eq!(Some(12), x.get(h).cloned()); + + x.clear(); + x[h] = 7; + assert_eq!(Some(7), x.get(h).cloned()); +} diff --git a/stack-graphs/tests/it/c/can_find_local_nodes.rs b/stack-graphs/tests/it/c/can_find_local_nodes.rs index 0a5d3a402..9fbf31859 100644 --- a/stack-graphs/tests/it/c/can_find_local_nodes.rs +++ b/stack-graphs/tests/it/c/can_find_local_nodes.rs @@ -22,6 +22,7 @@ use stack_graphs::c::sg_partial_path_list_free; use stack_graphs::c::sg_partial_path_list_new; use stack_graphs::c::sg_partial_path_list_paths; use stack_graphs::c::sg_stack_graph_nodes; +use stack_graphs::c::sg_stitcher_config; use stack_graphs::graph::Node; use crate::c::test_graph::TestGraph; @@ -33,11 +34,15 @@ fn check_local_nodes(graph: &TestGraph, file: &str, expected_local_nodes: &[&str let partials = sg_partial_path_arena_new(); let path_list = sg_partial_path_list_new(); + let stitcher_config = sg_stitcher_config { + detect_similar_paths: false, + }; sg_partial_path_arena_find_partial_paths_in_file( graph.graph, partials, file.as_u32(), path_list, + &stitcher_config, std::ptr::null(), ); diff --git a/stack-graphs/tests/it/c/can_find_partial_paths_in_file.rs b/stack-graphs/tests/it/c/can_find_partial_paths_in_file.rs index b39bc524a..ae93f84be 100644 --- a/stack-graphs/tests/it/c/can_find_partial_paths_in_file.rs +++ b/stack-graphs/tests/it/c/can_find_partial_paths_in_file.rs @@ -22,6 +22,7 @@ use stack_graphs::c::sg_partial_path_list_new; use stack_graphs::c::sg_partial_path_list_paths; use stack_graphs::c::sg_partial_scope_stack; use stack_graphs::c::sg_partial_symbol_stack; +use stack_graphs::c::sg_stitcher_config; use stack_graphs::c::SG_LIST_EMPTY_HANDLE; use stack_graphs::c::SG_NULL_HANDLE; use stack_graphs::partial::PartialPath; @@ -77,11 +78,15 @@ fn check_partial_paths_in_file(graph: &TestGraph, file: &str, expected_paths: &[ let partials = sg_partial_path_arena_new(); let path_list = sg_partial_path_list_new(); + let stitcher_config = sg_stitcher_config { + detect_similar_paths: false, + }; sg_partial_path_arena_find_partial_paths_in_file( graph.graph, partials, file.as_u32(), path_list, + &stitcher_config, std::ptr::null(), ); diff --git a/stack-graphs/tests/it/c/can_find_qualified_definitions_with_phased_partial_path_stitching.rs b/stack-graphs/tests/it/c/can_find_qualified_definitions_with_phased_partial_path_stitching.rs index dcc276c86..613a54556 100644 --- a/stack-graphs/tests/it/c/can_find_qualified_definitions_with_phased_partial_path_stitching.rs +++ b/stack-graphs/tests/it/c/can_find_qualified_definitions_with_phased_partial_path_stitching.rs @@ -28,6 +28,7 @@ use stack_graphs::c::sg_partial_path_list_free; use stack_graphs::c::sg_partial_path_list_new; use stack_graphs::c::sg_partial_path_list_paths; use stack_graphs::c::sg_stack_graph; +use stack_graphs::c::sg_stitcher_config; use stack_graphs::copious_debugging; use stack_graphs::graph::StackGraph; use stack_graphs::partial::PartialPath; @@ -55,11 +56,15 @@ impl StorageLayer { let rust_graph = unsafe { &(*graph).inner }; let path_list = sg_partial_path_list_new(); for file in rust_graph.iter_files() { + let stitcher_config = sg_stitcher_config { + detect_similar_paths: false, + }; sg_partial_path_arena_find_partial_paths_in_file( graph, partials, file.as_u32(), path_list, + &stitcher_config, std::ptr::null(), ); } diff --git a/stack-graphs/tests/it/c/can_jump_to_definition.rs b/stack-graphs/tests/it/c/can_jump_to_definition.rs index 800ce06e8..aeea163f6 100644 --- a/stack-graphs/tests/it/c/can_jump_to_definition.rs +++ b/stack-graphs/tests/it/c/can_jump_to_definition.rs @@ -15,6 +15,7 @@ use stack_graphs::c::sg_partial_path_list_count; use stack_graphs::c::sg_partial_path_list_free; use stack_graphs::c::sg_partial_path_list_new; use stack_graphs::c::sg_partial_path_list_paths; +use stack_graphs::c::sg_stitcher_config; use stack_graphs::partial::PartialPath; use crate::c::test_graph::TestGraph; @@ -28,12 +29,16 @@ fn check_jump_to_definition(graph: &TestGraph, expected_paths: &[&str]) { .iter_nodes() .filter(|handle| rust_graph[*handle].is_reference()) .collect::>(); + let stitcher_config = sg_stitcher_config { + detect_similar_paths: false, + }; sg_partial_path_arena_find_all_complete_paths( graph.graph, paths, references.len(), references.as_ptr() as *const _, path_list, + &stitcher_config, std::ptr::null(), ); diff --git a/stack-graphs/tests/it/c/can_jump_to_definition_with_phased_partial_path_stitching.rs b/stack-graphs/tests/it/c/can_jump_to_definition_with_phased_partial_path_stitching.rs index 615ffedd5..ce20ae378 100644 --- a/stack-graphs/tests/it/c/can_jump_to_definition_with_phased_partial_path_stitching.rs +++ b/stack-graphs/tests/it/c/can_jump_to_definition_with_phased_partial_path_stitching.rs @@ -27,6 +27,7 @@ use stack_graphs::c::sg_partial_path_list_free; use stack_graphs::c::sg_partial_path_list_new; use stack_graphs::c::sg_partial_path_list_paths; use stack_graphs::c::sg_stack_graph; +use stack_graphs::c::sg_stitcher_config; use stack_graphs::copious_debugging; use stack_graphs::partial::PartialPath; use stack_graphs::partial::PartialScopeStackBindings; @@ -49,11 +50,15 @@ impl StorageLayer { let rust_graph = unsafe { &(*graph).inner }; let path_list = sg_partial_path_list_new(); for file in rust_graph.iter_files() { + let stitcher_config = sg_stitcher_config { + detect_similar_paths: false, + }; sg_partial_path_arena_find_partial_paths_in_file( graph, partials, file.as_u32(), path_list, + &stitcher_config, std::ptr::null(), ); } diff --git a/stack-graphs/tests/it/can_find_local_nodes.rs b/stack-graphs/tests/it/can_find_local_nodes.rs index 105e5a795..4f471d88f 100644 --- a/stack-graphs/tests/it/can_find_local_nodes.rs +++ b/stack-graphs/tests/it/can_find_local_nodes.rs @@ -12,6 +12,7 @@ use stack_graphs::graph::StackGraph; use stack_graphs::partial::PartialPaths; use stack_graphs::stitching::Database; use stack_graphs::stitching::ForwardPartialPathStitcher; +use stack_graphs::stitching::StitcherConfig; use stack_graphs::NoCancellation; use crate::test_graphs; @@ -24,6 +25,7 @@ fn check_local_nodes(graph: &StackGraph, file: &str, expected_local_nodes: &[&st graph, &mut partials, file, + StitcherConfig::default(), &NoCancellation, |graph, partials, path| { database.add_partial_path(graph, partials, path.clone()); diff --git a/stack-graphs/tests/it/can_find_node_partial_paths_in_database.rs b/stack-graphs/tests/it/can_find_node_partial_paths_in_database.rs index 3ccea9548..4e825c47e 100644 --- a/stack-graphs/tests/it/can_find_node_partial_paths_in_database.rs +++ b/stack-graphs/tests/it/can_find_node_partial_paths_in_database.rs @@ -15,6 +15,7 @@ use stack_graphs::partial::PartialPath; use stack_graphs::partial::PartialPaths; use stack_graphs::stitching::Database; use stack_graphs::stitching::ForwardPartialPathStitcher; +use stack_graphs::stitching::StitcherConfig; use stack_graphs::NoCancellation; use crate::test_graphs; @@ -33,6 +34,7 @@ fn check_node_partial_paths( graph, &mut partials, file, + StitcherConfig::default(), &NoCancellation, |graph, partials, path| { db.add_partial_path(graph, partials, path.clone()); diff --git a/stack-graphs/tests/it/can_find_partial_paths_in_file.rs b/stack-graphs/tests/it/can_find_partial_paths_in_file.rs index abba6951b..adef36ae3 100644 --- a/stack-graphs/tests/it/can_find_partial_paths_in_file.rs +++ b/stack-graphs/tests/it/can_find_partial_paths_in_file.rs @@ -10,7 +10,7 @@ use std::collections::BTreeSet; use pretty_assertions::assert_eq; use stack_graphs::graph::StackGraph; use stack_graphs::partial::PartialPaths; -use stack_graphs::stitching::ForwardPartialPathStitcher; +use stack_graphs::stitching::{ForwardPartialPathStitcher, StitcherConfig}; use stack_graphs::NoCancellation; use crate::test_graphs; @@ -23,6 +23,7 @@ fn check_partial_paths_in_file(graph: &StackGraph, file: &str, expected_paths: & graph, &mut partials, file, + StitcherConfig::default(), &NoCancellation, |graph, partials, path| { results.insert(path.display(graph, partials).to_string()); diff --git a/stack-graphs/tests/it/can_find_root_partial_paths_in_database.rs b/stack-graphs/tests/it/can_find_root_partial_paths_in_database.rs index 97dca4788..e8beff576 100644 --- a/stack-graphs/tests/it/can_find_root_partial_paths_in_database.rs +++ b/stack-graphs/tests/it/can_find_root_partial_paths_in_database.rs @@ -17,7 +17,7 @@ use stack_graphs::partial::PartialScopedSymbol; use stack_graphs::partial::PartialSymbolStack; use stack_graphs::stitching::Database; use stack_graphs::stitching::ForwardPartialPathStitcher; -use stack_graphs::stitching::SymbolStackKey; +use stack_graphs::stitching::StitcherConfig; use stack_graphs::NoCancellation; use crate::test_graphs; @@ -35,6 +35,7 @@ fn check_root_partial_paths( graph, &mut partials, file, + StitcherConfig::default(), &NoCancellation, |graph, partials, path| { db.add_partial_path(graph, partials, path.clone()); @@ -53,8 +54,12 @@ fn check_root_partial_paths( } let mut results = Vec::>::new(); - let key = SymbolStackKey::from_partial_symbol_stack(&mut partials, &mut db, symbol_stack); - db.find_candidate_partial_paths_from_root(graph, &mut partials, Some(key), &mut results); + db.find_candidate_partial_paths_from_root( + graph, + &mut partials, + Some(symbol_stack), + &mut results, + ); let actual_partial_paths = results .into_iter() diff --git a/stack-graphs/tests/it/can_jump_to_definition.rs b/stack-graphs/tests/it/can_jump_to_definition.rs index 965207f4c..70158f495 100644 --- a/stack-graphs/tests/it/can_jump_to_definition.rs +++ b/stack-graphs/tests/it/can_jump_to_definition.rs @@ -12,6 +12,7 @@ use stack_graphs::graph::StackGraph; use stack_graphs::partial::PartialPaths; use stack_graphs::stitching::ForwardPartialPathStitcher; use stack_graphs::stitching::GraphEdgeCandidates; +use stack_graphs::stitching::StitcherConfig; use stack_graphs::NoCancellation; use crate::test_graphs; @@ -25,6 +26,7 @@ fn check_jump_to_definition(graph: &StackGraph, expected_paths: &[&str]) { ForwardPartialPathStitcher::find_all_complete_partial_paths( &mut GraphEdgeCandidates::new(graph, &mut paths, None), references, + StitcherConfig::default(), &NoCancellation, |graph, paths, path| { results.insert(path.display(graph, paths).to_string()); diff --git a/stack-graphs/tests/it/can_jump_to_definition_with_forward_partial_path_stitching.rs b/stack-graphs/tests/it/can_jump_to_definition_with_forward_partial_path_stitching.rs index d45e96afc..7f5996e3e 100644 --- a/stack-graphs/tests/it/can_jump_to_definition_with_forward_partial_path_stitching.rs +++ b/stack-graphs/tests/it/can_jump_to_definition_with_forward_partial_path_stitching.rs @@ -13,6 +13,7 @@ use stack_graphs::partial::PartialPaths; use stack_graphs::stitching::Database; use stack_graphs::stitching::DatabaseCandidates; use stack_graphs::stitching::ForwardPartialPathStitcher; +use stack_graphs::stitching::StitcherConfig; use stack_graphs::NoCancellation; use crate::test_graphs; @@ -27,6 +28,7 @@ fn check_jump_to_definition(graph: &StackGraph, expected_partial_paths: &[&str]) graph, &mut partials, file, + StitcherConfig::default(), &NoCancellation, |graph, partials, path| { db.add_partial_path(graph, partials, path.clone()); @@ -42,6 +44,7 @@ fn check_jump_to_definition(graph: &StackGraph, expected_partial_paths: &[&str]) ForwardPartialPathStitcher::find_all_complete_partial_paths( &mut DatabaseCandidates::new(graph, &mut partials, &mut db), references, + StitcherConfig::default(), &NoCancellation, |_, _, p| { complete_partial_paths.push(p.clone()); diff --git a/stack-graphs/tests/it/cycles.rs b/stack-graphs/tests/it/cycles.rs index 12c8b5672..df840eba2 100644 --- a/stack-graphs/tests/it/cycles.rs +++ b/stack-graphs/tests/it/cycles.rs @@ -16,6 +16,7 @@ use stack_graphs::partial::PartialPaths; use stack_graphs::stitching::Database; use stack_graphs::stitching::ForwardPartialPathStitcher; use stack_graphs::stitching::GraphEdges; +use stack_graphs::stitching::StitcherConfig; use stack_graphs::CancelAfterDuration; use std::time::Duration; @@ -199,6 +200,7 @@ fn finding_simple_identity_cycle_is_detected() { &graph, &mut partials, file, + StitcherConfig::default(), &cancellation_flag, |_, _, _| path_count += 1, ); @@ -292,6 +294,7 @@ fn finding_composite_identity_cycle_is_detected() { &graph, &mut partials, file, + StitcherConfig::default(), &cancellation_flag, |_, _, _| path_count += 1, ); @@ -353,6 +356,7 @@ fn appending_eliminating_cycle_terminates() { &graph, &mut partials, file, + StitcherConfig::default(), &cancellation_flag, |_, _, _| path_count += 1, ); diff --git a/stack-graphs/tests/it/graph.rs b/stack-graphs/tests/it/graph.rs index 353850c6f..620d066c9 100644 --- a/stack-graphs/tests/it/graph.rs +++ b/stack-graphs/tests/it/graph.rs @@ -8,7 +8,7 @@ use std::collections::HashSet; use maplit::hashset; -use stack_graphs::graph::StackGraph; +use stack_graphs::graph::{Degree, StackGraph}; use crate::test_graphs; use crate::test_graphs::CreateStackGraph; @@ -157,14 +157,6 @@ fn can_add_and_remove_edges() { .collect::>(), hashset! { (h2, 0), (h3, 0), (h4, 0) } ); - graph.remove_edge(h1, h3); - assert_eq!( - graph - .outgoing_edges(h1) - .map(|edge| edge.sink) - .collect::>(), - hashset! { h2, h4 } - ); } #[test] @@ -204,3 +196,22 @@ fn can_add_graph_to_empty_graph() { ); } } + +#[test] +fn can_get_incoming_edges() { + let mut graph = StackGraph::new(); + let file = graph.get_or_create_file("test.py"); + let h1 = graph.internal_scope(file, 0); + let h2 = graph.internal_scope(file, 1); + let h3 = graph.internal_scope(file, 2); + assert_eq!(Degree::Zero, graph.incoming_edge_degree(h1)); + assert_eq!(Degree::Zero, graph.incoming_edge_degree(h2)); + assert_eq!(Degree::Zero, graph.incoming_edge_degree(h3)); + graph.add_edge(h1, h2, 0); + graph.add_edge(h3, h2, 0); + assert_eq!(Degree::Zero, graph.incoming_edge_degree(h1)); + assert_eq!(Degree::Multiple, graph.incoming_edge_degree(h2)); + assert_eq!(Degree::Zero, graph.incoming_edge_degree(h3)); + graph.add_edge(h3, h1, 0); + assert_eq!(Degree::One, graph.incoming_edge_degree(h1)); +} diff --git a/stack-graphs/tests/it/main.rs b/stack-graphs/tests/it/main.rs index bea8a6030..e5a0e6edd 100644 --- a/stack-graphs/tests/it/main.rs +++ b/stack-graphs/tests/it/main.rs @@ -24,4 +24,8 @@ mod graph; mod partial; #[cfg(feature = "serde")] mod serde; +mod stats; +mod stitching; +#[cfg(feature = "storage")] +mod storage; mod util; diff --git a/stack-graphs/tests/it/serde.rs b/stack-graphs/tests/it/serde.rs index 04679f353..3ecc1c48f 100644 --- a/stack-graphs/tests/it/serde.rs +++ b/stack-graphs/tests/it/serde.rs @@ -12,7 +12,7 @@ use stack_graphs::graph; use stack_graphs::graph::StackGraph; use stack_graphs::partial::PartialPaths; use stack_graphs::serde; -use stack_graphs::stitching::{Database, ForwardPartialPathStitcher}; +use stack_graphs::stitching::{Database, ForwardPartialPathStitcher, StitcherConfig}; use stack_graphs::NoCancellation; use crate::test_graphs; @@ -984,6 +984,7 @@ fn can_serialize_partial_paths() { &graph, &mut partials, file, + StitcherConfig::default(), &NoCancellation, |g, ps, p| { db.add_partial_path(g, ps, p.clone()); diff --git a/stack-graphs/tests/it/stats.rs b/stack-graphs/tests/it/stats.rs new file mode 100644 index 000000000..4d83d0c1d --- /dev/null +++ b/stack-graphs/tests/it/stats.rs @@ -0,0 +1,52 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use itertools::Itertools; +use pretty_assertions::assert_eq; + +use stack_graphs::stats::*; + +#[test] +fn empty_distribution() { + let hist: FrequencyDistribution = FrequencyDistribution::default(); + + assert_eq!(0, hist.unique()); + assert_eq!(0, hist.count()); + + let result = hist.quantiles(0).into_iter().cloned().collect_vec(); + let expected: Vec = vec![]; + assert_eq!(expected, result); +} + +#[test] +fn singleton_distribution() { + let mut hist = FrequencyDistribution::default(); + hist.record(42); + + assert_eq!(1, hist.unique()); + assert_eq!(1, hist.count()); + + let result = hist.quantiles(4).into_iter().cloned().collect_vec(); + let expected: Vec = vec![42, 42, 42, 42, 42]; + assert_eq!(expected, result); +} + +#[test] +fn four_value_distribution() { + let mut hist = FrequencyDistribution::default(); + hist.record(3); + hist.record(4); + hist.record(1); + hist.record(2); + + assert_eq!(4, hist.unique()); + assert_eq!(4, hist.count()); + + let result = hist.quantiles(4).into_iter().cloned().collect_vec(); + let expected: Vec = vec![1, 1, 2, 3, 4]; + assert_eq!(expected, result); +} diff --git a/stack-graphs/tests/it/stitching.rs b/stack-graphs/tests/it/stitching.rs new file mode 100644 index 000000000..ddb878606 --- /dev/null +++ b/stack-graphs/tests/it/stitching.rs @@ -0,0 +1,104 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use itertools::Itertools; +use stack_graphs::graph::StackGraph; +use stack_graphs::partial::PartialPaths; +use stack_graphs::stitching::Database; + +use crate::util::create_partial_path_and_edges; +use crate::util::create_pop_symbol_node; +use crate::util::create_push_symbol_node; + +fn test_foo_bar_root_candidate_paths(symbols: &[&str], variable: bool) -> usize { + let mut graph = StackGraph::new(); + let file = graph.add_file("test").unwrap(); + let mut partials = PartialPaths::new(); + + let r = StackGraph::root_node(); + let foo = create_pop_symbol_node(&mut graph, file, "foo", true); + let bar = create_pop_symbol_node(&mut graph, file, "bar", true); + + let path_with_variable = + create_partial_path_and_edges(&mut graph, &mut partials, &[r, foo, bar]).unwrap(); + + let mut path_without_variable = path_with_variable.clone(); + path_without_variable.eliminate_precondition_stack_variables(&mut partials); + + let mut db = Database::new(); + db.add_partial_path(&graph, &mut partials, path_with_variable); + db.add_partial_path(&graph, &mut partials, path_without_variable); + + let r = StackGraph::root_node(); + let refs = symbols + .into_iter() + .map(|r| create_push_symbol_node(&mut graph, file, *r, true)) + .chain(std::iter::once(r)) + .collect_vec(); + let mut path = create_partial_path_and_edges(&mut graph, &mut partials, &refs).unwrap(); + if !variable { + path.eliminate_precondition_stack_variables(&mut partials); + } + + let mut results = Vec::new(); + db.find_candidate_partial_paths_from_root( + &mut graph, + &mut partials, + Some(path.symbol_stack_postcondition), + &mut results, + ); + + results.len() +} + +#[test] +fn find_candidates_for_exact_symbol_stack_with_variable() { + // <"foo","bar",%2> ~ <"foo","bar",%1> | yes, %2 = %1 + // <"foo","bar",%2> ~ <"foo","bar"> | yes, %2 = <> + let results = test_foo_bar_root_candidate_paths(&["bar", "foo"], true); + assert_eq!(2, results); +} + +#[test] +fn find_candidates_for_exact_symbol_stack_without_variable() { + // <"foo","bar"> ~ <"foo","bar",%1> | yes, %1 = <> + // <"foo","bar"> ~ <"foo","bar"> | yes + let results = test_foo_bar_root_candidate_paths(&["bar", "foo"], false); + assert_eq!(2, results); +} + +#[test] +fn find_candidates_for_longer_symbol_stack_with_variable() { + // <"foo","bar","quz",%2> ~ <"foo","bar",%1> | yes, %1 = <"quz",%2> + // <"foo","bar","quz",%2> ~ <"foo","bar"> | no + let results = test_foo_bar_root_candidate_paths(&["quz", "bar", "foo"], true); + assert_eq!(1, results); +} + +#[test] +fn find_candidates_for_longer_symbol_stack_without_variable() { + // <"foo","bar","quz"> ~ <"foo","bar",%1> | yes, %1 = <"quz"> + // <"foo","bar","quz"> ~ <"foo","bar"> | no + let results = test_foo_bar_root_candidate_paths(&["quz", "bar", "foo"], false); + assert_eq!(1, results); +} + +#[test] +fn find_candidates_for_shorter_symbol_stack_with_variable() { + // <"foo",%2> ~ <"foo","bar",%1> | yes, %2 = <"bar",%1> + // <"foo",%2> ~ <"foo","bar"> | yes, %2 = <"bar"> + let results = test_foo_bar_root_candidate_paths(&["foo"], true); + assert_eq!(2, results); +} + +#[test] +fn find_candidates_for_shorter_symbol_stack_without_variable() { + // <"foo"> ~ <"foo","bar",%1> | no + // <"foo"> ~ <"foo","bar"> | no + let results = test_foo_bar_root_candidate_paths(&["foo"], false); + assert_eq!(0, results); +} diff --git a/stack-graphs/tests/it/storage.rs b/stack-graphs/tests/it/storage.rs new file mode 100644 index 000000000..464700414 --- /dev/null +++ b/stack-graphs/tests/it/storage.rs @@ -0,0 +1,127 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use itertools::Itertools; +use stack_graphs::graph::StackGraph; +use stack_graphs::partial::PartialPaths; +use stack_graphs::storage::SQLiteWriter; +use stack_graphs::NoCancellation; + +use crate::util::create_partial_path_and_edges; +use crate::util::create_pop_symbol_node; +use crate::util::create_push_symbol_node; + +fn test_foo_bar_root_candidate_paths(symbols: &[&str], variable: bool) -> usize { + let mut reader = { + let mut writer = SQLiteWriter::open_in_memory().unwrap(); + + let mut graph = StackGraph::new(); + let file = graph.add_file("test1").unwrap(); + let mut partials = PartialPaths::new(); + + let r = StackGraph::root_node(); + let foo = create_pop_symbol_node(&mut graph, file, "foo", true); + let bar = create_pop_symbol_node(&mut graph, file, "bar", true); + + let path_with_variable = + create_partial_path_and_edges(&mut graph, &mut partials, &[r, foo, bar]).unwrap(); + + let mut path_without_variable = path_with_variable.clone(); + path_without_variable.eliminate_precondition_stack_variables(&mut partials); + + writer + .store_result_for_file( + &graph, + file, + "", + &mut partials, + vec![&path_with_variable, &path_without_variable], + ) + .unwrap(); + + writer.into_reader() + }; + + { + let (graph, partials, _) = reader.get(); + let file = graph.add_file("test2").unwrap(); + + let r = StackGraph::root_node(); + let refs = symbols + .into_iter() + .map(|r| create_push_symbol_node(graph, file, *r, true)) + .chain(std::iter::once(r)) + .collect_vec(); + let mut path = create_partial_path_and_edges(graph, partials, &refs).unwrap(); + if !variable { + path.eliminate_precondition_stack_variables(partials); + } + + reader + .load_partial_path_extensions(&path, &NoCancellation) + .unwrap(); + + let (graph, partials, db) = reader.get(); + let mut results = Vec::new(); + db.find_candidate_partial_paths_from_root( + graph, + partials, + Some(path.symbol_stack_postcondition), + &mut results, + ); + + results.len() + } +} + +#[test] +fn find_candidates_for_exact_symbol_stack_with_variable() { + // <"foo","bar",%2> ~ <"foo","bar",%1> | yes, %2 = %1 + // <"foo","bar",%2> ~ <"foo","bar"> | yes, %2 = <> + let results = test_foo_bar_root_candidate_paths(&["bar", "foo"], true); + assert_eq!(2, results); +} + +#[test] +fn find_candidates_for_exact_symbol_stack_without_variable() { + // <"foo","bar"> ~ <"foo","bar",%1> | yes, %1 = <> + // <"foo","bar"> ~ <"foo","bar"> | yes + let results = test_foo_bar_root_candidate_paths(&["bar", "foo"], false); + assert_eq!(2, results); +} + +#[test] +fn find_candidates_for_longer_symbol_stack_with_variable() { + // <"foo","bar","quz",%2> ~ <"foo","bar",%1> | yes, %1 = <"quz",%2> + // <"foo","bar","quz",%2> ~ <"foo","bar"> | no + let results = test_foo_bar_root_candidate_paths(&["quz", "bar", "foo"], true); + assert_eq!(1, results); +} + +#[test] +fn find_candidates_for_longer_symbol_stack_without_variable() { + // <"foo","bar","quz"> ~ <"foo","bar",%1> | yes, %1 = <"quz"> + // <"foo","bar","quz"> ~ <"foo","bar"> | no + let results = test_foo_bar_root_candidate_paths(&["quz", "bar", "foo"], false); + assert_eq!(1, results); +} + +#[test] +fn find_candidates_for_shorter_symbol_stack_with_variable() { + // <"foo",%2> ~ <"foo","bar",%1> | yes, %2 = <"bar",%1> + // <"foo",%2> ~ <"foo","bar"> | yes, %2 = <"bar"> + let results = test_foo_bar_root_candidate_paths(&["foo"], true); + assert_eq!(2, results); +} + +#[test] +fn find_candidates_for_shorter_symbol_stack_without_variable() { + // <"foo"> ~ <"foo","bar",%1> | no + // <"foo"> ~ <"foo","bar"> | no + let results = test_foo_bar_root_candidate_paths(&["foo"], false); + assert_eq!(0, results); +} diff --git a/tree-sitter-stack-graphs/CHANGELOG.md b/tree-sitter-stack-graphs/CHANGELOG.md index 8d8ef7ca1..e1df552e4 100644 --- a/tree-sitter-stack-graphs/CHANGELOG.md +++ b/tree-sitter-stack-graphs/CHANGELOG.md @@ -5,7 +5,39 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## v0.10.0 -- 2024-12-12 + +Upgraded `tree-sitter` dependency to version 0.24. + +## v0.9.0 -- 2024-07-09 + +### Library + +- New crate-level constants `FILE_PATH_VAR` and `ROOT_PATH_VAR` standardize the TSG global variable names to use for the file and root path. +- The file path variable will only use the filename set in the stack graph if no value was explicitly set. + +### CLI + +#### Added + +- Tests run faster for languages with builtins sources by caching the partial paths for the builtins. +- Indexing will set a value for the root path variable that is passed to TSG. The value is based on the directory that was provided on the command line. + +#### Changed + +- Failure to index a file will not abort indexing anymore, but simply mark the file as failed, as we already do for files with parse errors. + +#### Removed + +- The NPM distribution has been removed. + +## v0.8.1 -- 2024-03-06 + +The `stack-graphs` dependency was updated to `v0.13` to fix the build problems of the `v0.8.0` release. + +## v0.8.0 -- 2024-03-05 + +The `tree-sitter` dependency version was updated to fix install problems. ### Library diff --git a/tree-sitter-stack-graphs/Cargo.toml b/tree-sitter-stack-graphs/Cargo.toml index 2d6dabbd1..3c1c01a80 100644 --- a/tree-sitter-stack-graphs/Cargo.toml +++ b/tree-sitter-stack-graphs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tree-sitter-stack-graphs" -version = "0.7.1" +version = "0.10.0" description = "Create stack graphs using tree-sitter parsers" homepage = "https://github.com/github/stack-graphs/tree/main/tree-sitter-stack-graphs" repository = "https://github.com/github/stack-graphs/" @@ -49,12 +49,12 @@ lsp = [ ] [dependencies] -anyhow = "1.0" +anyhow = "1.0.4" base64 = { version = "0.21", optional = true } capture-it = { version = "0.3", optional = true } clap = { version = "4", optional = true, features = ["derive"] } colored = { version = "2.0", optional = true } -controlled-option = ">=0.4" +controlled-option = "0.4.1" crossbeam-channel = { version = "0.5", optional = true } dialoguer = { version = "0.10", optional = true } dirs = { version = "5", optional = true } @@ -62,24 +62,25 @@ env_logger = { version = "0.9", optional = true } indoc = { version = "1.0", optional = true } itertools = "0.10" log = "0.4" -lsp-positions = { version="0.3", path="../lsp-positions" } +lsp-positions = { version="0.3.4", path="../lsp-positions", features=["tree-sitter"] } # explicit version is required to be able to publish crate once_cell = "1" pathdiff = { version = "0.2.1", optional = true } regex = "1" rust-ini = "0.18" serde_json = { version="1.0", optional=true } sha1 = { version="0.10", optional=true } -stack-graphs = { version=">=0.11, <=0.12", path="../stack-graphs" } +stack-graphs = { version = "0.14", path="../stack-graphs" } # explicit version is required to be able to publish crate thiserror = "1.0" time = { version = "0.3", optional = true } tokio = { version = "1.26", optional = true, features = ["io-std", "rt", "rt-multi-thread"] } tower-lsp = { version = "0.19", optional = true } -tree-sitter = ">= 0.19" -tree-sitter-config = { version = "0.19", optional = true } -tree-sitter-graph = "0.11" -tree-sitter-loader = "0.20" +tree-sitter = "0.24" # keep the same minor version as the tree-sitter dependency + # of tree-sitter-graph to prevent install problems +tree-sitter-config = { version = "0.24", optional = true } +tree-sitter-graph = "0.12" +tree-sitter-loader = "0.24" walkdir = { version = "2.3", optional = true } [dev-dependencies] pretty_assertions = "0.7" -tree-sitter-python = "0.19.1" +tree-sitter-python = "=0.23.5" diff --git a/tree-sitter-stack-graphs/README.md b/tree-sitter-stack-graphs/README.md index 01ce41971..56f61e131 100644 --- a/tree-sitter-stack-graphs/README.md +++ b/tree-sitter-stack-graphs/README.md @@ -1,7 +1,6 @@ # tree-sitter-stack-graphs -The `tree-sitter-stack-graphs` crate lets you create stack graphs using the -[tree-sitter][] grammar for a language. +The `tree-sitter-stack-graphs` crate lets you create stack graphs using the [tree-sitter][] grammar for a language. [tree-sitter]: https://tree-sitter.github.io/ @@ -9,50 +8,82 @@ The `tree-sitter-stack-graphs` crate lets you create stack graphs using the - [Examples](https://github.com/github/stack-graphs/blob/main/tree-sitter-stack-graphs/examples/) - [Release notes](https://github.com/github/stack-graphs/blob/main/tree-sitter-stack-graphs/CHANGELOG.md) -## Usage +## Using the API To use this library, add the following to your `Cargo.toml`: -``` toml +```toml [dependencies] -tree-sitter-stack-graphs = "0.7" +tree-sitter-stack-graphs = "0.10" ``` -Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs/*/) for -more details on how to use this library. +Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs/*/) for more details on how to use this library. -## Command-line Program +## Using the Command-line Program -The command-line program for `tree-sitter-stack-graphs` lets you do stack -graph based analysis and lookup from the command line. +The command-line program for `tree-sitter-stack-graphs` lets you do stack graph based analysis and lookup from the command line. -Install the program using `cargo install` as follows: +The CLI can be run as follows: -``` sh -$ cargo install --features cli tree-sitter-stack-graphs -$ tree-sitter-stack-graphs --help -``` +1. _(Installed)_ Install the CLI using Cargo as follows: -Alternatively, the program can be invoked via NPM as follows: + ```sh + cargo install --features cli tree-sitter-stack-graphs + ``` -``` sh -$ npx tree-sitter-stack-graphs -``` + After this, the CLI should be available as `tree-sitter-stack-graphs`. + +2. _(From source)_ Instead of installing the CLI, it can also be run directly from the crate directory, as a replacement for a `tree-sitter-stack-graphs` invocation, as follows: + + ```sh + cargo run --features cli -- + ``` + +The basic CLI workflow for the command-line program is to index source code and issue queries against the resulting database: + +1. Index a source folder as follows: + + ```sh + tree-sitter-stack-graphs index SOURCE_DIR + ``` + + _Indexing will skip any files that have already be indexed. To force a re-index, add the `-f` flag._ + + To check the status if a source folder, run: + + ```sh + tree-sitter-stack-graphs status SOURCE_DIR + ``` -## Getting Started + To clean the database and start with a clean slate, run: -Starting a new project to develop stack graph definitions for your favourite language -is as easy as running the `init` command: + ```sh + tree-sitter-stack-graphs clean + ``` -``` sh -$ tree-sitter-stack-graphs init PROJECT_DIR + _Pass the `--delete` flag to not just empty the database, but also delete it. This is useful to resolve `unsupported database version` errors that may occur after a version update._ + +2. Run a query to find the definition(s) for a reference on a given line and column, run: + + ```sh + tree-sitter-stack-graphs query definition SOURCE_PATH:LINE:COLUMN + ``` + + Resulting definitions are printed, including a source line if the source file is available. + +Discover all available commands and flags by passing the `-h` flag to the CLI directly, or to any of the subcommands. + +## Getting Started on a new Language + +Starting a new project to develop stack graph definitions for your favourite language is as easy as running the `init` command: + +```sh +tree-sitter-stack-graphs init PROJECT_DIR ``` -Answer the questions to provide information about the language, the grammar dependency, and -the project and hit `Generate` to generate the new project. Check out `PROJECT_DIR/README.md` -to find out how to start developing. +Answer the questions to provide information about the language, the grammar dependency, and the project and hit `Generate` to generate the new project. Check out `PROJECT_DIR/README.md` to find out how to start developing. -Also check out our [examples][] for more details on how to work with the CLI. +Check out [examples][] of stack graph rules for typical language features. [examples]: https://github.com/github/stack-graphs/blob/main/tree-sitter-stack-graphs/examples/ @@ -65,14 +96,14 @@ Rust can be installed and updated using [rustup][]. Build the project by running: -``` -$ cargo build +```sh +cargo build ``` Run the tests by running: -``` -$ cargo test +```sh +cargo test ``` The project consists of a library and a CLI. @@ -81,22 +112,22 @@ To run `cargo` commands on the CLI as well, add `--features cli` or `--all-featu Run the CLI from source as follows: -``` sh -$ cargo run --features cli -- ARGS +```sh +cargo run --features cli -- ARGS ``` Sources are formatted using the standard Rust formatted, which is applied by running: -``` -$ cargo fmt +```sh +cargo fmt ``` ## License Licensed under either of - - [Apache License, Version 2.0][apache] ([LICENSE-APACHE](LICENSE-APACHE)) - - [MIT license][mit] ([LICENSE-MIT](LICENSE-MIT)) +- [Apache License, Version 2.0][apache] ([LICENSE-APACHE](LICENSE-APACHE)) +- [MIT license][mit] ([LICENSE-MIT](LICENSE-MIT)) at your option. diff --git a/tree-sitter-stack-graphs/examples/README.md b/tree-sitter-stack-graphs/examples/README.md index 901616422..5d5186ad5 100644 --- a/tree-sitter-stack-graphs/examples/README.md +++ b/tree-sitter-stack-graphs/examples/README.md @@ -7,19 +7,19 @@ Each directory contains a `stack-graphs.tsg` file that describes at the top what Running the examples requires the Python grammar to be available. This can be installed (in this directory) by executing: ```bash -$ ./bootstrap +./bootstrap ``` Run the tests for an example by executing: ```bash -$ ./run EXAMPLE_DIR +./run EXAMPLE_DIR ``` or, from within the example's directory: ```bash -$ ../run +../run ``` To render HTML visualizations of the stack graphs for the tests in an example, add the `-V` flag to run. @@ -27,7 +27,7 @@ To render HTML visualizations of the stack graphs for the tests in an example, a Print the parse tree of an example file by executing: ```bash -$ ./parse EXAMPLE_FILE +./parse EXAMPLE_FILE ``` The following examples are available: diff --git a/tree-sitter-stack-graphs/npm/.gitignore b/tree-sitter-stack-graphs/npm/.gitignore deleted file mode 100644 index 02b3615e1..000000000 --- a/tree-sitter-stack-graphs/npm/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/.crates.toml -/.crates2.json -/bin/ -/package-lock.json diff --git a/tree-sitter-stack-graphs/npm/README.md b/tree-sitter-stack-graphs/npm/README.md deleted file mode 100644 index 27dd321be..000000000 --- a/tree-sitter-stack-graphs/npm/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# tree-sitter-stack-graphs - -This package provides a convenient way to install the [tree-sitter-stack-graphs](https://crates.io/crates/tree-sitter-stack-graphs) CLI in an NPM project. - -Add it as a dev dependency to an existing project using: - - npm i -D tree-sitter-stack-graphs - -It is also possible to invoke it directly using: - - npx tree-sitter-stack-graphs - -See the tree-sitter-stack-graphs [documentation](https://crates.io/crates/tree-sitter-stack-graphs) for details on usage. diff --git a/tree-sitter-stack-graphs/npm/cli.js b/tree-sitter-stack-graphs/npm/cli.js deleted file mode 100755 index f4d654e15..000000000 --- a/tree-sitter-stack-graphs/npm/cli.js +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env node - -const spawn = require("child_process").spawn; -const path = require("path"); - -const tssg = process.platform === "win32" - ? "tree-sitter-stack-graphs.exe" - : "tree-sitter-stack-graphs"; - -spawn( - path.join(__dirname, "bin", tssg), process.argv.slice(2), - { - "stdio": "inherit" - }, -).on('close', process.exit); diff --git a/tree-sitter-stack-graphs/npm/install.js b/tree-sitter-stack-graphs/npm/install.js deleted file mode 100644 index 6ac20756b..000000000 --- a/tree-sitter-stack-graphs/npm/install.js +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env node - -const child_process = require("child_process"); -const packageJSON = require("./package.json"); - -const cargo = process.platform === "win32" - ? "cargo.exe" - : "cargo"; - -try { - child_process.execSync(cargo); -} catch (error) { - console.error(error.message); - console.error("Failed to execute Cargo. Cargo needs to be available to install this package!"); - process.exit(1); -} - -child_process.spawn( - cargo, [ - "install", - "--quiet", - "--root", ".", - "--version", "^"+packageJSON.version, - "--features", "cli", - packageJSON.name, - ], - { - "stdio": "inherit" - }, -).on('close', process.exit); diff --git a/tree-sitter-stack-graphs/npm/package.json b/tree-sitter-stack-graphs/npm/package.json deleted file mode 100644 index a49ec5fad..000000000 --- a/tree-sitter-stack-graphs/npm/package.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "name": "tree-sitter-stack-graphs", - "version": "0.7.0", - "description": "Create stack graphs using tree-sitter parsers", - "homepage": "https://github.com/github/stack-graphs/tree/main/tree-sitter-stack-graphs", - "repository": { - "type": "git", - "url": "https://github.com/github/stack-graphs.git" - }, - "keywords": [ - "tree-sitter", - "stack-graphs" - ], - "license": "MIT OR Apache-2.0", - "author": "GitHub ", - "contributors": [ - "Douglas Creager ", - "Hendrik van Antwerpen " - ], - "bin": { - "tree-sitter-stack-graphs": "./cli.js" - }, - "scripts": { - "install": "node install.js" - } -} diff --git a/tree-sitter-stack-graphs/src/cli/index.rs b/tree-sitter-stack-graphs/src/cli/index.rs index a90056f4b..6425d8ca7 100644 --- a/tree-sitter-stack-graphs/src/cli/index.rs +++ b/tree-sitter-stack-graphs/src/cli/index.rs @@ -11,7 +11,10 @@ use stack_graphs::arena::Handle; use stack_graphs::graph::File; use stack_graphs::graph::StackGraph; use stack_graphs::partial::PartialPaths; +use stack_graphs::stats::FrequencyDistribution; use stack_graphs::stitching::ForwardPartialPathStitcher; +use stack_graphs::stitching::Stats as StitchingStats; +use stack_graphs::stitching::StitcherConfig; use stack_graphs::storage::FileStatus; use stack_graphs::storage::SQLiteWriter; use std::collections::HashMap; @@ -23,6 +26,7 @@ use tree_sitter_graph::Variables; use crate::cli::util::duration_from_seconds_str; use crate::cli::util::iter_files_and_directories; +use crate::cli::util::print_indexing_stats; use crate::cli::util::reporter::ConsoleReporter; use crate::cli::util::reporter::Level; use crate::cli::util::reporter::Reporter; @@ -38,6 +42,7 @@ use crate::BuildError; use crate::CancelAfterDuration; use crate::CancellationFlag; use crate::NoCancellation; +use crate::{FILE_PATH_VAR, ROOT_PATH_VAR}; #[derive(Args)] pub struct IndexArgs { @@ -78,6 +83,9 @@ pub struct IndexArgs { )] pub max_file_time: Option, + #[clap(long)] + pub stats: bool, + /// Wait for user input before starting analysis. Useful for profiling. #[clap(long)] pub wait_at_start: bool, @@ -93,6 +101,7 @@ impl IndexArgs { hide_error_details: false, max_file_time: None, wait_at_start: false, + stats: false, } } @@ -105,6 +114,7 @@ impl IndexArgs { let mut indexer = Indexer::new(&mut db, &mut loader, &reporter); indexer.force = self.force; indexer.max_file_time = self.max_file_time; + indexer.set_collect_stats(self.stats); let source_paths = self .source_paths @@ -112,6 +122,11 @@ impl IndexArgs { .map(|p| p.canonicalize()) .collect::, _>>()?; indexer.index_all(source_paths, self.continue_from, &NoCancellation)?; + + if self.stats { + println!(); + print_indexing_stats(indexer.into_stats()); + } Ok(()) } @@ -145,6 +160,7 @@ pub struct Indexer<'a> { db: &'a mut SQLiteWriter, loader: &'a mut Loader, reporter: &'a dyn Reporter, + stats: Option, /// Index files, even if they already exist in the database. pub force: bool, /// Maximum time per file. @@ -163,6 +179,15 @@ impl<'a> Indexer<'a> { reporter, force: false, max_file_time: None, + stats: None, + } + } + + pub fn set_collect_stats(&mut self, collect_stats: bool) { + if !collect_stats { + self.stats = None; + } else if self.stats.is_none() { + self.stats = Some(IndexingStats::default()); } } @@ -279,6 +304,9 @@ impl<'a> Indexer<'a> { } Err(e) => return Err(IndexError::LoadError(e)), }; + let stitcher_config = StitcherConfig::default() + .with_detect_similar_paths(!lcs.no_similar_paths_in_file()) + .with_collect_stats(self.stats.is_some()); let source = file_reader.get(source_path)?; let tag = sha1(source); @@ -328,26 +356,32 @@ impl<'a> Indexer<'a> { if let Err(err) = result { match err.inner { BuildError::Cancelled(_) => { - file_status.warning("parsing timed out", None); + file_status.warning("timed out", None); self.db - .store_error_for_file(source_path, &tag, "parsing timed out")?; - return Ok(()); - } - BuildError::ParseErrors { .. } => { - file_status.failure("parsing failed", Some(&err.display_pretty())); - self.db.store_error_for_file( - source_path, - &tag, - &format!("parsing failed: {}", err.inner), - )?; + .store_error_for_file(source_path, &tag, "timed out")?; return Ok(()); } _ => { - file_status.failure("failed to build stack graph", Some(&err.display_pretty())); - return Err(IndexError::StackGraph); + file_status.failure("failed", Some(&err.display_pretty())); + self.db.store_error_for_file(source_path, &tag, "failed")?; + return Ok(()); } } }; + if let Some(stats) = &mut self.stats { + stats.total_graph_nodes.record(graph.iter_nodes().count()); + let mut total_edges = 0; + for n in graph.iter_nodes() { + let edge_count = graph.outgoing_edges(n).count(); + if graph[n].is_root() { + stats.root_out_degree = edge_count; + } else { + stats.node_out_degrees.record(edge_count); + total_edges += edge_count; + } + } + stats.total_graph_edges.record(total_edges); + } let mut partials = PartialPaths::new(); let mut paths = Vec::new(); @@ -355,12 +389,17 @@ impl<'a> Indexer<'a> { &graph, &mut partials, file, + stitcher_config, &(&cancellation_flag as &dyn CancellationFlag), |_g, _ps, p| { paths.push(p.clone()); }, ) { - Ok(_) => {} + Ok(stitching_stats) => { + if let Some(stats) = &mut self.stats { + stats.stitching_stats += stitching_stats; + } + } Err(_) => { file_status.warning("path computation timed out", None); self.db.store_error_for_file( @@ -391,7 +430,16 @@ impl<'a> Indexer<'a> { ) -> std::result::Result<(), BuildErrorWithSource<'b>> { let relative_source_path = source_path.strip_prefix(source_root).unwrap(); if let Some(lc) = lcs.primary { - let globals = Variables::new(); + let mut globals = Variables::new(); + + globals + .add(FILE_PATH_VAR.into(), source_path.to_str().unwrap().into()) + .expect("failed to add file path variable"); + + globals + .add(ROOT_PATH_VAR.into(), source_root.to_str().unwrap().into()) + .expect("failed to add root path variable"); + lc.sgl .build_stack_graph_into(graph, file, source, &globals, cancellation_flag) .map_err(|inner| BuildErrorWithSource { @@ -438,6 +486,10 @@ impl<'a> Indexer<'a> { *continue_from = None; false } + + pub fn into_stats(self) -> IndexingStats { + self.stats.unwrap_or_default() + } } #[derive(Debug, Error)] @@ -448,7 +500,7 @@ pub enum IndexError { LoadError(#[source] crate::loader::LoadError<'static>), #[error("failed to read file")] ReadError(#[from] std::io::Error), - #[error("failed to build stank graph")] + #[error("failed to build stack graph")] StackGraph, #[error(transparent)] StorageError(#[from] stack_graphs::storage::StorageError), @@ -461,3 +513,17 @@ impl From for IndexError { } type Result = std::result::Result; + +#[derive(Clone, Debug, Default)] +pub struct IndexingStats { + // The distribution of the total number of nodes per file graph. + pub total_graph_nodes: FrequencyDistribution, + // The distribution of the total number of edges per file graph. + pub total_graph_edges: FrequencyDistribution, + // The distribution of the out-degrees of non-root nodes. + pub node_out_degrees: FrequencyDistribution, + // The root node's out-degree. + pub root_out_degree: usize, + // The stitching statistics. + pub stitching_stats: StitchingStats, +} diff --git a/tree-sitter-stack-graphs/src/cli/init.rs b/tree-sitter-stack-graphs/src/cli/init.rs index e87785477..e513e5acd 100644 --- a/tree-sitter-stack-graphs/src/cli/init.rs +++ b/tree-sitter-stack-graphs/src/cli/init.rs @@ -455,33 +455,79 @@ impl ProjectSettings<'_> { fn generate_readme(&self, project_path: &Path) -> anyhow::Result<()> { let mut file = File::create(project_path.join("README.md"))?; writedoc! {file, r####" - # tree-sitter-stack-graphs definition for {} + # tree-sitter-stack-graphs definition for {language_name} - This project defines tree-sitter-stack-graphs rules for {} using the [{}][] grammar. + This project defines tree-sitter-stack-graphs rules for {language_name} using the [{grammar_crate_name}][] grammar. - [{}]: https://crates.io/crates/{} + [{grammar_crate_name}]: https://crates.io/crates/{grammar_crate_name} - ## Usage + - [API documentation](https://docs.rs/{crate_name}/) + - [Release notes](https://github.com/github/stack-graphs/blob/main/languages/{crate_name}/CHANGELOG.md) + + ## Using the API To use this library, add the following to your `Cargo.toml`: - ``` toml + ```toml [dependencies] - {} = "{}" + {crate_name} = "{crate_version}" ``` - Check out our [documentation](https://docs.rs/{}/*/) for more details on how to use this library. + Check out our [documentation](https://docs.rs/{crate_name}/*/) for more details on how to use this library. - ## Command-line Program + ## Using the Command-line Program - The command-line program for `{}` lets you do stack graph based analysis and lookup from the command line. + The command-line program for `{crate_name}` lets you do stack graph based analysis and lookup from the command line. - Install the program using `cargo install` as follows: + The CLI can be run as follows: - ``` sh - $ cargo install --features cli {} - $ {} --help - ``` + 1. _(Installed)_ Install the CLI using Cargo as follows: + + ```sh + cargo install --features cli {crate_name} + ``` + + After this, the CLI should be available as `{crate_name}`. + + 2. _(From source)_ Instead of installing the CLI, it can also be run directly from the crate directory, as a replacement for a `{crate_name}` invocation, as follows: + + ```sh + cargo run --features cli -- + ``` + + The basic CLI workflow for the command-line program is to index source code and issue queries against the resulting database: + + 1. Index a source folder as follows: + + ```sh + {crate_name} index SOURCE_DIR + ``` + + _Indexing will skip any files that have already be indexed. To force a re-index, add the `-f` flag._ + + To check the status if a source folder, run: + + ```sh + {crate_name} status SOURCE_DIR + ``` + + To clean the database and start with a clean slate, run: + + ```sh + {crate_name} clean + ``` + + _Pass the `--delete` flag to not just empty the database, but also delete it. This is useful to resolve `unsupported database version` errors that may occur after a version update._ + + 2. Run a query to find the definition(s) for a reference on a given line and column, run: + + ```sh + {crate_name} query definition SOURCE_PATH:LINE:COLUMN + ``` + + Resulting definitions are printed, including a source line if the source file is available. + + Discover all available commands and flags by passing the `-h` flag to the CLI directly, or to any of the subcommands. ## Development @@ -492,35 +538,29 @@ impl ProjectSettings<'_> { The project is organized as follows: - The stack graph rules are defined in `src/stack-graphs.tsg`. - - Builtins sources and configuration are defined in `src/builtins.{}` and `builtins.cfg` respectively. + - Builtins sources and configuration are defined in `src/builtins.{language_file_extension}` and `builtins.cfg` respectively. - Tests are put into the `test` directory. - ### Building and Running Tests - - Build the project by running: - - ``` sh - $ cargo build - ``` + ### Running Tests Run the tests as follows: - ``` sh - $ cargo test + ```sh + cargo test ``` The project consists of a library and a CLI. By default, running `cargo` only applies to the library. To run `cargo` commands on the CLI as well, add `--features cli` or `--all-features`. Run the CLI from source as follows: - ``` sh - $ cargo run --features cli -- ARGS + ```sh + cargo run --features cli -- ARGS ``` Sources are formatted using the standard Rust formatted, which is applied by running: - ``` sh - $ cargo fmt + ```sh + cargo fmt ``` ### Writing TSG @@ -535,34 +575,30 @@ impl ProjectSettings<'_> { Parse and test a single file by executing the following commands: - ``` sh - $ cargo run --features cli -- parse FILES... - $ cargo run --features cli -- test TESTFILES... + ```sh + cargo run --features cli -- parse FILES... + cargo run --features cli -- test TESTFILES... ``` Generate a visualization to debug failing tests by passing the `-V` flag: - ``` sh - $ cargo run --features cli -- test -V TESTFILES... + ```sh + cargo run --features cli -- test -V TESTFILES... ``` To generate the visualization regardless of test outcome, execute: - ``` sh - $ cargo run --features cli -- test -V --output-mode=always TESTFILES... + ```sh + cargo run --features cli -- test -V --output-mode=always TESTFILES... ``` - Go to https://crates.io/crates/tree-sitter-stack-graphs for links to examples and documentation. + Go to for links to examples and documentation. "####, - self.language_name, - self.language_name, self.grammar_crate_name(), - self.grammar_crate_name(), self.grammar_crate_name(), - self.crate_name(), self.crate_version(), - self.crate_name(), - self.crate_name(), - self.crate_name(), - self.crate_name(), - self.language_file_extension, + language_name=self.language_name, + grammar_crate_name=self.grammar_crate_name(), + crate_name=self.crate_name(), + crate_version=self.crate_version(), + language_file_extension=self.language_file_extension, }?; Ok(()) } @@ -718,7 +754,6 @@ impl ProjectSettings<'_> { let mut file = File::create(project_path.join("rust/lib.rs"))?; self.write_license_header(&mut file, "// ")?; writedoc! {file, r#" - use tree_sitter_stack_graphs::loader::FileAnalyzers; use tree_sitter_stack_graphs::loader::LanguageConfiguration; use tree_sitter_stack_graphs::loader::LoadError; use tree_sitter_stack_graphs::CancellationFlag; @@ -735,9 +770,6 @@ impl ProjectSettings<'_> { /// The stack graphs builtins source for this language. pub const STACK_GRAPHS_BUILTINS_SOURCE: &str = include_str!("../src/builtins.{}"); - /// The name of the file path global variable. - pub const FILE_PATH_VAR: &str = "FILE_PATH"; - pub fn language_configuration(cancellation_flag: &dyn CancellationFlag) -> LanguageConfiguration {{ try_language_configuration(cancellation_flag).unwrap_or_else(|err| panic!("{{}}", err)) }} @@ -746,7 +778,7 @@ impl ProjectSettings<'_> { cancellation_flag: &dyn CancellationFlag, ) -> Result {{ LanguageConfiguration::from_sources( - {}::language(), + {}::LANGUAGE.into(), Some(String::from("source.{}")), None, vec![String::from("{}")], @@ -757,7 +789,6 @@ impl ProjectSettings<'_> { STACK_GRAPHS_BUILTINS_SOURCE, )), Some(STACK_GRAPHS_BUILTINS_CONFIG), - FileAnalyzers::new(), cancellation_flag, ) }} diff --git a/tree-sitter-stack-graphs/src/cli/load.rs b/tree-sitter-stack-graphs/src/cli/load.rs index b32332f9d..1fd22fa2b 100644 --- a/tree-sitter-stack-graphs/src/cli/load.rs +++ b/tree-sitter-stack-graphs/src/cli/load.rs @@ -67,7 +67,7 @@ impl PathLoaderArgs { builtins_paths, )? } else { - let loader_config = TsConfig::load() + let loader_config = TsConfig::load(None) .and_then(|v| v.get()) .map_err(LoadError::TreeSitter)?; Loader::from_tree_sitter_configuration( diff --git a/tree-sitter-stack-graphs/src/cli/lsp.rs b/tree-sitter-stack-graphs/src/cli/lsp.rs index 03f1c8060..57ed770bb 100644 --- a/tree-sitter-stack-graphs/src/cli/lsp.rs +++ b/tree-sitter-stack-graphs/src/cli/lsp.rs @@ -560,25 +560,6 @@ impl FromStdError for std::result::Result { } } -trait FromAnyhowError { - #[must_use] - fn from_error(self) -> Result; -} - -impl FromAnyhowError for std::result::Result { - #[must_use] - fn from_error(self) -> Result { - match self { - Ok(value) => Ok(value), - Err(err) => Err(Error { - code: ErrorCode::ServerError(-1), - message: err.to_string(), - data: None, - }), - } - } -} - #[derive(Debug)] pub enum Job { IndexPath(PathBuf), diff --git a/tree-sitter-stack-graphs/src/cli/match.rs b/tree-sitter-stack-graphs/src/cli/match.rs index 108c8450e..889d2374a 100644 --- a/tree-sitter-stack-graphs/src/cli/match.rs +++ b/tree-sitter-stack-graphs/src/cli/match.rs @@ -49,7 +49,7 @@ impl MatchArgs { None => return Err(anyhow!("No stack graph language found")), }; let source = file_reader.get(&self.source_path)?; - let tree = parse(lc.language, &self.source_path, source)?; + let tree = parse(&lc.language, &self.source_path, source)?; if self.stanza.is_empty() { lc.sgl.tsg.try_visit_matches(&tree, source, true, |mat| { print_matches(lc.sgl.tsg_path(), &self.source_path, source, mat) diff --git a/tree-sitter-stack-graphs/src/cli/parse.rs b/tree-sitter-stack-graphs/src/cli/parse.rs index aff7681b1..64275b915 100644 --- a/tree-sitter-stack-graphs/src/cli/parse.rs +++ b/tree-sitter-stack-graphs/src/cli/parse.rs @@ -40,14 +40,14 @@ impl ParseArgs { None => return Err(anyhow!("No stack graph language found")), }; let source = file_reader.get(&self.source_path)?; - let tree = parse(lang, &self.source_path, source)?; + let tree = parse(&lang, &self.source_path, source)?; print_tree(tree); Ok(()) } } pub(super) fn parse( - language: tree_sitter::Language, + language: &tree_sitter::Language, path: &Path, source: &str, ) -> anyhow::Result { diff --git a/tree-sitter-stack-graphs/src/cli/query.rs b/tree-sitter-stack-graphs/src/cli/query.rs index 08ede0b93..dd5526570 100644 --- a/tree-sitter-stack-graphs/src/cli/query.rs +++ b/tree-sitter-stack-graphs/src/cli/query.rs @@ -5,18 +5,23 @@ // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ +use std::path::Path; +use std::path::PathBuf; + use clap::Args; use clap::Parser; use clap::Subcommand; use clap::ValueHint; use stack_graphs::stitching::ForwardPartialPathStitcher; +use stack_graphs::stitching::Stats as StitchingStats; +use stack_graphs::stitching::StitcherConfig; use stack_graphs::storage::FileStatus; use stack_graphs::storage::SQLiteReader; -use std::path::Path; -use std::path::PathBuf; use thiserror::Error; use tree_sitter_graph::parse_error::Excerpt; +use crate::cli::util::print_database_stats; +use crate::cli::util::print_stitching_stats; use crate::cli::util::reporter::ConsoleReporter; use crate::cli::util::reporter::Reporter; use crate::cli::util::sha1; @@ -33,6 +38,9 @@ pub struct QueryArgs { #[clap(long)] pub wait_at_start: bool, + #[clap(long)] + pub stats: bool, + #[clap(subcommand)] target: Target, } @@ -43,7 +51,14 @@ impl QueryArgs { wait_for_input()?; } let mut db = SQLiteReader::open(&db_path)?; - self.target.run(&mut db) + let stitching_stats = self.target.run(&mut db, self.stats)?; + if self.stats { + println!(); + print_stitching_stats(stitching_stats); + println!(); + print_database_stats(db.stats()); + } + Ok(()) } } @@ -53,12 +68,14 @@ pub enum Target { } impl Target { - pub fn run(self, db: &mut SQLiteReader) -> anyhow::Result<()> { + fn run(self, db: &mut SQLiteReader, collect_stats: bool) -> anyhow::Result { let reporter = ConsoleReporter::details(); let mut querier = Querier::new(db, &reporter); + querier.set_collect_stats(collect_stats); match self { - Self::Definition(cmd) => cmd.run(&mut querier), + Self::Definition(cmd) => cmd.run(&mut querier)?, } + Ok(querier.into_stats()) } } @@ -116,7 +133,7 @@ impl Definition { n => println!("{}has {} definitions", " ".repeat(indent), n), } for definition in definitions.into_iter() { - println!( + print!( "{}", Excerpt::from_source( &definition.path, @@ -136,11 +153,24 @@ impl Definition { pub struct Querier<'a> { db: &'a mut SQLiteReader, reporter: &'a dyn Reporter, + stats: Option, } impl<'a> Querier<'a> { pub fn new(db: &'a mut SQLiteReader, reporter: &'a dyn Reporter) -> Self { - Self { db, reporter } + Self { + db, + reporter, + stats: None, + } + } + + pub fn set_collect_stats(&mut self, collect_stats: bool) { + if !collect_stats { + self.stats = None; + } else if self.stats.is_none() { + self.stats = Some(StitchingStats::default()); + } } pub fn definitions( @@ -160,7 +190,7 @@ impl<'a> Querier<'a> { _ => { self.reporter.started(&log_path); self.reporter.failed(&log_path, "file not indexed", None); - return Ok(Vec::default()); + return Ok(Default::default()); } } @@ -174,7 +204,7 @@ impl<'a> Querier<'a> { if starting_nodes.is_empty() { self.reporter .cancelled(&log_path, "no references at location", None); - return Ok(Vec::default()); + return Ok(Default::default()); } let mut result = Vec::new(); @@ -185,16 +215,29 @@ impl<'a> Querier<'a> { }; let mut reference_paths = Vec::new(); - if let Err(err) = ForwardPartialPathStitcher::find_all_complete_partial_paths( + let stitcher_config = StitcherConfig::default() + // always detect similar paths, we don't know the language configurations for the data in the database + .with_detect_similar_paths(true) + .with_collect_stats(self.stats.is_some()); + let ref_result = ForwardPartialPathStitcher::find_all_complete_partial_paths( self.db, std::iter::once(node), + stitcher_config, &cancellation_flag, |_g, _ps, p| { reference_paths.push(p.clone()); }, - ) { - self.reporter.failed(&log_path, "query timed out", None); - return Err(err.into()); + ); + match ref_result { + Ok(ref_stats) => { + if let Some(stats) = &mut self.stats { + *stats += ref_stats + } + } + Err(err) => { + self.reporter.failed(&log_path, "query timed out", None); + return Err(err.into()); + } } let (graph, partials, _) = self.db.get(); @@ -246,6 +289,10 @@ impl<'a> Querier<'a> { Ok(result) } + + pub fn into_stats(self) -> StitchingStats { + self.stats.unwrap_or_default() + } } #[derive(Debug, Error)] diff --git a/tree-sitter-stack-graphs/src/cli/test.rs b/tree-sitter-stack-graphs/src/cli/test.rs index 4bcb47171..e3034cbbf 100644 --- a/tree-sitter-stack-graphs/src/cli/test.rs +++ b/tree-sitter-stack-graphs/src/cli/test.rs @@ -18,9 +18,14 @@ use stack_graphs::serde::Filter; use stack_graphs::stitching::Database; use stack_graphs::stitching::DatabaseCandidates; use stack_graphs::stitching::ForwardPartialPathStitcher; +use stack_graphs::stitching::StitcherConfig; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; use std::time::Duration; +use tree_sitter::Language; use tree_sitter_graph::Variables; use crate::cli::util::duration_from_seconds_str; @@ -38,6 +43,7 @@ use crate::test::Test; use crate::test::TestResult; use crate::CancelAfterDuration; use crate::CancellationFlag; +use crate::FILE_PATH_VAR; #[derive(Args)] #[clap(after_help = r#"PATH SPECIFICATIONS: @@ -175,10 +181,16 @@ impl TestArgs { pub fn run(self, mut loader: Loader) -> anyhow::Result<()> { let reporter = self.get_reporter(); let mut total_result = TestResult::new(); + let mut cache = HashMap::new(); for (test_root, test_path, _) in iter_files_and_directories(self.test_paths.clone()) { let mut file_status = CLIFileReporter::new(&reporter, &test_path); - let test_result = - self.run_test(&test_root, &test_path, &mut loader, &mut file_status)?; + let test_result = self.run_test( + &test_root, + &test_path, + &mut loader, + &mut file_status, + &mut cache, + )?; file_status.assert_reported(); total_result.absorb(test_result); } @@ -210,14 +222,15 @@ impl TestArgs { } /// Run test file. Takes care of the output when an error is returned. - fn run_test( + fn run_test<'a>( &self, test_root: &Path, test_path: &Path, - loader: &mut Loader, + loader: &'a mut Loader, file_status: &mut CLIFileReporter, + cache: &mut HashMap, ) -> anyhow::Result { - match self.run_test_inner(test_root, test_path, loader, file_status) { + match self.run_test_inner(test_root, test_path, loader, file_status, cache) { ok @ Ok(_) => ok, err @ Err(_) => { file_status.failure_if_processing("error", None); @@ -226,12 +239,13 @@ impl TestArgs { } } - fn run_test_inner( + fn run_test_inner<'a>( &self, test_root: &Path, test_path: &Path, - loader: &mut Loader, + loader: &'a mut Loader, file_status: &mut CLIFileReporter, + cache: &mut HashMap, ) -> anyhow::Result { let cancellation_flag = CancelAfterDuration::from_option(self.max_test_time); @@ -262,11 +276,24 @@ impl TestArgs { file_status.processing(); + let stitcher_config = + StitcherConfig::default().with_detect_similar_paths(!lc.no_similar_paths_in_file); + let mut partials = PartialPaths::new(); + let mut db = Database::new(); + let source = file_reader.get(test_path)?; let default_fragment_path = test_path.strip_prefix(test_root).unwrap(); let mut test = Test::from_source(test_path, source, default_fragment_path)?; if !self.no_builtins { - self.load_builtins_into(&lc, &mut test.graph)?; + self.load_builtins_into( + &lc, + &mut test.graph, + &mut partials, + &mut db, + stitcher_config, + cancellation_flag.as_ref(), + cache, + )?; } let mut globals = Variables::new(); for test_fragment in &test.fragments { @@ -290,7 +317,16 @@ impl TestArgs { &mut Some(test_fragment.source.as_ref()), )? { globals.clear(); + test_fragment.add_globals_to(&mut globals); + + globals + .add( + FILE_PATH_VAR.into(), + test_fragment.path.to_str().unwrap().into(), + ) + .unwrap_or_default(); + lc.sgl.build_stack_graph_into( &mut test.graph, test_fragment.file, @@ -324,20 +360,24 @@ impl TestArgs { Ok(_) => {} } } - let mut partials = PartialPaths::new(); - let mut db = Database::new(); - for file in test.graph.iter_files() { + for fragment in &test.fragments { ForwardPartialPathStitcher::find_minimal_partial_path_set_in_file( &test.graph, &mut partials, - file, + fragment.file, + stitcher_config, &cancellation_flag.as_ref(), |g, ps, p| { db.add_partial_path(g, ps, p.clone()); }, )?; } - let result = test.run(&mut partials, &mut db, cancellation_flag.as_ref())?; + let result = test.run( + &mut partials, + &mut db, + stitcher_config, + cancellation_flag.as_ref(), + )?; let success = result.failure_count() == 0; let outputs = if self.output_mode.test(!success) { let files = test.fragments.iter().map(|f| f.file).collect::>(); @@ -349,6 +389,7 @@ impl TestArgs { &mut db, &|_: &StackGraph, h: &Handle| files.contains(h), success, + stitcher_config, cancellation_flag.as_ref(), )? } else { @@ -377,14 +418,45 @@ impl TestArgs { Ok(result) } - fn load_builtins_into( + fn load_builtins_into<'a>( &self, - lc: &LanguageConfiguration, + lc: &'a LanguageConfiguration, graph: &mut StackGraph, + partials: &mut PartialPaths, + db: &mut Database, + stitcher_config: StitcherConfig, + cancellation_flag: &dyn CancellationFlag, + cache: &mut HashMap, ) -> anyhow::Result<()> { - if let Err(h) = graph.add_from_graph(&lc.builtins) { - return Err(anyhow!("Duplicate builtin file {}", &graph[h])); - } + let files = graph + .add_from_graph(&lc.builtins) + .map_err(|h| anyhow!("Duplicate builtin file {}", &graph[h]))?; + let files = files.into_iter().collect::>(); + match cache.entry(lc.language.clone()) { + Entry::Occupied(o) => { + o.get().load_into(graph, partials, db)?; + } + Entry::Vacant(v) => { + for file in &files { + ForwardPartialPathStitcher::find_minimal_partial_path_set_in_file( + graph, + partials, + *file, + stitcher_config, + &cancellation_flag, + |g, ps, p| { + db.add_partial_path(g, ps, p.clone()); + }, + )?; + } + v.insert(db.to_serializable_filter( + graph, + partials, + &|_: &StackGraph, f: &Handle| files.contains(f), + )); + } + }; + Ok(()) } @@ -397,6 +469,7 @@ impl TestArgs { db: &mut Database, filter: &dyn Filter, success: bool, + stitcher_config: StitcherConfig, cancellation_flag: &dyn CancellationFlag, ) -> anyhow::Result> { let mut outputs = Vec::with_capacity(3); @@ -425,7 +498,14 @@ impl TestArgs { } let mut db = if save_paths.is_some() || save_visualization.is_some() { - self.compute_paths(graph, partials, db, filter, cancellation_flag)? + self.compute_paths( + graph, + partials, + db, + filter, + stitcher_config, + cancellation_flag, + )? } else { Database::new() }; @@ -474,6 +554,7 @@ impl TestArgs { partials: &mut PartialPaths, db: &mut Database, filter: &dyn Filter, + stitcher_config: StitcherConfig, cancellation_flag: &dyn CancellationFlag, ) -> anyhow::Result { let references = graph @@ -484,6 +565,7 @@ impl TestArgs { ForwardPartialPathStitcher::find_all_complete_partial_paths( &mut DatabaseCandidates::new(graph, partials, db), references.clone(), + stitcher_config, &cancellation_flag, |_, _, p| { paths.push(p.clone()); diff --git a/tree-sitter-stack-graphs/src/cli/util.rs b/tree-sitter-stack-graphs/src/cli/util.rs index 66181a1fd..6b7deab4c 100644 --- a/tree-sitter-stack-graphs/src/cli/util.rs +++ b/tree-sitter-stack-graphs/src/cli/util.rs @@ -18,8 +18,13 @@ use sha1::Sha1; use stack_graphs::arena::Handle; use stack_graphs::graph::Node; use stack_graphs::graph::StackGraph; +use stack_graphs::stats::FrequencyDistribution; +use stack_graphs::stitching::Stats as StitchingStats; +use stack_graphs::storage::Stats as StorageStats; use std::ffi::OsStr; use std::ffi::OsString; +use std::fmt::Display; +use std::hash::Hash; use std::io::Write; use std::ops::Range; use std::path::Path; @@ -27,7 +32,8 @@ use std::path::PathBuf; use std::time::Duration; use walkdir::WalkDir; -use self::reporter::Reporter; +use crate::cli::index::IndexingStats; +use crate::cli::util::reporter::Reporter; pub mod reporter; @@ -437,7 +443,7 @@ impl<'a> CLIFileReporter<'a> { status: &str, details: Option<&dyn std::fmt::Display>, ) { - if !self.path_logged { + if !self.path_logged || self.status_logged { return; } self.failure(status, details); @@ -495,3 +501,94 @@ impl std::fmt::Display for DisplayBuildErrorPretty<'_> { ) } } + +pub(super) fn print_indexing_stats(stats: IndexingStats) { + print_quartiles_header("graph stats"); + print_quartiles_row("total graph nodes", stats.total_graph_nodes); + print_quartiles_row("total graph edges", stats.total_graph_edges); + print_quartiles_row("node out degrees", stats.node_out_degrees); + print_value_row("root out degree", stats.root_out_degree); + println!(); + print_stitching_stats(stats.stitching_stats); +} + +pub(super) fn print_stitching_stats(stats: StitchingStats) { + print_quartiles_header("stitching stats"); + print_quartiles_row("initial paths", stats.initial_paths); + print_quartiles_row("queued paths per phase", stats.queued_paths_per_phase); + print_quartiles_row("processed paths per phase", stats.processed_paths_per_phase); + print_quartiles_row("accepted path length", stats.accepted_path_length); + print_quartiles_row("terminal path length", stats.terminal_path_lengh); + print_quartiles_row("node path candidates", stats.candidates_per_node_path); + print_quartiles_row("node path extensions", stats.extensions_per_node_path); + print_quartiles_row("root path candidates", stats.candidates_per_root_path); + print_quartiles_row("root path extensions", stats.extensions_per_root_path); + print_quartiles_row("node visits", stats.node_visits.frequencies()); + print_value_row("root visits", stats.root_visits); + print_quartiles_row( + "similar path counts", + stats.similar_paths_stats.similar_path_count, + ); + print_quartiles_row( + "similar path bucket sizes", + stats.similar_paths_stats.similar_path_bucket_size, + ); +} + +pub(super) fn print_database_stats(stats: StorageStats) { + println!( + "| {:^29} | {:^9} | {:^9} |", + "database stats", "loads", "cached", + ); + println!("|-------------------------------|-----------|-----------|"); + println!( + "| {:>29} | {:>9} | {:>9} |", + "files", stats.file_loads, stats.file_cached + ); + println!( + "| {:>29} | {:>9} | {:>9} |", + "node paths", stats.node_path_loads, stats.node_path_cached + ); + println!( + "| {:>29} | {:>9} | {:>9} |", + "rootpaths", stats.root_path_loads, stats.root_path_cached + ); +} + +fn print_quartiles_header(title: &str) { + println!( + "| {:^29} | {:^9} | {:^9} | {:^9} | {:^9} | {:^9} | {:^9} |", + title, "min", "p25", "p50", "p75", "max", "count", + ); + println!( + "|-------------------------------|-----------|-----------|-----------|-----------|-----------|-----------|" + ); +} + +fn print_quartiles_row(title: &str, hist: FrequencyDistribution) { + let qs = hist.quantiles(4); + if qs.is_empty() { + println!( + "| {:>29} | {:>9} | {:>9} | {:>9} | {:>9} | {:>9} | {:>9} |", + title, "-", "-", "-", "-", "-", 0 + ); + } else { + println!( + "| {:>29} | {:>9} | {:>9} | {:>9} | {:>9} | {:>9} | {:>9} |", + title, + qs[0], + qs[1], + qs[2], + qs[3], + qs[4], + hist.count(), + ); + } +} + +fn print_value_row(title: &str, value: X) { + println!( + "| {:>29} | {:>9} | {:>9} | {:>9} | {:>9} | {:>9} | {:>9} |", + title, "-", "-", "-", "-", "-", value + ); +} diff --git a/tree-sitter-stack-graphs/src/cli/visualize.rs b/tree-sitter-stack-graphs/src/cli/visualize.rs index e0665c353..f17f23b88 100644 --- a/tree-sitter-stack-graphs/src/cli/visualize.rs +++ b/tree-sitter-stack-graphs/src/cli/visualize.rs @@ -10,6 +10,7 @@ use clap::ValueHint; use stack_graphs::serde::NoFilter; use stack_graphs::stitching::Database; use stack_graphs::stitching::ForwardPartialPathStitcher; +use stack_graphs::stitching::StitcherConfig; use stack_graphs::storage::SQLiteReader; use stack_graphs::NoCancellation; use std::path::Path; @@ -54,9 +55,13 @@ impl VisualizeArgs { .filter(|n| graph[*n].is_reference()) .collect::>(); let mut complete_paths_db = Database::new(); + let stitcher_config = StitcherConfig::default() + // always detect similar paths, we don't know the language configurations for the data in the database + .with_detect_similar_paths(true); ForwardPartialPathStitcher::find_all_complete_partial_paths( &mut db, starting_nodes, + stitcher_config, cancellation_flag, |g, ps, p| { complete_paths_db.add_partial_path(g, ps, p.clone()); diff --git a/tree-sitter-stack-graphs/src/lib.rs b/tree-sitter-stack-graphs/src/lib.rs index 86651b6be..6319d1cb9 100644 --- a/tree-sitter-stack-graphs/src/lib.rs +++ b/tree-sitter-stack-graphs/src/lib.rs @@ -324,7 +324,7 @@ //! import sys //! print(sys.path) //! "#; -//! let grammar = tree_sitter_python::language(); +//! let grammar = tree_sitter_python::LANGUAGE.into(); //! let tsg_source = STACK_GRAPH_RULES; //! let mut language = StackGraphLanguage::from_str(grammar, tsg_source)?; //! let mut stack_graph = StackGraph::new(); @@ -435,9 +435,16 @@ static SCOPE_ATTRS: Lazy> = static PRECEDENCE_ATTR: &'static str = "precedence"; // Global variables -static ROOT_NODE_VAR: &'static str = "ROOT_NODE"; -static JUMP_TO_SCOPE_NODE_VAR: &'static str = "JUMP_TO_SCOPE_NODE"; -static FILE_PATH_VAR: &'static str = "FILE_PATH"; +/// Name of the variable used to pass the root node. +pub const ROOT_NODE_VAR: &'static str = "ROOT_NODE"; +/// Name of the variable used to pass the jump-to-scope node. +pub const JUMP_TO_SCOPE_NODE_VAR: &'static str = "JUMP_TO_SCOPE_NODE"; +/// Name of the variable used to pass the file path. +/// If a root path is given, it should be a descendant of the root path. +pub const FILE_PATH_VAR: &'static str = "FILE_PATH"; +/// Name of the variable used to pass the root path. +/// If given, should be an ancestor of the file path. +pub const ROOT_PATH_VAR: &'static str = "ROOT_PATH"; /// Holds information about how to construct stack graphs for a particular language. pub struct StackGraphLanguage { @@ -472,7 +479,7 @@ impl StackGraphLanguage { language: tree_sitter::Language, tsg_source: &str, ) -> Result { - let tsg = tree_sitter_graph::ast::File::from_str(language, tsg_source)?; + let tsg = tree_sitter_graph::ast::File::from_str(language.clone(), tsg_source)?; Ok(StackGraphLanguage { language, tsg, @@ -511,8 +518,8 @@ impl StackGraphLanguage { &mut self.functions } - pub fn language(&self) -> tree_sitter::Language { - self.language + pub fn language(&self) -> &tree_sitter::Language { + &self.language } /// Returns the original TSG path, if it was provided at construction or set with @@ -617,7 +624,7 @@ impl<'a> Builder<'a> { ) -> Result<(), BuildError> { let tree = { let mut parser = Parser::new(); - parser.set_language(self.sgl.language)?; + parser.set_language(&self.sgl.language)?; let ts_cancellation_flag = TreeSitterCancellationFlag::from(cancellation_flag); // The parser.set_cancellation_flag` is unsafe, because it does not tie the // lifetime of the parser to the lifetime of the cancellation flag in any way. @@ -635,16 +642,17 @@ impl<'a> Builder<'a> { let tree = parse_errors.into_tree(); let mut globals = Variables::nested(globals); - if globals.get(&ROOT_NODE_VAR.into()).is_none() { - let root_node = self.inject_node(NodeID::root()); - globals - .add(ROOT_NODE_VAR.into(), root_node.into()) - .expect("Failed to set ROOT_NODE"); - } + + let root_node = self.inject_node(NodeID::root()); + globals + .add(ROOT_NODE_VAR.into(), root_node.into()) + .unwrap_or_default(); + let jump_to_scope_node = self.inject_node(NodeID::jump_to()); globals .add(JUMP_TO_SCOPE_NODE_VAR.into(), jump_to_scope_node.into()) .expect("Failed to set JUMP_TO_SCOPE_NODE"); + if globals.get(&FILE_PATH_VAR.into()).is_none() { let file_name = self.stack_graph[self.file].to_string(); globals diff --git a/tree-sitter-stack-graphs/src/loader.rs b/tree-sitter-stack-graphs/src/loader.rs index 1620334ce..861e4ff95 100644 --- a/tree-sitter-stack-graphs/src/loader.rs +++ b/tree-sitter-stack-graphs/src/loader.rs @@ -29,6 +29,9 @@ use tree_sitter_loader::Loader as TsLoader; use crate::CancellationFlag; use crate::FileAnalyzer; use crate::StackGraphLanguage; +use crate::FILE_PATH_VAR; + +const BUILTINS_FILENAME: &str = ""; pub static DEFAULT_TSG_PATHS: Lazy> = Lazy::new(|| vec![LoadPath::Grammar("queries/stack-graphs".into())]); @@ -44,11 +47,16 @@ pub struct LanguageConfiguration { pub sgl: StackGraphLanguage, pub builtins: StackGraph, pub special_files: FileAnalyzers, + /// Can be set to true if the stack graph rules ensure that there can be no similar + /// paths in a file, in which case it is safe to turn of similar path detection. If + /// incorrectly set to true, performance of path finding suffers from exponential + /// blow up. + pub no_similar_paths_in_file: bool, } impl LanguageConfiguration { /// Build a language configuration from tsg and builtins sources. The tsg path - /// is kept for informational only, see [`StackGraphLanguage::from_source`][]. + /// is kept for informational use only, see [`StackGraphLanguage::from_source`][]. pub fn from_sources<'a>( language: Language, scope: Option, @@ -58,23 +66,27 @@ impl LanguageConfiguration { tsg_source: &'a str, builtins_source: Option<(PathBuf, &'a str)>, builtins_config: Option<&str>, - special_files: FileAnalyzers, cancellation_flag: &dyn CancellationFlag, ) -> Result> { - let sgl = StackGraphLanguage::from_source(language, tsg_path.clone(), tsg_source).map_err( - |err| LoadError::SglParse { - inner: err, - tsg_path, - tsg: Cow::from(tsg_source), - }, - )?; + let sgl = StackGraphLanguage::from_source(language.clone(), tsg_path.clone(), tsg_source) + .map_err(|err| LoadError::SglParse { + inner: err, + tsg_path, + tsg: Cow::from(tsg_source), + })?; let mut builtins = StackGraph::new(); if let Some((builtins_path, builtins_source)) = builtins_source { let mut builtins_globals = Variables::new(); + if let Some(builtins_config) = builtins_config { Loader::load_globals_from_config_str(builtins_config, &mut builtins_globals)?; } - let file = builtins.add_file("").unwrap(); + + builtins_globals + .add(FILE_PATH_VAR.into(), BUILTINS_FILENAME.into()) + .unwrap_or_default(); + + let file = builtins.add_file(BUILTINS_FILENAME).unwrap(); sgl.build_stack_graph_into( &mut builtins, file, @@ -97,7 +109,8 @@ impl LanguageConfiguration { file_types, sgl, builtins, - special_files, + special_files: FileAnalyzers::new(), + no_similar_paths_in_file: false, }) } @@ -143,7 +156,7 @@ impl FileAnalyzers { } } - pub fn add( + pub fn with( mut self, file_name: String, analyzer: impl FileAnalyzer + Send + Sync + 'static, @@ -152,6 +165,15 @@ impl FileAnalyzers { self } + pub fn add( + &mut self, + file_name: String, + analyzer: impl FileAnalyzer + Send + Sync + 'static, + ) -> &mut Self { + self.file_analyzers.insert(file_name, Arc::new(analyzer)); + self + } + pub fn get(&self, file_name: &str) -> Option> { self.file_analyzers.get(file_name).cloned() } @@ -250,7 +272,7 @@ impl Loader { &mut self, path: &Path, content: &mut dyn ContentProvider, - ) -> Result, LoadError<'static>> { + ) -> Result, LoadError<'static>> { match &mut self.0 { LoaderImpl::Paths(loader) => loader.load_tree_sitter_language_for_file(path, content), LoaderImpl::Provided(loader) => { @@ -311,9 +333,16 @@ impl Loader { graph: &mut StackGraph, cancellation_flag: &dyn CancellationFlag, ) -> Result<(), LoadError<'a>> { - let file = graph.add_file(&path.to_string_lossy()).unwrap(); + let file_name = path.to_string_lossy(); + let file = graph.add_file(&file_name).unwrap(); let mut globals = Variables::new(); + Self::load_globals_from_config_str(&config, &mut globals)?; + + globals + .add(FILE_PATH_VAR.into(), BUILTINS_FILENAME.into()) + .unwrap_or_default(); + sgl.build_stack_graph_into(graph, file, &source, &globals, cancellation_flag) .map_err(|err| LoadError::Builtins { inner: err, @@ -358,6 +387,17 @@ impl FileLanguageConfigurations<'_> { pub fn has_some(&self) -> bool { self.primary.is_some() || !self.secondary.is_empty() } + + pub fn no_similar_paths_in_file(&self) -> bool { + let mut no_similar_paths_in_file = true; + if let Some(lc) = &self.primary { + no_similar_paths_in_file &= lc.no_similar_paths_in_file; + } + for (lc, _) in &self.secondary { + no_similar_paths_in_file &= lc.no_similar_paths_in_file; + } + return no_similar_paths_in_file; + } } #[derive(Debug, Error)] @@ -455,10 +495,10 @@ impl LanguageConfigurationsLoader { &mut self, path: &Path, content: &mut dyn ContentProvider, - ) -> Result, LoadError<'static>> { + ) -> Result, LoadError<'static>> { for configuration in self.configurations.iter() { if configuration.matches_file(path, content)? { - return Ok(Some(configuration.language)); + return Ok(Some(&configuration.language)); } } Ok(None) @@ -526,9 +566,9 @@ impl PathLoader { &mut self, path: &Path, content: &mut dyn ContentProvider, - ) -> Result, LoadError<'static>> { + ) -> Result, LoadError<'static>> { if let Some(selected_language) = self.select_language_for_file(path, content)? { - return Ok(Some(selected_language.language)); + return Ok(Some(&selected_language.language)); } Ok(None) } @@ -550,7 +590,7 @@ impl PathLoader { Some(index) => index, None => { let tsg = self.load_tsg_from_paths(&language)?; - let sgl = StackGraphLanguage::new(language.language, tsg); + let sgl = StackGraphLanguage::new(language.language.clone(), tsg); let mut builtins = StackGraph::new(); self.load_builtins_from_paths_into( @@ -561,13 +601,15 @@ impl PathLoader { )?; let lc = LanguageConfiguration { - language: language.language, + language: language.language.clone(), scope: language.scope, content_regex: language.content_regex, file_types: language.file_types, sgl, builtins, special_files: FileAnalyzers::new(), + // always detect similar paths, we don't know the language configuration when loading from the file system + no_similar_paths_in_file: false, }; self.cache.push((language.language, lc)); @@ -653,7 +695,7 @@ impl PathLoader { } if tsg_path.exists() { let tsg_source = std::fs::read_to_string(tsg_path)?; - return Loader::load_tsg(language.language, Cow::from(tsg_source)); + return Loader::load_tsg(language.language.clone(), Cow::from(tsg_source)); } } return Err(LoadError::NoTsgFound); @@ -700,11 +742,11 @@ impl PathLoader { graph: &mut StackGraph, cancellation_flag: &dyn CancellationFlag, ) -> Result<(), LoadError<'static>> { - let source = std::fs::read_to_string(builtins_path.clone())?; + let source = std::fs::read_to_string(builtins_path)?; let mut config_path = builtins_path.to_path_buf(); config_path.set_extension("cfg"); let config = if config_path.exists() { - std::fs::read_to_string(builtins_path.clone())? + std::fs::read_to_string(builtins_path)? } else { "".into() }; @@ -743,10 +785,11 @@ impl SupplementedTsLoader { .map_err(LoadError::TreeSitter)?; let configurations = self .0 - .find_language_configurations_at_path(&path) + .find_language_configurations_at_path(&path, true) .map_err(LoadError::TreeSitter)?; let languages = languages .into_iter() + .map(|(l, _)| l) .zip(configurations.into_iter()) .map(SupplementedLanguage::from) .filter(|language| scope.map_or(true, |scope| language.matches_scope(scope))) diff --git a/tree-sitter-stack-graphs/src/test.rs b/tree-sitter-stack-graphs/src/test.rs index 19abda61d..236e254c5 100644 --- a/tree-sitter-stack-graphs/src/test.rs +++ b/tree-sitter-stack-graphs/src/test.rs @@ -74,6 +74,7 @@ use stack_graphs::graph::SourceInfo; use stack_graphs::graph::StackGraph; use stack_graphs::partial::PartialPaths; use stack_graphs::stitching::Database; +use stack_graphs::stitching::StitcherConfig; use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; @@ -623,13 +624,20 @@ impl Test { &mut self, partials: &mut PartialPaths, db: &mut Database, + stitcher_config: StitcherConfig, cancellation_flag: &dyn CancellationFlag, ) -> Result { let mut result = TestResult::new(); for fragment in &self.fragments { for assertion in &fragment.assertions { match assertion - .run(&self.graph, partials, db, &cancellation_flag) + .run( + &self.graph, + partials, + db, + stitcher_config, + &cancellation_flag, + ) .map_or_else(|e| self.from_error(e), |v| Ok(v)) { Ok(_) => result.add_success(), diff --git a/tree-sitter-stack-graphs/tests/it/builder.rs b/tree-sitter-stack-graphs/tests/it/builder.rs index 661e8b973..719e25d71 100644 --- a/tree-sitter-stack-graphs/tests/it/builder.rs +++ b/tree-sitter-stack-graphs/tests/it/builder.rs @@ -9,6 +9,7 @@ use stack_graphs::graph::StackGraph; use tree_sitter_graph::Variables; use tree_sitter_stack_graphs::NoCancellation; use tree_sitter_stack_graphs::StackGraphLanguage; +use tree_sitter_stack_graphs::FILE_PATH_VAR; use crate::edges::check_stack_graph_edges; use crate::nodes::check_stack_graph_nodes; @@ -22,13 +23,19 @@ fn can_support_preexisting_nodes() { "#; let python = "pass"; + let file_name = "test.py"; + let mut graph = StackGraph::new(); - let file = graph.get_or_create_file("test.py"); + let file = graph.get_or_create_file(file_name); let node_id = graph.new_node_id(file); let _preexisting_node = graph.add_scope_node(node_id, true).unwrap(); - let globals = Variables::new(); - let language = StackGraphLanguage::from_str(tree_sitter_python::language(), tsg).unwrap(); + let mut globals = Variables::new(); + globals + .add(FILE_PATH_VAR.into(), file_name.into()) + .expect("failed to add file path variable"); + + let language = StackGraphLanguage::from_str(tree_sitter_python::LANGUAGE.into(), tsg).unwrap(); language .build_stack_graph_into(&mut graph, file, python, &globals, &NoCancellation) .expect("Failed to build graph"); @@ -45,15 +52,21 @@ fn can_support_injected_nodes() { "#; let python = "pass"; + let file_name = "test.py"; + let mut graph = StackGraph::new(); - let file = graph.get_or_create_file("test.py"); + let file = graph.get_or_create_file(file_name); let node_id = graph.new_node_id(file); let _preexisting_node = graph.add_scope_node(node_id, true).unwrap(); - let language = StackGraphLanguage::from_str(tree_sitter_python::language(), tsg).unwrap(); + let language = StackGraphLanguage::from_str(tree_sitter_python::LANGUAGE.into(), tsg).unwrap(); let mut builder = language.builder_into_stack_graph(&mut graph, file, python); let mut globals = Variables::new(); + globals + .add(FILE_PATH_VAR.into(), file_name.into()) + .expect("failed to add file path variable"); + globals .add("EXT_NODE".into(), builder.inject_node(node_id).into()) .expect("Failed to add EXT_NODE variable"); diff --git a/tree-sitter-stack-graphs/tests/it/loader.rs b/tree-sitter-stack-graphs/tests/it/loader.rs index 8fcf0ccac..401af46e5 100644 --- a/tree-sitter-stack-graphs/tests/it/loader.rs +++ b/tree-sitter-stack-graphs/tests/it/loader.rs @@ -9,6 +9,7 @@ use once_cell::sync::Lazy; use pretty_assertions::assert_eq; use stack_graphs::graph::StackGraph; use std::path::PathBuf; +use tree_sitter::Language; use tree_sitter_stack_graphs::loader::FileAnalyzers; use tree_sitter_stack_graphs::loader::LanguageConfiguration; use tree_sitter_stack_graphs::loader::Loader; @@ -25,16 +26,17 @@ static TSG: Lazy = Lazy::new(|| { #[test] fn can_load_from_provided_language_configuration() { - let language = tree_sitter_python::language(); - let sgl = StackGraphLanguage::from_str(language, &TSG).unwrap(); + let language: Language = tree_sitter_python::LANGUAGE.into(); + let sgl = StackGraphLanguage::from_str(language.clone(), &TSG).unwrap(); let lc = LanguageConfiguration { - language: language, + language: language.clone(), scope: Some("source.py".into()), content_regex: None, file_types: vec!["py".into()], sgl, builtins: StackGraph::new(), special_files: FileAnalyzers::new(), + no_similar_paths_in_file: false, }; let mut loader = Loader::from_language_configurations(vec![lc], None).expect("Expected loader to succeed"); @@ -42,10 +44,10 @@ fn can_load_from_provided_language_configuration() { let tsl = loader .load_tree_sitter_language_for_file(&PATH, &mut None) .expect("Expected loading tree-sitter language to succeed"); - assert_eq!(tsl, Some(language)); + assert_eq!(tsl, Some(&language)); let lc = loader .load_for_file(&PATH, &mut None, &NoCancellation) .expect("Expected loading stack graph language to succeed"); - assert_eq!(lc.primary.map(|lc| lc.language), Some(language)); + assert_eq!(lc.primary.map(|lc| &lc.language), Some(&language)); } diff --git a/tree-sitter-stack-graphs/tests/it/main.rs b/tree-sitter-stack-graphs/tests/it/main.rs index c1e37a40e..01cfd9754 100644 --- a/tree-sitter-stack-graphs/tests/it/main.rs +++ b/tree-sitter-stack-graphs/tests/it/main.rs @@ -5,6 +5,8 @@ // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ +use std::path::Path; + use stack_graphs::arena::Handle; use stack_graphs::graph::File; use stack_graphs::graph::StackGraph; @@ -12,6 +14,7 @@ use tree_sitter_graph::Variables; use tree_sitter_stack_graphs::BuildError; use tree_sitter_stack_graphs::NoCancellation; use tree_sitter_stack_graphs::StackGraphLanguage; +use tree_sitter_stack_graphs::FILE_PATH_VAR; mod builder; mod edges; @@ -23,11 +26,18 @@ pub(self) fn build_stack_graph( python_source: &str, tsg_source: &str, ) -> Result<(StackGraph, Handle), BuildError> { + let file_name = "test.py"; let language = - StackGraphLanguage::from_str(tree_sitter_python::language(), tsg_source).unwrap(); + StackGraphLanguage::from_str(tree_sitter_python::LANGUAGE.into(), tsg_source).unwrap(); let mut graph = StackGraph::new(); - let file = graph.get_or_create_file("test.py"); - let globals = Variables::new(); + let file = graph.get_or_create_file(file_name); + let mut globals = Variables::new(); + let source_path = Path::new(file_name); + + globals + .add(FILE_PATH_VAR.into(), source_path.to_str().unwrap().into()) + .expect("failed to add file path variable"); + language.build_stack_graph_into(&mut graph, file, python_source, &globals, &NoCancellation)?; Ok((graph, file)) } diff --git a/tree-sitter-stack-graphs/tests/it/test.rs b/tree-sitter-stack-graphs/tests/it/test.rs index b281f283b..fae14fe76 100644 --- a/tree-sitter-stack-graphs/tests/it/test.rs +++ b/tree-sitter-stack-graphs/tests/it/test.rs @@ -5,6 +5,7 @@ // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ +use crate::FILE_PATH_VAR; use once_cell::sync::Lazy; use pretty_assertions::assert_eq; use stack_graphs::arena::Handle; @@ -13,6 +14,7 @@ use stack_graphs::graph::StackGraph; use stack_graphs::partial::PartialPaths; use stack_graphs::stitching::Database; use stack_graphs::stitching::ForwardPartialPathStitcher; +use stack_graphs::stitching::StitcherConfig; use std::path::Path; use std::path::PathBuf; use tree_sitter_graph::Variables; @@ -71,7 +73,7 @@ fn build_stack_graph_into( globals: &Variables, ) -> Result<(), BuildError> { let language = - StackGraphLanguage::from_str(tree_sitter_python::language(), tsg_source).unwrap(); + StackGraphLanguage::from_str(tree_sitter_python::LANGUAGE.into(), tsg_source).unwrap(); language.build_stack_graph_into(graph, file, python_source, globals, &NoCancellation)?; Ok(()) } @@ -93,10 +95,20 @@ fn check_test( expected_successes + expected_failures, assertion_count, ); + let mut globals = Variables::new(); for fragments in &test.fragments { globals.clear(); + fragments.add_globals_to(&mut globals); + + globals + .add( + FILE_PATH_VAR.into(), + fragments.path.to_str().unwrap().into(), + ) + .unwrap_or_default(); + build_stack_graph_into( &mut test.graph, fragments.file, @@ -113,6 +125,7 @@ fn check_test( &test.graph, &mut partials, fragment.file, + StitcherConfig::default(), &stack_graphs::NoCancellation, |graph, partials, path| { db.add_partial_path(graph, partials, path.clone()); @@ -122,7 +135,12 @@ fn check_test( } let results = test - .run(&mut partials, &mut db, &NoCancellation) + .run( + &mut partials, + &mut db, + StitcherConfig::default(), + &NoCancellation, + ) .expect("should never be cancelled"); assert_eq!( expected_successes, diff --git a/valgrind.supp b/valgrind.supp new file mode 100644 index 000000000..25c3cfbda --- /dev/null +++ b/valgrind.supp @@ -0,0 +1,8 @@ +{ + rust-1.83-false-positive-1 + Memcheck:Leak + match-leak-kinds: possible + fun:malloc + ... + fun:main +}