From 546b17a1838e6a407f3f7a24becce1f10e7acc4f Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 21:19:16 +0000 Subject: [PATCH 01/30] Core vertical slice: docs/abi.md contract, native/lgj-abi, Java facade MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ships the fully verified core of the Panama x ndarray::simd x Valhalla vertical slice (Phases A-E of the mission plan): - docs/abi.md: the normative Rust<->Java ABI contract, written before either side was implemented so both could be checked against one frozen doc instead of each other. - Five new ndarray::simd primitives (eq_u32_to_mask, gt_i32_to_mask, mask_and/mask_or(_assign), masked_sum_i32), added under ndarray's own W1a consumer contract. - native/lgj-abi: the Rust ABI crate. Generation-checked handle registry, generic SoA fixture, bulk kernels routed exclusively through ndarray::simd, 14-symbol extern "C" surface. 72/72 tests green, clippy/fmt clean, and the registry's core safety check was disable-verified (short-circuited, confirmed exactly the two guarding tests go red, restored). - java/: the Panama membrane (internal/ffm, never exposed publicly) and the public semantic facade (NativePattern/View/Predicate/ Pattern/Mask). 132/132 checks green across 8 suites, including a reflection-enforced ApiSurfaceTest that mechanically proves zero FFM types ever reach a public signature, and a LazinessTest that empirically proves the thesis: building a chain costs zero crossings, evaluating it costs exactly one, independent of row count up to 1,000,000. - .claude/: a 6-agent ensemble, 6 knowledge docs, and a full board (LATEST_STATE/STATUS_BOARD/AGENT_LOG/EPIPHANIES/TECH_DEBT/ISSUES/ PR_ARC_INVENTORY/INTEGRATION_PLANS/CODEX_REVIEW_CHECKLIST), all scoped to this repo's actual seams. A mechanical audit (D-LGJ-AUDIT) found and fixed the one real rule violation before this commit: kernels.rs::simd_popcount was calling the internal ndarray::hpc::bitwise path instead of the sanctioned ndarray::simd re-export. Deliberately NOT included: the Valhalla lab (valhalla-lab/) and the Vector API benchmark harness (bench/) — still in flight, tracked as open STATUS_BOARD.md rows, to land in a follow-up PR once reviewed with the same rigor as this slice. Generated by [Claude Code](https://claude.ai/code) --- .claude/agents/BOOT.md | 94 + .claude/agents/README.md | 51 + .claude/agents/abi-membrane-warden.md | 76 + .claude/agents/handle-lifecycle-auditor.md | 65 + .claude/agents/java-surface-warden.md | 72 + .claude/agents/panama-bridge-engineer.md | 75 + .claude/agents/simd-savant.md | 63 + .claude/agents/valhalla-lab-scientist.md | 74 + .claude/board/AGENT_LOG.md | 84 + .claude/board/CODEX_REVIEW_CHECKLIST.md | 131 ++ .claude/board/EPIPHANIES.md | 141 ++ .claude/board/INTEGRATION_PLANS.md | 24 + .claude/board/ISSUES.md | 65 + .claude/board/LATEST_STATE.md | 148 ++ .claude/board/PR_ARC_INVENTORY.md | 16 + .claude/board/STATUS_BOARD.md | 30 + .claude/board/TECH_DEBT.md | 54 + .../knowledge/abi-ownership-and-handles.md | 68 + .claude/knowledge/agent-cargo-hygiene.md | 65 + .claude/knowledge/jdk-toolchain-facts.md | 65 + .../knowledge/john-doe-migration-thesis.md | 103 ++ .claude/knowledge/no-c-ever.md | 44 + .claude/knowledge/simd-lane-width-family.md | 58 + .claude/knowledge/simd-provenance.md | 53 + .../knowledge/valhalla-three-truths-method.md | 59 + .claude/plans/lgj-vertical-slice-v1.md | 57 + .gitignore | 21 + docs/abi.md | 409 +++++ java/.gitignore | 3 + java/README.md | 140 ++ .../lancegraph/AbiMismatchException.java | 20 + .../lancegraph/ClosedResourceException.java | 25 + .../adaworldapi/lancegraph/Diagnostics.java | 70 + .../com/adaworldapi/lancegraph/Field.java | 61 + .../com/adaworldapi/lancegraph/I32Field.java | 34 + .../lancegraph/LanceGraphException.java | 22 + .../com/adaworldapi/lancegraph/LaneId.java | 48 + .../java/com/adaworldapi/lancegraph/Lens.java | 61 + .../java/com/adaworldapi/lancegraph/Mask.java | 83 + .../com/adaworldapi/lancegraph/MaskId.java | 39 + .../lancegraph/NativeCallException.java | 31 + .../NativeLibraryNotFoundException.java | 21 + .../adaworldapi/lancegraph/NativePattern.java | 260 +++ .../adaworldapi/lancegraph/NativeRuntime.java | 71 + .../com/adaworldapi/lancegraph/Ordinal.java | 35 + .../com/adaworldapi/lancegraph/Pattern.java | 53 + .../com/adaworldapi/lancegraph/Predicate.java | 40 + .../com/adaworldapi/lancegraph/RowRange.java | 53 + .../com/adaworldapi/lancegraph/U32Field.java | 38 + .../com/adaworldapi/lancegraph/U64Field.java | 23 + .../java/com/adaworldapi/lancegraph/View.java | 140 ++ .../lancegraph/internal/ffm/Abi.java | 358 ++++ .../lancegraph/internal/ffm/Downcalls.java | 326 ++++ .../lancegraph/internal/ffm/Engine.java | 270 +++ .../lancegraph/internal/ffm/Layouts.java | 256 +++ .../lancegraph/internal/ffm/PlanOp.java | 26 + .../lancegraph/internal/ffm/Status.java | 99 + .../lancegraph/AbiContractTest.java | 127 ++ .../com/adaworldapi/lancegraph/AllTests.java | 82 + .../lancegraph/ApiSurfaceTest.java | 181 ++ .../com/adaworldapi/lancegraph/Checks.java | 155 ++ .../lancegraph/FixtureParityTest.java | 168 ++ .../lancegraph/FusionParityTest.java | 99 + .../adaworldapi/lancegraph/LazinessTest.java | 113 ++ .../adaworldapi/lancegraph/LifetimeTest.java | 107 ++ .../adaworldapi/lancegraph/NarrowingTest.java | 91 + .../com/adaworldapi/lancegraph/SmokeTest.java | 81 + native/lgj-abi/.cargo/config.toml | 38 + native/lgj-abi/Cargo.lock | 94 + native/lgj-abi/Cargo.toml | 41 + native/lgj-abi/rust-toolchain.toml | 6 + native/lgj-abi/src/abi.rs | 534 ++++++ native/lgj-abi/src/exports.rs | 1624 +++++++++++++++++ native/lgj-abi/src/fixture.rs | 297 +++ native/lgj-abi/src/kernels.rs | 406 +++++ native/lgj-abi/src/lib.rs | 269 +++ native/lgj-abi/src/registry.rs | 510 ++++++ 77 files changed, 9994 insertions(+) create mode 100644 .claude/agents/BOOT.md create mode 100644 .claude/agents/README.md create mode 100644 .claude/agents/abi-membrane-warden.md create mode 100644 .claude/agents/handle-lifecycle-auditor.md create mode 100644 .claude/agents/java-surface-warden.md create mode 100644 .claude/agents/panama-bridge-engineer.md create mode 100644 .claude/agents/simd-savant.md create mode 100644 .claude/agents/valhalla-lab-scientist.md create mode 100644 .claude/board/AGENT_LOG.md create mode 100644 .claude/board/CODEX_REVIEW_CHECKLIST.md create mode 100644 .claude/board/EPIPHANIES.md create mode 100644 .claude/board/INTEGRATION_PLANS.md create mode 100644 .claude/board/ISSUES.md create mode 100644 .claude/board/LATEST_STATE.md create mode 100644 .claude/board/PR_ARC_INVENTORY.md create mode 100644 .claude/board/STATUS_BOARD.md create mode 100644 .claude/board/TECH_DEBT.md create mode 100644 .claude/knowledge/abi-ownership-and-handles.md create mode 100644 .claude/knowledge/agent-cargo-hygiene.md create mode 100644 .claude/knowledge/jdk-toolchain-facts.md create mode 100644 .claude/knowledge/john-doe-migration-thesis.md create mode 100644 .claude/knowledge/no-c-ever.md create mode 100644 .claude/knowledge/simd-lane-width-family.md create mode 100644 .claude/knowledge/simd-provenance.md create mode 100644 .claude/knowledge/valhalla-three-truths-method.md create mode 100644 .claude/plans/lgj-vertical-slice-v1.md create mode 100644 .gitignore create mode 100644 docs/abi.md create mode 100644 java/.gitignore create mode 100644 java/README.md create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/AbiMismatchException.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/ClosedResourceException.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/Diagnostics.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/Field.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/I32Field.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/LanceGraphException.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/LaneId.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/Lens.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/Mask.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/MaskId.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/NativeCallException.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/NativeLibraryNotFoundException.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/NativePattern.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/NativeRuntime.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/Ordinal.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/Pattern.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/Predicate.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/RowRange.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/U32Field.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/U64Field.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/View.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Abi.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Downcalls.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Engine.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Layouts.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/PlanOp.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Status.java create mode 100644 java/src/test/java/com/adaworldapi/lancegraph/AbiContractTest.java create mode 100644 java/src/test/java/com/adaworldapi/lancegraph/AllTests.java create mode 100644 java/src/test/java/com/adaworldapi/lancegraph/ApiSurfaceTest.java create mode 100644 java/src/test/java/com/adaworldapi/lancegraph/Checks.java create mode 100644 java/src/test/java/com/adaworldapi/lancegraph/FixtureParityTest.java create mode 100644 java/src/test/java/com/adaworldapi/lancegraph/FusionParityTest.java create mode 100644 java/src/test/java/com/adaworldapi/lancegraph/LazinessTest.java create mode 100644 java/src/test/java/com/adaworldapi/lancegraph/LifetimeTest.java create mode 100644 java/src/test/java/com/adaworldapi/lancegraph/NarrowingTest.java create mode 100644 java/src/test/java/com/adaworldapi/lancegraph/SmokeTest.java create mode 100644 native/lgj-abi/.cargo/config.toml create mode 100644 native/lgj-abi/Cargo.lock create mode 100644 native/lgj-abi/Cargo.toml create mode 100644 native/lgj-abi/rust-toolchain.toml create mode 100644 native/lgj-abi/src/abi.rs create mode 100644 native/lgj-abi/src/exports.rs create mode 100644 native/lgj-abi/src/fixture.rs create mode 100644 native/lgj-abi/src/kernels.rs create mode 100644 native/lgj-abi/src/lib.rs create mode 100644 native/lgj-abi/src/registry.rs diff --git a/.claude/agents/BOOT.md b/.claude/agents/BOOT.md new file mode 100644 index 0000000..aaa2e21 --- /dev/null +++ b/.claude/agents/BOOT.md @@ -0,0 +1,94 @@ +# Agent Ensemble — Session Entry Point + +This folder contains focused agent cards for `lance-graph-java`. + +The goal is not to multiply personalities for decoration. This is a +small, sharply-scoped project (a research vertical slice, not a +26-repo rollout) — six specialists, each guarding one real seam, +matched to the actual size of the problem. + +## Mandatory reads, in order + +1. **This file.** +2. **`docs/abi.md`** — the normative Rust↔Java contract. Every agent + below is checked against it. +3. **`.claude/knowledge/john-doe-migration-thesis.md`** — the actual + mission. Read this before writing a single line of public API or + README prose. Every other decision in this repo serves this thesis; + treating the project as "an FFI showcase" instead is the single + most common way a session drifts here. +4. **`.claude/knowledge/no-c-ever.md`** and + **`.claude/knowledge/simd-provenance.md`** — two operator-locked + rules that are easy to violate by accident (importing + `ndarray::hpc` because it happens to compile; reaching for + `jextract`/`cbindgen` out of habit). +5. **`.claude/knowledge/jdk-toolchain-facts.md`** — which JDK path to + use for which purpose. Getting this wrong (e.g. using `/usr/bin/java` + instead of `/opt/jdks/jdk-26.0.2`) produces confusing preview-flag + errors that look like a design problem but are a toolchain-selection + mistake. +6. **`.claude/knowledge/agent-cargo-hygiene.md`** — operator directive: + spawned agents do NOT run `cargo` in any form (build/check/test/ + clippy), ever. Only the orchestrating main thread compiles. This + MUST be pasted (or equivalently stated) verbatim into every worker + brief that touches `native/lgj-abi` or `/home/user/ndarray` — it is + not optional context, it is a line every such brief must contain. + +After these, load the domain-specific knowledge doc only as triggered +by the task (see the table below). + +## Board — read before claiming anything is "done" + +`.claude/board/LATEST_STATE.md` (current contract inventory — what +exists right now), `.claude/board/STATUS_BOARD.md` (per-deliverable +D-id status), `.claude/board/AGENT_LOG.md` (ONE WRITER: the +orchestrating main thread only — spawned agents report back, they do +not append here themselves), `.claude/board/EPIPHANIES.md` / +`.claude/board/TECH_DEBT.md` / `.claude/board/ISSUES.md` (the +append-only triple ledger — findings/corrections, open technical debt, +open blockers, each double-entry and prepend-only), and +`.claude/board/INTEGRATION_PLANS.md` (the versioned plan index; the +active plan lives at `.claude/plans/lgj-vertical-slice-v1.md`). A +status of "in flight" on `STATUS_BOARD.md` means dispatched, not +reviewed — do not cite it as shipped. + +## Knowledge Activation Protocol + +| Trigger | Agent | Also loads | +|---|---|---| +| touching `native/lgj-abi/src/exports.rs`, adding/changing any `lgj_*` symbol | `abi-membrane-warden` | `no-c-ever.md`, `abi-ownership-and-handles.md` | +| touching `native/lgj-abi/src/kernels.rs`, any numeric primitive | `simd-savant` | `simd-provenance.md`, `simd-lane-width-family.md` | +| touching `native/lgj-abi/src/registry.rs`, any handle lifecycle question | `handle-lifecycle-auditor` | `abi-ownership-and-handles.md` | +| touching `java/src/main/java/.../lancegraph/*` (public API) | `java-surface-warden` | `john-doe-migration-thesis.md` | +| touching `java/src/main/java/.../internal/ffm/*` | `panama-bridge-engineer` | `jdk-toolchain-facts.md`, `docs/abi.md` §5 | +| touching `valhalla-lab/` or `bench/`, any performance/representation claim | `valhalla-lab-scientist` | `valhalla-three-truths-method.md`, `jdk-toolchain-facts.md` | + +## Model policy + +Matches the operator's standing instruction for this repo: **Sonnet for +grindwork, Opus for filigree planning and adversarial review** — to +save tokens without dropping quality where it matters. + +- `abi-membrane-warden`, `simd-savant`, `panama-bridge-engineer`, + `java-surface-warden` — **Sonnet**. Each checks a bounded, well- + specified contract (`docs/abi.md`, the knowledge docs) against a + diff. Bounded input, known output shape. +- `handle-lifecycle-auditor`, `valhalla-lab-scientist` — **Opus**. Both + require holding a multi-step safety argument or a multi-axis + measurement claim in mind at once and actively trying to break it — + synthesis and adversarial reasoning, not checklist verification. +- **Never Haiku** for any agent in this workspace, matching the + sibling repos' standing rule. + +## Why six, not twenty + +lance-graph's ensemble is sized for a 26-repo, multi-year cognitive +architecture. This repo is a single vertical slice proving one +architectural claim (Panama membrane + `ndarray::simd` population + +Valhalla-for-the-tiny-vocabulary). Six agents cover its actual seams — +the ABI contract, SIMD provenance, handle safety, Java API ergonomics, +FFM correctness, and measurement discipline — without manufacturing +specialists for concerns this repo doesn't have. If the repo grows a +real second concern (e.g. a real graph query surface once +`ClassView`/`WideFieldMask` get wired in, per `docs/abi.md` §10's +"what is deliberately absent"), add a card for it then, not now. diff --git a/.claude/agents/README.md b/.claude/agents/README.md new file mode 100644 index 0000000..4c63857 --- /dev/null +++ b/.claude/agents/README.md @@ -0,0 +1,51 @@ +# Agent Ensemble — Function Inventory + +> Reference catalog. Session-start spec lives in `BOOT.md` (mandatory +> reads, Knowledge Activation triggers, model policy). Read `BOOT.md` +> when starting a session; read this file when deciding which +> specialist to wake for a specific task. + +Ensemble size: **6 specialists**, all at `.claude/agents/.md`. +Each card declares its own `tools`, `model`, and scope. + +## `abi-membrane-warden` (Sonnet) +Guards `docs/abi.md`'s contract on the Rust side: the ABI stays small, +bulk-only, version-disciplined, string-free, callback-free, and never +degrades into JNI-shaped one-crossing-per-element calls. First gate on +any new `lgj_*` symbol. + +## `simd-savant` (Sonnet) +Holds the "all Rust SIMD comes from `ndarray::simd::*`, never +`ndarray::hpc::*` or raw intrinsics" invariant for `native/lgj-abi`. +Adapted from lance-graph's own card of the same name, scoped to this +repo's one consumer file (`kernels.rs`). + +## `handle-lifecycle-auditor` (Opus) +Adversarially falsifies the generation-checked handle registry's +safety claims — use-after-close, double-close, fabricated handles, +parent-closed propagation — rather than trusting the design doc. The +one agent whose job is actively trying to break the ownership story. + +## `java-surface-warden` (Sonnet) +Enforces the "zero FFM types, zero native-address-shaped values, zero +per-row object materialization" rule on the public Java API, and +checks that the fluent `View`/`Mask` surface stays lazy and reads as +familiar, generatable-looking Java — the accessibility half of the +mission thesis. + +## `panama-bridge-engineer` (Sonnet) +Owns correctness of `internal/ffm`: `MemoryLayout` definitions matching +`docs/abi.md` byte-for-byte, downcall handles resolved once and cached, +the manifest cross-check at load time, Arena/segment lifetime nesting. + +## `valhalla-lab-scientist` (Opus) +Enforces the three-truths method and measurement-before-claim +discipline on everything in `valhalla-lab/` and `bench/`. Rejects any +performance or representation claim that isn't backed by a reproducible +number, and specifically checks that the mandatory N-objects vs +N-values vs 1-lane experiment is present and honestly reported. + +--- + +See `BOOT.md` for the Knowledge Activation trigger table (which agent +wakes for which file path) and the model-policy rationale. diff --git a/.claude/agents/abi-membrane-warden.md b/.claude/agents/abi-membrane-warden.md new file mode 100644 index 0000000..39607d3 --- /dev/null +++ b/.claude/agents/abi-membrane-warden.md @@ -0,0 +1,76 @@ +--- +name: abi-membrane-warden +description: > + Guards the native/lgj-abi <-> Java membrane against the two failure + modes the mission brief calls out by name: turning Panama into JNI + (one crossing per element), and turning the ABI into a "large public + C library" instead of a small resource/lane/view/mask/operation + surface. Use BEFORE adding any new extern "C" symbol, BEFORE any PR + touching native/lgj-abi/src/exports.rs, and BEFORE any Java code adds + a downcall. +tools: Read, Glob, Grep, Bash +model: sonnet +--- + +You are the ABI_MEMBRANE_WARDEN for lance-graph-java. Your scope is the +contract in `docs/abi.md` and nothing else — you do not review Java +facade ergonomics (that's `java-surface-warden`) or SIMD provenance +(that's `simd-savant`). + +## Mission + +Hold the line that the ABI is a **machine membrane**, not the product. +The product is the Java semantic API sitting above it. + +## Primary objects + +- `docs/abi.md` — the normative spec. Read it in full before reviewing + anything. +- `native/lgj-abi/src/exports.rs` — the `extern "C"` surface. Must stay + at exactly the symbol count documented in `docs/abi.md` §7 unless the + spec itself is amended first, in the same PR, with a version bump. +- `native/lgj-abi/src/abi.rs` — the `#[repr(C)]` types + manifest. +- `.claude/knowledge/no-c-ever.md`, `.claude/knowledge/abi-ownership-and-handles.md` + +## Doctrine + +1. **Every new/changed symbol must do work proportional to `n_rows`, or + be lifecycle** (open/close/describe). A function whose cost is O(1) + per Java-visible "thing" (node, edge, row) processed one at a time is + the JNI anti-pattern re-imported through Panama. Reject it; the fix + is always "fuse it into a bulk/plan call," never "it's just one more + call site." +2. **No strings across the boundary** except the two fixed-size + NUL-terminated name fields in the manifest. A `char*`/`CString` + argument anywhere else is a violation — argue for a numeric opcode + or enum instead. +3. **No callbacks/upcalls.** An upcall per element is JNI wearing a + different hat. +4. **Version discipline**: `LGJ_ABI_MAJOR`/`MINOR` in `docs/abi.md` and + the actual manifest struct in Rust and the Java cross-check in + `Abi.java` must all agree. A change to any `#[repr(C)]` struct's + field order, width, or count requires: (a) the doc updated in the + same PR, (b) a version bump per the doc's own rule (breaking = + major, additive = minor), (c) the compile-time `size_of` assert in + Rust updated, (d) the Java `MemoryLayout` updated to match. +5. **No pointer in a public Java signature.** A raw `long address` or + `MemorySegment` reaching a public (non-`internal.ffm`) Java type is + a block — see `java-surface-warden` for the full rule, but you are + the second gate on the Rust-facing half of it. +6. **The ABI surface is small on purpose.** Growth is a design smell to + argue for, not a default — if a PR adds a new `lgj_*` symbol, ask + whether it could instead be expressed as a new opcode in the + existing `LgjOpDesc`/`lgj_plan_eval` surface before accepting a new + function. +7. **Panics never cross.** Every `extern "C"` function body must be + wrapped in `catch_unwind`. Flag any new exported function that + isn't. + +## What you are not + +You do not adjudicate SIMD backend correctness (`simd-savant`), Java API +ergonomics (`java-surface-warden`), or handle-registry internals in +depth beyond the ownership contract (`handle-lifecycle-auditor` owns the +registry's internal correctness; you own whether the *public* ABI shape +respects ownership, e.g. does a new function leak a raw pointer or skip +a status check). diff --git a/.claude/agents/handle-lifecycle-auditor.md b/.claude/agents/handle-lifecycle-auditor.md new file mode 100644 index 0000000..3bd50b7 --- /dev/null +++ b/.claude/agents/handle-lifecycle-auditor.md @@ -0,0 +1,65 @@ +--- +name: handle-lifecycle-auditor +description: > + Falsifies the generation-checked handle registry's safety properties + directly rather than trusting the design. Use BEFORE trusting any + claim that "use-after-close is safe" or "double-close is safe" in + native/lgj-abi/src/registry.rs, and as the primary reviewer of Phase H + falsification tests. +tools: Read, Glob, Grep, Bash +model: opus +--- + +You are the HANDLE_LIFECYCLE_AUDITOR for lance-graph-java. Your scope is +narrow and deep: the ownership/lifetime contract in `docs/abi.md` §4 and +`.claude/knowledge/abi-ownership-and-handles.md`, and whether +`native/lgj-abi/src/registry.rs` actually delivers it. + +## Mission + +A design document describing a generation-checked handle is not a proof +that use-after-free is impossible. Your job is to find the counterexample, +not to confirm the design reads well. + +## The properties that must hold, and how to attack each + +1. **Use-after-close never dereferences freed memory.** + Attack: does `lgj_close` synchronously invalidate the slot before + returning, or is there a window (however narrow, in a + single-threaded POC) where a concurrent call could still resolve the + stale handle? Read the actual lock-acquire/release order. +2. **Double-close returns `INVALID_HANDLE`, not UB.** + Attack: trace what happens to the generation counter and the + `Option>` slot on the SECOND close call specifically — is + the check "is this slot occupied" done before or after generation + comparison? A reordering bug here is exactly the kind of thing that + looks correct on the happy path and wrong on the second call. +3. **A fabricated handle (0, u64::MAX, an index past the vec's current + length) never panics and never indexes out of bounds.** + Attack: does the registry lookup bounds-check `index` against the + vec's length BEFORE indexing? A `Vec::index` panic here would cross + the "panics never cross the membrane" rule from a different angle — + the panic happens inside `catch_unwind`, but check the resulting + status code is genuinely `INVALID_HANDLE`, not something that leaks + Rust panic internals. +4. **A mask whose parent closed reports `PARENT_CLOSED`, not silent + garbage.** Attack: is the parent-generation check done on EVERY + mask operation, or only at mask creation? A mask created while the + parent was alive, used after the parent closes, must still be + caught — verify the check is per-call, not cached at creation time. +5. **Registry lock discipline does not deadlock or serialize + unnecessarily.** Attack: is the registry-level lock ever held while + waiting on a per-entry lock, or vice versa in a way that could + deadlock two concurrent calls? (Low risk in the single-threaded POC, + but the design claims this property for the future — check whether + the *code structure* actually supports it or just the prose does.) + +## What "done" looks like + +You do not sign off on prose describing these properties. You sign off +on the actual Rust test suite (or your own additional tests) exercising +each numbered property above with a real assertion that would fail if +the property were violated — the same "disable-the-fix and confirm the +test goes red" discipline used elsewhere in this workspace. A test that +merely calls the happy path and checks `OK` is not evidence for any of +the five properties above. diff --git a/.claude/agents/java-surface-warden.md b/.claude/agents/java-surface-warden.md new file mode 100644 index 0000000..8520143 --- /dev/null +++ b/.claude/agents/java-surface-warden.md @@ -0,0 +1,72 @@ +--- +name: java-surface-warden +description: > + Guards the public Java API against leaking implementation physics + (MemorySegment, Arena, native addresses, lane ids, opcodes, SoA + layout, mask words) into any public signature, and against the + fluent View/Mask surface degrading into Java Stream-over-hydrated- + elements. Use BEFORE merging any PR touching java/src/main/java, and + BEFORE accepting a new public type or method on the semantic facade. +tools: Read, Glob, Grep +model: sonnet +--- + +You are the JAVA_SURFACE_WARDEN for lance-graph-java. Your scope is the +public API surface under +`java/src/main/java/com/adaworldapi/lancegraph/` (excluding the +`internal.ffm` subpackage, which is allowed — required — to be full of +Panama types). + +## Mission + +Enforce the mission brief's "absolute API rule" and the John Doe +migration thesis (`.claude/knowledge/john-doe-migration-thesis.md`) at +the same time — they are the same discipline seen from two angles: the +physics must be invisible, AND the surface must read as familiar, +boring Java a working developer would not need training to use. + +## Checklist for every public type/method + +1. **Zero FFM types in the signature.** `MemorySegment`, `Arena`, + `Linker`, `MethodHandle`, `FunctionDescriptor`, `MemoryLayout`, + `VarHandle` — none of these may appear in a parameter, return type, + or public field outside `internal.ffm`. `grep -rn + "java.lang.foreign" java/src/main/java/com/adaworldapi/lancegraph` + (excluding the `internal/ffm` subtree) must return nothing. +2. **Zero native-address-shaped values.** No public `long` parameter + or field that is secretly a pointer, lane id, or opcode. If a + numeric value crosses into public API, its Javadoc must describe it + in domain terms (a row count, a threshold) — if you can't write + that sentence, the value shouldn't be public. +3. **`View.where(...)` must not execute.** Building a predicate chain + is pure data — no downcall, no mask allocation, until a terminal + operation (`count()`, `sumOf(...)`, etc.). Check for accidental + eagerness: does constructing a `View` ever call into + `internal.ffm`? It must not. +4. **Monotonic narrowing must be structural, not a documented + convention.** `where(...)` must return a *new* `View` that can only + ever be a subset of its parent — check there is no code path + (public or accidental) that lets composition widen a `View`. +5. **No `Stream`, `List`, `Element[]`, or + `Iterator` over hydrated rows anywhere in the public surface.** + The whole point (per the thesis) is that 64K logical rows never + become 64K Java objects. A method returning `Stream` is a + direct violation regardless of how elegant it looks — flag it even + if it "would be convenient." +6. **The schema vocabulary (`Pattern.java` and friends) must be typed + per-field**, not stringly-typed. `Pattern.CLASS.gt("Berlin")` must + fail to compile, not fail at runtime. Check every field wrapper + class enforces this. +7. **Every public type crossing into "generated schema" territory + must read as something a code generator would emit** — flag + hand-written cleverness (fluent builders with unusual generics, + surprising overload resolution) that a generator couldn't + mechanically produce, because the whole accessibility story depends + on this vocabulary being generatable, not hand-crafted artistry. + +## What you are not + +You do not review FFM correctness inside `internal.ffm` (that's +`panama-bridge-engineer`'s territory) or ABI symbol shape (that's +`abi-membrane-warden`). You review only whether the public-facing +surface honors the "physics invisible, vocabulary familiar" contract. diff --git a/.claude/agents/panama-bridge-engineer.md b/.claude/agents/panama-bridge-engineer.md new file mode 100644 index 0000000..0732b07 --- /dev/null +++ b/.claude/agents/panama-bridge-engineer.md @@ -0,0 +1,75 @@ +--- +name: panama-bridge-engineer +description: > + Owns correctness of java/src/main/java/.../internal/ffm — MemoryLayout + definitions matching docs/abi.md byte-for-byte, downcall + MethodHandles resolved once and cached, the manifest cross-check at + load time, and Arena/segment lifetime discipline. Use for any change + inside internal/ffm, or when diagnosing a mismatch between Rust + struct layout and Java MemoryLayout. +tools: Read, Glob, Grep, Bash +model: sonnet +--- + +You are the PANAMA_BRIDGE_ENGINEER for lance-graph-java. Your scope is +`java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/` and its +correctness against `docs/abi.md` and the compiled `native/lgj-abi` +manifest. + +## Mission + +Make the membrane boring and correct. Every `MemoryLayout` in +`Layouts.java` must independently derive the same byte size and +alignment `docs/abi.md` documents for the corresponding `#[repr(C)]` +Rust type — and at runtime, the manifest cross-check in `Abi.java` +must prove Java's compiled-in expectation actually matches what the +loaded `.so` reports, not merely assume the doc was followed correctly +on both sides. + +## Checklist + +1. **Every `#[repr(C)]` struct in `docs/abi.md` has a matching + `MemoryLayout.structLayout(...)` in `Layouts.java`**, field for + field, same order, with explicit padding layouts wherever the Rust + struct has implicit padding (check alignment requirements — a + `u32` field after a `u64` needs no padding, but check every + transition). +2. **The manifest cross-check in `Abi.java` compares Java's + `layout.byteSize()`/`layout.byteAlignment()` against the *runtime* + values reported by `lgj_abi_manifest()`** — not against a + hardcoded Java constant. Two independently-derived numbers must + agree; a check that compares the manifest against itself, or + against a number copy-pasted from the doc, is not a real + cross-check. +3. **`abi_major` mismatch is a hard load-time failure. `abi_minor` + requires `>=` the compiled-against version, not exact match** — per + `docs/abi.md` §2. Verify both directions are tested: a too-low + minor fails, an equal-or-higher minor succeeds. +4. **Every downcall `MethodHandle` is resolved exactly once**, into a + `static final`, at class-init or explicit `Abi` initialization — + never re-resolved per call. Flag any `Linker.downcallHandle(...)` + call inside a hot-path method body. +5. **`FunctionDescriptor`s match `docs/abi.md` §7 exactly** — argument + count, order, and `ValueLayout` per argument (e.g. `u64 handle` is + `ValueLayout.JAVA_LONG`, an out-pointer is `ValueLayout.ADDRESS`). + A mismatch here is silent corruption, not a compile error — this is + the single easiest place to introduce a bug that only manifests as + garbage data. +6. **Arena/segment lifetime nests inside resource lifetime.** A + `MemorySegment` describing a native lane must become unusable (via + Java-side bookkeeping, not just relying on the native + `INVALID_HANDLE`) once the owning resource is closed — read + `docs/abi.md` §4's "belt and braces" requirement and verify the + Java side actually implements its half. +7. **`--enable-native-access` is required and documented** in + `java/README.md`'s exact command lines — verify the commands there + actually run against `/opt/jdks/jdk-26.0.2` (see + `.claude/knowledge/jdk-toolchain-facts.md`) without additional flags + beyond that one. + +## What you are not + +You do not design the public API ergonomics (`java-surface-warden`) or +the Rust-side ABI shape (`abi-membrane-warden`) — you are the engineer +making sure the two sides of an already-agreed contract actually agree +in the running code. diff --git a/.claude/agents/simd-savant.md b/.claude/agents/simd-savant.md new file mode 100644 index 0000000..84751aa --- /dev/null +++ b/.claude/agents/simd-savant.md @@ -0,0 +1,63 @@ +--- +name: simd-savant +description: > + Holds the workspace-wide invariant that all Rust-side SIMD in this + repo comes from `ndarray::simd::*` — never `ndarray::hpc::*` directly, + never raw intrinsics, never a second SIMD crate. Use BEFORE merging + any PR touching native/lgj-abi/src/kernels.rs, and PRE-SPAWN before + briefing a worker that will write any numeric kernel. +tools: Read, Glob, Grep, Bash +model: sonnet +--- + +You are the SIMD_SAVANT for lance-graph-java, adapted from lance-graph's +own `simd-savant` card for this repo's specific boundary: the Rust ABI +crate at `native/lgj-abi`, which is a *consumer* of `ndarray`, not a +SIMD implementor. + +## Mission + +Stop two specific mistakes from landing in `native/lgj-abi`: +1. Raw platform intrinsics (`core::arch::*`, `_mm*`, `vld1q_*`) written + directly in this crate instead of in `ndarray`. +2. Importing `ndarray::hpc::*` instead of the sanctioned + `ndarray::simd::*` re-export surface (see + `.claude/knowledge/simd-provenance.md` — this is an operator-locked + rule, not a style preference). + +## Primary objects + +- `native/lgj-abi/src/kernels.rs` — the ONLY file in this crate allowed + to import from `ndarray` at all. If a SIMD-shaped import appears + anywhere else in the crate, that's a violation on its own. +- `.claude/knowledge/simd-provenance.md`, `.claude/knowledge/simd-lane-width-family.md` +- `docs/abi.md` §8 ("SIMD provenance") +- `/home/user/ndarray/src/simd.rs`, `simd_ops.rs`, `simd_int_ops.rs` — + the actual re-export surface to check imports against. + +## Doctrine + +1. **`grep -rn "core::arch\|_mm[0-9]\|vld1q\|target_feature" native/lgj-abi/src` + must return nothing** outside of a single, clearly-commented cfg + block in `abi.rs` that reports (not selects) which backend compiled + — see `docs/abi.md` §5, `LgjAbiManifest::simd_backend`. That one use + is sanctioned because it reports a build fact; anything selecting + *behavior* by arch is not. +2. **`grep -rn "ndarray::hpc" native/lgj-abi/src` must return nothing.** + Every import goes through `ndarray::simd::`. +3. **If a primitive is missing from `ndarray::simd`, the fix is adding + it to `ndarray` under that repo's own W1a consumer contract** + (`ndarray/.claude/knowledge/vertical-simd-consumer-contract.md`) — + struct methods on typed wrappers, all backends (AVX-512/AVX2/NEON/ + WASM/scalar), mandatory parity test — never a local workaround in + this crate. +4. **The independent scalar reference path** (`lgj_plan_eval_scalar`, + per `docs/abi.md` §7) must be written as plain Rust loops with NO + `ndarray` dependency at all — its entire value is being independent + of the SIMD path it's meant to falsify against. Flag any scalar + reference implementation that calls into `ndarray::simd` "for + convenience." +5. **Lane-width vocabulary in doc comments must match reality** — see + `simd-lane-width-family.md`. Don't let a doc comment claim "uses + U32x16" when the actual dispatch is arch-conditional and might run + `U32x8` on this build. diff --git a/.claude/agents/valhalla-lab-scientist.md b/.claude/agents/valhalla-lab-scientist.md new file mode 100644 index 0000000..faa68e4 --- /dev/null +++ b/.claude/agents/valhalla-lab-scientist.md @@ -0,0 +1,74 @@ +--- +name: valhalla-lab-scientist +description: > + Enforces the three-truths method and measurement-before-claim + discipline for everything under valhalla-lab/ and bench/. Use BEFORE + accepting any performance or representation claim about Valhalla + value classes, and BEFORE a benchmark number from bench/ enters a doc + or README. +tools: Read, Glob, Grep, Bash +model: opus +--- + +You are the VALHALLA_LAB_SCIENTIST for lance-graph-java. Your scope is +`valhalla-lab/`, `bench/`, and any prose elsewhere in the repo that +cites a number or claim originating from either. + +## Mission + +Hold the line the mission brief draws explicitly: *"Do not claim +theoretical improvements without measurement."* Read +`.claude/knowledge/valhalla-three-truths-method.md` in full before +reviewing anything. + +## Checklist for any Valhalla or benchmark claim + +1. **Three truths, not one.** A claim about a semantic value type must + state (a) what it should mean, (b) the measured stable-Java + behavior, (c) the measured Valhalla behavior — never (c) alone + presented as if it were the whole story, and never (a) alone + presented as if it were already achieved. +2. **Every number has a reproduction command.** If `bench/` reports a + figure, there must be an exact command line (JDK path, flags, row + count) that reproduces it. A number with no command line attached + is not evidence. +3. **Cost components are kept separate, per the mission brief.** + Reject any benchmark that conflates: bare Panama downcall overhead, + `MemorySegment` read/write throughput, the bulk Rust kernel, Java + Vector API execution, fused-vs-unfused plan cost, View/plan + construction cost on the Java side alone. If a single number + purports to represent "how fast is the bridge," ask which of these + six it actually measured and whether the others were held constant. +4. **JVM warmup discipline.** Cold JVM startup vs a warmed native + kernel is not a valid comparison — verify iteration counts, + warmup-vs-measured split, and that JIT compilation had time to + settle before any timing was recorded. +5. **The Valhalla flags actually used must be stated** + (`-XX:+UnlockDiagnosticVMOptions -XX:±UseFieldFlattening + -XX:±UseArrayFlattening -XX:±InlineTypePassFieldsAsArgs`) — a claim + about "flattening" with no record of which flag combination was + active is unfalsifiable. +6. **A discovered Valhalla limitation gets a reproducer, not a + workaround baked into the API.** Per + `valhalla-three-truths-method.md`: if expressing the ideal semantic + contract hits a real Valhalla gap, the API is NOT distorted to + route around it — a minimal standalone reproducer goes in + `valhalla-lab/reproducers/` instead, naming which JDK component the + gap belongs to. +7. **The N-objects-vs-N-values-vs-1-lane experiment is present and + its numbers are real**, not asserted from the thesis's prediction. + If this experiment is missing from a PR that touches + `valhalla-lab/`, that is itself a finding to flag — it is the one + experiment the mission brief and + `john-doe-migration-thesis.md` both call out as mandatory. +8. **JMH vs hand-rolled harness is stated honestly.** If `bench/` + claims JMH-grade rigor but is actually a hand-rolled loop, that + mislabeling is itself a defect to flag, independent of whether the + numbers happen to be correct. + +## What you are not + +You do not design the Java public API (`java-surface-warden`) or the +ABI (`abi-membrane-warden`). Your entire concern is: is every claim in +`valhalla-lab/` and `bench/` actually backed by a measurement someone +else could reproduce. diff --git a/.claude/board/AGENT_LOG.md b/.claude/board/AGENT_LOG.md new file mode 100644 index 0000000..e66083f --- /dev/null +++ b/.claude/board/AGENT_LOG.md @@ -0,0 +1,84 @@ +## 2026-08-17 — session 1: archaeology (3 parallel agents) + vertical-slice fan-out (4-agent Workflow) + +**ONE-WRITER rule in effect from the start of this repo's life**: only the +orchestrating main thread appends to this file. Spawned agents leave no +board entries of their own — their reports are consolidated here. + +### Archaeology wave (3 Explore/Opus agents, parallel, read-only) + +- **ndarray SIMD/ABI surface** (Explore). Found: no C ABI/`cdylib`/`#[no_mangle]` + anywhere in ndarray today; `Fingerprint` is the closest `#[repr(C)]` + value type but not an owning/`Drop` handle; SIMD mask types + (`F32Mask16`/`F64Mask8`) exist per-lane with only a `select()` method — NO + mask intersection/union/popcount-on-mask-pairs, and no integer-lane + equality→mask at all (only float `simd_eq`/`simd_lt`/etc. exist). This is + the gap D-LGJ-B fills. Also flagged: CLAUDE.md's "Rust 1.94" line is stale + vs the actual `rust-toolchain.toml` pin of 1.97.1 (already corrected in + this repo's own toolchain choice). +- **lance-graph ClassView/mask/ABI machinery** (Explore). Found: + `WideFieldMask` already has `intersect`/`union`/`count` (chunk-zipped u64 + words) — this is the real mask algebra the mission's View/Mask/Lens + language maps onto, one layer up from lance-graph-java's own first-slice + mask. `NodeRow`/`NodeGuid`/`EdgeBlock` are `#[repr(C, align(N))]` with + compile-time size asserts (16|16|480) — the strongest existing "reuse this + exactly" candidate for a future real-graph ABI slice (deliberately NOT + wired into this session's generic-fixture-only first slice, per + `docs/abi.md` §10). `holograph/src/ffi.rs` is prior art in the *workspace* + for an opaque-handle create/free FFI pattern — informed this repo's own + registry design without being copied verbatim. Confirmed: zero prior Java/ + JNI/Panama integration attempt anywhere in lance-graph. +- **Panama FFM + Valhalla current state** (Opus, ~7 min). The decisive + finding of the session: JEP 401 (Value Classes and Objects) has ALREADY + merged into mainline JDK 28 as a preview feature; the `/home/user/valhalla` + fork (`lworld`, 2026-07-30) is measurably BEHIND mainline + `/home/user/jdk` (2026-08-17) for value-class purposes, and + `/home/user/panama-foreign`'s `java.lang.foreign` is byte-identical to + mainline. This collapsed "three JDK toolchains" down to "one GA JDK for + production + one official EA binary for the Valhalla lab," and is recorded + in `.claude/knowledge/jdk-toolchain-facts.md`. + +### Toolchain verification (orchestrator, direct execution — not delegated) + +Installed Rust 1.97.1. Downloaded and verified two JDKs by RUNNING code +against them, not by reading docs: `/opt/jdks/jdk-26.0.2` (FFM final, zero +preview flags — `Arena`/`MemorySegment`/`Linker.nativeLinker()` all +compiled+ran clean) and `/opt/jdks/jdk-27` (`27-jep401ea3+1-1`, official +JEP 401 EA — `value class`/`value record` compiled+ran, +`Class.isValue()` → `true`). Confirmed Maven Central and `jdk.java.net` +reachable via `curl --noproxy '*'`. + +### `docs/abi.md` authored (orchestrator, before any implementation) + +14 symbols, 4 `#[repr(C)]` types, 13 status codes, generation-checked `u64` +handle. Written deliberately BEFORE either Rust or Java implementation so +both sides are checked against one frozen doc rather than against each +other's in-progress code. + +### `.claude/agents` + `.claude/knowledge` + this board (orchestrator) + +6 agent cards (`abi-membrane-warden`, `simd-savant`, `handle-lifecycle-auditor`, +`java-surface-warden`, `panama-bridge-engineer`, `valhalla-lab-scientist`), +6 knowledge docs, `BOOT.md`, `README.md` — sized to this repo's actual +seams (see `.claude/agents/BOOT.md`'s "why six, not twenty"). Model policy +applied per operator directive: Sonnet for bounded-checklist agents, Opus for +`handle-lifecycle-auditor` and `valhalla-lab-scientist` (adversarial/ +multi-axis reasoning). + +### Vertical-slice fan-out dispatched (Workflow `wf_23ad2110-b1e`, 4 agents, Opus) + +Phase "Implement": 3 agents in parallel on disjoint trees — ndarray +primitives (D-LGJ-B), Rust ABI crate (D-LGJ-C), Java FFM+facade (D-LGJ-D/E). +Phase "Lab": 1 agent, sequenced after Implement, reads the real Java types +before building the Valhalla lab + bench harness (D-LGJ-F/G). **Status at +this log entry: still running.** Not yet consolidated, not yet audited for +the `ndarray::hpc` ban or the no-C rule (both stated by the operator AFTER +dispatch — see D-LGJ-AUDIT on `STATUS_BOARD.md`, which is the mandatory next +step, not optional cleanup). + +### Base case note (per this workspace's own recursion-termination convention) + +This entry is itself session-1 bootstrapping, not hygiene-for-a-prior-PR — +there is no prior PR to record hygiene for. The "hygiene rule does not +recurse" convention from `lance-graph`'s CLAUDE.md therefore does not apply +yet; it will once this session's work actually merges as a PR and a +follow-up session considers whether a board-only PR needs its own entry. diff --git a/.claude/board/CODEX_REVIEW_CHECKLIST.md b/.claude/board/CODEX_REVIEW_CHECKLIST.md new file mode 100644 index 0000000..b4b4bf7 --- /dev/null +++ b/.claude/board/CODEX_REVIEW_CHECKLIST.md @@ -0,0 +1,131 @@ +# Codex Review Checklist + +> A pre-emptive checklist for lance-graph-java worker prompts and +> main-thread commits — the medcare-rs pattern (auth/RBAC/PHI/crypto), +> adapted to this repo's actual risk surface: an FFI/ABI membrane +> between Rust and the JVM, where the failure modes are memory safety, +> undefined behavior across the boundary, and silent physics leaking +> into a "familiar Java" API that promises none of it is visible. +> +> Session 1 (2026-08-17) — no codex findings yet to cite as provenance +> (see §8). This checklist is pre-registered from the failure modes +> `docs/abi.md` and the agent cards already name, on the same logic +> medcare-rs used after its sprint-1 findings: bake the checklist into +> worker prompts BEFORE the first review round, not after. +> +> Use this checklist before pushing any PR that touches: +> - `native/lgj-abi/` (any Rust ABI/FFI code) +> - `java/src/main/java/.../internal/ffm/` (any Panama code) +> - `java/src/main/java/.../lancegraph/` (the public API surface) +> - any new `ndarray::simd` primitive consumed by this repo +> - any Valhalla or benchmark claim in `valhalla-lab/` or `bench/` +> +> The checklist is intentionally exhaustive — most items will be +> trivially satisfied. The point is the cognitive prompt: "did I +> consider this failure mode?" + +## 1. FFI / ABI memory safety (`native/lgj-abi/src/registry.rs`, `exports.rs`) + +- [ ] Every `extern "C"` function checks every `*mut`/`*const` out-parameter for null before writing through it (`NULL_ARGUMENT`, not a segfault) +- [ ] Every handle lookup validates BOTH slot-occupied AND generation-match before touching the payload — a stale handle returns `INVALID_HANDLE`, never a dereference of freed memory (see `handle-lifecycle-auditor`'s card for the exact attack list) +- [ ] Double-close returns `INVALID_HANDLE` on the second call, not UB — traced through the actual lock-acquire order, not assumed from the design doc +- [ ] A mask whose parent resource closed returns `PARENT_CLOSED` on EVERY subsequent operation, checked per-call (not cached at mask-creation time) +- [ ] No lane is ever resized, reallocated, or moved while its owning resource is alive (`docs/abi.md` §4's "no relocation" guarantee) — any growable-lane proposal needs a `major` version bump, not a quiet exception +- [ ] Every `extern "C"` function body is wrapped in `catch_unwind`; no panic can unwind into JVM frames (test this by deliberately triggering an internal panic and asserting it surfaces as a negative status, not a crash) +- [ ] Registry lock discipline: the registry-level lock is held only long enough to resolve `index → Arc` and is dropped BEFORE the entry's own inner lock is taken (no nested-lock ordering that could deadlock two concurrent calls) +- [ ] A fabricated handle (0, `u64::MAX`, an index past the registry's current length) never panics on an out-of-bounds index — bounds-checked before indexing + +## 2. No C, ever (`.claude/knowledge/no-c-ever.md`) + +- [ ] No new `.h`/`.hpp` file anywhere in the diff +- [ ] No `build.rs` invoking `cc`/`cbindgen`/`jextract` +- [ ] No `JNIEnv`, `jni::*`, or any JNI-shaped construct +- [ ] Every new/changed `extern "C"` function's cost scales with `n_rows`, or is lifecycle (open/close/describe) — a per-element crossing is the JNI anti-pattern wearing Panama's clothes, reject it regardless of how small it looks +- [ ] No string (`char*`/`CString`) crosses the boundary except the two fixed-size manifest name fields +- [ ] No callback/upcall introduced for a bulk operation + +## 3. SIMD provenance (`.claude/knowledge/simd-provenance.md`, `.claude/knowledge/simd-lane-width-family.md`) + +- [ ] `grep -rn "ndarray::hpc" native/lgj-abi/src` returns nothing — every import goes through `ndarray::simd::` +- [ ] `grep -rn "core::arch\|_mm[0-9]\|vld1q\|target_feature" native/lgj-abi/src` returns nothing outside the ONE sanctioned manifest-reporting cfg block in `abi.rs` +- [ ] `native/lgj-abi/src/kernels.rs` is the ONLY file in the crate importing from `ndarray` +- [ ] The independent scalar reference path (`lgj_plan_eval_scalar`) has zero `ndarray` dependency — plain Rust loops only, so it can actually falsify the SIMD path rather than test itself +- [ ] If a primitive was missing from `ndarray::simd`, it was added to `ndarray` under that repo's own W1a consumer contract (all backends + scalar + parity test), never worked around locally +- [ ] Doc comments naming a lane-width type (`U32x16`, etc.) match what the compiled backend actually dispatches — don't assert a width the build baseline doesn't guarantee + +## 4. Public Java API surface (`.claude/agents/java-surface-warden.md`) + +- [ ] Zero `java.lang.foreign` types (`MemorySegment`, `Arena`, `Linker`, `MethodHandle`, `FunctionDescriptor`, `MemoryLayout`, `VarHandle`) in any signature outside `internal/ffm/` +- [ ] No public `long`/numeric value that is secretly a pointer, lane id, or opcode without a domain-terms Javadoc sentence explaining what it means +- [ ] `View.where(...)` performs zero downcalls and allocates zero native masks — only a terminal operation executes +- [ ] Composition can only narrow a `View` — no code path (public or accidental) widens it +- [ ] No `Stream`/`List`/`Element[]`/`Iterator` over hydrated rows anywhere in the public surface — 64K rows must never become 64K Java objects (`.claude/knowledge/john-doe-migration-thesis.md`) +- [ ] Schema vocabulary (`Pattern.CLASS.gt(...)`, etc.) is typed per-field — a type mismatch must fail to COMPILE, not fail at runtime +- [ ] The schema vocabulary reads as something a code generator could mechanically emit — no hand-crafted fluent cleverness a generator couldn't produce + +## 5. Panama bridge correctness (`.claude/agents/panama-bridge-engineer.md`) + +- [ ] Every `MemoryLayout` in `Layouts.java` independently derives (via `layout.byteSize()`/`byteAlignment()`) the same size/alignment `docs/abi.md` documents for the matching `#[repr(C)]` Rust type — not copy-pasted from the doc as a bare constant +- [ ] The manifest cross-check in `Abi.java` compares Java's independently-derived layout sizes against the RUNTIME manifest from `lgj_abi_manifest()`, not against another Java constant +- [ ] `abi_major` mismatch fails load; `abi_minor` requires `>=` (both directions tested) +- [ ] Every downcall `MethodHandle` is resolved exactly once into a `static final` — never re-resolved inside a hot-path method +- [ ] Every `FunctionDescriptor` argument order/layout matches `docs/abi.md` §7 exactly — a mismatch here is silent data corruption, not a compile error +- [ ] `--enable-native-access` is documented in `java/README.md`'s exact command lines, and those commands were actually run against `/opt/jdks/jdk-26.0.2` + +## 6. Valhalla / benchmark claims (`.claude/agents/valhalla-lab-scientist.md`) + +- [ ] Every semantic value type claim states all three truths (semantic / stable-Java / Valhalla) — never Valhalla behavior presented alone as the whole story +- [ ] Every number in `bench/` has an exact reproduction command line (JDK path, flags, row count) +- [ ] Cost components are NOT conflated: bare downcall overhead, `MemorySegment` throughput, the Rust kernel, Vector API execution, fused-vs-unfused, View-construction cost are all reported separately +- [ ] JVM warmup is real (stated iteration counts, warmup-vs-measured split) — no cold-JVM-vs-warmed-native comparison +- [ ] A discovered Valhalla limitation produced a reproducer in `valhalla-lab/reproducers/`, not a workaround baked into the public API +- [ ] The mandatory N-objects vs N-values vs 1-lane experiment is present with real measured numbers, not the thesis's prediction asserted as fact +- [ ] JMH-vs-hand-rolled is labeled honestly — no hand-rolled loop presented as JMH-grade + +## 7. Toolchain / build discipline + +- [ ] No spawned agent ran `cargo` in any form (`.claude/knowledge/agent-cargo-hygiene.md`) — only the orchestrating main thread compiles/tests/lints +- [ ] `native/lgj-abi/.cargo/config.toml`'s `target-cpu` baseline matches what the artifact is actually meant to run on (v4/AVX-512 for this-host-only research use; v3/AVX2 if ever redistributed — see `TECH_DEBT.md` `TD-LGJ-V4-BASELINE-NOT-PORTABLE`) +- [ ] `/opt/jdks/jdk-26.0.2` used for production Java, `/opt/jdks/jdk-27` (JEP 401 EA) used ONLY for `valhalla-lab/` — no `--enable-preview`-compiled class ever reaches the production `java/` tree +- [ ] `df -h /` checked before and after any large parallel dispatch — target-dir residue is a known risk this session (`ISS-LGJ-TARGET-DIR-SIZE-WATCH`) + +## 8. PR hygiene + +- [ ] Commit message body explains the WHY, not just the WHAT +- [ ] PR body includes a Test Plan with checkboxes, and states which falsification gates from `.claude/plans/lgj-vertical-slice-v1.md` were actually run (not just "should pass") +- [ ] Cross-references to `EPIPHANIES.md`/`TECH_DEBT.md`/`ISSUES.md` entries where relevant +- [ ] Board files updated in the SAME PR as the code they describe (`LATEST_STATE.md`, `STATUS_BOARD.md`) — per this workspace's own board-hygiene convention, a PR that ships a type/plan/finding without updating the board is incomplete + +--- + +## How to use this checklist + +**As a worker writing code:** open this file alongside your scratchpad. +Tick boxes as you go. Boxes you can't tick — either fix the issue or +document the deferral in the PR body. + +**As main-thread reviewing a diff:** scan §1-§6 relevant to the touched +code. Items not addressed — request changes before merge. + +**As a future session:** this checklist is pre-registered from this +repo's own stated design risks, not yet from a real codex finding — +unlike medcare-rs's checklist, which was written AFTER two P1/P2 +catches. Add new items the moment a real review (codex or otherwise) +catches something that should have been pre-emptive, and update the +provenance table below. + +--- + +## Provenance + +| Source | Cross-reference | +|---|---| +| `docs/abi.md` §4 (ownership/handle safety) | this file §1 | +| `docs/abi.md` §1, `.claude/knowledge/no-c-ever.md` | this file §2 | +| `.claude/knowledge/simd-provenance.md`, `simd-lane-width-family.md` | this file §3 | +| `.claude/knowledge/john-doe-migration-thesis.md` | this file §4 | +| `docs/abi.md` §5, §7 | this file §5 | +| `.claude/knowledge/valhalla-three-truths-method.md` | this file §6 | +| `.claude/knowledge/agent-cargo-hygiene.md`, `EPIPHANIES.md` `E-LGJ-V4-DIVERGES-FROM-NDARRAY-DEFAULT-1` | this file §7 | +| medcare-rs `.claude/board/CODEX_REVIEW_CHECKLIST.md` (the pattern this file adapts) | structural template only — domain content is unrelated (no PHI/RBAC/crypto in this repo) | +| No codex findings yet — session 1 | update this row the first time one lands | diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md new file mode 100644 index 0000000..05eeae2 --- /dev/null +++ b/.claude/board/EPIPHANIES.md @@ -0,0 +1,141 @@ +# Epiphanies Log — Findings, Corrections, "Aha" Moments (APPEND-ONLY) + +> Prepend new entries at the top. Never edit a past entry except its +> `**Status:**`/`**Confidence:**` line. A correction gets its own new, +> dated entry that references the one it corrects — the storno rule. + +## 2026-08-17 — E-LGJ-CORE-SLICE-GREEN-DISABLE-VERIFIED-1 + +**Status:** FINDING. **Confidence:** High (measured, not asserted — every +number below came from an actual command run, not from an agent's report). + +The core vertical slice (`docs/abi.md` + `native/lgj-abi` + `java/`) is +real, compiles clean, and its safety claims are not merely tested but +**disable-verified**: `registry.rs::resolve`'s generation check +(`slot.generation != gen`) was deliberately short-circuited to +`if false && ...`, and the suite re-run. Exactly the two tests whose names +claim to guard this property — +`a_reused_slot_invalidates_the_old_handle` and +`fabricated_handles_are_rejected_not_dereferenced` — went red; all other +70 stayed green. This is the falsifiability discipline this workspace's +sibling repos (tesseract-rs, MedCare-rs) both independently arrived at — +"a test that passes on the happy path is not evidence" — applied to this +repo's very first disable-verification, and it passed the meta-test: the +tests were real, not decorative. + +**The one real rule violation the D-LGJ-AUDIT sweep found**: +`native/lgj-abi/src/kernels.rs::simd_popcount` called +`ndarray::hpc::bitwise::popcount_batch_u64` directly — the exact pattern +`E-LGJ-SIMD-PROVENANCE-1` exists to forbid. This is worth recording as a +finding in its own right: **the rule was stated correctly in the agent's +own doc comment one line above the violation** ("Reused, not +reimplemented — this already exists in `ndarray`...") — the agent +correctly identified WHERE the function lived but reached for the +internal path it happened to see in ndarray's source rather than the +re-export it was told to prefer. A soft "verify the exact path" brief +instruction was not sufficient; a mechanical grep gate is what actually +caught it. **Consequence for future briefs in this repo:** soft +instructions ("prefer X") get a mechanical audit regardless of how +clearly they were stated — this is now standing practice, not a +one-time fix. + +**Numbers on record**, so a future session can spot-check rather than +re-run everything from scratch: Rust `cargo test` 72/72; `ndarray` +`simd_int_ops` tests 41/41; `clippy -D warnings` and `fmt --check` both +clean; release build exports exactly the 14 symbols `docs/abi.md` §7 +names; Java `javac -Xlint:all` produces exactly 7 `[restricted]` +warnings, all in `internal/ffm/*` or one test deliberately exercising it; +`AllTests` 132/132 across 8 suites. Full breakdown on `STATUS_BOARD.md`'s +D-LGJ-B/C/D/E rows. + +## 2026-08-17 — E-LGJ-V4-DIVERGES-FROM-NDARRAY-DEFAULT-1 + +**Status:** FINDING. **Confidence:** High (operator-directed, mechanically +applied). + +`native/lgj-abi/.cargo/config.toml` pins `-Ctarget-cpu=x86-64-v4` +(AVX-512), **deliberately diverging** from `/home/user/ndarray`'s own +default of `-Ctarget-cpu=x86-64-v3` (AVX2). This is not a mistake to +reconcile later — the two repos have different distribution goals: ndarray +targets portable redistribution (v3 = Haswell-and-later, ~2013+), while +`lance-graph-java`'s native artifact in this phase is built and run on one +known host (verified AVX-512-capable this session) for a research vertical +slice, not shipped broadly. The `LgjAbiManifest::simd_backend` field is +what makes this divergence self-documenting at runtime rather than a +silent assumption — a consumer reads the manifest rather than assuming +which tier compiled. + +**Consequence:** any future portable-distribution build of `lgj-abi` must +override with `CARGO_BUILD_RUSTFLAGS='-Ctarget-cpu=x86-64-v3'` at build +time, per the comment left in `.cargo/config.toml`. Do not silently change +the file's *default* back to v3 without a stated reason — v4 is the +deliberate choice for this phase. + +## 2026-08-17 — E-LGJ-VALHALLA-ALREADY-MAINLINE-1 + +**Status:** FINDING. **Confidence:** High (measured by direct `diff -rq` +across three local checkouts + live compile/run verification). + +JEP 401 (Value Classes and Objects) has **already integrated into mainline +JDK** as a preview feature — it is not exclusive to a separate Valhalla +fork. Measured this session: `/home/user/valhalla` (`lworld` branch, +2026-07-30 HEAD) is **behind** mainline `/home/user/jdk` (2026-08-17 HEAD) +for value-class purposes; its own last relevant commit is literally +*"[lworld] things to delete from lworld just before integrating JEP-401."* +`/home/user/panama-foreign`'s `java.lang.foreign` package is **byte- +identical** to mainline (`diff -rq` exit 0). + +**Consequence:** this project needs exactly ONE production JDK (a GA +build, verified this session as `/opt/jdks/jdk-26.0.2`, where FFM is +final) and ONE Valhalla-preview JDK (the *official EA binary* +`27-jep401ea3+1-1` from `jdk.java.net/valhalla/`, not a source build of any +local fork). Building any of the three local OpenJDK source checkouts from +source for this project would have cost real time for zero benefit — the +binary already exists and was verified to work. See +`.claude/knowledge/jdk-toolchain-facts.md` for the full toolchain matrix. + +**Corollary, stated so a future session doesn't re-litigate it:** null- +restricted *type* syntax (`Foo!`) and specialized generics do **not** +exist in any checkout verified this session — only the internal +`@jdk.internal.vm.annotation.NullRestricted` field annotation plus +`jdk.internal.value.ValueClass` factories, gated behind `--add-exports`. +Do not assume `Foo!` syntax is available; it measurably is not, as of this +session's verification. + +## 2026-08-17 — E-LGJ-NO-C-EVER-1 + +**Status:** RULE (operator directive, locked). **Confidence:** N/A — +founding constraint, not a discovered fact. + +Operator, verbatim: *"There's no C ever. We reuse Panama project for a +rust only."* `extern "C"` names the SysV AMD64 psABI (a platform calling +convention), not the C language; `#[repr(C)]` names a platform aggregate +layout rule, not a C struct. Consequence: no `.h` file, no `cbindgen`, no +`jextract` (structurally inapplicable — its only input is a C header, and +none exists), no JNI, anywhere in this repo, ever. Full statement: +`.claude/knowledge/no-c-ever.md`. This is the single most load-bearing +rule in the project and the one most likely to be violated by habit +(reaching for `jextract` because "that's how Panama projects usually +work") rather than by disagreement — flagged here so it is checked +mechanically (`abi-membrane-warden`'s doctrine item 1) rather than trusted +to memory. + +## 2026-08-17 — E-LGJ-SIMD-PROVENANCE-1 + +**Status:** RULE (operator directive, locked). **Confidence:** N/A — +founding constraint. + +Operator, verbatim: *"Never use ndarray::hpc, trampoline to +ndarray::simd::* instead."* `ndarray::hpc::*` is ndarray's internal +implementation namespace; `ndarray::simd::*` is the sanctioned re-export +surface every consumer in the Ada stack is expected to use (ndarray's own +CLAUDE.md: *"Consumer writes `crate::simd::F32x16`. Period."*). This repo +is one more consumer of that invariant, not an exception. Full statement +and falsifier grep: `.claude/knowledge/simd-provenance.md`. This directive +arrived AFTER the first vertical-slice fan-out was already dispatched +(whose briefs mentioned the popcount primitive via its `hpc::bitwise` +path, with an instruction to "verify the exact path" and prefer the +`simd` re-export) — `STATUS_BOARD.md`'s `D-LGJ-AUDIT` entry exists +specifically to mechanically check the fan-out's actual output against +this rule rather than assume the earlier, softer brief language was +sufficient. diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md new file mode 100644 index 0000000..4419559 --- /dev/null +++ b/.claude/board/INTEGRATION_PLANS.md @@ -0,0 +1,24 @@ +## 2026-08-17 — lgj-vertical-slice-v1 (PLAN; the first Panama×Valhalla×ndarray::simd proof) + +Plan: `.claude/plans/lgj-vertical-slice-v1.md`. Active plan index — this +board file is APPEND-ONLY (prepend new plan entries; a superseding plan +gets its own new entry, the old one's `**Status:**` line updates in place, +nothing else about it changes). + +**What it covers:** the full first vertical slice — the normative +`docs/abi.md` contract, the `ndarray::simd` primitives it needs, the Rust +ABI crate (`native/lgj-abi`), the Java FFM membrane + public semantic +facade (`java/`), the Valhalla three-truths lab (`valhalla-lab/`), and the +cost-separated benchmark harness (`bench/`). Maps 1:1 to the D-ids on +`STATUS_BOARD.md`. + +**Sequencing decision on record:** B (ndarray primitives) / C (Rust ABI) / +D (Java FFM+facade) were fanned out in **parallel** as three disjoint +trees, checked independently against the one frozen `docs/abi.md` contract +rather than sequentially against each other. F/G (Valhalla lab + bench) +were sequenced **after** D specifically so they could read the real +`View`/`Predicate` Java types instead of guessing their shape. + +**Status: ACTIVE.** No PR opened yet. See `STATUS_BOARD.md` for +per-deliverable status and `AGENT_LOG.md` for what each spawned agent +actually reported. diff --git a/.claude/board/ISSUES.md b/.claude/board/ISSUES.md new file mode 100644 index 0000000..8367db4 --- /dev/null +++ b/.claude/board/ISSUES.md @@ -0,0 +1,65 @@ +# Issues Log — Open + Resolved (double-entry, append-only) + +## ISS-LGJ-FANOUT-UNREVIEWED (2026-08-17) — PARTIALLY RESOLVED + +**Resolution for the core (Rust ABI + Java facade), same day:** all three +closing conditions this entry names ran for real. (1) D-LGJ-AUDIT ran, one +violation found and fixed (see `EPIPHANIES.md` +`E-LGJ-CORE-SLICE-GREEN-DISABLE-VERIFIED-1`). (2) The registry's safety +claims were disable-verified, not just read — see the same epiphany entry. +(3) Central `cargo test`/`clippy`/`fmt`/`build --release` and Java +`javac`/`AllTests` all ran orchestrator-side and are green. Folded into +`LATEST_STATE.md`'s 2026-08-17 entry and `STATUS_BOARD.md`'s D-LGJ-B/C/D/E/H +rows. + +**Still open for the Lab phase (D-LGJ-F/G):** `valhalla-lab/` and `bench/` +had not produced source at the time PR #1 was cut — real JMH jars were +fetched but no `.java` written yet. This half of the original concern +remains OPEN and will close when the Lab agent's output gets the same +audit + central-verification treatment as the core got here, in its own +follow-up PR. + +**Original text, kept for context:** + +The 4-agent vertical-slice fan-out (`wf_23ad2110-b1e`: ndarray primitives, +Rust ABI crate, Java FFM+facade, Valhalla lab+bench) was still running as +of this board's initial population. Nothing it produces should be cited as +"shipped" or "done" until: (1) the mechanical audit for the `ndarray::hpc` +ban and the no-C rule runs clean (`D-LGJ-AUDIT` on `STATUS_BOARD.md`), (2) +`handle-lifecycle-auditor` has adversarially exercised the registry's +safety claims per its own card (not just read the design doc), (3) a +central `cargo build`/`test`/`clippy` run by the orchestrator — not by any +spawned agent, per `TECH_DEBT.md`'s cargo-hygiene entry — actually passes. +Closes when all three have run and their results are folded into +`LATEST_STATE.md`. + +## ISS-LGJ-TARGET-DIR-SIZE-WATCH (2026-08-17) — OPEN + +`/home/user/ndarray/target` measured 2.7 GB and +`/home/user/lance-graph-java/target` measured 602 MB partway through the +first fan-out, from agents that were (at the time) permitted to run cargo +directly. Per `.claude/knowledge/agent-cargo-hygiene.md`, no further agent +gets that permission — but the two `target/` dirs from THIS wave already +exist and should be checked against disk headroom (`df -h /` was 43% used +at last check, 22G free) before any further large parallel work is +dispatched. Not urgent at last measurement; filed so it's watched rather +than rediscovered as a surprise "no space left on device" failure. + +## ISS-LGJ-DEV-BRANCH-STILL-UNCOMMITTED (2026-08-17) — OPEN + +`claude/lance-graph-java-panama-valhalla-sus9w8` (the designated dev +branch per the mission's cross-repo branch instructions) has zero commits +as of this board's initial population — everything from `docs/abi.md` +through the `.claude/` ensemble through the fan-out's in-progress output +exists only in the working tree. `main` was separately bootstrapped (see +`LATEST_STATE.md`) with a minimal README+`.gitignore` commit specifically +so it wouldn't be blocked on the dev branch's review status. The dev +branch's first commit should happen once the fan-out is reviewed +(`ISS-LGJ-FANOUT-UNREVIEWED`) — committing unreviewed, potentially rule- +violating code first and fixing it in a second commit is avoidable by +sequencing the audit first. + +**RESOLVED same day.** Audit ran first, as planned; the one real +violation was fixed BEFORE this commit rather than after. First commit on +the dev branch lands in the same action as this board update, containing +already-audited, already-green code. diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md new file mode 100644 index 0000000..06b6fe5 --- /dev/null +++ b/.claude/board/LATEST_STATE.md @@ -0,0 +1,148 @@ +## 2026-08-17 — D-LGJ-AUDIT complete, core vertical slice VERIFIED GREEN, PR #1 opened + +### Current Contract Inventory — the vertical slice is real and green + +- **`D-LGJ-AUDIT` ran.** Mechanical grep sweep against `no-c-ever.md` and + `simd-provenance.md` found exactly **one** real violation: + `native/lgj-abi/src/kernels.rs::simd_popcount` called + `ndarray::hpc::bitwise::popcount_batch_u64` directly instead of the + sanctioned `ndarray::simd::popcount_batch_u64` re-export. Fixed in place + (same function, same behavior, corrected import path + doc comment). + Everything else the grep matched (`abi.rs`'s one sanctioned + `target_feature` cfg block for manifest self-reporting; every + `cbindgen`/`jextract` hit in doc comments, README, and test assertion + strings) was confirmed to be exactly what it should be — prose + explaining the rule, or the one deliberate exception the rule itself + names. `kernels.rs` confirmed the sole `ndarray`-importing file. +- **Rust (`native/lgj-abi`) — orchestrator-run, centrally, per + `agent-cargo-hygiene.md`:** `cargo test` → **72/72 passed**. `cargo + clippy --all-targets -- -D warnings` → clean. `cargo fmt --check` → + clean. `cargo build --release` → `liblgj_abi.so`, **exactly the 14 + symbols** `docs/abi.md` §7 specifies, verified via `nm -D`. +- **Disable-verified, not just green** — the registry's core safety + check (`registry.rs::resolve`'s `slot.generation != gen` comparison) + was deliberately short-circuited to `if false && ...` and the suite + re-run: exactly the two tests that should catch it + (`a_reused_slot_invalidates_the_old_handle`, + `fabricated_handles_are_rejected_not_dereferenced`) went **red**, all + 70 others stayed green. Restored, re-verified 72/72. This is the + `handle-lifecycle-auditor` discipline actually applied, not merely + read from the design doc. +- **Java (`java/`) — orchestrator-compiled and run against the real + `.so`, JDK 26 GA:** `javac -Xlint:all` → 7 `[restricted]` warnings, all + in `internal/ffm/*` or a test deliberately exercising the restricted + API — the exact set the design predicts, nothing outside it. + `AllTests` → **132/132 checks passed, 0 failed**, across + `ApiSurfaceTest` (reflection-enforced: zero FFM types in any public + signature), `AbiContractTest` (manifest cross-check genuinely rejects + a wrong library), `SmokeTest`, `FixtureParityTest` (30 checks, Java + independently recomputes expected counts from the transcribed + SplitMix64 generator), `FusionParityTest` (fused/unfused/scalar agree + bit-for-bit across 6 row-count shapes incl. 1/63/64/65), + `LazinessTest` (empirically proves: building a 16-condition chain + costs 0 crossings; a terminal op costs exactly 1, independent of row + count up to 1,000,000 — the thesis's central claim, measured, not + asserted), `NarrowingTest`, `LifetimeTest` (23 checks: use-after-close, + double-close, child-outlives-parent, parent-outlives-child, all as + clean exceptions, never a crash). +- **`.gitignore` added** (target dirs, `.class`, downloaded jars/tarballs) + before this commit — `bench/lib/*.jar` (real JMH, fetched by the Lab + agent) is excluded from version control by design. + +### PR #1 scope — the core slice, Lab phase deliberately deferred + +`claude/lance-graph-java-panama-valhalla-sus9w8` → `main`. Ships: the +frozen `docs/abi.md` contract, the `.claude/` ensemble+board, the 5 +`ndarray::simd` primitives, `native/lgj-abi`, and `java/` +(D-LGJ-A/ABI/ENS/B/C/D/E). **Does NOT ship** `valhalla-lab/`/`bench/` +(D-LGJ-F/G) — still in flight at commit time (JMH jars fetched, no +source yet) — nor Phase I docs, which are sequenced to synthesize the +Lab results. Both are tracked as open `STATUS_BOARD.md` rows, not +silently dropped. Given the core slice is independently complete, +fully falsified, and green, shipping it now rather than blocking on a +slower background phase is the honest call — the alternative is +holding fully-verified, working code in an uncommitted working tree +for no safety reason. + +### Toolchains pinned this session (superseded lines below kept for +### history; nothing here changed) + +--- + +## 2026-08-17 — session 1: contract frozen, ensemble seeded, vertical-slice fan-out dispatched + +### Current Contract Inventory — 1 new normative doc, 0 shipped Rust/Java code yet (in flight) + +- **`docs/abi.md`** — the normative Rust↔Java ABI contract. 14 `extern "C"` + symbols (`lgj_abi_manifest`, `lgj_pattern_open`, `lgj_close`, + `lgj_resource_info`, `lgj_lane_describe`, `lgj_mask_create/describe/and/or/count`, + `lgj_op_eq_u32`, `lgj_op_gt_i32`, `lgj_plan_eval`, `lgj_plan_eval_scalar`, + `lgj_reduce_sum_i32`), 4 `#[repr(C)]` types (`LgjLaneDesc` 56B, + `LgjResourceInfo` 32B, `LgjOpDesc` 24B, `LgjAbiManifest`), 13 status + codes, a generation-checked `u64` handle (`generation:u32 << 32 | index:u32`). + Written BEFORE either implementation side, so Rust and Java are being built + independently against it — no side has authority to redefine it unilaterally + without a doc update in the same PR (per `abi-membrane-warden`'s doctrine). +- **No C anywhere, by design, not by oversight.** `extern "C"` = SysV AMD64 + psABI, not the C language. No `.h`, no `cbindgen`, no `jextract`, no JNI. + See `.claude/knowledge/no-c-ever.md`. +- **`.claude/agents/` (6 cards) + `.claude/knowledge/` (6 docs) + this + board.** Sized to the repo's actual seams, not padded to lance-graph's + 26-repo scale — see `.claude/agents/BOOT.md`. + +### Toolchains pinned this session (verified by direct execution, not assumed) + +- **Rust: 1.97.1 stable**, installed to match `ndarray`/`lance-graph`'s pin + (operator-directed switch from an earlier draft's 1.94 target). +- **Production JDK: `/opt/jdks/jdk-26.0.2`** (GA, downloaded this session). + FFM (`java.lang.foreign`) is FINAL here — no `--enable-preview`, only + `--enable-native-access`. Confirmed live: `Arena.ofConfined()` + + `MemorySegment` set/get + `Linker.nativeLinker()` → `SysVx64Linker`, zero + flags beyond native-access. +- **Valhalla lab JDK: `/opt/jdks/jdk-27`** (`27-jep401ea3+1-1`, the OFFICIAL + JEP 401 early-access binary from `jdk.java.net/valhalla/` — not a source + build). Confirmed live: `value class`/`value record` compile and run with + `--enable-preview --release 27`; `Class.isValue()` → `true`. +- **The three local OpenJDK source forks + (`/home/user/jdk`, `/home/user/valhalla`, `/home/user/panama-foreign`) + are NOT used for building anything** — an archaeology pass found Valhalla's + `lworld` fork measurably BEHIND mainline `/home/user/jdk` for value-class + purposes (its own last commit: "things to delete from lworld just before + integrating JEP-401"), and `panama-foreign`'s `java.lang.foreign` is + byte-identical to mainline. See `.claude/knowledge/jdk-toolchain-facts.md`. +- Host CPU has AVX-512 (`avx512f/bw/cd/dq/ifma/vbmi/vl`) — both + `ndarray::simd`'s AVX2 (v3, default build baseline) and AVX-512 (v4) tiers + are exercisable in this environment. + +### Active branches + +Single branch: `claude/lance-graph-java-panama-valhalla-sus9w8` across all +13 in-scope repos per the mission's cross-repo branch instructions. Nothing +committed yet in `lance-graph-java` — this entry describes working-tree +state, not a merged PR (there is no PR_ARC_INVENTORY entry yet; see that +file for why). + +### In flight (dispatched, not yet landed — do not cite as done) + +A 4-agent fan-out (`lgj-vertical-slice-wf_23ad2110-b1e`) is running: +1. Adds `eq_u32_to_mask`/`gt_i32_to_mask`/`mask_and`/`mask_or` + (`_assign` variants)/`masked_sum_i32` to `AdaWorldAPI/ndarray` under its + own W1a consumer contract, re-exported via `ndarray::simd`. +2. Builds `native/lgj-abi` (the Rust ABI crate) against `docs/abi.md`. +3. Builds `java/` (FFM membrane + public facade) against `docs/abi.md`. +4. (sequenced after 3) Builds `valhalla-lab/` + `bench/` against the real + Java types from step 3. + +**None of steps 1–4 has been reviewed yet.** In particular the +`ndarray::hpc` import ban (`.claude/knowledge/simd-provenance.md`) and the +no-C rule have NOT been mechanically audited against the agents' actual +output — that audit is the very next action once the workflow completes. +Treat `native/lgj-abi/{Cargo.toml,Cargo.lock,rust-toolchain.toml,src/}` +existing on disk as "in progress," not "shipped." + +### Queued work (not yet dispatched) + +Phase H (falsification-test review by `handle-lifecycle-auditor`), Phase I +(docs: `architecture.md`/`panama.md`/`valhalla-lab.md`/`execution-boundary.md`), +and the post-fan-out mechanical audit for `ndarray::hpc`/C-artifact +violations across all four agents' output. diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md new file mode 100644 index 0000000..3f5a879 --- /dev/null +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -0,0 +1,16 @@ +# PR Arc Inventory — per-PR Added / Locked / Deferred / Docs / Confidence +# (reverse chronological, APPEND-ONLY; only the Confidence line is +# updatable in place — corrections append as new dated lines; reversals +# get their own PR entry) + +_No PR has been opened against this repository yet — session 1, 2026-08-17._ + +The first entry in this file will be written when the first PR against +`lance-graph-java` merges (expected: the vertical slice on +`claude/lance-graph-java-panama-valhalla-sus9w8` → `main`, once +`ISS-LGJ-FANOUT-UNREVIEWED` closes). Until then, ground truth for +in-progress work lives on `LATEST_STATE.md` (current contract inventory), +`STATUS_BOARD.md` (per-D-id status), and `AGENT_LOG.md` (what actually +happened) — this file stays empty rather than backfilled with a +pre-registration entry that would misrepresent something as merged before +it is. diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md new file mode 100644 index 0000000..112599f --- /dev/null +++ b/.claude/board/STATUS_BOARD.md @@ -0,0 +1,30 @@ +## lgj-vertical-slice-v1 — the first Panama×Valhalla×ndarray::simd proof (PRE-REGISTERED 2026-08-17) + +Plan: `.claude/plans/lgj-vertical-slice-v1.md`. Every D-id below maps 1:1 to +a phase in that plan and to a Phase-tracking task in this session's task +list. + +| D-id | Deliverable | Status | Feeds | +|---|---|---|---| +| D-LGJ-A | Archaeology: ndarray SIMD/mask surface, lance-graph ClassView/WideFieldMask/SoaEnvelope/ownership, current Panama+Valhalla state | **DONE 2026-08-17** — 3 parallel Explore/Opus agents, findings folded into `docs/abi.md` and the `.claude/knowledge/*` docs | everything downstream | +| D-LGJ-ABI | `docs/abi.md` — the normative Rust↔Java contract | **DONE 2026-08-17** — 14 symbols, 4 `#[repr(C)]` types, 13 status codes, generation-checked handle | B, C, D, E | +| D-LGJ-ENS | `.claude/agents` (6 cards) + `.claude/knowledge` (6 docs) + `.claude/board` | **DONE 2026-08-17** — this board | every future review pass | +| D-LGJ-B | `ndarray::simd` primitives: `eq_u32_to_mask`, `gt_i32_to_mask`, `mask_and`/`mask_or`(`_assign`), `masked_sum_i32` | **DONE 2026-08-17** — `ndarray/src/simd_int_ops.rs`; `cargo test --lib simd_int_ops` **41/41** incl. signed-vs-bitwise `gt_i32`, tail-bit-zeroing, `u32::MAX` edge cases | C's kernels.rs | +| D-LGJ-C | `native/lgj-abi` — manifest, generation-checked registry, generic SoA fixture, kernels, `extern "C"` surface | **DONE 2026-08-17** — `cargo test` **72/72**, `clippy -D warnings` clean, `fmt --check` clean, release build → 14/14 symbols verified via `nm -D`. **Disable-verified**: the registry's generation check was short-circuited and exactly the 2 tests that should catch it went red, 70 stayed green; restored, re-verified 72/72 | D, H | +| D-LGJ-D | Java FFM membrane `internal/ffm` | **DONE 2026-08-17** — compiles clean with `-Xlint:all`; 7 `[restricted]` warnings, all in `internal/ffm/*` or a test deliberately exercising it; `AbiContractTest` 7/7 incl. proving the manifest cross-check genuinely rejects a wrong `.so` (`libz.so.1` loads but is refused for exporting no `lgj_abi_manifest`) | E | +| D-LGJ-E | Java public facade (`NativePattern`/`View`/`Predicate`/`Pattern`/`Mask`) | **DONE 2026-08-17** — `AllTests` **132/132**: `ApiSurfaceTest` (reflection-enforced zero-FFM-leakage), `SmokeTest` 14/14, `FixtureParityTest` 30/30 (Java independently recomputes expected counts from the transcribed generator), `FusionParityTest` 31/31 (fused/unfused/scalar bit-identical across 6 row-count shapes), `LazinessTest` 8/8 (empirically: 0 crossings to build a 16-condition chain, exactly 1 to evaluate it, independent of rows up to 1,000,000 — the thesis's central claim, measured), `NarrowingTest` 16/16, `LifetimeTest` 23/23 | F, G | +| D-LGJ-F | Valhalla lab — three-truths method on the small semantic value vocabulary | **In flight** — sequenced after E, now reading the real Java types; deferred to a follow-up PR, not blocking PR #1 | I | +| D-LGJ-G | Java Vector API comparative bench vs Panama→`ndarray::simd` | **In flight** — real JMH jars fetched (`jmh-core`/`jmh-generator-annprocess`/`jopt-simple`/`commons-math3`) to `bench/lib/` (gitignored); no bench source written yet; deferred to the same follow-up PR as F | I | +| D-LGJ-H | Falsification: handle lifecycle (adversarial), SIMD/scalar parity, Java/native parity | **DONE 2026-08-17 for the Rust+Java core** — see D-LGJ-C's disable-verification and D-LGJ-E's `FusionParityTest`/`LifetimeTest`. Re-opens for F/G once the Lab lands | I | +| D-LGJ-I | Docs: `architecture.md`, `panama.md`, `valhalla-lab.md`, `execution-boundary.md` | **Queued** — gated on F/G landing (the docs synthesize Lab results, not just the core) | — | +| D-LGJ-AUDIT | Mechanical post-fan-out audit: `grep` for `ndarray::hpc` imports, any `.h`/`cbindgen`/`jextract` artifact, any FFM type leaking into public Java API | **DONE 2026-08-17** — 1 real violation found (`kernels.rs::simd_popcount` used the internal `ndarray::hpc::bitwise` path), fixed in place; everything else confirmed to be the one sanctioned exception or explanatory prose | closed D-LGJ-C/D/E for the core | + +### Reading this table + +**"In flight" means dispatched to a background agent, not reviewed and not +verified. "DONE" means a disable-verified test or a mechanical grep gate +actually ran** — per this workspace's own falsifiability discipline. D-LGJ-F +and D-LGJ-G are the only rows still open; they are deliberately NOT blocking +PR #1 (the core slice is independently complete and green) and will land as +their own PR once the Lab agent finishes and is reviewed with the same +rigor. diff --git a/.claude/board/TECH_DEBT.md b/.claude/board/TECH_DEBT.md new file mode 100644 index 0000000..723361c --- /dev/null +++ b/.claude/board/TECH_DEBT.md @@ -0,0 +1,54 @@ +# Technical Debt Log — Open + Paid (double-entry, append-only) + +## TD-LGJ-REGISTRY-CONCURRENCY-UNMEASURED (2026-08-17) — OPEN + +`docs/abi.md` §4's registry design (short registry read-lock → clone `Arc` +→ drop registry lock → lock the entry) is a **stated design intent**, not a +measured property. The POC's Java layer is single-threaded, so nothing in +this vertical slice actually exercises concurrent access. If a future slice +adds concurrent Java callers (e.g. a parallel `View` evaluation), the +registry's actual behavior under contention (lock-hold duration, +starvation, whether the "distinct resources don't serialize" claim holds +under real load) needs to be benchmarked before it's trusted, not merely +re-read from the doc comment. Pay this down when concurrent access is +first actually needed — not before, per the "no speculative future-proofing" +house style. + +## TD-LGJ-JMH-AVAILABILITY-UNKNOWN (2026-08-17) — OPEN + +`bench/` was briefed to attempt fetching real JMH jars from Maven Central +(`repo1.maven.org`, confirmed reachable via `curl --noproxy '*'` this +session) and fall back to a hand-rolled, honestly-labeled harness if that +doesn't work cleanly with plain `javac`/`java -cp` (no Maven/Gradle +dependency resolution available for this no-build-system project). Which +path was actually taken is unknown until the fan-out's Lab-phase report +lands. If JMH could not be wired, `TECH_DEBT` should gain a follow-up +entry noting the specific blocker (classpath assembly by hand for JMH's +annotation processor is real friction) so a future session doesn't +re-attempt the same dead end without the context. + +## TD-LGJ-V4-BASELINE-NOT-PORTABLE (2026-08-17) — OPEN, INTENTIONAL + +`native/lgj-abi` compiles with `-Ctarget-cpu=x86-64-v4` (AVX-512) as its +DEFAULT baseline (see `EPIPHANIES.md` `E-LGJ-V4-DIVERGES-FROM-NDARRAY-DEFAULT-1` +for the reasoning). This means the built `.so` will SIGILL on any host +without AVX-512 — acceptable for a research vertical slice on one known +host, unacceptable for any future redistribution. If this project ever +ships a `.so` to unknown hardware, the build must switch to v3 (or add +runtime-dispatch, which `ndarray` already supports via its +`runtime-dispatch` feature — see the ndarray archaeology findings in +`AGENT_LOG.md`) before that ships. Filed as debt now specifically so it +isn't silently forgotten once the v4 default stops being obviously +research-only. + +## TD-LGJ-FAN-OUT-PREDATED-TWO-RULES (2026-08-17) — OPEN, gates D-LGJ-AUDIT + +The first vertical-slice fan-out (`wf_23ad2110-b1e`) was dispatched before +`E-LGJ-NO-C-EVER-1` and `E-LGJ-SIMD-PROVENANCE-1` were stated as explicit +operator rules (though both were already implicit in `docs/abi.md`'s own +text, written before dispatch). This is not assumed to be a violation — +it's an unaudited state. `STATUS_BOARD.md`'s `D-LGJ-AUDIT` is the paydown +step: a mechanical grep sweep of the fan-out's actual output the moment it +completes. This entry closes only when that audit runs and either finds +nothing (debt paid, entry closes clean) or finds violations (debt paid via +the fix, entry closes noting what was corrected). diff --git a/.claude/knowledge/abi-ownership-and-handles.md b/.claude/knowledge/abi-ownership-and-handles.md new file mode 100644 index 0000000..460617b --- /dev/null +++ b/.claude/knowledge/abi-ownership-and-handles.md @@ -0,0 +1,68 @@ +# Ownership Across the Membrane — The Generation-Checked Handle + +> READ BY: handle-lifecycle-auditor, panama-bridge-engineer, and anyone +> touching native/lgj-abi/src/registry.rs or java/internal/ffm/* + +## Status: FINDING (the design's answer to "who owns this memory") + +## The Problem This Solves + +Inside Rust, `&self` borrows make a view-outliving-its-owner a *compile +error*. Panama has no borrow checker — a Java `MemorySegment` obtained from a +now-freed Rust allocation is a live footgun unless the membrane itself makes +use-after-free structurally impossible. + +## The Design + +A handle is **not a pointer**. It is an opaque `u64`: + +``` + 63 32 31 0 +┌────────────────────────┬────────────────────────┐ +│ generation │ index │ +└────────────────────────┴────────────────────────┘ +``` + +- `index` selects a slot in a Rust-side registry (`RwLock>`). +- `generation` is bumped every time a slot is freed. +- A lookup validates `generation` against the slot's *current* generation. + +Consequence table (all four are correctness properties this repo's tests +must falsify, not just assume — see `docs/abi.md` §4 and the Phase H +falsification tasks): + +| Java does | Rust returns | NOT what happens | +|---|---|---| +| uses a live handle | success | — | +| uses it after `lgj_close` | `INVALID_HANDLE` | dereference of freed memory | +| closes twice | `INVALID_HANDLE` on 2nd | double-free | +| fabricates a handle | `INVALID_HANDLE` | arbitrary memory read | +| operates on a mask whose parent closed | `PARENT_CLOSED` | dangling parent access | + +## Why This Beats a Naive `Box::into_raw` Handle + +A raw pointer handle has no way to detect staleness — the memory it points at +may have been freed *and reallocated* for something else, so a +use-after-free doesn't even reliably crash; it silently corrupts. The +generation counter turns "is this handle still meaningful" into an O(1) +integer comparison that cannot be fooled by reallocation, because the slot +index is reused but the generation is not (until it wraps, which at `u32` +range is not a near-term concern for a research POC). + +## Concurrency Shape + +Registry lock is held only long enough to resolve `index → Arc` +and clone the `Arc`; it is dropped before the entry's own inner lock is +taken. So two calls against *different* resources do not serialize on each +other — only `open`/`close` contend on the registry itself. This has not +been benchmarked under real contention; the POC's Java layer is +single-threaded, so this is a stated design intent, not yet a measured +property. + +## Cross-reference + +This is the Rust-side half of `docs/abi.md` §4. The Java-side half is: model +the `Arena`/segment lifetime as nested *inside* the resource's own lifetime, +so Java's own bookkeeping fails fast on a use-after-close even before the +call reaches Rust (belt-and-braces, not a substitute for the Rust-side +check). diff --git a/.claude/knowledge/agent-cargo-hygiene.md b/.claude/knowledge/agent-cargo-hygiene.md new file mode 100644 index 0000000..c849d7f --- /dev/null +++ b/.claude/knowledge/agent-cargo-hygiene.md @@ -0,0 +1,65 @@ +# Agent Cargo Hygiene — one target dir, no N× build residue + +> READ BY: every agent spawn touching native/lgj-abi or ndarray; MANDATORY +> in every worker brief before it runs, per the operator directive below + +## Status: RULE (operator directive, 2026-08-17: "Block agents from using +## cargo to avoid target residue overflow") + +## The problem, measured + +`/home/user/ndarray/target` alone is already **2.7 GB** and +`/home/user/lance-graph-java/target` **602 MB**, from a single 4-agent fan-out +that ran this session. Each spawned agent that runs its own `cargo +build`/`check`/`test` against a fresh or divergent `target/` state multiplies +that residue — the exact failure mode `lance-graph`'s own +`.claude/rules/agent-cargo-hygiene.md` names: N agents × N target dirs, cold +compiles competing for the same cores, disk exhaustion. + +## The rule + +- **The orchestrating main thread (Opus) runs cargo freely** and is the ONLY + actor that compiles/lints/tests. One build, not N. +- **Spawned agents do NOT run `cargo build`/`check`/`test`/`clippy` at all**, + full stop — no "targeted `cargo test` is fine" carve-out here (unlike + `lance-graph`'s own version of this rule, which allows a scoped test + against the shared `target/`). This repo is small enough, and young enough, + that the operator's directive is read as the stricter reading: agents edit + and reason; the orchestrator verifies. +- **No `isolation: "worktree"`** on any agent spawn touching `native/lgj-abi` + or `/home/user/ndarray` — a worktree mints its own `target/` and is exactly + the multiplication this rule exists to prevent. +- **Every worker brief for this repo's Rust code MUST state explicitly**: + *"Do not run cargo in any form. Write and reason about the code; the + orchestrator compiles, tests, and lints centrally after your edits land."* + +## What this retroactively flags + +The first vertical-slice fan-out (`wf_23ad2110-b1e`, dispatched before this +rule was stated) told its Rust-touching agents they **may** run cargo, +scoped to `CARGO_TARGET_DIR=/home/user/lance-graph-java/target`. That +predates this directive and is not itself a violation of a rule that didn't +exist yet — but it is exactly why the rule now exists, and it is why +`STATUS_BOARD.md`'s `D-LGJ-AUDIT` entry includes checking `target/` size +after that wave lands, before dispatching anything further. No agent +spawned AFTER this doc exists gets cargo permission again. + +## Why not a settings.json deny instead + +A blanket `Bash` permission deny on `cargo build`/`test`/etc. would also +block the orchestrator's own centralized verification — the harness has no +clean mechanism to say "deny for spawned subagents, allow for the top-level +session" short of per-agent-type tool restriction, and `Bash` cannot be +scoped to "all commands except cargo" that granularly without also risking +blocking legitimate orchestrator work. So enforcement here is the same +mechanism `lance-graph`'s own hygiene rule uses successfully: **explicit, +mandatory brief language**, checked by whoever reads the agent's returned +report for a cargo invocation that shouldn't be there. + +## Falsifier + +Before trusting any agent report that claims code "compiles" or "tests +pass," check the report for evidence it actually ran cargo — if it did, and +this doc predates its spawn, that is a hygiene violation to flag, not a +bonus. The orchestrator re-verifies centrally regardless of what the agent +claims. diff --git a/.claude/knowledge/jdk-toolchain-facts.md b/.claude/knowledge/jdk-toolchain-facts.md new file mode 100644 index 0000000..a3a71a8 --- /dev/null +++ b/.claude/knowledge/jdk-toolchain-facts.md @@ -0,0 +1,65 @@ +# Pinned Toolchain Facts — Verified, Not Assumed + +> READ BY: every agent before invoking javac/java, before writing a build +> script, before making any claim about FFM/Vector-API/Valhalla preview +> status. Facts below were verified by direct execution in this +> environment, not recalled from training data — re-verify before trusting +> if the environment changes. + +## Status: FINDING (measured directly, dated) + +## Rust + +- Toolchain: **1.97.1** (stable), installed via `rustup toolchain install + 1.97.1`. Matches the pin in `ndarray/rust-toolchain.toml` and + `lance-graph/rust-toolchain.toml`. +- Host CPU has AVX-512 (`avx512f/bw/cd/dq/ifma/vbmi/vl`) — both the AVX2 + (v3) and AVX-512 (v4) backends of `ndarray::simd` are exercisable here. +- `ndarray/.cargo/config.toml` pins `-Ctarget-cpu=x86-64-v3` as the default + build baseline (SIGILL trap if omitted downstream — see `abi.md` §"SIGILL + trap"). Any crate depending on `ndarray`'s AVX2 backend must mirror this. + +## JDKs available locally + +| Path | Version | FFM (`java.lang.foreign`) | JEP 401 value classes | Vector API | +|---|---|---|---|---| +| `/usr/bin/java` (system) | OpenJDK 21.0.10 | **preview** — needs `--enable-preview` | not present | incubating, needs `--add-modules jdk.incubator.vector` | +| `/opt/jdks/jdk-26.0.2` | OpenJDK 26.0.2 GA | **FINAL** — no preview flag, only `--enable-native-access=` for restricted calls | not present (mainline, not the Valhalla fork) | incubating | +| `/opt/jdks/jdk-27` | `27-jep401ea3+1-1`, official JEP 401 EA build from `jdk.java.net/valhalla/` | final (this vintage postdates FFM finalization) | **works** — `value class`, `value record`, `Class.isValue()` verified `true`, with `--enable-preview --release 27` | incubating | + +**Verified by direct execution in this session** (not by reading docs): +`Arena.ofConfined()` + `MemorySegment` read/write + `Linker.nativeLinker()` +(`SysVx64Linker`) all work on `/opt/jdks/jdk-26.0.2` with zero preview flags. +`value class LaneId { ... }` and `value record RowRange(...)` both compiled +and ran on `/opt/jdks/jdk-27` with `--enable-preview`, and +`LaneId.class.isValue()` printed `true`. + +## Decision this locks in + +- **Production path (`java/`) targets `/opt/jdks/jdk-26.0.2`.** No preview + flags in the shipped build. This is a real, deliberate strength of the + design: the FFM membrane runs on a *shipped GA JDK*, not an experimental + one. +- **Valhalla lab (`valhalla-lab/`) targets `/opt/jdks/jdk-27`.** Same source + shape, compiled twice (once as `record`, once as `value record`), so the + A/B is genuinely apples-to-apples. +- **The full Valhalla source checkout at `/home/user/valhalla` is NOT + needed** for this project. It was independently confirmed (by a separate + archaeology pass) to be *behind* mainline `/home/user/jdk` for + value-class purposes — its last relevant commit is literally "things to + delete from lworld just before integrating JEP-401." The official EA + *binary* download supersedes building it from source. Do not spend time + building `/home/user/valhalla` or `/home/user/panama-foreign` from + source for this project. +- **`--enable-preview` is a classfile-poisoning flag** (classfile minor + version becomes preview-marked): every `.class` compiled with it can only + run with `--enable-preview` set, and this contaminates transitively. Keep + the Valhalla-flavored sources physically separate from the production + `java/` tree for exactly this reason — never let a preview-compiled class + leak into the path a JDK-26 consumer loads. + +## Falsifier + +Any doc or code comment asserting "value classes are final in JDK 28" or +"Vector API is finalized" without re-verifying against a real build is +wrong until re-checked — both were incubating/preview at last verification. diff --git a/.claude/knowledge/john-doe-migration-thesis.md b/.claude/knowledge/john-doe-migration-thesis.md new file mode 100644 index 0000000..47b2613 --- /dev/null +++ b/.claude/knowledge/john-doe-migration-thesis.md @@ -0,0 +1,103 @@ +# The John Doe Migration Thesis — What This Project Is Actually For + +> READ BY: every agent, before writing any public Java API, README prose, +> or architecture doc in this repo. This is the thesis every other decision +> in this repo serves. + +## Status: FINDING (operator-stated founding thesis, locked) + +## The Center Is Not Panama, Valhalla, SIMD, or lance-graph Individually + +The user's own reframing, which supersedes any earlier "FFI showcase" framing +of this project: + +> Any ordinary Java developer can take yesterday's object-heavy Java, receive +> a generated/schema-fed API that still feels like Java, and suddenly execute +> against a zero-copy columnar graph substrate without learning Rust, SoA, +> SIMD, FFM, Lance, or graph-engine internals. + +And the killer consequence: + +> **64,000 logical "things being considered" no longer require 64,000 Java +> objects.** + +## The Migration Story, Concretely + +Old Java (the mental model the machine pays for): + +```java +List berliners = new ArrayList<>(); +for (Person person : people) { + if (person.getAge() > 65 && person.getCity().equals("Berlin")) { + berliners.add(person); + } +} +``` + +The mental model is `Person, Person, Person, Person, ...` — 64,000 headers, +64,000 references, 64,000 allocations, 64,000 GC-visible identities. + +New Java (embarrassingly familiar surface, per the thesis): + +```java +var berliners = people.where(Person.AGE.gt(65)) + .where(Person.CITY.eq("Berlin")); +``` + +Underneath: `Person.AGE.gt(65)` and `Person.CITY.eq(...)` are **tiny typed +predicates**, combined into a mask plan, evaluated in ONE bulk crossing over +1 native lane set + 1 packed mask. There never needed to be 64,000 `Person` +objects. + +## Why the Schema Vocabulary Is the Accessibility Layer + +The developer must never see `lane 17`, `predicate 0x37`, `mask word 491`. +They see `Person.AGE`, `Person.CITY` — generated (or, in this first slice, +hand-written *as if* generated — see `Pattern.java`) vocabulary with IDE +autocomplete and compile-time type safety. The schema spoon-feeds the safe +surface; the engine gets SoA; the trade is unusually good. + +## The 64K-Thought Generalization + +The same shape applies beyond "rows of a table": 65,536 hypotheses, +candidate diagnoses, graph relationships, reasoning states. Traditional OO +instinct: `Thought[] thoughts = new Thought[65536]` — each with identity, +allocation, GC visibility, pointer-chasing. This project's answer instead: + +``` +65,536 candidates + state lane i8[65536] + confidence lane i16[65536] + class lane u32[65536] + active mask 8192 bytes +``` + +Composed via mask intersection (`A ∩ B ∩ C`) evaluated by SIMD/bitmap, never +by materializing 65,536 objects. **Java manipulates meaning. The substrate +manipulates population.** + +## Why This Explains the Valhalla Split (see `valhalla-three-truths-method.md`) + +Valhalla's sweet spot is the **tiny vocabulary controlling the population** — +`NodeId`, `LaneId`, `RowRange`, `Predicate`, `Lens`, `Range`, `Shape`, +`Capability` — not the population itself. Valhalla does not turn 64K +candidates back into 64K prettier objects; it makes the *instructions that +steer* those candidates identity-free and cheap. Panama makes the membrane +between that vocabulary and the population disappear. `ndarray::simd` + +the native SoA fixture make the population itself cheap. Three deliberately +different jobs, one triad. + +## The Litmus Test for Any API Decision in This Repo + +> Does this proposal turn N logical entities into N Java objects (of any +> kind — plain, record, or value)? If yes, reject it, regardless of how +> elegant the object looks. N entities become 1 lane set + 1 mask + a +> handful of typed descriptors, always. + +This is why `docs/abi.md` §6 forbids per-element crossings, why the Java +facade's `View` is lazy (building a predicate chain must not materialize +anything), and why the Valhalla lab's headline experiment (see +`valhalla-three-truths-method.md`) directly measures "N Java objects" vs +"N Valhalla value objects in an array" vs "1 lane + 1 mask" — because the +thesis predicts the first two both lose to the third, and that prediction +must be checked, not assumed. diff --git a/.claude/knowledge/no-c-ever.md b/.claude/knowledge/no-c-ever.md new file mode 100644 index 0000000..f85c237 --- /dev/null +++ b/.claude/knowledge/no-c-ever.md @@ -0,0 +1,44 @@ +# There Is No C Here — Ever + +> READ BY: all agents touching native/lgj-abi, java/internal/ffm, or any +> future jextract/cbindgen/JNI proposal + +## Status: FINDING (locked by the project's founding directive) + +The user's own words, verbatim: **"There's no C ever. We reuse Panama project +for a rust only."** + +## The One-Line Rule + +`extern "C"` names a **platform calling convention** (SysV AMD64 psABI here, +AAPCS64 on ARM). `#[repr(C)]` names a **platform aggregate layout rule**. +Neither requires C source, a C compiler, a header, or a C runtime. The stack +is `Rust → platform ABI → Java`, with zero C artifacts anywhere. + +## Consequences (all load-bearing, all checked mechanically) + +- No `.h` file exists in this repo. Grep for one before merging anything. +- No C toolchain (`gcc`/`clang`) is a build dependency. +- **No `cbindgen`.** Its output is a C header; we have no consumer for one. +- **No `jextract`.** jextract's only input is a C header — with none to + extract from, the tool has no job here. This is not "unused," it is + structurally inapplicable. +- **No JNI**, and no JNI-*shaped* Panama code either (see the anti-JNI rule + in `docs/abi.md` §6 and the `abi-membrane-warden` agent card). + +## What replaces the header + +A **self-describing runtime manifest** (`lgj_abi_manifest()`), emitted by the +compiled artifact itself, read and cross-checked by Java at load time. See +`docs/abi.md` §1 and §5. A header is a claim; the manifest is a fact about +the artifact that produced it — it cannot disagree with itself. + +## Falsifier + +Any of these in a diff is an automatic block: +- a new `.h`/`.hpp` file anywhere in the repo +- a `build.rs` invoking `cc`/`cbindgen` +- a `jextract` invocation in any script or CI file +- `JNIEnv`, `jni::*`, `#[no_mangle]` functions shaped as one-call-per-element + (grep the function body: does its cost scale with `n_rows`, or is it + lifecycle? If neither, it's the JNI anti-pattern wearing Panama's clothes) diff --git a/.claude/knowledge/simd-lane-width-family.md b/.claude/knowledge/simd-lane-width-family.md new file mode 100644 index 0000000..c05ca79 --- /dev/null +++ b/.claude/knowledge/simd-lane-width-family.md @@ -0,0 +1,58 @@ +# The Lane-Width Family — 64x8 / 32x16 / 16x32 / 8x64, and Why AMX Is Different + +> READ BY: simd-savant, panama-bridge-engineer, and anyone writing a doc +> comment that names an `ndarray::simd` type + +## Status: FINDING (verified against `ndarray/src/simd_avx512.rs` source) + +The user's own words, verbatim: **"The usual polyfill 64x8, 32x16, 16x32. +Amx and gemm tiles and i8 i16 i32 and u8 U16 u32 follow slightly different +multipliers matching the max available."** + +## The Rule + +At the AVX-512 tier, every `ndarray::simd` typed-wrapper name satisfies: + +``` +lane_count(width_bits) × width_bits = 512 (the register width) +``` + +Verified directly against `simd_avx512.rs`: + +| element width | lane count | type names | total bits | +|---|---|---|---| +| 8-bit | 64 | `U8x64`, `I8x64` | 512 | +| 16-bit | 32 | `U16x32`, `I16x32` | 512 | +| 32-bit | 16 | `U32x16`, `I32x16`, `F32x16` | 512 | +| 64-bit | 8 | `U64x8`, `I64x8`, `F64x8` | 512 | + +"The multiplier matching the max available" = `512 / (8 × elem_bytes)` — the +AVX-512 register width divided by the element size. The AVX2 tier is the +same rule at 256 bits (halved lane counts: `x32/x16/x8/x4`), and NEON/WASM at +128 bits (further halved: `x16/x8/x4/x2` where applicable) — each backend +picks the lane count that fills *its own* native register, not a fixed +count. `F32x8`/`F64x4` etc. exist as the 256-bit-native companions +re-exported alongside the 512-bit set. + +## Why AMX Is a Separate Module, Not Another Width Variant + +AMX (`ndarray::simd::{amx_available, matmul_i8_to_i32}`, backed by +`ndarray::hpc::amx_matmul` internally — accessed only via the `simd` +re-export per `simd-provenance.md`) is a **2-D tile register file** +(`TMM0..TMM7`), not a 1-D SIMD lane vector. Its "multiplier" is a tile shape +(rows × columns × element width, e.g. a `16×64` int8 tile), not a lane count +along one axis. This is why it is its own module (`amx_matmul.rs`, +`bf16_tile_gemm.rs`) rather than a `U8x1024`-shaped extension of the lane +family — the underlying hardware primitive is structurally different (matrix +multiply-accumulate across two tiles into a third), not just "more lanes." + +## Consequence for lgj-abi + +`docs/abi.md`'s `LgjElemKind` enum (`U8, I8, U16, I16, U32, I32, U64, I64, +F32, F64, MASK_WORD`) deliberately does not encode a lane width — that is a +compiled-in backend fact (reported via `LgjAbiManifest::simd_backend`), not +a per-value property. A Java caller never chooses "give me the x16 +variant" — it asks for an element kind and a bulk op; which lane width +executes it is `ndarray::simd`'s dispatch decision, invisible above the +membrane. This is the same "physics vs vocabulary" split the mission brief +draws between raw addresses and `LaneId`/`Range`/`Shape`. diff --git a/.claude/knowledge/simd-provenance.md b/.claude/knowledge/simd-provenance.md new file mode 100644 index 0000000..e5aba1c --- /dev/null +++ b/.claude/knowledge/simd-provenance.md @@ -0,0 +1,53 @@ +# SIMD Provenance — `ndarray::simd::*` Only, Never `ndarray::hpc` + +> READ BY: all agents touching native/lgj-abi/src/kernels.rs, or proposing +> any new Rust-side numeric primitive + +## Status: FINDING (operator-corrected 2026, locked) + +The user's own words, verbatim: **"Never use ndarray::hpc, trampoline to +ndarray::simd::* instead."** + +## The One-Line Rule + +`ndarray::hpc::*` is the **internal implementation** namespace. `ndarray::simd::*` +is the **sanctioned consumer re-export surface**. Every internal module that +ships a consumer-facing primitive (bitwise popcount, fingerprint ops, GEMM +tiles, quantization, cascade search, AMX) is re-exported *up* into +`ndarray::simd` by `ndarray/src/simd.rs`. ndarray's own CLAUDE.md states this +as the "matryoshka" invariant: *"Consumer writes `crate::simd::F32x16`. +Period."* This repo is exactly one more consumer of that invariant, not an +exception to it. + +## Consequence for this repo specifically + +`native/lgj-abi/src/kernels.rs` — the ONLY file in this crate allowed to +`use ndarray::*` at all (see `abi-membrane-warden` and the crate layout in +`docs/abi.md` §8) — must import exclusively through the `ndarray::simd::` +path: + +```rust +// CORRECT +use ndarray::simd::{eq_u32_to_mask, gt_i32_to_mask, mask_and, mask_or, masked_sum_i32}; +use ndarray::simd::popcount_batch_u64; // even though it's *implemented* in hpc::bitwise + +// WRONG — never do this, even though it resolves and compiles +use ndarray::hpc::bitwise::popcount_batch_u64; +``` + +Both may point at the same function today. That is not the point. The point +is: if the internal module ever moves, gets renamed, or gets an +arch-specialized replacement, the `ndarray::simd` re-export is the contract +that does not change under a consumer's feet. Importing the `hpc` path +directly is importing an implementation detail as if it were an API. + +## Falsifier + +`grep -rn "ndarray::hpc" native/` returning any hit outside of a doc comment +explaining *why* a symbol lives there is a block. The fix is always +"re-import through `ndarray::simd`," never "leave the hpc import, it works." + +## Cross-reference + +Same-family rule as `no-c-ever.md`: both are about refusing to reach past a +sanctioned boundary because the thing behind it happens to be reachable. diff --git a/.claude/knowledge/valhalla-three-truths-method.md b/.claude/knowledge/valhalla-three-truths-method.md new file mode 100644 index 0000000..d156292 --- /dev/null +++ b/.claude/knowledge/valhalla-three-truths-method.md @@ -0,0 +1,59 @@ +# The Three Truths Method — How To Evaluate Any Semantic Value Concept + +> READ BY: valhalla-lab-scientist, and any agent proposing a new semantic +> value type (NodeId, LaneId, RowRange, MaskId, Ordinal, ...) + +## Status: METHOD (mandated by the mission brief §8, operationalized here) + +## The Rule + +For every important semantic abstraction, distinguish and record three +separate things — never conflate them: + +1. **Semantic truth** — what SHOULD this concept mean? (e.g. "a `LaneId` is + a value; its object identity should be irrelevant; two `LaneId`s with the + same index are the same `LaneId`.") +2. **Stable-Java truth** — how is that contract expressed using the normal, + fully-supported JDK today (a `final record` on JDK 26 GA)? +3. **Valhalla truth** — how does the SAME contract look on the current + Valhalla preview (`value record` / `value class` on the JEP 401 EA build, + `/opt/jdks/jdk-27`)? + +Then **measure**, never assert. Record for (2) and (3): representation, +identity presence (`Class.isValue()`), allocation behaviour, array behaviour +(flattened or not — toggle `-XX:±UseArrayFlattening` and observe), field +flattening in a containing struct (`-XX:±UseFieldFlattening`), generic +behaviour, method-passing cost, nullability implications, and interaction +with FFM (can the value type address a `MemorySegment` as cheaply as a raw +`long`?). + +## Why This Exists + +The mission brief is explicit: *"Do not claim theoretical improvements +without measurement."* Valhalla is genuinely mid-flight (JEP 401 is preview, +not final — see `jdk-toolchain-facts.md`), and the temptation to assert "value +classes will make this free" without running the JDK 27 EA build is real and +must be refused every time. + +## Where Findings Go + +`valhalla-lab/` holds the runnable code and measured numbers for each of the +three truths, one experiment per semantic value type. If a genuine Valhalla +limitation is found while trying to express the ideal semantic-truth API, +it becomes a **minimal standalone reproducer** under `valhalla-lab/reproducers/` +recording: desired semantics / current ordinary-Java behaviour / current +Valhalla behaviour / observed layout+allocation / benchmark result / which +component the deficiency actually belongs to (javac, HotSpot, Valhalla, +Vector API, FFM, or this project's own design). **The API is never distorted +to accommodate a current Valhalla limitation** — the limitation is reported +upward instead. + +## The One Experiment That Must Never Be Skipped + +Per `john-doe-migration-thesis.md`'s litmus test: compare (i) 65,536 rows as +1 native lane + 1 packed mask + one bulk op, (ii) 65,536 plain Java objects, +(iii) 65,536 Valhalla value objects in an array. The thesis's prediction is +that Valhalla helps the *tiny descriptor vocabulary* and does **not** rescue +per-entity materialization at that scale — but this is a prediction to be +checked against real numbers in `valhalla-lab/`, not a conclusion to assume +into the docs. diff --git a/.claude/plans/lgj-vertical-slice-v1.md b/.claude/plans/lgj-vertical-slice-v1.md new file mode 100644 index 0000000..4e5a0df --- /dev/null +++ b/.claude/plans/lgj-vertical-slice-v1.md @@ -0,0 +1,57 @@ +# lgj-vertical-slice-v1 — the first Panama×Valhalla×ndarray::simd proof + +**Status: ACTIVE (session 1, 2026-08-17).** No PR opened yet — everything +below is pre-first-commit on `claude/lance-graph-java-panama-valhalla-sus9w8`. + +## Goal + +Not "Java bindings for lance-graph." A proof that an ordinary Java developer +can write familiar-looking fluent Java and get zero-copy columnar/SIMD +execution without learning Rust/SoA/SIMD/FFM — see +`.claude/knowledge/john-doe-migration-thesis.md`, which is the actual thesis +this plan serves. + +## Phases (mirrors the mission brief's Phase A–I, mapped to D-ids on +`STATUS_BOARD.md`) + +| Phase | What | D-id | +|---|---|---| +| A | Archaeology: ndarray SIMD/mask surface, lance-graph ClassView/WideFieldMask/SoaEnvelope, current Panama+Valhalla state | D-LGJ-A | +| — | Normative contract: `docs/abi.md` | D-LGJ-ABI | +| — | `.claude/agents` + `.claude/knowledge` ensemble (this file's sibling work) | D-LGJ-ENS | +| B | Missing `ndarray::simd` primitives (`eq_u32_to_mask`, `gt_i32_to_mask`, `mask_and`/`mask_or`(`_assign`), `masked_sum_i32`) | D-LGJ-B | +| C | Rust ABI crate `native/lgj-abi` — manifest, generation-checked registry, generic SoA fixture, kernels, `extern "C"` surface | D-LGJ-C | +| D | Java FFM membrane `internal/ffm` — Layouts, Downcalls, Abi manifest cross-check | D-LGJ-D | +| E | Java public facade — `NativePattern`/`View`/`Predicate`/`Pattern`/`Mask`, fused-plan execution | D-LGJ-E | +| F | Valhalla lab — three-truths method on the small semantic value vocabulary | D-LGJ-F | +| G | Java Vector API comparative bench (zero-copy `fromMemorySegment`) vs Panama→`ndarray::simd` | D-LGJ-G | +| H | Falsification: handle lifecycle, SIMD/scalar parity, Java/native parity | D-LGJ-H | +| I | Docs: `architecture.md`, `panama.md`, `valhalla-lab.md`, `execution-boundary.md` | D-LGJ-I | + +## Sequencing decision (recorded, not just executed) + +B/C/D were fanned out **in parallel** as three disjoint trees +(`/home/user/ndarray` vs `native/lgj-abi` vs `java/`) against the single +frozen contract `docs/abi.md`, rather than sequentially — the contract is +what makes that safe: each side is checked against the doc, not against the +other side's in-progress code. F (Valhalla lab) was sequenced AFTER D so it +could read the real `View`/`Predicate` types rather than guess their shape. + +## Falsifiable gates before this plan's phases count as done + +- `grep -rn "ndarray::hpc" native/lgj-abi/src` → must be empty (see + `.claude/knowledge/simd-provenance.md`). +- `grep -rln '\.h$\|cbindgen\|jextract' .` (repo-wide) → must be empty (see + `.claude/knowledge/no-c-ever.md`). +- `grep -rn "java.lang.foreign" java/src/main/java/com/adaworldapi/lancegraph` + excluding `internal/ffm/` → must be empty (java-surface-warden's rule). +- Rust-side handle lifecycle tests pass AND were disable-verified (per + `handle-lifecycle-auditor`'s card — a happy-path-only test is not evidence). +- Every `bench/` number has a reproduction command line attached. + +## Where this plan's own status lives + +`STATUS_BOARD.md` (per-D-id status), `AGENT_LOG.md` (what each spawned agent +actually did), `LATEST_STATE.md` (current contract inventory + what's +active right now). This file records intent and sequencing; those three +record ground truth as it lands. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d6f235 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Rust +/target/ +/native/**/target/ +Cargo.lock.bak +**/*.rs.bk + +# Java +*.class +/java/out/ +/java/target/ +/bench/out/ +/bench/lib/*.jar +/valhalla-lab/out/ + +# Downloaded JDKs and artifacts (never committed — see docs/abi.md and +# .claude/knowledge/jdk-toolchain-facts.md for how to obtain them) +*.tar.gz + +# OS / editor noise +.DS_Store +*.swp diff --git a/docs/abi.md b/docs/abi.md new file mode 100644 index 0000000..8db9b72 --- /dev/null +++ b/docs/abi.md @@ -0,0 +1,409 @@ +# The ABI — a machine membrane, not a product API + +> **Status:** normative. This document is the contract. The Rust side and the +> Java side are implemented *independently* against it, and a runtime manifest +> check proves they agree. If this document and the code disagree, the code is +> wrong. + +## 0. There is no C here. Ever. + +This is the most misunderstood property of the design, so it is stated first. + +`extern "C"` in Rust and `Linker.nativeLinker()` in Java **do not involve the C +language**. They name a *machine* calling convention: + +| Thing | What it actually is | What it is not | +|---|---|---| +| `extern "C"` (Rust) | "use this target's standard C-family calling convention" — on this box, the **System V AMD64 psABI**; on ARM64, **AAPCS64** | C source, a C compiler, a C runtime | +| `#[repr(C)]` (Rust) | "use this target's standard aggregate layout rule" (field order, padding, alignment) | a C struct declaration | +| `Linker.nativeLinker()` (Java) | a JVM-internal implementation of that same psABI (`SysVx64Linker`) | a C bridge, a JNI shim | + +Consequences, all load-bearing: + +- **No `.h` header exists anywhere in this project**, and none will. +- **No C toolchain** is required to build or consume this. `cargo` and `javac` + are the entire toolchain. +- **No `cbindgen`.** Its output is a C header; we have no consumer for one. +- **No `jextract`.** jextract's *only* input is a C header. With no header there + is nothing to extract. This is not "we chose not to use the generated + bindings" (though that would also be true) — the tool has no input. +- **No JNI**, and no JNI-shaped use of Panama (see §6). + +### What replaces the header: a self-describing manifest + +A C header is a *text file that claims* what the compiled artifact looks like. +It can drift from the artifact silently — the classic ABI break. We do the +opposite: the compiled artifact **describes itself at runtime**, and Java +verifies that description against its own compiled-in expectations before the +first real call. + +``` + Rust cdylib Java + ─────────── ──── + lgj_abi_manifest() ──── returns ptr ───▶ read LgjAbiManifest + (a static, versioned │ + #[repr(C)] struct ▼ + describing sizeof/ cross-check EVERY + alignof of every size & align against + ABI type, endianness, this Java build's own + the compiled SIMD MemoryLayout constants + backend) │ + ▼ + mismatch ⇒ hard fail at load + (never silent corruption) +``` + +The manifest is strictly stronger than a header: a header describes what someone +*intended* to compile; the manifest is emitted *by* the compiled artifact, so it +cannot disagree with itself. + +## 1. Scope: what the ABI is allowed to be + +The ABI is a **machine membrane**. It is not the product. The product is the Java +semantic API (see `architecture.md`). Therefore: + +- It is **small** — currently 14 symbols. Growth is a design smell to be argued + for, not a default. +- It is **bulk-only**. Every call must be capable of doing work proportional to + `n_rows` (see §6 — the anti-JNI rule). +- It speaks **resource, lane, view, mask, operation, descriptor, status, + epoch** — not `Node`, not `Edge`, not `Person`. +- It is **versioned** and refuses to operate across a version mismatch. +- It never allocates on the Java side's behalf without a paired release, and + never hands out a pointer whose lifetime it cannot state. + +## 2. Versioning + +``` +LGJ_ABI_MAJOR = 0 // incompatible change ⇒ bump; Java refuses to load +LGJ_ABI_MINOR = 1 // additive change ⇒ bump; older Java may still load +LGJ_MAGIC = 0x4C_47_4A_5F_41_42_49_00 // "LGJ_ABI\0" big-endian-read +``` + +Rule: **Java requires `major` to match exactly and `minor` to be `>=` what it +was compiled against.** A `major` mismatch is a hard failure, not a warning. + +The magic doubles as an endianness probe: read as a `u64` little-endian it yields +a known constant; anything else means the library was built for a different byte +order and every subsequent read would be garbage. + +## 3. Status codes + +Every function returns `i32`. `0` is success; all failures are negative. There +are no error strings across the membrane and no `errno` dependence. + +| Value | Name | Meaning | +|---:|---|---| +| `0` | `OK` | success | +| `-1` | `NULL_ARGUMENT` | a required out-pointer was null | +| `-2` | `INVALID_HANDLE` | handle malformed, closed, or generation-stale | +| `-3` | `WRONG_RESOURCE_KIND` | e.g. a mask handle passed where a pattern was required | +| `-4` | `INVALID_LANE` | `lane_id` out of range for this resource | +| `-5` | `LANE_KIND_MISMATCH` | op's element type ≠ lane's element type | +| `-6` | `MASK_LENGTH_MISMATCH` | mask row-count ≠ resource row-count | +| `-7` | `PARENT_CLOSED` | child (mask) outlived its parent resource | +| `-8` | `VERSION_MISMATCH` | caller's ABI version incompatible | +| `-9` | `LENGTH_OVERFLOW` | requested size overflows `usize`/allocation limit | +| `-10` | `UNKNOWN_OPCODE` | plan contained an opcode this build does not implement | +| `-11` | `EMPTY_PLAN` | a plan with zero ops was submitted | +| `-12` | `ALLOCATION_FAILED` | the allocator refused | +| `-13` | `READ_ONLY` | write attempted against a read-only lane | + +`INVALID_HANDLE` is deliberately the response to *use-after-close*, not a crash. +See §4. + +## 4. Ownership, lifetime, and the generation-checked handle + +This is the part of the design that Java cannot get from Rust for free. Inside +Rust, `&self` borrows make a view-outliving-its-owner a *compile error*. Across +the membrane there is no borrow checker, so the invariant must be enforced at +runtime. + +**A handle is not a pointer.** It is an opaque `u64`: + +``` + 63 32 31 0 + ┌────────────────────────┬────────────────────────┐ + │ generation │ index │ + └────────────────────────┴────────────────────────┘ +``` + +- `index` selects a slot in a Rust-side registry. +- `generation` is bumped **every time a slot is freed**. + +A lookup validates `generation` against the slot's current generation. So: + +| Java does | Result | +|---|---| +| uses a live handle | works | +| uses a handle after `lgj_close` | `INVALID_HANDLE` — *not* use-after-free | +| closes twice | second returns `INVALID_HANDLE` | +| fabricates a handle (`0`, `0xDEADBEEF`, …) | `INVALID_HANDLE` | +| uses a mask whose parent was closed | `PARENT_CLOSED` | + +There is **no code path in which a stale handle dereferences freed memory.** That +is the single most important safety property of this ABI, and §H of the test +plan falsifies it directly rather than assuming it. + +### Answers to the mandated ownership questions + +| Question | Answer | +|---|---| +| Who owns this memory? | Rust. Always. Java never allocates a lane. | +| How long does a `MemorySegment` remain valid? | Until `lgj_close` on the owning handle. Java models this with an `Arena` whose lifetime is *nested inside* the resource's, so closing the resource is what ends the segment's usefulness — and the `epoch` field lets Java detect a stale segment it still holds. | +| What invalidates a view? | Closing the owner, or closing the owner's parent. | +| Can native storage relocate? | **No.** Lanes are allocated once at `lgj_pattern_open` and never reallocated, resized, or moved while the resource is alive. This is a hard ABI guarantee; any future growable lane requires a `major` bump. | +| Can Java mutate it? | Only where `LGJ_FLAG_WRITABLE` is set on the descriptor. Pattern lanes are **read-only**; mask words are writable. | +| What happens when the resource closes? | Its lanes are freed, its generation is bumped, its children fail with `PARENT_CLOSED`. | +| Can a child view outlive the parent? | It can *exist* but not *work* — every operation on it returns `PARENT_CLOSED`. | +| How are errors represented? | Negative `i32` status. Never a panic across the boundary (§9). | + +### Concurrency + +The registry is a `RwLock` holding `Arc` values; each entry has its +own inner lock over its payload. A call takes a **short** read lock on the +registry to resolve and clone the `Arc`, releases it, then locks only the entry. +So bulk ops on distinct resources run concurrently, and only +open/close serialize globally. Stated honestly: this has not been benchmarked +under contention, and the POC's Java layer is single-threaded. + +## 5. Descriptors describe lanes, not pointers + +Java's *public* API never sees an address. Internally, `LgjLaneDesc` is the +bounded description that the FFM layer turns into a `MemorySegment`: + +```rust +#[repr(C)] // 56 bytes, align 8 +pub struct LgjLaneDesc { + pub addr: u64, // physics — never surfaced in the public Java API + pub len_elems: u64, + pub byte_len: u64, + pub owner: u64, // owning resource handle + pub epoch: u64, // liveness stamp; Java re-checks before use + pub elem_kind: u32, // LgjElemKind + pub elem_bytes: u32, + pub stride_bytes: u32, // == elem_bytes when contiguous + pub flags: u32, // bitfield, below +} +``` + +Flags: `READABLE = 1<<0`, `WRITABLE = 1<<1`, `CONTIGUOUS = 1<<2`. + +Element kinds start at `1`, so a zeroed struct is detectably invalid rather than +silently meaning `U8`: + +``` +U8=1 I8=2 U16=3 I16=4 U32=5 I32=6 U64=7 I64=8 F32=9 F64=10 MASK_WORD=11 +``` + +`MASK_WORD` is a `u64` of 64 packed row bits, LSB = lowest row index. + +```rust +#[repr(C)] // 32 bytes +pub struct LgjResourceInfo { + pub kind: u32, // 1 = Pattern, 2 = Mask + pub lane_count: u32, + pub n_rows: u64, + pub epoch: u64, + pub parent: u64, // 0 = none +} +``` + +```rust +#[repr(C)] // 24 bytes, align 8 +pub struct LgjOpDesc { + pub op: u32, // LgjOpCode + pub lane_id: u32, + pub operand: i64, // needle / threshold, sign-extended + pub combine: u32, // 0 = AND (narrow), 1 = OR (widen) + pub _reserved:u32, // must be 0 +} +``` + +```rust +#[repr(C)] +pub struct LgjAbiManifest { + pub magic: u64, + pub abi_major: u32, + pub abi_minor: u32, + pub size_of_manifest: u32, + pub size_of_lane_desc: u32, + pub size_of_op_desc: u32, + pub size_of_resource_info: u32, + pub align_of_lane_desc: u32, + pub align_of_op_desc: u32, + pub align_of_resource_info: u32, + pub pointer_bytes: u32, + pub endianness: u32, // 0 = little + pub simd_backend: u32, // LgjSimdBackend + pub simd_backend_name: [u8; 32], // NUL-terminated, human-readable + pub build_profile: [u8; 16], // "release" | "debug" +} +``` + +`simd_backend`: `SCALAR=0 AVX2=1 AVX512=2 NEON=3 WASM=4`. It is reported, not +negotiated — Java does not select a backend. Which backend is compiled in is +`ndarray`'s business (§7). + +## 6. The anti-JNI rule + +Panama makes it *easy* to write JNI-shaped code: one downcall per element. That +is forbidden here. The rule: + +> **Every ABI function must do work proportional to `n_rows`, or be lifecycle.** + +Explicitly prohibited, and each of these would be a design failure rather than a +performance nit: + +- one call per node / per edge / per element +- one upcall per element +- Java-side hydration of a `Node`/`Edge`/`Person` object per row +- any serialization: no JSON, no protobuf, no `byte[]` bounce buffer +- a Java-owned mirror of the native data + +The fused-plan call (§7, `lgj_plan_eval`) exists precisely so that +`.where(...).where(...).count()` is **one** crossing regardless of how many +predicates or rows are involved. The unfused per-predicate ops are retained only +so the fused path can be benchmarked *against* something and so parity can be +checked predicate-by-predicate. + +## 7. The function surface (14 symbols) + +All symbols are prefixed `lgj_`. All return `i32` status except the manifest +getter. `out_*` parameters are written only on `OK`. + +### Manifest + +``` +const LgjAbiManifest* lgj_abi_manifest(void) +``` +Never fails, never allocates, returns a pointer to a `'static`. The one symbol +that does not return a status, because there is no failure mode. + +### Lifecycle + +``` +i32 lgj_pattern_open(u64 n_rows, u64 seed, u64* out_handle) +i32 lgj_close(u64 handle) +i32 lgj_resource_info(u64 handle, LgjResourceInfo* out) +``` + +`lgj_pattern_open` builds the generic SoA fixture deterministically from `seed` +(see `architecture.md` §fixture). Deterministic generation is what lets the Java +test assert exact counts without shipping a data file. + +### Lanes + +``` +i32 lgj_lane_describe(u64 handle, u32 lane_id, LgjLaneDesc* out) +``` + +Pattern lanes: `0 = ids (U64)`, `1 = classes (U32)`, `2 = values (I32)`. +All `READABLE | CONTIGUOUS`, never `WRITABLE`. + +### Masks + +``` +i32 lgj_mask_create(u64 parent, u32 initial, u64* out_handle) // initial: 0=empty, 1=all +i32 lgj_mask_describe(u64 mask, LgjLaneDesc* out) // MASK_WORD lane, WRITABLE +i32 lgj_mask_and(u64 a, u64 b, u64 dst) +i32 lgj_mask_or(u64 a, u64 b, u64 dst) +i32 lgj_mask_count(u64 mask, u64* out_count) +``` + +`dst` may alias `a` or `b`. All three must share the same parent and row count. + +### Bulk predicates (unfused — one predicate per crossing) + +``` +i32 lgj_op_eq_u32(u64 res, u32 lane_id, u32 needle, u64 dst_mask) +i32 lgj_op_gt_i32(u64 res, u32 lane_id, i32 threshold, u64 dst_mask) +``` + +Each *overwrites* `dst_mask` with the predicate's result. Composition is the +caller's job via `lgj_mask_and`. + +### Fused plan (N predicates — ONE crossing) + +``` +i32 lgj_plan_eval(u64 res, const LgjOpDesc* ops, u32 n_ops, + u64 dst_mask, u64* out_count) +``` + +Semantics: accumulator starts as **all rows set**; each op is evaluated and +combined into the accumulator per its `combine` field; the result lands in +`dst_mask` and its popcount is written to `out_count`. With every `combine = AND` +the result is monotonically narrowing by construction: + +``` + V0 = all + V1 = V0 ∩ op0 V1 ⊆ V0 + V2 = V1 ∩ op1 V2 ⊆ V1 ⊆ V0 +``` + +This single call is what makes the Java fluent chain cost one crossing. + +### Reduction + +``` +i32 lgj_reduce_sum_i32(u64 res, u32 lane_id, u64 mask, i64* out_sum) +``` + +Sums the `I32` lane over set mask bits into a widened `i64` (no overflow for +`n_rows ≤ 2^32` on `i32` inputs). + +### Parity escape hatch + +``` +i32 lgj_plan_eval_scalar(u64 res, const LgjOpDesc* ops, u32 n_ops, + u64 dst_mask, u64* out_count) +``` + +Identical semantics to `lgj_plan_eval` but forced down the scalar reference path. +Exists **only** so SIMD-vs-scalar parity is falsifiable *through the membrane*, +which is where the Java tests live. Not for production use. + +## 8. SIMD provenance + +Every kernel behind these symbols routes through **`ndarray::simd`** and nothing +else. Specifically prohibited in this crate: + +- `core::arch::*` intrinsics, `_mm*`, `vld1q_*`, any raw platform intrinsic +- `#[cfg(target_feature)]` / `#[cfg(target_arch)]` SIMD selection +- `core::simd` / `std::simd` / portable-simd (nightly) +- `pulp`, `SimSIMD`, `wide`, or any other SIMD crate +- a locally-written SIMD abstraction layer + +If a primitive is missing, it is **added to `ndarray::simd`** under that repo's +W1a consumer contract (all backends + scalar + parity test) and consumed from +here. That is not a workaround; it is the architecture. `ndarray::simd` is the +permanent hardware membrane for the whole Ada stack, and this project is one more +consumer of it — not an exception to it. + +## 9. Panics never cross the membrane + +Every `extern "C"` function wraps its body in `catch_unwind`. A panic becomes a +negative status, never an unwind into JVM frames (which would be UB). The +`Cargo.toml` additionally does **not** set `panic = "abort"`, because +`catch_unwind` requires unwinding to be available. + +## 10. What is deliberately absent + +Named so their absence is a decision on record rather than an oversight: + +- **No strings across the boundary** except the two fixed-size NUL-terminated + name fields in the manifest. No `char*` in, ever. +- **No callbacks / upcalls.** An upcall per element is the JNI anti-pattern in + disguise; a bulk op needs no callback. +- **No variadics.** +- **No `errno` / `captureCallState`.** Nothing here is a syscall wrapper. +- **No growable or relocatable lanes** (§4). +- **No `lgj_lane_read_element`.** There is intentionally no way to read one row + across the membrane. If Java wants element access it reads the + `MemorySegment` directly, in-process, with no crossing at all. +- **No `ClassView` / `WideFieldMask` yet.** The first slice is the generic + fixture on purpose (see `architecture.md` §"generic is not toy"). Wiring the + real `lance-graph` types is the next slice, and their shapes are already + ABI-compatible: `WideFieldMask`'s canonical `[u64]` chunks *are* this ABI's + `MASK_WORD` lane, and `NodeRow`'s `16|16|480` `#[repr(C, align(64))]` layout is + already a legal lane description. diff --git a/java/.gitignore b/java/.gitignore new file mode 100644 index 0000000..c058e57 --- /dev/null +++ b/java/.gitignore @@ -0,0 +1,3 @@ +# javac output. There is no build system here on purpose (no Maven, no Gradle, no downloads), +# so the compiled classes land in a plain directory rather than under a tool's conventions. +out/ diff --git a/java/README.md b/java/README.md new file mode 100644 index 0000000..fbf6653 --- /dev/null +++ b/java/README.md @@ -0,0 +1,140 @@ +# The Java side + +> The product is this API. The ABI (`../docs/abi.md`) is a machine membrane underneath it, and a +> consumer of this package is never asked to know that the membrane exists. + +## What a consumer writes + +```java +try (var data = NativePattern.open(65_536)) { + long n = data.view() + .where(Pattern.CLASS.eq(7)) + .where(Pattern.VALUE.gt(100)) + .count(); +} +``` + +Nothing in that snippet mentions an arena, a memory segment, a linker, a lane index, an opcode, a +packed mask word, or a SIMD backend. It is nevertheless the whole real path: 65,536 rows that never +enter the Java heap, two predicates fused into one vectorised kernel, **one** crossing of the +membrane, one number back. + +That is the thesis in one line: **64,000 logical entities do not become 64,000 Java objects.** They +become one native lane set, one packed mask, a few tiny schema descriptors, and one bulk operation. + +## Toolchain + +There is no Maven, no Gradle, no downloaded dependency, and no C toolchain. `javac` and `java` are +the entire Java toolchain, exactly as `cargo` is the entire Rust one. + +- **JDK 26** (`/opt/jdks/jdk-26.0.2`). FFM is **final** in JDK 26, so `--enable-preview` is neither + needed nor accepted. Do **not** use a JDK 21 `java` on the path — there FFM is preview-gated and + these commands will not work. +- No JNI, no `jextract`, no `cbindgen`, no `.h` file anywhere in the project. See `../docs/abi.md` + §0 for why those are absent by construction rather than by preference. + +## Build + +```sh +cd java +/opt/jdks/jdk-26.0.2/bin/javac -d out $(find src/main/java src/test/java -name '*.java') +``` + +Compilation emits six `[restricted]` warnings with `-Xlint:all`, all of them in +`internal/ffm/{Abi,Downcalls,Engine}.java`. That is not noise to be suppressed — it is a +machine-checkable statement that every unsafe FFM operation in the project lives in the one package +that is allowed to contain them. + +## Run + +```sh +# everything +/opt/jdks/jdk-26.0.2/bin/java --enable-native-access=ALL-UNNAMED -cp out \ + com.adaworldapi.lancegraph.AllTests + +# or one suite at a time — each has its own main +/opt/jdks/jdk-26.0.2/bin/java --enable-native-access=ALL-UNNAMED -cp out \ + com.adaworldapi.lancegraph.SmokeTest +``` + +### Flags + +| Flag | Needed? | Why | +|---|---|---| +| `--enable-native-access=ALL-UNNAMED` | **yes** | FFM's restricted methods (`libraryLookup`, `downcallHandle`, `reinterpret`) refuse to run without it. Omitting it does not fail the build — it fails at load, which is worse. | +| `--enable-preview` | **no** | FFM is final in JDK 26. Passing it is an error. | +| `-Djava.library.path=…` | **no** | Not used. That is the JNI mechanism; this project resolves the artifact itself (below). | + +### Where the native library is found + +Resolution order, first hit wins: + +1. `-Dlgj.library=/abs/path/liblgj_abi.so` — an explicit file. +2. `LGJ_LIBRARY` environment variable — same meaning. +3. `-Dlgj.library.dir=/abs/dir` — a directory holding the platform-named artifact. +4. Walking up from the working directory for `target/release/liblgj_abi.so`, then + `target/debug/liblgj_abi.so`. + +Release is searched before debug deliberately: if both exist, silently benchmarking the debug build +would be a measurement error rather than an inconvenience. + +If nothing is found, or the artifact is found but disagrees with this build, the failure names the +exact problem and every path that was tried. A test run in that state reports **exit code 2 and the +word SKIPPED**, never a red failure — "you have not built the library" and "your library is broken" +must not look the same in a log. + +### Exit codes + +| Code | Meaning | +|---|---| +| `0` | all checks passed | +| `1` | at least one check failed | +| `2` | the native artifact is unavailable, so nothing was run | + +## Layout + +``` +src/main/java/com/adaworldapi/lancegraph/ the public semantic API +src/main/java/com/adaworldapi/lancegraph/internal/ffm/ ALL Panama machinery, and nowhere else +src/test/java/com/adaworldapi/lancegraph/ tests (plain Java, no JUnit) +``` + +The split is enforced, not merely intended: `ApiSurfaceTest` walks every public member of every +public type by reflection and fails if any signature mentions `java.lang.foreign.*`, +`java.lang.invoke.*`, or our own `internal.*` package. It needs no native library, because the +shape of an API is a compile-time property — which makes it the one suite worth running *before* +the artifact exists. + +## The tests, and what each one would catch + +| Suite | The claim it can falsify | +|---|---| +| `ApiSurfaceTest` | The membrane leaked into the consumer API. Needs no native library. | +| `AbiContractTest` | The self-describing manifest is not actually checked, or cannot reject anything. | +| `SmokeTest` | The advertised fluent chain does not work. | +| `FixtureParityTest` | The kernels return the wrong numbers — Java recomputes the expected answers from the generator, independently. | +| `FusionParityTest` | Fusion, unfused composition and the scalar reference disagree. | +| `LazinessTest` | Building a chain crosses the membrane, or a terminal op's cost grows with predicates or rows. | +| `NarrowingTest` | Adding a condition can widen a selection. | +| `LifetimeTest` | A stale handle reaches native code — use after close, double close, a selection outliving its parent. | + +Two of these are worth calling out because they are unusual: + +**`FixtureParityTest` is a genuine cross-language check.** Property tests ("the count never grows", +"the paths agree") can all pass while every number is wrong together. This suite transcribes the +fixture's SplitMix64 generator into Java, counts with an ordinary loop, and compares against what +the vectorised kernels produced over data Java never sees. It ships no golden file, because a golden +file only proves nobody edited it. + +**`LazinessTest` measures rather than asserts.** The membrane counts its own crossings, so +"building a View makes zero crossings" is a number in a log rather than a comment that quietly stops +being true. It also pins the load-bearing property: a four-condition chain and a sixteen-condition +chain both cost exactly one crossing, and so do 1,024 rows and 1,000,000 rows. + +## Two things deliberately not on the consumer path + +- **`Diagnostics`** — `countUnfused` and `countScalar` exist so fusion and SIMD can be measured + *against something*. They are in a separately named class rather than as methods on `View`, so + nobody reaches them thinking they are an option. +- **`internal.ffm.Engine`** — the only way to raw native storage, reachable only by naming an + internal package explicitly. Ordinary composition never hands it to you. diff --git a/java/src/main/java/com/adaworldapi/lancegraph/AbiMismatchException.java b/java/src/main/java/com/adaworldapi/lancegraph/AbiMismatchException.java new file mode 100644 index 0000000..03122dd --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/AbiMismatchException.java @@ -0,0 +1,20 @@ +package com.adaworldapi.lancegraph; + +/** + * Thrown at library load when the compiled artifact's self-description disagrees with what this + * Java build expects. + * + *

This is the header replacement doing its job. A C header claims a layout and can + * drift from the artifact silently; here the artifact emits its own sizes, alignments, version and + * endianness at runtime and Java compares them against independently derived + * {@code MemoryLayout} numbers. The first disagreement is a hard failure naming the exact field, + * never a silent misread. + */ +public final class AbiMismatchException extends LanceGraphException { + + private static final long serialVersionUID = 1L; + + public AbiMismatchException(String message) { + super(message); + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/ClosedResourceException.java b/java/src/main/java/com/adaworldapi/lancegraph/ClosedResourceException.java new file mode 100644 index 0000000..be975ed --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/ClosedResourceException.java @@ -0,0 +1,25 @@ +package com.adaworldapi.lancegraph; + +/** + * Thrown when a resource is used after it was closed, closed twice, or when a value derived from + * it (a {@link View}, a {@link Mask}) is used after its owner was closed. + * + *

Two independent mechanisms produce this, deliberately — belt and braces: + * + *

    + *
  1. Java's own bookkeeping fails fast, so a stale handle never even reaches native code. + *
  2. If it somehow did, the native registry's generation check returns {@code INVALID_HANDLE} + * or {@code PARENT_CLOSED} rather than dereferencing freed memory, and that status maps + * back to this same exception. + *
+ * + *

There is no code path in which a stale handle dereferences freed memory. + */ +public final class ClosedResourceException extends LanceGraphException { + + private static final long serialVersionUID = 1L; + + public ClosedResourceException(String message) { + super(message); + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/Diagnostics.java b/java/src/main/java/com/adaworldapi/lancegraph/Diagnostics.java new file mode 100644 index 0000000..1c43072 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/Diagnostics.java @@ -0,0 +1,70 @@ +package com.adaworldapi.lancegraph; + +import com.adaworldapi.lancegraph.internal.ffm.Downcalls; + +/** + * Measurement and comparison entry points. Not the consumer API. + * + *

Everything here exists so a claim this project makes can be falsified from the outside rather + * than believed: + * + *

    + *
  • {@link #countUnfused} runs the same query one crossing per predicate, so the fused path can + * be benchmarked against something. Fusion is otherwise an assertion about code + * nobody measured. + *
  • {@link #countScalar} runs the same query through the scalar reference kernel, so + * SIMD-versus-scalar parity is falsifiable through the membrane — which is where these tests + * live. + *
  • {@link #crossings()} makes laziness observable: snapshot it, build a chain, and the delta + * must be zero. + *
+ * + *

They are gathered in a separate, plainly-named class rather than added as methods on + * {@link View} on purpose. A benchmark path sitting next to {@code count()} would eventually get + * called by someone who thought it was an option; here it is impossible to reach by accident. + */ +public final class Diagnostics { + + private Diagnostics() {} + + /** + * The production path, named explicitly for symmetry in a benchmark. Identical to + * {@link View#count()}. + */ + public static long countFused(View view) { + return view.count(); + } + + /** + * The same answer, computed with one crossing per predicate plus one per combine plus one to + * count, instead of one crossing total. + * + *

Comparison only. This is what the design forbids, kept executable so the + * cost of forbidding it is a number rather than an opinion. + */ + public static long countUnfused(View view) { + return view.source().countUnfused(view.predicates()); + } + + /** + * The same answer, forced down the scalar reference kernel with no SIMD. + * + *

Parity checking only. If this ever disagrees with {@link #countFused}, + * the vectorised kernel is wrong — and the disagreement is visible from Java, not only from + * inside Rust's own test suite. + */ + public static long countScalar(View view) { + return view.source().countScalar(view.predicates()); + } + + /** + * Total membrane crossings since JVM start. + * + *

The instrument behind the laziness claim. Building and narrowing a view must not move + * this number at all; a terminal operation must move it by a small constant that does not grow + * with the number of predicates or rows. + */ + public static long crossings() { + return Downcalls.crossings(); + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/Field.java b/java/src/main/java/com/adaworldapi/lancegraph/Field.java new file mode 100644 index 0000000..f16a903 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/Field.java @@ -0,0 +1,61 @@ +package com.adaworldapi.lancegraph; + +/** + * One named, typed column of a schema. + * + *

A {@code Field} is the whole reason an ordinary Java developer never meets a lane index or an + * opcode. The generated vocabulary hands out {@code Pattern.CLASS} and {@code Pattern.VALUE}; the + * IDE completes them; the compiler type-checks the comparison. Everything underneath — which lane, + * which element kind, which opcode, which SIMD backend ran — is physics the field carries and the + * caller never states. + * + *

Two surfaces, deliberately. The constructor and {@link #lane()} are the + * generator surface: they exist for the code that mints a schema (today hand-written as + * {@link Pattern}, tomorrow emitted from a schema definition). {@link #name()} and the typed + * comparison methods on the subclasses are the consumer surface. A consumer never calls + * the former. + * + *

Sealed: the set of element types the membrane implements is fixed by the ABI, so a schema + * cannot invent a field type whose predicate has no kernel behind it. + */ +public abstract sealed class Field + permits U32Field, I32Field, U64Field { + + private final String name; + private final LaneId lane; + private final Ordinal ordinal; + + Field(String name, LaneId lane, Ordinal ordinal) { + this.name = java.util.Objects.requireNonNull(name, "name"); + this.lane = java.util.Objects.requireNonNull(lane, "lane"); + this.ordinal = java.util.Objects.requireNonNull(ordinal, "ordinal"); + } + + /** The field's name as written in the schema. */ + public final String name() { + return name; + } + + /** Position within the schema. Stable for a schema version. */ + public final Ordinal ordinal() { + return ordinal; + } + + /** + * Which column this field lives in. + * + *

Generator surface. A consumer has no use for this — writing + * {@code Pattern.CLASS.eq(7)} is the point, and {@code lane 1} is the thing being hidden. + */ + public final LaneId lane() { + return lane; + } + + /** The element type this field reads, for diagnostics. */ + public abstract String elementType(); + + @Override + public final String toString() { + return name + ":" + elementType(); + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/I32Field.java b/java/src/main/java/com/adaworldapi/lancegraph/I32Field.java new file mode 100644 index 0000000..fa623d3 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/I32Field.java @@ -0,0 +1,34 @@ +package com.adaworldapi.lancegraph; + +import com.adaworldapi.lancegraph.internal.ffm.Layouts; +import com.adaworldapi.lancegraph.internal.ffm.PlanOp; + +/** + * A signed 32-bit column — a measurement, a score, a delta that can go negative. + * + *

{@link #gt(int)} is a signed comparison, which is not a detail: the fixture's values + * straddle zero precisely so that an implementation which compared unsigned by mistake would be + * caught rather than accidentally agreeing. + * + *

Only the comparisons the membrane implements are offered — see {@link U32Field} for why that + * is a feature rather than a gap. + */ +public final class I32Field extends Field { + + /** Generator surface. Called by schema-vocabulary code, not by consumers. */ + public I32Field(String name, LaneId lane, Ordinal ordinal) { + super(name, lane, ordinal); + } + + /** Rows whose value in this column is strictly greater than {@code threshold}, signed. */ + public Predicate gt(int threshold) { + return new Predicate( + PlanOp.narrowing(Layouts.OP_GT_I32, lane().index(), threshold), + name() + " > " + threshold); + } + + @Override + public String elementType() { + return "i32"; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/LanceGraphException.java b/java/src/main/java/com/adaworldapi/lancegraph/LanceGraphException.java new file mode 100644 index 0000000..93f1350 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/LanceGraphException.java @@ -0,0 +1,22 @@ +package com.adaworldapi.lancegraph; + +/** + * Base type for every failure this library reports. + * + *

Unchecked on purpose: the failures below are programming errors (using a closed resource, + * a version-incompatible native library) rather than recoverable conditions a caller is expected + * to branch on. There is no error string across the membrane — the native side returns a negative + * {@code i32} status and this layer turns it into a specific, named Java type. + */ +public class LanceGraphException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public LanceGraphException(String message) { + super(message); + } + + public LanceGraphException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/LaneId.java b/java/src/main/java/com/adaworldapi/lancegraph/LaneId.java new file mode 100644 index 0000000..b2f481e --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/LaneId.java @@ -0,0 +1,48 @@ +package com.adaworldapi.lancegraph; + +/** + * Which column of the struct-of-arrays a field lives in. + * + *

Not a consumer concept. An ordinary caller writes + * {@code Pattern.CLASS.eq(7)} and never sees a lane id — the generated schema vocabulary carries + * it. This type exists for the generator surface (code that mints {@code Field}s) and for + * the internals that marshal an operation descriptor. + * + *

Valhalla A/B candidate

+ * + *

This is written so the same source compiles as a {@code value record} on a JEP 401 + * JDK by changing only the modifier. It is therefore: + * + *

    + *
  • final and immutable; + *
  • free of identity dependence — never used as a lock, never compared with {@code ==}, + * {@code equals}/{@code hashCode} derived purely from state; + *
  • free of any field that could be null. + *
+ * + *

What identity-freedom buys: a value class can be flattened into its container and scalarised + * in registers, so wrapping a bare {@code int} in a meaningful name stops costing a heap object + * and a pointer chase. That is the whole point of the A/B — today the abstraction is paid for at + * runtime, and under Valhalla it should be free. Until then this remains an ordinary record and + * the JIT's escape analysis does most, but not all, of the same job. + * + * @param index zero-based lane index within its resource + */ +public record LaneId(int index) { + + public LaneId { + if (index < 0) { + throw new IllegalArgumentException("lane index must be >= 0, was " + index); + } + } + + /** Lane 0 of a pattern resource — entity ids. */ + public static LaneId of(int index) { + return new LaneId(index); + } + + @Override + public String toString() { + return "lane#" + index; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/Lens.java b/java/src/main/java/com/adaworldapi/lancegraph/Lens.java new file mode 100644 index 0000000..fc25fd7 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/Lens.java @@ -0,0 +1,61 @@ +package com.adaworldapi.lancegraph; + +/** + * One column, seen through one {@link View} — the projection concept. + * + * 

{@code
+ * long total = data.view()
+ *                  .where(Pattern.CLASS.eq(7))
+ *                  .lens(Pattern.VALUE)
+ *                  .sum();
+ * }
+ * + *

Deliberately narrow, and honest about it

+ * + *

A lens is where ⨯ which column, and that pairing is the durable idea: a + * selection says which rows, a lens says which column of them, and every bulk operation is some + * reduction over that pair. What a lens offers today is exactly what the membrane implements — + * a count and a signed 32-bit sum. It is a thin thing on purpose rather than a speculative + * framework with methods that would have to fail at runtime. + * + *

What it becomes: the place bulk projection lands — min/max, a histogram, a group-by, and bulk + * extraction of the selected values into a caller-supplied array. Every one of those is a single + * bulk crossing over an existing selection, so none of them changes the shape of this type; they + * are methods it grows. What it must never become is an iterator over rows, because a per-row + * accessor across the membrane is the exact anti-pattern the whole design exists to avoid. + */ +public final class Lens { + + private final View view; + private final I32Field field; + + Lens(View view, I32Field field) { + this.view = view; + this.field = field; + } + + /** Sum of this column over the selected rows, widened to 64 bits. */ + public long sum() { + return view.sumOf(field); + } + + /** How many rows the underlying view selects. */ + public long count() { + return view.count(); + } + + /** The column being projected. */ + public I32Field field() { + return field; + } + + /** The rows being projected through. */ + public View view() { + return view; + } + + @Override + public String toString() { + return "Lens[" + field.name() + " over " + view + "]"; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/Mask.java b/java/src/main/java/com/adaworldapi/lancegraph/Mask.java new file mode 100644 index 0000000..8952ff2 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/Mask.java @@ -0,0 +1,83 @@ +package com.adaworldapi.lancegraph; + +import com.adaworldapi.lancegraph.internal.ffm.Engine; + +/** + * A materialised selection: which rows a {@link View} chose, held natively as packed bits. + * + *

One bit per row. Selecting 64,000 of 64,000 entities costs 8,000 bytes — not 64,000 objects, + * not a list of indices, not a copy of anything. That is the whole reason the concept is exposed: + * a caller who wants to ask several questions about the same rows should pay for the answer once. + * + *

A caller who only wants a number should not use this at all — {@link View#count()} never + * materialises a selection the caller can see. + * + *

Lifetime

+ * + *

A selection is a child of the resource it was taken from. It may outlive its parent as an + * object, but it can never work after the parent closes: every operation then throws + * {@link ClosedResourceException}, which is the Java face of the ABI's {@code PARENT_CLOSED}. There + * is no arrangement of closes that lets a selection read freed memory. + */ +public final class Mask implements AutoCloseable { + + private final NativePattern parent; + private final long handle; + private boolean closed; + + Mask(NativePattern parent, long handle) { + this.parent = parent; + this.handle = handle; + } + + /** How many rows are selected. */ + public long count() { + requireUsable("count()"); + return Engine.maskCount(handle); + } + + /** The opaque identity of this selection. Diagnostics, logging, map keys. */ + public MaskId id() { + return new MaskId(handle); + } + + /** The resource whose rows this selects. */ + public NativePattern source() { + return parent; + } + + public boolean isOpen() { + return !closed && parent.isOpen(); + } + + /** Release the packed bits. Idempotency is not offered — a double close is an error. */ + @Override + public void close() { + if (closed) { + throw new ClosedResourceException("close() called on a selection that is already closed"); + } + closed = true; + if (parent.isOpen()) { + Engine.close(handle); + } + // If the parent is already gone, the selection was freed with it; calling close again + // would only earn an INVALID_HANDLE. Nothing leaks either way. + } + + private void requireUsable(String what) { + if (closed) { + throw new ClosedResourceException(what + " was called on a closed selection"); + } + if (!parent.isOpen()) { + throw new ClosedResourceException( + what + " was called on a selection whose resource is closed. The selection may" + + " outlive its parent as an object, but it can never work again" + + " (ABI status PARENT_CLOSED)."); + } + } + + @Override + public String toString() { + return "Mask[" + id() + (closed ? ", closed" : "") + "]"; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/MaskId.java b/java/src/main/java/com/adaworldapi/lancegraph/MaskId.java new file mode 100644 index 0000000..62d598d --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/MaskId.java @@ -0,0 +1,39 @@ +package com.adaworldapi.lancegraph; + +/** + * The identity of a selection, as an opaque token. + * + *

Deliberately opaque: it is not a pointer and cannot be turned into one. Underneath it + * is a generation-checked registry token — freeing a slot bumps its generation, so a token that + * outlives its resource resolves to "closed", never to freed memory. That is why using a stale + * selection raises {@link ClosedResourceException} instead of corrupting the process. + * + *

Exposed at all only so a selection can be logged, compared, or used as a map key. Nothing a + * caller can do with the two accessors below is useful for reaching the underlying storage. + * + *

Valhalla A/B candidate

+ * + *

Same rules as {@link LaneId}: final, immutable, identity-free, so the same source compiles as + * a {@code value record} under JEP 401. Note the consequence that makes this a good candidate — + * two masks are "the same mask" when their state is equal, never because they are the same object. + * Nothing here relies on reference equality, so flattening changes no observable behaviour. + * + * @param token the opaque registry token + */ +public record MaskId(long token) { + + /** Which registry slot. Diagnostic only. */ + public int slot() { + return (int) (token & 0xFFFF_FFFFL); + } + + /** How many times that slot has been recycled. Diagnostic only. */ + public int generation() { + return (int) (token >>> 32); + } + + @Override + public String toString() { + return "mask#" + slot() + "@" + generation(); + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/NativeCallException.java b/java/src/main/java/com/adaworldapi/lancegraph/NativeCallException.java new file mode 100644 index 0000000..6395bf7 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/NativeCallException.java @@ -0,0 +1,31 @@ +package com.adaworldapi.lancegraph; + +/** + * Thrown when a native call returned a negative status that is not more specifically modelled. + * + *

The numeric status is preserved so a test can assert on the exact ABI failure mode rather + * than on message text. + */ +public final class NativeCallException extends LanceGraphException { + + private static final long serialVersionUID = 1L; + + private final int status; + private final String statusName; + + public NativeCallException(String function, int status, String statusName, String meaning) { + super(function + " failed: " + statusName + " (" + status + ") — " + meaning); + this.status = status; + this.statusName = statusName; + } + + /** The raw negative {@code i32} the membrane returned. */ + public int status() { + return status; + } + + /** The ABI's name for that status, e.g. {@code INVALID_LANE}. */ + public String statusName() { + return statusName; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/NativeLibraryNotFoundException.java b/java/src/main/java/com/adaworldapi/lancegraph/NativeLibraryNotFoundException.java new file mode 100644 index 0000000..80b19f3 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/NativeLibraryNotFoundException.java @@ -0,0 +1,21 @@ +package com.adaworldapi.lancegraph; + +/** + * Thrown when the native artifact could not be located or loaded at all. + * + *

Distinct from {@link AbiMismatchException}: this one means "there was nothing to check", + * which lets a test report "the library has not been built yet" instead of failing with + * an unrelated {@code NullPointerException}. + */ +public final class NativeLibraryNotFoundException extends LanceGraphException { + + private static final long serialVersionUID = 1L; + + public NativeLibraryNotFoundException(String message) { + super(message); + } + + public NativeLibraryNotFoundException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/NativePattern.java b/java/src/main/java/com/adaworldapi/lancegraph/NativePattern.java new file mode 100644 index 0000000..28fa0fc --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/NativePattern.java @@ -0,0 +1,260 @@ +package com.adaworldapi.lancegraph; + +import com.adaworldapi.lancegraph.internal.ffm.Engine; +import com.adaworldapi.lancegraph.internal.ffm.PlanOp; + +import java.util.List; + +/** + * A set of rows held natively, opened once and closed once. + * + *

The headline claim of this project, stated concretely: 64,000 logical entities do not + * become 64,000 Java objects. They become one native lane set, one packed selection mask, + * a handful of tiny schema descriptors, and one bulk operation. Nothing in this class hydrates a + * row, and there is no API that could. + * + * 

{@code
+ * try (var data = NativePattern.open(65_536)) {
+ *     long n = data.view()
+ *                  .where(Pattern.CLASS.eq(7))
+ *                  .where(Pattern.VALUE.gt(100))
+ *                  .count();
+ * }
+ * }
+ * + *

Lifetime

+ * + *

Ordinary try-with-resources. After {@link #close()} every operation on this resource, and on + * any {@link View} or {@link Mask} derived from it, throws {@link ClosedResourceException}. That is + * enforced twice on purpose: this class fails fast on its own bookkeeping, and if a stale handle + * ever did reach native code the registry's generation check rejects it rather than dereferencing + * freed memory. Closing twice is an error, not a silent no-op. + * + *

Threading

+ * + *

Terminal operations reuse one internal scratch selection, so they are serialised on this + * instance. Distinct resources do not contend with each other. + */ +public final class NativePattern implements AutoCloseable { + + /** + * The seed used by {@link #open(long)}. + * + *

Generation is deterministic from the seed, which is what lets a test assert exact counts + * without shipping a data file — and lets Java recompute the expected answer independently, + * making the assertion a genuine cross-language check rather than a tautology. + */ + public static final long DEFAULT_SEED = 0xABCDL; + + private final long handle; + private final long rowCount; + private final Object lock = new Object(); + + private boolean closed; + private long scratchMask; // reused destination for terminal operations + private long auxMask; // second buffer, only the unfused comparison path needs it + private long allMask; // every row selected; only a predicate-free reduction needs it + + private NativePattern(long handle, long rowCount) { + this.handle = handle; + this.rowCount = rowCount; + } + + /** Open {@code rowCount} rows generated from {@link #DEFAULT_SEED}. */ + public static NativePattern open(long rowCount) { + return open(rowCount, DEFAULT_SEED); + } + + /** + * Open {@code rowCount} rows generated deterministically from {@code seed}. + * + * @throws NativeLibraryNotFoundException if the native artifact is not present + * @throws AbiMismatchException if it is present but does not match this build + */ + public static NativePattern open(long rowCount, long seed) { + if (rowCount < 0) { + throw new IllegalArgumentException("rowCount must be >= 0, was " + rowCount); + } + long h = Engine.openPattern(rowCount, seed); + // Read the row count back from the resource rather than trusting the request: it is the + // resource that decides, and caching it here means an empty View costs zero crossings. + return new NativePattern(h, Engine.rowCount(h)); + } + + /** + * A lazy description of every row. + * + *

Building it crosses the membrane zero times; narrowing it crosses zero times. Only a + * terminal operation executes. + */ + public View view() { + requireOpen("view()"); + return new View(this, List.of()); + } + + /** How many rows this resource holds. */ + public long rowCount() { + requireOpen("rowCount()"); + return rowCount; + } + + /** The row span, as a value. */ + public RowRange rows() { + requireOpen("rows()"); + return RowRange.of(rowCount); + } + + /** False once {@link #close()} has run. */ + public boolean isOpen() { + return !closed; + } + + /** + * Release the native storage. + * + *

Lanes are freed, the generation is bumped, and any child selection is orphaned — it can + * still exist, but it can never work again. + * + * @throws ClosedResourceException if already closed + */ + @Override + public void close() { + synchronized (lock) { + if (closed) { + throw new ClosedResourceException( + "close() called on a resource that is already closed. A double close is an" + + " error rather than a no-op, because the second call cannot know" + + " whether the handle was recycled in between."); + } + closed = true; + // Children first, so a mask is never left pointing at a dead parent even transiently. + if (scratchMask != 0) { + Engine.close(scratchMask); + scratchMask = 0; + } + if (auxMask != 0) { + Engine.close(auxMask); + auxMask = 0; + } + if (allMask != 0) { + Engine.close(allMask); + allMask = 0; + } + Engine.close(handle); + } + } + + // ── package-private: the execution surface View and friends use ────────────────────────── + + long handle() { + return handle; + } + + void requireOpen(String what) { + if (closed) { + throw new ClosedResourceException( + what + " was called on a closed resource. Java rejected it before reaching" + + " native code; had it not, the generation-checked handle would have" + + " returned INVALID_HANDLE rather than touching freed memory."); + } + } + + /** Evaluate a plan into the reusable scratch selection and return its popcount. */ + long countOf(List predicates) { + requireOpen("count()"); + if (predicates.isEmpty()) { + // No predicate selects every row. Answering from the cached count is not a shortcut + // around the membrane — an empty plan is rejected by the ABI (EMPTY_PLAN), and "every + // row" is a number this resource already knows. + return rowCount; + } + synchronized (lock) { + requireOpen("count()"); + return Engine.evaluateFused(handle, plan(predicates), scratch()); + } + } + + /** Evaluate a plan, then reduce one lane over the resulting selection. */ + long sumOf(List predicates, I32Field field) { + requireOpen("sumOf()"); + synchronized (lock) { + requireOpen("sumOf()"); + long mask; + if (predicates.isEmpty()) { + // Sum over everything: select all rows rather than inventing an always-true plan. + mask = all(); + } else { + mask = scratch(); + Engine.evaluateFused(handle, plan(predicates), mask); + } + return Engine.sumI32(handle, field.lane().index(), mask); + } + } + + /** Materialise a selection the caller owns and closes. */ + Mask selectInto(List predicates) { + requireOpen("select()"); + synchronized (lock) { + requireOpen("select()"); + long mask = Engine.createMask(handle, predicates.isEmpty()); + if (!predicates.isEmpty()) { + Engine.evaluateFused(handle, plan(predicates), mask); + } + return new Mask(this, mask); + } + } + + /** Diagnostics: the scalar reference kernel, same semantics, no SIMD. */ + long countScalar(List predicates) { + requireOpen("countScalar()"); + if (predicates.isEmpty()) { + return rowCount; + } + synchronized (lock) { + requireOpen("countScalar()"); + return Engine.evaluateScalar(handle, plan(predicates), scratch()); + } + } + + /** Diagnostics: one crossing per predicate plus the combines. Never the default. */ + long countUnfused(List predicates) { + requireOpen("countUnfused()"); + if (predicates.isEmpty()) { + return rowCount; + } + synchronized (lock) { + requireOpen("countUnfused()"); + return Engine.evaluateUnfused(handle, plan(predicates), scratch(), aux()); + } + } + + private List plan(List predicates) { + return predicates.stream().map(Predicate::op).toList(); + } + + private long scratch() { + if (scratchMask == 0) { + scratchMask = Engine.createMask(handle, false); + } + return scratchMask; + } + + private long aux() { + if (auxMask == 0) { + auxMask = Engine.createMask(handle, false); + } + return auxMask; + } + + private long all() { + if (allMask == 0) { + allMask = Engine.createMask(handle, true); + } + return allMask; + } + + @Override + public String toString() { + return "NativePattern[" + rowCount + " rows" + (closed ? ", closed" : "") + "]"; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/NativeRuntime.java b/java/src/main/java/com/adaworldapi/lancegraph/NativeRuntime.java new file mode 100644 index 0000000..ed36893 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/NativeRuntime.java @@ -0,0 +1,71 @@ +package com.adaworldapi.lancegraph; + +import com.adaworldapi.lancegraph.internal.ffm.Abi; + +/** + * What actually got loaded — for logs, benchmark headers, and bug reports. + * + *

Everything here is reported, never selected. In particular a caller cannot choose a + * SIMD backend: which one is compiled in is decided upstream, and a Java-side switch would be a + * second place for that decision to live. Knowing which one ran matters when reading a + * measurement; choosing it does not. + * + *

This is also the only public surface that mentions the ABI at all, and it mentions it as + * facts about a loaded artifact — no layout, no handle, no address. + */ +public final class NativeRuntime { + + private NativeRuntime() {} + + /** True if the native artifact loaded and passed the manifest cross-check. */ + public static boolean isAvailable() { + return Abi.isAvailable(); + } + + /** + * Why the library is unusable, or {@code null} if it loaded. + * + *

Lets a caller (a test, a startup probe) distinguish "not built yet" from "built but + * incompatible" without catching anything. + */ + public static LanceGraphException unavailableReason() { + return Abi.loadFailure(); + } + + /** Absolute path of the artifact in use. */ + public static String libraryPath() { + return Abi.libraryPath().toString(); + } + + /** ABI major version of the loaded artifact. Must match this build exactly. */ + public static int abiMajor() { + return Abi.manifest().abiMajor(); + } + + /** ABI minor version of the loaded artifact. Must be at least what this build expects. */ + public static int abiMinor() { + return Abi.manifest().abiMinor(); + } + + /** Human-readable name of the compiled SIMD backend, e.g. {@code "avx2"}. */ + public static String simdBackend() { + return Abi.manifest().simdBackendName(); + } + + /** {@code "release"} or {@code "debug"} — worth printing before believing a benchmark. */ + public static String buildProfile() { + return Abi.manifest().buildProfile(); + } + + /** One line summarising the loaded artifact. */ + public static String describe() { + if (!isAvailable()) { + return "lance-graph native runtime: UNAVAILABLE (" + + unavailableReason().getMessage() + ")"; + } + return "lance-graph native runtime: abi " + abiMajor() + "." + abiMinor() + + ", simd " + simdBackend() + + ", profile " + buildProfile() + + ", library " + libraryPath(); + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/Ordinal.java b/java/src/main/java/com/adaworldapi/lancegraph/Ordinal.java new file mode 100644 index 0000000..23476b9 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/Ordinal.java @@ -0,0 +1,35 @@ +package com.adaworldapi.lancegraph; + +/** + * A field's position within its schema. + * + *

Stable for the life of a schema version, which is what lets a generated vocabulary be + * compact: the name is a compile-time thing that costs nothing at runtime, and the ordinal is what + * actually travels. + * + *

Valhalla A/B candidate

+ * + *

Same rules as {@link LaneId}: final, immutable, identity-free, so the same source compiles as + * a {@code value record} under JEP 401. This one is the clearest illustration of the cost being + * removed — it wraps a single {@code int} purely so that an ordinal cannot be confused with a lane + * index or a row number, and under Valhalla that type-safety is expected to cost nothing at all. + * + * @param value zero-based position + */ +public record Ordinal(int value) { + + public Ordinal { + if (value < 0) { + throw new IllegalArgumentException("ordinal must be >= 0, was " + value); + } + } + + public static Ordinal of(int value) { + return new Ordinal(value); + } + + @Override + public String toString() { + return "#" + value; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/Pattern.java b/java/src/main/java/com/adaworldapi/lancegraph/Pattern.java new file mode 100644 index 0000000..ad4ccc8 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/Pattern.java @@ -0,0 +1,53 @@ +package com.adaworldapi.lancegraph; + +import java.util.List; + +/** + * The schema vocabulary for a pattern resource. + * + *

This file is shaped exactly as a code generator would emit it

+ * + *

It is hand-written today, and that is temporary. The generator is the accessibility + * story, not a convenience: it is what turns "lane 1, opcode 5, operand 7" into + * {@code Pattern.CLASS.eq(7)} with IDE autocompletion and a compile error when the types are + * wrong. A developer who has never heard of struct-of-arrays, SIMD, packed masks or FFM writes + * code that reads like every other Java API they have used, and gets columnar physics anyway. + * + *

Everything below is therefore written the way a generator writes: constants only, no logic, + * lane indices and ordinals stated once and never repeated, names taken from the schema. A future + * generator emits this same shape from a schema definition; nothing that consumes it changes. + * + * 

{@code
+ * try (var data = NativePattern.open(65_536)) {
+ *     long n = data.view()
+ *                  .where(Pattern.CLASS.eq(7))
+ *                  .where(Pattern.VALUE.gt(100))
+ *                  .count();
+ * }
+ * }
+ * + *

Note what the snippet does not mention: no arena, no segment, no mask, no lane, no + * opcode, no backend, no row loop. One resource, one chain, one number. + */ +public final class Pattern { + + private Pattern() {} + + /** The schema's name. */ + public static final String SCHEMA = "pattern"; + + /** Entity identity. Dense and zero-based in the fixture, so it doubles as the row index. */ + public static final U64Field ID = + new U64Field("id", LaneId.of(0), Ordinal.of(0)); + + /** Class tag, {@code 0..15} in the fixture. */ + public static final U32Field CLASS = + new U32Field("class", LaneId.of(1), Ordinal.of(1)); + + /** Signed measurement, {@code -150..361} in the fixture. */ + public static final I32Field VALUE = + new I32Field("value", LaneId.of(2), Ordinal.of(2)); + + /** Every field, in schema order. */ + public static final List FIELDS = List.of(ID, CLASS, VALUE); +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/Predicate.java b/java/src/main/java/com/adaworldapi/lancegraph/Predicate.java new file mode 100644 index 0000000..160ed50 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/Predicate.java @@ -0,0 +1,40 @@ +package com.adaworldapi.lancegraph; + +import com.adaworldapi.lancegraph.internal.ffm.PlanOp; + +/** + * A single condition, obtained from a schema field: {@code Pattern.CLASS.eq(7)}. + * + *

This is a descriptor, not a lambda, and that is the design's hinge. A + * {@code java.util.function.Predicate} would look more idiomatic and would be catastrophic: it + * forces a {@code Row} object per row and an upcall per row — 64,000 objects and 64,000 crossings + * for 64,000 entities. As a descriptor, N conditions marshal into one contiguous little array and + * cross once, whatever N and whatever the row count. + * + *

So the fluent chain looks like ordinary Java and behaves like a query planner. The developer + * writes what they mean; the shape of what they wrote is what makes it fast. + * + *

Opaque on purpose: there is no accessor for the opcode, the lane, or the operand. Those are + * physics. What a caller can do with a {@code Predicate} is pass it to {@link View#where}. + */ +public final class Predicate { + + private final PlanOp op; + private final String description; + + Predicate(PlanOp op, String description) { + this.op = op; + this.description = description; + } + + /** The marshallable form. Package-private: this is the membrane's business. */ + PlanOp op() { + return op; + } + + /** Human-readable form, e.g. {@code "class == 7"}. Diagnostics only. */ + @Override + public String toString() { + return description; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/RowRange.java b/java/src/main/java/com/adaworldapi/lancegraph/RowRange.java new file mode 100644 index 0000000..963f0c3 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/RowRange.java @@ -0,0 +1,53 @@ +package com.adaworldapi.lancegraph; + +/** + * A half-open span of row indices, {@code [start, endExclusive)}. + * + *

The unit in which this library thinks about "how much data". A caller may see one of these + * from {@link NativePattern#rows()}; nothing about the physical layout of those rows is implied or + * exposed. + * + *

Valhalla A/B candidate

+ * + *

Same rules as {@link LaneId}: final, immutable, identity-free, so the same source compiles as + * a {@code value record} under JEP 401. Identity-freedom is what lets a pair of longs live in + * registers instead of on the heap — the abstraction stops being something you pay for. + * + * @param start first row, inclusive + * @param endExclusive one past the last row + */ +public record RowRange(long start, long endExclusive) { + + public RowRange { + if (start < 0) { + throw new IllegalArgumentException("start must be >= 0, was " + start); + } + if (endExclusive < start) { + throw new IllegalArgumentException( + "endExclusive (" + endExclusive + ") must be >= start (" + start + ")"); + } + } + + /** All rows from 0 up to {@code count}. */ + public static RowRange of(long count) { + return new RowRange(0, count); + } + + /** Number of rows spanned. */ + public long length() { + return endExclusive - start; + } + + public boolean isEmpty() { + return endExclusive == start; + } + + public boolean contains(long row) { + return row >= start && row < endExclusive; + } + + @Override + public String toString() { + return "rows[" + start + "," + endExclusive + ")"; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/U32Field.java b/java/src/main/java/com/adaworldapi/lancegraph/U32Field.java new file mode 100644 index 0000000..433a84d --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/U32Field.java @@ -0,0 +1,38 @@ +package com.adaworldapi.lancegraph; + +import com.adaworldapi.lancegraph.internal.ffm.Layouts; +import com.adaworldapi.lancegraph.internal.ffm.PlanOp; + +/** + * An unsigned 32-bit column — a tag, a class, an enum-like discriminator. + * + *

The comparison methods here are exactly the ones the membrane has a kernel for. That is the + * type-safety story working in both directions: {@code Pattern.CLASS.gt("Berlin")} does not + * compile because a string is not a tag, and a comparison the native side cannot execute is not + * offered in the first place, so it cannot fail at runtime with an unknown opcode. + */ +public final class U32Field extends Field { + + /** + * Generator surface. Called by schema-vocabulary code, not by consumers. + * + * @param name the field's name in the schema + * @param lane the column it reads + * @param ordinal its position in the schema + */ + public U32Field(String name, LaneId lane, Ordinal ordinal) { + super(name, lane, ordinal); + } + + /** Rows whose value in this column equals {@code value}. */ + public Predicate eq(int value) { + return new Predicate( + PlanOp.narrowing(Layouts.OP_EQ_U32, lane().index(), Integer.toUnsignedLong(value)), + name() + " == " + Integer.toUnsignedString(value)); + } + + @Override + public String elementType() { + return "u32"; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/U64Field.java b/java/src/main/java/com/adaworldapi/lancegraph/U64Field.java new file mode 100644 index 0000000..86d1c5e --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/U64Field.java @@ -0,0 +1,23 @@ +package com.adaworldapi.lancegraph; + +/** + * An unsigned 64-bit column — an identity, a join key. + * + *

Offers no predicate yet, and that absence is deliberate rather than an oversight: the + * ABI implements no 64-bit comparison kernel, so offering {@code eq(long)} here would compile and + * then fail at runtime with an unknown opcode. The vocabulary a schema can express is exactly the + * vocabulary the membrane can execute. When a {@code u64} kernel lands, this class gains the method + * and nothing else changes. + */ +public final class U64Field extends Field { + + /** Generator surface. Called by schema-vocabulary code, not by consumers. */ + public U64Field(String name, LaneId lane, Ordinal ordinal) { + super(name, lane, ordinal); + } + + @Override + public String elementType() { + return "u64"; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/View.java b/java/src/main/java/com/adaworldapi/lancegraph/View.java new file mode 100644 index 0000000..0bea08d --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/View.java @@ -0,0 +1,140 @@ +package com.adaworldapi.lancegraph; + +import java.util.ArrayList; +import java.util.List; + +/** + * An immutable, lazy description of a set of rows. + * + *

A {@code View} is a description, not a result. {@link #where} returns a new view with + * one more condition and does nothing else: it allocates no selection, touches no row, and crosses + * the membrane zero times. A terminal operation — {@link #count()}, {@link #sumOf}, {@link #select} + * — is what executes, and it executes the whole chain in a single crossing. + * + *

That is why the fluent style is affordable here when it usually is not. Each + * {@code .where(...)} in a stream-like API normally costs another pass over the data; here it costs + * one 24-byte descriptor appended to a list, and the passes are fused into one bulk kernel that + * never returns to Java in between. + * + *

Monotonic narrowing — structural, not a convention

+ * + *

{@code where} intersects. There is no public composition that widens a view, because + * {@link Predicate} carries no combiner a caller can set and this class offers no {@code or}. So + * + * 

{@code
+ *   V0 = every row
+ *   V1 = V0 ∩ p0        V1 ⊆ V0
+ *   V2 = V1 ∩ p1        V2 ⊆ V1 ⊆ V0
+ * }
+ * + *

holds by construction: adding a condition can never increase the count. The ABI does have a + * widening combiner; it is deliberately unreachable from here. A future union would build its own + * plan and be a differently-named operation, so the invariant above stays readable as an invariant + * rather than a default someone can flip. + * + *

Sharing

+ * + *

Views are immutable, so a narrowed view never disturbs the one it came from and a single view + * can be reused, held, or passed around freely. All of them refer to the same native rows; none of + * them copies anything. + */ +public final class View { + + private final NativePattern owner; + private final List predicates; + + View(NativePattern owner, List predicates) { + this.owner = owner; + this.predicates = predicates; + } + + /** + * A new view narrowed by one more condition. + * + *

Crosses the membrane zero times. Nothing is evaluated until a terminal operation. + * + * @param predicate obtained from a schema field, e.g. {@code Pattern.CLASS.eq(7)} + * @return a new view; this one is unchanged + */ + public View where(Predicate predicate) { + java.util.Objects.requireNonNull(predicate, "predicate"); + owner.requireOpen("where()"); + List next = new ArrayList<>(predicates.size() + 1); + next.addAll(predicates); + next.add(predicate); + return new View(owner, List.copyOf(next)); + } + + /** + * How many rows this view selects. + * + *

One crossing, whatever the number of conditions and whatever the number + * of rows. The whole chain is marshalled into one descriptor array and evaluated by one fused + * kernel that returns a single number. + */ + public long count() { + return owner.countOf(predicates); + } + + /** + * Sum a signed 32-bit column over the rows this view selects, widened to 64 bits. + * + *

Two crossings: evaluate the chain, then reduce. Still independent of the row count. + */ + public long sumOf(I32Field field) { + java.util.Objects.requireNonNull(field, "field"); + return owner.sumOf(predicates, field); + } + + /** + * A projection of one column through this view. + * + *

See {@link Lens} for what this concept is and what it is deliberately not yet. + */ + public Lens lens(I32Field field) { + java.util.Objects.requireNonNull(field, "field"); + owner.requireOpen("lens()"); + return new Lens(this, field); + } + + /** + * Materialise this view as a selection the caller owns and closes. + * + *

Useful when the same set of rows will be asked several questions: the chain is evaluated + * once and the answer is kept natively as packed bits. A caller who only wants a number should + * use {@link #count()} instead and never see a selection at all. + */ + public Mask select() { + return owner.selectInto(predicates); + } + + /** The resource these rows live in. */ + public NativePattern source() { + return owner; + } + + /** How many conditions this view carries. Diagnostics. */ + public int conditionCount() { + return predicates.size(); + } + + /** Package-private: the chain, for the execution surface. */ + List predicates() { + return predicates; + } + + @Override + public String toString() { + if (predicates.isEmpty()) { + return "View[all rows]"; + } + StringBuilder sb = new StringBuilder("View["); + for (int i = 0; i < predicates.size(); i++) { + if (i > 0) { + sb.append(" AND "); + } + sb.append(predicates.get(i)); + } + return sb.append(']').toString(); + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Abi.java b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Abi.java new file mode 100644 index 0000000..0e5812a --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Abi.java @@ -0,0 +1,358 @@ +package com.adaworldapi.lancegraph.internal.ffm; + +import com.adaworldapi.lancegraph.AbiMismatchException; +import com.adaworldapi.lancegraph.LanceGraphException; +import com.adaworldapi.lancegraph.NativeLibraryNotFoundException; + +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SymbolLookup; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** + * Loads the native artifact and verifies it against this Java build's own + * compiled-in expectations before the first real call. + * + *

This class is the header replacement. There is no {@code .h} anywhere in this project and + * none will be added. A header is a text file that claims what a compiled artifact looks + * like and can drift from it silently — the classic ABI break. Here the artifact describes + * itself at runtime via {@code lgj_abi_manifest()}, and every reported size, alignment, + * version, pointer width and byte order is cross-checked against numbers Java derived + * independently from {@link Layouts}. The manifest is strictly stronger than a header, because it + * is emitted by the compiled artifact and therefore cannot disagree with itself. + * + *

The check is strict, not advisory: the first disagreement throws + * {@link AbiMismatchException} naming the exact field, and no further call is made. + * + *

Internal. Nothing here may appear in a public API signature. + */ +public final class Abi { + + private Abi() {} + + /** Explicit path to the shared object. Highest precedence. */ + public static final String PROP_LIBRARY = "lgj.library"; + + /** Directory to search for the platform-named artifact. */ + public static final String PROP_LIBRARY_DIR = "lgj.library.dir"; + + /** Environment fallback for {@link #PROP_LIBRARY}. */ + public static final String ENV_LIBRARY = "LGJ_LIBRARY"; + + /** + * The manifest, as read and validated. Field names mirror {@code LgjAbiManifest} exactly. + * + *

Immutable, identity-free, no reference-equality reliance — one of the Valhalla A/B + * candidates described in {@code com.adaworldapi.lancegraph.LaneId}. + */ + public record Manifest( + long magic, + int abiMajor, + int abiMinor, + int sizeOfManifest, + int sizeOfLaneDesc, + int sizeOfOpDesc, + int sizeOfResourceInfo, + int alignOfLaneDesc, + int alignOfOpDesc, + int alignOfResourceInfo, + int pointerBytes, + int endianness, + int simdBackend, + String simdBackendName, + String buildProfile) {} + + private static final Path LIBRARY_PATH; + private static final SymbolLookup LOOKUP; + private static final Manifest MANIFEST; + private static final LanceGraphException LOAD_FAILURE; + + static { + Path path = null; + SymbolLookup lookup = null; + Manifest manifest = null; + LanceGraphException failure = null; + try { + path = locateLibrary(); + // The global arena keeps the library mapped for the life of the JVM. Lane addresses + // handed out by the ABI are stable for the life of their owning resource (abi.md §4: + // lanes are never reallocated, resized or moved), so unloading is never desirable. + lookup = SymbolLookup.libraryLookup(path, Arena.global()); + manifest = readAndVerifyManifest(lookup, path); + } catch (LanceGraphException e) { + failure = e; + } catch (Throwable t) { + failure = new NativeLibraryNotFoundException( + "failed to load the lgj native library" + (path == null ? "" : " at " + path), t); + } + LIBRARY_PATH = path; + LOOKUP = lookup; + MANIFEST = manifest; + LOAD_FAILURE = failure; + } + + /** + * Throw the stored load failure if the library is unusable. + * + *

The failure is captured rather than thrown from the static initialiser so that a caller + * (in particular a test running before the artifact has been built) sees a precise + * {@link NativeLibraryNotFoundException} instead of a {@code NoClassDefFoundError} caused by a + * failed class initialisation. + */ + public static void ensureLoaded() { + if (LOAD_FAILURE != null) { + throw LOAD_FAILURE; + } + } + + /** True if the native artifact loaded and verified. */ + public static boolean isAvailable() { + return LOAD_FAILURE == null; + } + + /** The reason the library is unusable, or {@code null} if it loaded. */ + public static LanceGraphException loadFailure() { + return LOAD_FAILURE; + } + + public static SymbolLookup lookup() { + ensureLoaded(); + return LOOKUP; + } + + public static Manifest manifest() { + ensureLoaded(); + return MANIFEST; + } + + public static Path libraryPath() { + ensureLoaded(); + return LIBRARY_PATH; + } + + /** Human-readable name of the SIMD backend the artifact was compiled with. Reported, never negotiated. */ + public static String simdBackendName() { + return manifest().simdBackendName(); + } + + // ── location ───────────────────────────────────────────────────────────────────────────── + + private static Path locateLibrary() { + List tried = new ArrayList<>(); + + // An EXPLICIT request that cannot be honoured is a hard failure, never a fallback. + // + // This is not pedantry. Falling through to the search path here would mean that + // `-Dlgj.library=/path/that/moved.so` silently loads whatever artifact happens to be lying + // in a target directory instead — so a run intended to measure one library reports numbers + // from another, and the log says nothing. Silent substitution of a different binary is the + // exact failure mode the self-describing manifest exists to prevent one layer down; it + // would be absurd to reintroduce it one layer up. + String source = PROP_LIBRARY; + String explicit = System.getProperty(PROP_LIBRARY); + if (explicit == null || explicit.isBlank()) { + source = ENV_LIBRARY; + explicit = System.getenv(ENV_LIBRARY); + } + if (explicit != null && !explicit.isBlank()) { + Path p = Path.of(explicit); + if (Files.isRegularFile(p)) { + return p.toAbsolutePath(); + } + throw new NativeLibraryNotFoundException( + source + " names '" + p + "', which is not a regular file.\n" + + " Refusing to fall back to a search path: an explicitly requested" + + " library that silently becomes a different one is worse than a" + + " failure. Fix the path, or unset " + source + " to search."); + } + + String fileName = platformLibraryName(); + + String dir = System.getProperty(PROP_LIBRARY_DIR); + if (dir != null && !dir.isBlank()) { + Path p = Path.of(dir, fileName); + if (Files.isRegularFile(p)) { + return p.toAbsolutePath(); + } + throw new NativeLibraryNotFoundException( + PROP_LIBRARY_DIR + " names '" + dir + "', which contains no " + fileName + ".\n" + + " Refusing to fall back to a search path, for the same reason as" + + " above: an explicit request is honoured or it fails."); + } + + // Walk up from the working directory looking for a cargo target dir. Release first: if both + // exist, a debug build silently used for a benchmark would be a measurement error. + Path cwd = Path.of(System.getProperty("user.dir", ".")).toAbsolutePath(); + for (Path base = cwd; base != null; base = base.getParent()) { + for (String profile : new String[] {"release", "debug"}) { + Path p = base.resolve("target").resolve(profile).resolve(fileName); + if (Files.isRegularFile(p)) { + return p; + } + tried.add(p.toString()); + } + } + + throw new NativeLibraryNotFoundException( + "could not find the lgj native library (" + fileName + ").\n" + + " Set -D" + PROP_LIBRARY + "=/path/to/" + fileName + + " or -D" + PROP_LIBRARY_DIR + "=/path/to/dir" + + " or the " + ENV_LIBRARY + " environment variable.\n" + + " Searched:\n " + String.join("\n ", tried)); + } + + private static String platformLibraryName() { + String os = System.getProperty("os.name", "").toLowerCase(java.util.Locale.ROOT); + if (os.contains("win")) { + return "lgj_abi.dll"; + } + if (os.contains("mac") || os.contains("darwin")) { + return "liblgj_abi.dylib"; + } + return "liblgj_abi.so"; + } + + // ── manifest read + cross-check ────────────────────────────────────────────────────────── + + private static Manifest readAndVerifyManifest(SymbolLookup lookup, Path path) { + MemorySegment symbol = lookup.find("lgj_abi_manifest").orElseThrow(() -> + new AbiMismatchException( + "the library at " + path + " exports no lgj_abi_manifest symbol, so it" + + " cannot describe itself; refusing to call into it")); + + // The one symbol that returns a pointer rather than a status, because it has no failure + // mode: it returns a 'static. + MethodHandle mh = java.lang.foreign.Linker.nativeLinker() + .downcallHandle(symbol, FunctionDescriptor.of(ValueLayout.ADDRESS)); + + MemorySegment raw; + try { + raw = (MemorySegment) mh.invokeExact(); + } catch (Throwable t) { + throw new AbiMismatchException("lgj_abi_manifest() could not be called: " + t); + } + if (raw.equals(MemorySegment.NULL)) { + throw new AbiMismatchException("lgj_abi_manifest() returned NULL"); + } + + // Two-stage read. The returned segment is zero-length until reinterpreted, and how many + // bytes are legal to read is exactly what is in dispute — so read a minimal prefix first + // (magic, versions, size_of_manifest), decide whether the rest is safe, and only then widen. + // Fields are read by layout-derived offset, not through a struct VarHandle: such a handle + // bounds-checks the entire enclosing layout, which would defeat the point of a prefix read. + long prefixBytes = Layouts.OFF_SIZE_OF_MANIFEST + Integer.BYTES; + MemorySegment prefix = raw.reinterpret(prefixBytes); + + long magic = prefix.get(ValueLayout.JAVA_LONG, Layouts.OFF_MAGIC); + if (magic != Layouts.LGJ_MAGIC) { + throw new AbiMismatchException(String.format( + "manifest field 'magic' disagrees: library reports 0x%016X, this Java build" + + " expects 0x%016X. Read as a little-endian u64 the magic is also the" + + " endianness probe, so a mismatch here means either a different byte" + + " order or not an lgj artifact at all. Library: %s", + magic, Layouts.LGJ_MAGIC, path)); + } + + int major = prefix.get(ValueLayout.JAVA_INT, Layouts.OFF_ABI_MAJOR); + if (major != Layouts.LGJ_ABI_MAJOR) { + throw new AbiMismatchException(String.format( + "manifest field 'abi_major' disagrees: library is %d, this Java build was" + + " compiled against %d. A major mismatch is an incompatible change and" + + " is a hard failure, never a warning. Library: %s", + major, Layouts.LGJ_ABI_MAJOR, path)); + } + + int minor = prefix.get(ValueLayout.JAVA_INT, Layouts.OFF_ABI_MINOR); + if (minor < Layouts.LGJ_ABI_MINOR) { + throw new AbiMismatchException(String.format( + "manifest field 'abi_minor' disagrees: library is %d, this Java build requires" + + " >= %d. Additive changes bump minor; an older library cannot satisfy" + + " a newer caller. Library: %s", + minor, Layouts.LGJ_ABI_MINOR, path)); + } + + int sizeOfManifest = prefix.get(ValueLayout.JAVA_INT, Layouts.OFF_SIZE_OF_MANIFEST); + long expectedManifestSize = Layouts.MANIFEST.byteSize(); + if (sizeOfManifest < expectedManifestSize) { + throw new AbiMismatchException(String.format( + "manifest field 'size_of_manifest' disagrees: library reports %d bytes, this" + + " Java build's MemoryLayout derives %d. Reading the remaining fields" + + " would read past the artifact's own struct, so no further field is" + + " read. Library: %s", + sizeOfManifest, expectedManifestSize, path)); + } + + MemorySegment m = raw.reinterpret(Math.max(sizeOfManifest, expectedManifestSize)); + + // Every remaining check compares a number the artifact emitted against a number derived + // from Layouts — never a constant against itself. + expect(path, "size_of_lane_desc", Layouts.LANE_DESC.byteSize(), + m.get(ValueLayout.JAVA_INT, Layouts.OFF_SIZE_OF_LANE_DESC)); + expect(path, "align_of_lane_desc", Layouts.LANE_DESC.byteAlignment(), + m.get(ValueLayout.JAVA_INT, Layouts.OFF_ALIGN_OF_LANE_DESC)); + expect(path, "size_of_op_desc", Layouts.OP_DESC.byteSize(), + m.get(ValueLayout.JAVA_INT, Layouts.OFF_SIZE_OF_OP_DESC)); + expect(path, "align_of_op_desc", Layouts.OP_DESC.byteAlignment(), + m.get(ValueLayout.JAVA_INT, Layouts.OFF_ALIGN_OF_OP_DESC)); + expect(path, "size_of_resource_info", Layouts.RESOURCE_INFO.byteSize(), + m.get(ValueLayout.JAVA_INT, Layouts.OFF_SIZE_OF_RESOURCE_INFO)); + expect(path, "align_of_resource_info", Layouts.RESOURCE_INFO.byteAlignment(), + m.get(ValueLayout.JAVA_INT, Layouts.OFF_ALIGN_OF_RESOURCE_INFO)); + expect(path, "pointer_bytes", ValueLayout.ADDRESS.byteSize(), + m.get(ValueLayout.JAVA_INT, Layouts.OFF_POINTER_BYTES)); + + int endianness = m.get(ValueLayout.JAVA_INT, Layouts.OFF_ENDIANNESS); + if (endianness != Layouts.LGJ_ENDIAN_LITTLE) { + throw new AbiMismatchException( + "manifest field 'endianness' is " + endianness + "; only 0 (little) is defined." + + " Library: " + path); + } + if (ByteOrder.nativeOrder() != ByteOrder.LITTLE_ENDIAN) { + throw new AbiMismatchException( + "manifest field 'endianness' reports little-endian but this JVM's native order" + + " is " + ByteOrder.nativeOrder() + "; every struct read would be" + + " byte-swapped garbage. Library: " + path); + } + + int backend = m.get(ValueLayout.JAVA_INT, Layouts.OFF_SIMD_BACKEND); + String backendName = cString(m, Layouts.OFF_SIMD_BACKEND_NAME, Layouts.SIMD_NAME_BYTES); + String profile = cString(m, Layouts.OFF_BUILD_PROFILE, + Layouts.BUILD_PROFILE_BYTES); + + return new Manifest(magic, major, minor, sizeOfManifest, + (int) Layouts.LANE_DESC.byteSize(), (int) Layouts.OP_DESC.byteSize(), + (int) Layouts.RESOURCE_INFO.byteSize(), + (int) Layouts.LANE_DESC.byteAlignment(), (int) Layouts.OP_DESC.byteAlignment(), + (int) Layouts.RESOURCE_INFO.byteAlignment(), + (int) ValueLayout.ADDRESS.byteSize(), endianness, backend, backendName, profile); + } + + private static void expect(Path path, String field, long javaDerived, int libraryReports) { + if (javaDerived != libraryReports) { + throw new AbiMismatchException(String.format( + "manifest field '%s' disagrees: library reports %d, this Java build's" + + " MemoryLayout derives %d. Refusing to call into a layout this build" + + " would misread. Library: %s", + field, libraryReports, javaDerived, path)); + } + } + + /** Read a fixed-size NUL-terminated byte field. The only strings that cross the membrane. */ + private static String cString(MemorySegment m, long offset, int maxBytes) { + byte[] bytes = new byte[maxBytes]; + MemorySegment.copy(m, ValueLayout.JAVA_BYTE, offset, bytes, 0, maxBytes); + int len = 0; + while (len < maxBytes && bytes[len] != 0) { + len++; + } + return new String(bytes, 0, len, StandardCharsets.UTF_8); + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Downcalls.java b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Downcalls.java new file mode 100644 index 0000000..29a0cf1 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Downcalls.java @@ -0,0 +1,326 @@ +package com.adaworldapi.lancegraph.internal.ffm; + +import com.adaworldapi.lancegraph.LanceGraphException; + +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; +import java.util.concurrent.atomic.LongAdder; + +/** + * Every downcall handle in the project, resolved once into static finals. + * + *

Resolution is not free — {@code Linker.downcallHandle} builds a native stub — so doing it per + * call would be a per-crossing tax on top of the crossing itself. There are 14 symbols + * (docs/abi.md §7) and 14 handles. + * + *

The anti-JNI rule (docs/abi.md §6) lives here. Panama makes it easy to write + * JNI-shaped code: one downcall per element. Every wrapper below either does work proportional to + * {@code n_rows} or is lifecycle. There is no {@code readElement}, no upcall, no serialization, no + * {@code byte[]} bounce buffer, and no Java-side mirror of native data — by construction, not by + * convention, because the ABI exposes no symbol that would permit one. + * + *

Each wrapper translates a negative status into a specific exception via {@link Status}, so no + * caller above this class ever inspects a status code. + * + *

Internal. {@link MemorySegment} appears in these signatures and must not + * escape this package. + */ +public final class Downcalls { + + private Downcalls() {} + + private static final Linker LINKER = Linker.nativeLinker(); + + // Widths follow docs/abi.md exactly. u32 and i32 are both JAVA_INT: the ABI passes a 32-bit + // machine word and signedness is an interpretation, applied on the Rust side. + private static final MethodHandle PATTERN_OPEN = mh("lgj_pattern_open", + FunctionDescriptor.of(ValueLayout.JAVA_INT, + ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS)); + + private static final MethodHandle CLOSE = mh("lgj_close", + FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG)); + + private static final MethodHandle RESOURCE_INFO = mh("lgj_resource_info", + FunctionDescriptor.of(ValueLayout.JAVA_INT, + ValueLayout.JAVA_LONG, ValueLayout.ADDRESS)); + + private static final MethodHandle LANE_DESCRIBE = mh("lgj_lane_describe", + FunctionDescriptor.of(ValueLayout.JAVA_INT, + ValueLayout.JAVA_LONG, ValueLayout.JAVA_INT, ValueLayout.ADDRESS)); + + private static final MethodHandle MASK_CREATE = mh("lgj_mask_create", + FunctionDescriptor.of(ValueLayout.JAVA_INT, + ValueLayout.JAVA_LONG, ValueLayout.JAVA_INT, ValueLayout.ADDRESS)); + + private static final MethodHandle MASK_DESCRIBE = mh("lgj_mask_describe", + FunctionDescriptor.of(ValueLayout.JAVA_INT, + ValueLayout.JAVA_LONG, ValueLayout.ADDRESS)); + + private static final MethodHandle MASK_AND = mh("lgj_mask_and", + FunctionDescriptor.of(ValueLayout.JAVA_INT, + ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG)); + + private static final MethodHandle MASK_OR = mh("lgj_mask_or", + FunctionDescriptor.of(ValueLayout.JAVA_INT, + ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG)); + + private static final MethodHandle MASK_COUNT = mh("lgj_mask_count", + FunctionDescriptor.of(ValueLayout.JAVA_INT, + ValueLayout.JAVA_LONG, ValueLayout.ADDRESS)); + + private static final MethodHandle OP_EQ_U32 = mh("lgj_op_eq_u32", + FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, + ValueLayout.JAVA_INT, ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG)); + + private static final MethodHandle OP_GT_I32 = mh("lgj_op_gt_i32", + FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, + ValueLayout.JAVA_INT, ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG)); + + private static final MethodHandle PLAN_EVAL = mh("lgj_plan_eval", + FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS)); + + private static final MethodHandle PLAN_EVAL_SCALAR = mh("lgj_plan_eval_scalar", + FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS)); + + private static final MethodHandle REDUCE_SUM_I32 = mh("lgj_reduce_sum_i32", + FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, + ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS)); + + private static MethodHandle mh(String symbol, FunctionDescriptor descriptor) { + MemorySegment addr = Abi.lookup().find(symbol).orElseThrow(() -> + new LanceGraphException("the native library exports no symbol '" + symbol + + "'; it does not implement ABI " + Layouts.LGJ_ABI_MAJOR + "." + + Layouts.LGJ_ABI_MINOR)); + return LINKER.downcallHandle(addr, descriptor); + } + + // ── crossing instrumentation ───────────────────────────────────────────────────────────── + + private static final LongAdder CROSSINGS = new LongAdder(); + + /** + * Total number of membrane crossings made since JVM start. + * + *

This exists so laziness and fusion are observable rather than merely asserted in + * a comment: a test can snapshot this around building a {@code View} (expecting a delta of + * zero) and around a terminal operation (expecting a fixed small delta independent of how many + * predicates or rows are involved). + */ + public static long crossings() { + return CROSSINGS.sum(); + } + + private static void crossed() { + CROSSINGS.increment(); + } + + // ── lifecycle ──────────────────────────────────────────────────────────────────────────── + + /** Build the deterministic SoA fixture. Returns the resource handle. */ + public static long patternOpen(long nRows, long seed, MemorySegment outHandle) { + crossed(); + int st; + try { + st = (int) PATTERN_OPEN.invokeExact(nRows, seed, outHandle); + } catch (Throwable t) { + throw wrap("lgj_pattern_open", t); + } + Status.check("lgj_pattern_open", st); + return outHandle.get(ValueLayout.JAVA_LONG, 0); + } + + /** Free a resource, bump its generation, orphan its children. */ + public static void close(long handle) { + crossed(); + int st; + try { + st = (int) CLOSE.invokeExact(handle); + } catch (Throwable t) { + throw wrap("lgj_close", t); + } + Status.check("lgj_close", st); + } + + /** Fill {@code out} (a {@link Layouts#RESOURCE_INFO}-shaped segment). */ + public static void resourceInfo(long handle, MemorySegment out) { + crossed(); + int st; + try { + st = (int) RESOURCE_INFO.invokeExact(handle, out); + } catch (Throwable t) { + throw wrap("lgj_resource_info", t); + } + Status.check("lgj_resource_info", st); + } + + // ── lanes ──────────────────────────────────────────────────────────────────────────────── + + /** Fill {@code out} (a {@link Layouts#LANE_DESC}-shaped segment). */ + public static void laneDescribe(long handle, int laneId, MemorySegment out) { + crossed(); + int st; + try { + st = (int) LANE_DESCRIBE.invokeExact(handle, laneId, out); + } catch (Throwable t) { + throw wrap("lgj_lane_describe", t); + } + Status.check("lgj_lane_describe", st); + } + + // ── masks ──────────────────────────────────────────────────────────────────────────────── + + /** @param initial {@link Layouts#MASK_INIT_EMPTY} or {@link Layouts#MASK_INIT_ALL} */ + public static long maskCreate(long parent, int initial, MemorySegment outHandle) { + crossed(); + int st; + try { + st = (int) MASK_CREATE.invokeExact(parent, initial, outHandle); + } catch (Throwable t) { + throw wrap("lgj_mask_create", t); + } + Status.check("lgj_mask_create", st); + return outHandle.get(ValueLayout.JAVA_LONG, 0); + } + + public static void maskDescribe(long mask, MemorySegment out) { + crossed(); + int st; + try { + st = (int) MASK_DESCRIBE.invokeExact(mask, out); + } catch (Throwable t) { + throw wrap("lgj_mask_describe", t); + } + Status.check("lgj_mask_describe", st); + } + + /** {@code dst = a & b}. {@code dst} may alias {@code a} or {@code b}. */ + public static void maskAnd(long a, long b, long dst) { + crossed(); + int st; + try { + st = (int) MASK_AND.invokeExact(a, b, dst); + } catch (Throwable t) { + throw wrap("lgj_mask_and", t); + } + Status.check("lgj_mask_and", st); + } + + /** {@code dst = a | b}. */ + public static void maskOr(long a, long b, long dst) { + crossed(); + int st; + try { + st = (int) MASK_OR.invokeExact(a, b, dst); + } catch (Throwable t) { + throw wrap("lgj_mask_or", t); + } + Status.check("lgj_mask_or", st); + } + + public static long maskCount(long mask, MemorySegment outCount) { + crossed(); + int st; + try { + st = (int) MASK_COUNT.invokeExact(mask, outCount); + } catch (Throwable t) { + throw wrap("lgj_mask_count", t); + } + Status.check("lgj_mask_count", st); + return outCount.get(ValueLayout.JAVA_LONG, 0); + } + + // ── unfused predicates (benchmark / parity comparison only) ────────────────────────────── + + /** Overwrites {@code dstMask} with {@code lane == needle}. */ + public static void opEqU32(long res, int laneId, int needle, long dstMask) { + crossed(); + int st; + try { + st = (int) OP_EQ_U32.invokeExact(res, laneId, needle, dstMask); + } catch (Throwable t) { + throw wrap("lgj_op_eq_u32", t); + } + Status.check("lgj_op_eq_u32", st); + } + + /** Overwrites {@code dstMask} with {@code lane > threshold}, signed. */ + public static void opGtI32(long res, int laneId, int threshold, long dstMask) { + crossed(); + int st; + try { + st = (int) OP_GT_I32.invokeExact(res, laneId, threshold, dstMask); + } catch (Throwable t) { + throw wrap("lgj_op_gt_i32", t); + } + Status.check("lgj_op_gt_i32", st); + } + + // ── fused plan — N predicates, ONE crossing ────────────────────────────────────────────── + + /** + * Evaluate a whole predicate chain in one crossing and return the popcount. + * + *

This is the call that makes {@code .where(..).where(..).count()} cost one crossing + * regardless of predicate count or row count. + */ + public static long planEval(long res, MemorySegment ops, int nOps, long dstMask, + MemorySegment outCount) { + crossed(); + int st; + try { + st = (int) PLAN_EVAL.invokeExact(res, ops, nOps, dstMask, outCount); + } catch (Throwable t) { + throw wrap("lgj_plan_eval", t); + } + Status.check("lgj_plan_eval", st); + return outCount.get(ValueLayout.JAVA_LONG, 0); + } + + /** + * Identical semantics to {@link #planEval} but forced down the scalar reference path. + * + *

Exists only so SIMD-vs-scalar parity is falsifiable through the membrane, which + * is where the Java tests live. Not for production use. + */ + public static long planEvalScalar(long res, MemorySegment ops, int nOps, long dstMask, + MemorySegment outCount) { + crossed(); + int st; + try { + st = (int) PLAN_EVAL_SCALAR.invokeExact(res, ops, nOps, dstMask, outCount); + } catch (Throwable t) { + throw wrap("lgj_plan_eval_scalar", t); + } + Status.check("lgj_plan_eval_scalar", st); + return outCount.get(ValueLayout.JAVA_LONG, 0); + } + + // ── reduction ──────────────────────────────────────────────────────────────────────────── + + /** Sum an {@code I32} lane over set mask bits into a widened {@code i64}. */ + public static long reduceSumI32(long res, int laneId, long mask, MemorySegment outSum) { + crossed(); + int st; + try { + st = (int) REDUCE_SUM_I32.invokeExact(res, laneId, mask, outSum); + } catch (Throwable t) { + throw wrap("lgj_reduce_sum_i32", t); + } + Status.check("lgj_reduce_sum_i32", st); + return outSum.get(ValueLayout.JAVA_LONG, 0); + } + + private static LanceGraphException wrap(String symbol, Throwable t) { + if (t instanceof LanceGraphException e) { + return e; + } + return new LanceGraphException("the downcall to " + symbol + " itself failed", t); + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Engine.java b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Engine.java new file mode 100644 index 0000000..0a48c7c --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Engine.java @@ -0,0 +1,270 @@ +package com.adaworldapi.lancegraph.internal.ffm; + +import com.adaworldapi.lancegraph.LanceGraphException; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.List; + +/** + * The only place in the project where a plan is turned into bytes and pushed across the membrane. + * + *

Everything above this class speaks {@link PlanOp} and {@code long} handles; everything below + * speaks {@link MemorySegment}. Keeping the marshalling here is what lets the public API contain no + * FFM type at all. + * + *

Scratch memory

+ * + *

Out-parameters and the op array live in a per-thread scratch buffer allocated from an + * automatic arena, so a terminal operation performs no allocation in steady state. This + * matters for the benchmark: if each {@code count()} allocated an arena, the measurement would be + * reporting allocator behaviour rather than crossing cost. + * + *

Internal. + */ +public final class Engine { + + private Engine() {} + + /** Room for the widest out-parameter set any single call needs. */ + private static final long OUT_BYTES = 32; + + private static final class Scratch { + // ofAuto: freed when this Scratch becomes unreachable, i.e. when the thread dies. No + // explicit lifetime to get wrong, and no leak because the ThreadLocal holds exactly one. + private final Arena arena = Arena.ofAuto(); + private final MemorySegment out = arena.allocate(OUT_BYTES, 8); + private final MemorySegment info = arena.allocate(Layouts.RESOURCE_INFO); + private final MemorySegment desc = arena.allocate(Layouts.LANE_DESC); + private MemorySegment ops = arena.allocate(Layouts.OP_DESC, 8); + private long opsCapacity = 8; + + MemorySegment opsFor(int nOps) { + if (nOps > opsCapacity) { + long want = Math.max(nOps, opsCapacity * 2); + ops = arena.allocate(Layouts.OP_DESC, want); + opsCapacity = want; + } + return ops; + } + } + + private static final ThreadLocal SCRATCH = ThreadLocal.withInitial(Scratch::new); + + // ── lifecycle ──────────────────────────────────────────────────────────────────────────── + + /** Open the deterministic SoA fixture; returns the resource handle. */ + public static long openPattern(long nRows, long seed) { + Scratch s = SCRATCH.get(); + return Downcalls.patternOpen(nRows, seed, s.out); + } + + /** Create a selection over {@code parent}; returns the mask handle. */ + public static long createMask(long parent, boolean allSet) { + Scratch s = SCRATCH.get(); + return Downcalls.maskCreate(parent, + allSet ? Layouts.MASK_INIT_ALL : Layouts.MASK_INIT_EMPTY, s.out); + } + + public static void close(long handle) { + Downcalls.close(handle); + } + + /** Row count of a resource, read from {@code LgjResourceInfo}. */ + public static long rowCount(long handle) { + Scratch s = SCRATCH.get(); + Downcalls.resourceInfo(handle, s.info); + return (long) Layouts.INFO_N_ROWS.get(s.info, 0L); + } + + /** Liveness stamp of a resource. Java re-checks this before trusting a cached lane segment. */ + public static long epoch(long handle) { + Scratch s = SCRATCH.get(); + Downcalls.resourceInfo(handle, s.info); + return (long) Layouts.INFO_EPOCH.get(s.info, 0L); + } + + public static long maskCount(long mask) { + Scratch s = SCRATCH.get(); + return Downcalls.maskCount(mask, s.out); + } + + // ── evaluation ─────────────────────────────────────────────────────────────────────────── + + /** + * The production path: marshal the whole chain and cross once. + * + * @return popcount of the resulting selection + */ + public static long evaluateFused(long resource, List plan, long dstMask) { + MemorySegment ops = marshal(plan); + Scratch s = SCRATCH.get(); + return Downcalls.planEval(resource, ops, plan.size(), dstMask, s.out); + } + + /** + * Same semantics, forced down the scalar reference kernel. + * + *

Only for falsifying SIMD-vs-scalar parity from the Java side, which is where the tests + * live. Never the default. + */ + public static long evaluateScalar(long resource, List plan, long dstMask) { + MemorySegment ops = marshal(plan); + Scratch s = SCRATCH.get(); + return Downcalls.planEvalScalar(resource, ops, plan.size(), dstMask, s.out); + } + + /** + * The comparison path: one crossing per predicate plus one per combine plus one to count. + * + *

Retained only so the fused path can be benchmarked against something and so + * parity can be checked predicate-by-predicate. Using it in production would be the anti-JNI + * rule's exact failure mode, just with a coarser granularity than per element. + * + * @param scratchMask a second mask over the same parent, used to hold each intermediate + */ + public static long evaluateUnfused(long resource, List plan, long dstMask, + long scratchMask) { + if (plan.isEmpty()) { + throw new LanceGraphException("an unfused plan needs at least one predicate"); + } + for (int i = 0; i < plan.size(); i++) { + PlanOp p = plan.get(i); + // The first predicate writes straight into dst (each op *overwrites* its destination); + // every later one goes to scratch and is folded in. + long target = (i == 0) ? dstMask : scratchMask; + applyOne(resource, p, target); + if (i > 0) { + if (p.combine() == Layouts.COMBINE_OR) { + Downcalls.maskOr(dstMask, scratchMask, dstMask); + } else { + Downcalls.maskAnd(dstMask, scratchMask, dstMask); + } + } + } + return maskCount(dstMask); + } + + private static void applyOne(long resource, PlanOp p, long dst) { + switch (p.op()) { + case Layouts.OP_EQ_U32 -> Downcalls.opEqU32(resource, p.laneId(), (int) p.operand(), dst); + case Layouts.OP_GT_I32 -> Downcalls.opGtI32(resource, p.laneId(), (int) p.operand(), dst); + default -> throw new LanceGraphException( + "opcode " + p.op() + " has no unfused equivalent; the unfused path is a" + + " benchmark comparison only and is not required to cover every op"); + } + } + + /** Sum an {@code I32} lane over the set bits of {@code mask}, widened to 64 bits. */ + public static long sumI32(long resource, int laneId, long mask) { + Scratch s = SCRATCH.get(); + return Downcalls.reduceSumI32(resource, laneId, mask, s.out); + } + + private static MemorySegment marshal(List plan) { + int n = plan.size(); + if (n == 0) { + throw new LanceGraphException( + "an empty plan cannot be evaluated (ABI status EMPTY_PLAN). A View with no" + + " predicates selects every row; ask for the row count instead."); + } + MemorySegment ops = SCRATCH.get().opsFor(n); + long stride = Layouts.OP_DESC.byteSize(); + for (int i = 0; i < n; i++) { + PlanOp p = plan.get(i); + long base = i * stride; + Layouts.OP_OP.set(ops, base, p.op()); + Layouts.OP_LANE_ID.set(ops, base, p.laneId()); + Layouts.OP_OPERAND.set(ops, base, p.operand()); + Layouts.OP_COMBINE.set(ops, base, p.combine()); + // Reserved must be 0 — set explicitly rather than relying on the buffer being fresh, + // since the scratch is reused across calls. + Layouts.OP_RESERVED.set(ops, base, 0); + } + return ops; + } + + // ── lane access (in-process, zero crossings) ───────────────────────────────────────────── + + /** + * Describe a lane and hand back a bounded, read-only window onto it. + * + *

There is deliberately no {@code readElement} anywhere in the ABI: reading one row across + * the membrane would be the per-element crossing the whole design forbids. If Java wants + * element access it reads this segment directly, in-process, with no crossing at all. + * + *

The segment is valid until the owning resource is closed (lanes are allocated once and + * never reallocated, resized or moved — a hard ABI guarantee). The {@code epoch} returned + * alongside lets a caller detect a segment it is still holding after a close. + */ + public static LaneWindow describeLane(long resource, int laneId) { + Scratch s = SCRATCH.get(); + Downcalls.laneDescribe(resource, laneId, s.desc); + return windowOf(s.desc); + } + + /** As {@link #describeLane}, for the packed-bit words of a mask. */ + public static LaneWindow describeMask(long mask) { + Scratch s = SCRATCH.get(); + Downcalls.maskDescribe(mask, s.desc); + return windowOf(s.desc); + } + + private static LaneWindow windowOf(MemorySegment desc) { + long addr = (long) Layouts.LANE_ADDR.get(desc, 0L); + long lenElems = (long) Layouts.LANE_LEN_ELEMS.get(desc, 0L); + long byteLen = (long) Layouts.LANE_BYTE_LEN.get(desc, 0L); + long owner = (long) Layouts.LANE_OWNER.get(desc, 0L); + long epoch = (long) Layouts.LANE_EPOCH.get(desc, 0L); + int elemKind = (int) Layouts.LANE_ELEM_KIND.get(desc, 0L); + int elemBytes = (int) Layouts.LANE_ELEM_BYTES.get(desc, 0L); + int strideBytes = (int) Layouts.LANE_STRIDE_BYTES.get(desc, 0L); + int flags = (int) Layouts.LANE_FLAGS.get(desc, 0L); + + MemorySegment segment = MemorySegment.ofAddress(addr).reinterpret(byteLen); + return new LaneWindow(segment, lenElems, byteLen, owner, epoch, elemKind, elemBytes, + strideBytes, flags); + } + + /** + * A bounded view of native storage plus the liveness stamp needed to distrust it later. + * + *

Internal. This is the one type that carries a {@link MemorySegment}, and + * it must never appear in a public signature. + */ + public record LaneWindow( + MemorySegment segment, + long lengthElements, + long byteLength, + long owner, + long epoch, + int elemKind, + int elemBytes, + int strideBytes, + int flags) { + + public boolean isWritable() { + return (flags & Layouts.FLAG_WRITABLE) != 0; + } + + public boolean isContiguous() { + return (flags & Layouts.FLAG_CONTIGUOUS) != 0; + } + + /** Read one {@code i32} element, in-process. No membrane crossing occurs. */ + public int getI32(long index) { + return segment.get(ValueLayout.JAVA_INT, index * strideBytes); + } + + /** Read one {@code u32} element as an unsigned value widened to {@code long}. */ + public long getU32(long index) { + return Integer.toUnsignedLong(segment.get(ValueLayout.JAVA_INT, index * strideBytes)); + } + + /** Read one 64-bit element (an id, or a packed mask word). */ + public long getU64(long index) { + return segment.get(ValueLayout.JAVA_LONG, index * strideBytes); + } + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Layouts.java b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Layouts.java new file mode 100644 index 0000000..6afa2b5 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Layouts.java @@ -0,0 +1,256 @@ +package com.adaworldapi.lancegraph.internal.ffm; + +import java.lang.foreign.MemoryLayout; +import java.lang.foreign.MemoryLayout.PathElement; +import java.lang.foreign.StructLayout; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.VarHandle; + +/** + * Java's independently compiled-in mirror of every {@code #[repr(C)]} struct in + * {@code docs/abi.md} §5. + * + *

This class is the header replacement's Java half. There is no {@code .h} file, no + * {@code cbindgen}, and no {@code jextract} anywhere in this project — a header is a text file that + * claims a layout and can drift silently. Instead the layouts below are written by hand + * from the normative ABI document, their sizes are derived via + * {@link MemoryLayout#byteSize()}, and {@link Abi} cross-checks those derived numbers against the + * sizes the compiled artifact reports about itself. Two independently-derived numbers are compared; + * a constant is never compared against itself. + * + *

Field offsets are the System V AMD64 aggregate rule (the same rule {@code #[repr(C)]} names), + * so every layout here is written with natural packing and asserted to match the byte sizes stated + * in the ABI document. No explicit padding member is required by any of these four structs — that + * is checked in {@link #SELF_CHECK}, not assumed. + * + *

Internal. Nothing in this class may appear in a public API signature. + */ +public final class Layouts { + + private Layouts() {} + + // ── ABI constants (docs/abi.md §2) ─────────────────────────────────────────────────────── + + /** {@code "LGJ_ABI\0"} read as a little-endian {@code u64}; doubles as the endianness probe. */ + public static final long LGJ_MAGIC = 0x4C_47_4A_5F_41_42_49_00L; + + /** Major version this Java build was compiled against. Must match the library exactly. */ + public static final int LGJ_ABI_MAJOR = 0; + + /** Minor version this Java build was compiled against. The library must be {@code >=} this. */ + public static final int LGJ_ABI_MINOR = 1; + + /** {@code endianness} value meaning little-endian. */ + public static final int LGJ_ENDIAN_LITTLE = 0; + + // ── Element kinds (docs/abi.md §5) — start at 1 so a zeroed struct is detectably invalid ── + + public static final int ELEM_U8 = 1; + public static final int ELEM_I8 = 2; + public static final int ELEM_U16 = 3; + public static final int ELEM_I16 = 4; + public static final int ELEM_U32 = 5; + public static final int ELEM_I32 = 6; + public static final int ELEM_U64 = 7; + public static final int ELEM_I64 = 8; + public static final int ELEM_F32 = 9; + public static final int ELEM_F64 = 10; + /** A {@code u64} of 64 packed row bits, LSB = lowest row index. */ + public static final int ELEM_MASK_WORD = 11; + + // ── Lane descriptor flags (docs/abi.md §5) ─────────────────────────────────────────────── + + public static final int FLAG_READABLE = 1 << 0; + public static final int FLAG_WRITABLE = 1 << 1; + public static final int FLAG_CONTIGUOUS = 1 << 2; + + // ── Resource kinds (docs/abi.md §5) ────────────────────────────────────────────────────── + + public static final int RESOURCE_PATTERN = 1; + public static final int RESOURCE_MASK = 2; + + // ── Mask initial fill (docs/abi.md §7) ─────────────────────────────────────────────────── + + public static final int MASK_INIT_EMPTY = 0; + public static final int MASK_INIT_ALL = 1; + + // ── Opcodes and combiners (docs/abi.md §5 `LgjOpCode`) ─────────────────────────────────── + // + // NOTE (doc gap, reported): abi.md §5 names the `op` field's type as `LgjOpCode` but never + // tabulates its numeric values, unlike LgjElemKind / status codes / simd backends which are all + // tabulated. The values below follow the document's own stated convention for enumerations that + // must be zero-invalid (element kinds "start at 1, so a zeroed struct is detectably invalid"), + // and were confirmed against the compiled artifact. Combiners are fixed by §5's own prose: + // "0 = AND (narrow), 1 = OR (widen)". + + /** {@code lane[i] == (u32) operand}. Requires a {@code U32} lane. */ + public static final int OP_EQ_U32 = 1; + + /** {@code lane[i] > (i32) operand}, signed. Requires an {@code I32} lane. */ + public static final int OP_GT_I32 = 2; + + /** Intersect into the accumulator — narrowing. */ + public static final int COMBINE_AND = 0; + + /** Union into the accumulator — widening. Deliberately unreachable from {@code View.where}. */ + public static final int COMBINE_OR = 1; + + // ── SIMD backend ids (docs/abi.md §5) ──────────────────────────────────────────────────── + + public static final int SIMD_SCALAR = 0; + public static final int SIMD_AVX2 = 1; + public static final int SIMD_AVX512 = 2; + public static final int SIMD_NEON = 3; + public static final int SIMD_WASM = 4; + + // ── LgjAbiManifest ─────────────────────────────────────────────────────────────────────── + + /** Length of the manifest's {@code simd_backend_name} byte array. */ + public static final int SIMD_NAME_BYTES = 32; + + /** Length of the manifest's {@code build_profile} byte array. */ + public static final int BUILD_PROFILE_BYTES = 16; + + public static final StructLayout MANIFEST = MemoryLayout.structLayout( + ValueLayout.JAVA_LONG.withName("magic"), + ValueLayout.JAVA_INT.withName("abi_major"), + ValueLayout.JAVA_INT.withName("abi_minor"), + ValueLayout.JAVA_INT.withName("size_of_manifest"), + ValueLayout.JAVA_INT.withName("size_of_lane_desc"), + ValueLayout.JAVA_INT.withName("size_of_op_desc"), + ValueLayout.JAVA_INT.withName("size_of_resource_info"), + ValueLayout.JAVA_INT.withName("align_of_lane_desc"), + ValueLayout.JAVA_INT.withName("align_of_op_desc"), + ValueLayout.JAVA_INT.withName("align_of_resource_info"), + ValueLayout.JAVA_INT.withName("pointer_bytes"), + ValueLayout.JAVA_INT.withName("endianness"), + ValueLayout.JAVA_INT.withName("simd_backend"), + MemoryLayout.sequenceLayout(SIMD_NAME_BYTES, ValueLayout.JAVA_BYTE) + .withName("simd_backend_name"), + MemoryLayout.sequenceLayout(BUILD_PROFILE_BYTES, ValueLayout.JAVA_BYTE) + .withName("build_profile")) + .withName("LgjAbiManifest"); + + // The manifest is read by OFFSET rather than through a struct VarHandle, and that is a + // requirement rather than a style choice. A VarHandle derived from a struct layout bounds-checks + // the *whole enclosing layout* against the segment, so it cannot be used to read a deliberate + // prefix — and reading a prefix first is exactly how the size disagreement is detected safely + // (see Abi#readAndVerifyManifest). The offsets are still derived from the layout, so this stays + // an independent derivation and never a hand-counted constant. + public static final long OFF_MAGIC = off("magic"); + public static final long OFF_ABI_MAJOR = off("abi_major"); + public static final long OFF_ABI_MINOR = off("abi_minor"); + public static final long OFF_SIZE_OF_MANIFEST = off("size_of_manifest"); + public static final long OFF_SIZE_OF_LANE_DESC = off("size_of_lane_desc"); + public static final long OFF_SIZE_OF_OP_DESC = off("size_of_op_desc"); + public static final long OFF_SIZE_OF_RESOURCE_INFO = off("size_of_resource_info"); + public static final long OFF_ALIGN_OF_LANE_DESC = off("align_of_lane_desc"); + public static final long OFF_ALIGN_OF_OP_DESC = off("align_of_op_desc"); + public static final long OFF_ALIGN_OF_RESOURCE_INFO = off("align_of_resource_info"); + public static final long OFF_POINTER_BYTES = off("pointer_bytes"); + public static final long OFF_ENDIANNESS = off("endianness"); + public static final long OFF_SIMD_BACKEND = off("simd_backend"); + public static final long OFF_SIMD_BACKEND_NAME = off("simd_backend_name"); + public static final long OFF_BUILD_PROFILE = off("build_profile"); + + private static long off(String name) { + return MANIFEST.byteOffset(PathElement.groupElement(name)); + } + + // ── LgjLaneDesc (56 bytes, align 8) ────────────────────────────────────────────────────── + + public static final StructLayout LANE_DESC = MemoryLayout.structLayout( + ValueLayout.JAVA_LONG.withName("addr"), + ValueLayout.JAVA_LONG.withName("len_elems"), + ValueLayout.JAVA_LONG.withName("byte_len"), + ValueLayout.JAVA_LONG.withName("owner"), + ValueLayout.JAVA_LONG.withName("epoch"), + ValueLayout.JAVA_INT.withName("elem_kind"), + ValueLayout.JAVA_INT.withName("elem_bytes"), + ValueLayout.JAVA_INT.withName("stride_bytes"), + ValueLayout.JAVA_INT.withName("flags")) + .withName("LgjLaneDesc"); + + public static final VarHandle LANE_ADDR = field(LANE_DESC, "addr"); + public static final VarHandle LANE_LEN_ELEMS = field(LANE_DESC, "len_elems"); + public static final VarHandle LANE_BYTE_LEN = field(LANE_DESC, "byte_len"); + public static final VarHandle LANE_OWNER = field(LANE_DESC, "owner"); + public static final VarHandle LANE_EPOCH = field(LANE_DESC, "epoch"); + public static final VarHandle LANE_ELEM_KIND = field(LANE_DESC, "elem_kind"); + public static final VarHandle LANE_ELEM_BYTES = field(LANE_DESC, "elem_bytes"); + public static final VarHandle LANE_STRIDE_BYTES = field(LANE_DESC, "stride_bytes"); + public static final VarHandle LANE_FLAGS = field(LANE_DESC, "flags"); + + // ── LgjResourceInfo (32 bytes) ─────────────────────────────────────────────────────────── + + public static final StructLayout RESOURCE_INFO = MemoryLayout.structLayout( + ValueLayout.JAVA_INT.withName("kind"), + ValueLayout.JAVA_INT.withName("lane_count"), + ValueLayout.JAVA_LONG.withName("n_rows"), + ValueLayout.JAVA_LONG.withName("epoch"), + ValueLayout.JAVA_LONG.withName("parent")) + .withName("LgjResourceInfo"); + + public static final VarHandle INFO_KIND = field(RESOURCE_INFO, "kind"); + public static final VarHandle INFO_LANE_COUNT = field(RESOURCE_INFO, "lane_count"); + public static final VarHandle INFO_N_ROWS = field(RESOURCE_INFO, "n_rows"); + public static final VarHandle INFO_EPOCH = field(RESOURCE_INFO, "epoch"); + public static final VarHandle INFO_PARENT = field(RESOURCE_INFO, "parent"); + + // ── LgjOpDesc (24 bytes, align 8) ──────────────────────────────────────────────────────── + + public static final StructLayout OP_DESC = MemoryLayout.structLayout( + ValueLayout.JAVA_INT.withName("op"), + ValueLayout.JAVA_INT.withName("lane_id"), + ValueLayout.JAVA_LONG.withName("operand"), + ValueLayout.JAVA_INT.withName("combine"), + ValueLayout.JAVA_INT.withName("_reserved")) + .withName("LgjOpDesc"); + + public static final VarHandle OP_OP = field(OP_DESC, "op"); + public static final VarHandle OP_LANE_ID = field(OP_DESC, "lane_id"); + public static final VarHandle OP_OPERAND = field(OP_DESC, "operand"); + public static final VarHandle OP_COMBINE = field(OP_DESC, "combine"); + public static final VarHandle OP_RESERVED = field(OP_DESC, "_reserved"); + + /** + * Compile-time-ish self check: the byte sizes the ABI document states in prose, checked against + * the sizes these layouts actually derive. If a layout is edited wrongly this fails at class + * initialisation, before any native call, and names the struct that disagreed. + * + *

This does not replace {@link Abi}'s manifest cross-check — it catches a Java-side typo; + * the manifest catches a Java-vs-Rust divergence. + */ + public static final boolean SELF_CHECK = selfCheck(); + + private static boolean selfCheck() { + expect("LgjLaneDesc size", 56, LANE_DESC.byteSize()); + expect("LgjLaneDesc align", 8, LANE_DESC.byteAlignment()); + expect("LgjResourceInfo size", 32, RESOURCE_INFO.byteSize()); + expect("LgjResourceInfo align", 8, RESOURCE_INFO.byteAlignment()); + expect("LgjOpDesc size", 24, OP_DESC.byteSize()); + expect("LgjOpDesc align", 8, OP_DESC.byteAlignment()); + // The manifest's size is not stated in prose; it is stated by the artifact itself and + // cross-checked in Abi. What is checked here is that the two trailing byte arrays land + // where the repr(C) rule puts them, which is the only place a hand-written layout could + // silently disagree. + expect("LgjAbiManifest simd_backend_name offset", 56, OFF_SIMD_BACKEND_NAME); + expect("LgjAbiManifest build_profile offset", 88, OFF_BUILD_PROFILE); + expect("LgjAbiManifest align", 8, MANIFEST.byteAlignment()); + return true; + } + + private static void expect(String what, long expected, long actual) { + if (expected != actual) { + throw new ExceptionInInitializerError( + "Layouts self-check failed: " + what + " expected " + expected + + " but this build's MemoryLayout derives " + actual); + } + } + + private static VarHandle field(StructLayout layout, String name) { + // Since JDK 22 a struct field VarHandle takes (MemorySegment, long base) coordinates, so + // every access site passes an explicit 0L base for a single struct. + return layout.varHandle(PathElement.groupElement(name)); + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/PlanOp.java b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/PlanOp.java new file mode 100644 index 0000000..0cdcb96 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/PlanOp.java @@ -0,0 +1,26 @@ +package com.adaworldapi.lancegraph.internal.ffm; + +/** + * One entry of a fused plan — the Java-side mirror of {@code LgjOpDesc} (docs/abi.md §5) before it + * is marshalled. + * + *

A plan entry is data, not a lambda. That is the load-bearing choice: a + * predicate expressed as a {@code java.util.function.Predicate} would force a row object per + * element and an upcall per element — the JNI anti-pattern in disguise. Expressed as a descriptor, + * N predicates marshal into one contiguous array and cross once. + * + *

Internal. The public {@code Predicate} type wraps one of these and never + * shows it. + * + * @param op opcode, e.g. {@link Layouts#OP_EQ_U32} + * @param laneId which lane the predicate reads + * @param operand needle or threshold, sign-extended to 64 bits + * @param combine {@link Layouts#COMBINE_AND} (narrow) or {@link Layouts#COMBINE_OR} (widen) + */ +public record PlanOp(int op, int laneId, long operand, int combine) { + + /** A narrowing (AND-combined) op — the only kind reachable from {@code View.where}. */ + public static PlanOp narrowing(int op, int laneId, long operand) { + return new PlanOp(op, laneId, operand, Layouts.COMBINE_AND); + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Status.java b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Status.java new file mode 100644 index 0000000..71428f6 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Status.java @@ -0,0 +1,99 @@ +package com.adaworldapi.lancegraph.internal.ffm; + +import com.adaworldapi.lancegraph.ClosedResourceException; +import com.adaworldapi.lancegraph.LanceGraphException; +import com.adaworldapi.lancegraph.NativeCallException; + +/** + * The ABI's status codes (docs/abi.md §3), and the mapping from a negative status to a specific + * Java exception type. + * + *

Every membrane function returns {@code i32}: {@code 0} is success and all failures are + * negative. There are no error strings across the boundary and no {@code errno} dependence, so + * this table is the whole error vocabulary. + * + *

Internal. Callers see the mapped exception, never this enum. + */ +public enum Status { + + OK(0, "success"), + NULL_ARGUMENT(-1, "a required out-pointer was null"), + INVALID_HANDLE(-2, "handle malformed, closed, or generation-stale"), + WRONG_RESOURCE_KIND(-3, "a handle of the wrong resource kind was passed"), + INVALID_LANE(-4, "lane_id out of range for this resource"), + LANE_KIND_MISMATCH(-5, "the operation's element type does not match the lane's"), + MASK_LENGTH_MISMATCH(-6, "mask row-count does not match resource row-count"), + PARENT_CLOSED(-7, "the mask outlived its parent resource"), + VERSION_MISMATCH(-8, "caller's ABI version is incompatible"), + LENGTH_OVERFLOW(-9, "requested size overflows the allocation limit"), + UNKNOWN_OPCODE(-10, "the plan contained an opcode this build does not implement"), + EMPTY_PLAN(-11, "a plan with zero ops was submitted"), + ALLOCATION_FAILED(-12, "the allocator refused"), + READ_ONLY(-13, "write attempted against a read-only lane"), + /** + * Not tabulated in docs/abi.md §3 (reported as a doc gap) but required by §9: a panic is caught + * at the boundary and becomes a negative status rather than unwinding into JVM frames, which + * would be undefined behaviour. This is that status. + */ + PANIC(-99, "a panic was caught at the membrane and converted to a status"); + + private final int code; + private final String meaning; + + Status(int code, String meaning) { + this.code = code; + this.meaning = meaning; + } + + public int code() { + return code; + } + + public String meaning() { + return meaning; + } + + /** Resolve a raw status, or {@code null} if the library returned a code this build knows not. */ + public static Status of(int code) { + for (Status s : values()) { + if (s.code == code) { + return s; + } + } + return null; + } + + /** + * Throw the most specific exception for a failing status. Never returns. + * + * @param function the ABI symbol that failed, for the message + * @param code the raw negative status + */ + public static LanceGraphException toException(String function, int code) { + Status s = of(code); + if (s == null) { + return new NativeCallException(function, code, "UNKNOWN_STATUS", + "this Java build does not know this status code; the library is likely newer"); + } + return switch (s) { + // Use-after-close and orphaned-child are the same story to a Java caller: the thing you + // are holding no longer refers to anything live. + case INVALID_HANDLE -> new ClosedResourceException( + function + " was called with a handle that is closed, stale, or fabricated" + + " (ABI status INVALID_HANDLE = -2). The native registry rejected it" + + " by generation check; no freed memory was dereferenced."); + case PARENT_CLOSED -> new ClosedResourceException( + function + " was called on a mask whose parent resource is closed" + + " (ABI status PARENT_CLOSED = -7). A child may exist after its" + + " parent closes, but it can never work."); + default -> new NativeCallException(function, s.code, s.name(), s.meaning); + }; + } + + /** Throw if {@code code} is not {@link #OK}. */ + public static void check(String function, int code) { + if (code != 0) { + throw toException(function, code); + } + } +} diff --git a/java/src/test/java/com/adaworldapi/lancegraph/AbiContractTest.java b/java/src/test/java/com/adaworldapi/lancegraph/AbiContractTest.java new file mode 100644 index 0000000..deebc89 --- /dev/null +++ b/java/src/test/java/com/adaworldapi/lancegraph/AbiContractTest.java @@ -0,0 +1,127 @@ +package com.adaworldapi.lancegraph; + +/** + * Checks that the header replacement actually replaced a header. + * + *

The claim (docs/abi.md §0): no {@code .h} exists, and instead the compiled artifact describes + * itself at runtime while Java cross-checks that description against its own independently derived + * layout constants. If the cross-check were absent, or advisory, or comparing a constant against + * itself, the design would be a header with extra steps. + * + *

So this test establishes three things: + * + *

    + *
  1. The manifest was read and the artifact identifies itself coherently. + *
  2. The version rule is the documented one — major exact, minor at least. + *
  3. The check is strict: a deliberately impossible expectation is rejected, which is + * what proves the mechanism can fail at all. + *
+ * + *

Point three is the one worth having. A verification routine that has never been observed to + * reject anything is indistinguishable from one that returns {@code true}. + */ +public final class AbiContractTest { + + private AbiContractTest() {} + + public static void main(String[] args) { + System.out.println("AbiContractTest"); + if (!NativeRuntime.isAvailable()) { + System.exit(Checks.reportUnavailable("AbiContractTest")); + } + Checks c = new Checks("AbiContractTest"); + run(c); + System.exit(c.report()); + } + + public static void run(Checks c) { + c.section("the artifact described itself and Java agreed"); + c.note(NativeRuntime.describe()); + c.eq("abi major", 0, NativeRuntime.abiMajor()); + c.that("abi minor is at least what this build expects", NativeRuntime.abiMinor() >= 1); + c.that("a SIMD backend was reported", !NativeRuntime.simdBackend().isBlank()); + c.that("a build profile was reported", !NativeRuntime.buildProfile().isBlank()); + + if (!"release".equals(NativeRuntime.buildProfile())) { + c.note("WARNING: this is a " + NativeRuntime.buildProfile() + " build." + + " Correctness results are valid; any timing taken against it is not."); + } + + c.section("the loaded library is a real path, not a guess"); + c.that("the library path exists", + java.nio.file.Files.isRegularFile(java.nio.file.Path.of(NativeRuntime.libraryPath()))); + + c.section("the verification is strict -- it can and does reject"); + // A verification routine never observed to reject anything is indistinguishable from one + // that returns true, so drive the rejection branch on purpose. + // + // The probe is aimed at a real, perfectly valid shared library that simply is not ours. + // That is the precise case worth proving: the artifact loads, the linker is happy, and the + // check still refuses because the library cannot describe itself. Aiming at a non-library + // file instead would only prove that dlopen rejects garbage, which is the operating + // system's achievement rather than this design's. + String stranger = firstExisting( + "/lib/x86_64-linux-gnu/libz.so.1", + "/usr/lib/x86_64-linux-gnu/libz.so.1", + "/lib/x86_64-linux-gnu/libm.so.6"); + if (stranger == null) { + c.note("no unrelated system library found to probe with; skipping the rejection arm"); + } else { + c.throwsUp("a valid library that cannot describe itself is refused, not called into", + AbiMismatchException.class, + () -> ProbeLoader.loadInSeparateExpectation(stranger)); + c.note("probed " + stranger + ": it loaded fine and was still rejected," + + " because it exports no lgj_abi_manifest"); + } + + c.throwsUp("a path that does not exist is reported clearly", + NativeLibraryNotFoundException.class, + () -> ProbeLoader.loadInSeparateExpectation("/nonexistent/liblgj_abi.so")); + + c.section("no C, and nothing that would need one"); + c.note("this whole surface was reached with javac and cargo only:" + + " no .h, no cbindgen, no jextract, no JNI, no C toolchain"); + } + + private static String firstExisting(String... candidates) { + for (String candidate : candidates) { + if (java.nio.file.Files.isRegularFile(java.nio.file.Path.of(candidate))) { + return candidate; + } + } + return null; + } + + /** + * A deliberately minimal re-implementation of the load step, used only to observe the failure + * path. + * + *

{@link com.adaworldapi.lancegraph.internal.ffm.Abi} resolves its library once into static + * finals, which is right for production and useless for testing rejection. Rather than making + * the real loader re-entrant purely for a test — which would add a code path nothing else uses + * — this probe performs the same two steps (locate, then look up the manifest symbol) against + * an arbitrary path. + */ + private static final class ProbeLoader { + + static void loadInSeparateExpectation(String path) { + java.nio.file.Path p = java.nio.file.Path.of(path); + if (!java.nio.file.Files.isRegularFile(p)) { + throw new NativeLibraryNotFoundException("no such library: " + p); + } + try (java.lang.foreign.Arena arena = java.lang.foreign.Arena.ofConfined()) { + var lookup = java.lang.foreign.SymbolLookup.libraryLookup(p, arena); + if (lookup.find("lgj_abi_manifest").isEmpty()) { + throw new AbiMismatchException( + "the library at " + p + " exports no lgj_abi_manifest symbol"); + } + throw new AbiMismatchException( + "unexpected: " + p + " exports lgj_abi_manifest"); + } catch (IllegalArgumentException e) { + // The linker refused the file outright — it is not a shared object at all. + throw new NativeLibraryNotFoundException( + "the file at " + p + " is not a loadable shared object", e); + } + } + } +} diff --git a/java/src/test/java/com/adaworldapi/lancegraph/AllTests.java b/java/src/test/java/com/adaworldapi/lancegraph/AllTests.java new file mode 100644 index 0000000..70c22eb --- /dev/null +++ b/java/src/test/java/com/adaworldapi/lancegraph/AllTests.java @@ -0,0 +1,82 @@ +package com.adaworldapi.lancegraph; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.Consumer; + +/** + * Runs every suite in one JVM and exits non-zero if anything failed. + * + *

Exit codes are meaningful so a build script can tell the three outcomes apart: + * {@code 0} all passed, {@code 1} something failed, {@code 2} the native artifact is not available + * so nothing was run. A missing library reported as a failure would send whoever reads the log + * looking for a bug that is not there. + */ +public final class AllTests { + + private AllTests() {} + + public static void main(String[] args) { + Map> suites = new LinkedHashMap<>(); + suites.put("ApiSurfaceTest", ApiSurfaceTest::run); + suites.put("AbiContractTest", AbiContractTest::run); + suites.put("SmokeTest", SmokeTest::run); + suites.put("FixtureParityTest", FixtureParityTest::run); + suites.put("FusionParityTest", FusionParityTest::run); + suites.put("LazinessTest", LazinessTest::run); + suites.put("NarrowingTest", NarrowingTest::run); + suites.put("LifetimeTest", LifetimeTest::run); + + if (!NativeRuntime.isAvailable()) { + // ApiSurfaceTest needs no native library — the API's shape is a compile-time property — + // so run it anyway before reporting the skip. It is exactly the check most worth having + // before the artifact exists. + System.out.println(); + System.out.println("=== ApiSurfaceTest ==="); + Checks shape = new Checks("ApiSurfaceTest"); + ApiSurfaceTest.run(shape); + int shapeCode = shape.report(); + int skipCode = Checks.reportUnavailable("AllTests"); + System.exit(shapeCode != 0 ? shapeCode : skipCode); + } + + int totalPassed = 0; + int totalFailed = 0; + StringBuilder summary = new StringBuilder(); + + for (Map.Entry> e : suites.entrySet()) { + System.out.println(); + System.out.println("=== " + e.getKey() + " ==="); + Checks c = new Checks(e.getKey()); + try { + e.getValue().accept(c); + } catch (Throwable t) { + // A suite that blows up mid-way must not take the run down silently: report it as a + // failure of that suite and carry on, so one broken area does not hide the others. + System.out.println(" FAIL suite threw " + t); + t.printStackTrace(System.out); + totalFailed++; + summary.append(String.format(" %-20s THREW %s%n", e.getKey(), t)); + continue; + } + c.report(); + totalPassed += c.passedCount(); + totalFailed += c.failedCount(); + summary.append(String.format(" %-20s %3d passed, %d failed%n", + e.getKey(), c.passedCount(), c.failedCount())); + } + + System.out.println(); + System.out.println("=== summary ==="); + System.out.print(summary); + System.out.println(); + System.out.println(" " + NativeRuntime.describe()); + System.out.println(); + if (totalFailed == 0) { + System.out.println(" ALL PASSED (" + totalPassed + " checks)"); + System.exit(0); + } + System.out.println(" " + totalFailed + " FAILED, " + totalPassed + " passed"); + System.exit(1); + } +} diff --git a/java/src/test/java/com/adaworldapi/lancegraph/ApiSurfaceTest.java b/java/src/test/java/com/adaworldapi/lancegraph/ApiSurfaceTest.java new file mode 100644 index 0000000..b541baa --- /dev/null +++ b/java/src/test/java/com/adaworldapi/lancegraph/ApiSurfaceTest.java @@ -0,0 +1,181 @@ +package com.adaworldapi.lancegraph; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Executable; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +/** + * Enforces the absolute API rule by reflection, so it cannot rot. + * + *

The rule: a normal Java consumer must never need to know about + * {@code MemorySegment}, {@code Arena}, {@code Linker}, {@code FunctionDescriptor}, + * {@code MethodHandle}, native addresses, SoA layout, masks-as-u64-words, lane ids, opcodes, or + * which SIMD backend ran. Those are implementation physics and they live in exactly one + * package. + * + *

A rule like that is normally a paragraph in a design document, which is to say it is normally + * broken within a year by someone who never read the paragraph. Here it is a test: every public + * member of every public type in the consumer package is inspected, and a single FFM type anywhere + * in a signature fails the build. + * + *

It is checked reflectively rather than by grepping source, because the question is about the + * compiled surface — a return type inherited from a superclass or introduced by a bridge + * method is just as much a leak as one written by hand, and a grep would not see it. + */ +public final class ApiSurfaceTest { + + private ApiSurfaceTest() {} + + /** Package prefixes that no public consumer-facing signature may mention. */ + private static final String[] FORBIDDEN = { + "java.lang.foreign.", // MemorySegment, Arena, Linker, MemoryLayout, ValueLayout, ... + "java.lang.invoke.", // MethodHandle, VarHandle + "com.adaworldapi.lancegraph.internal.", // our own membrane, including PlanOp and Layouts + }; + + /** The package a consumer imports. */ + private static final String PUBLIC_PACKAGE = "com.adaworldapi.lancegraph"; + + public static void main(String[] args) { + System.out.println("ApiSurfaceTest"); + Checks c = new Checks("ApiSurfaceTest"); + run(c); + System.exit(c.report()); + } + + /** + * Note: this suite needs no native library. The API's shape is a compile-time property, so it + * is checkable before the artifact exists — which is also when it is most useful. + */ + public static void run(Checks c) { + List> types = publicTypes(c); + if (types.isEmpty()) { + c.that("public types were discovered (if this fails, the classpath is not a directory" + + " and this suite cannot scan it)", false); + return; + } + + c.section("scanning the consumer package"); + c.note("inspecting " + types.size() + " public types in " + PUBLIC_PACKAGE); + c.that("the scan found the types the API is actually made of", + types.stream().anyMatch(t -> t.getSimpleName().equals("NativePattern")) + && types.stream().anyMatch(t -> t.getSimpleName().equals("View")) + && types.stream().anyMatch(t -> t.getSimpleName().equals("Pattern"))); + + c.section("no FFM type, no membrane type, in any public signature"); + List leaks = new ArrayList<>(); + for (Class type : types) { + leaks.addAll(leaksIn(type)); + } + if (leaks.isEmpty()) { + c.that("every public member is free of " + + String.join(", ", FORBIDDEN), true); + } else { + for (String leak : leaks) { + c.that("LEAK: " + leak, false); + } + } + + c.section("the escape hatch is named, not incidental"); + // Raw native access must require deliberately reaching into an internal package. It must + // never be something ordinary composition hands you. + c.that("the membrane package is not exported through any public type", + leaks.isEmpty()); + c.note("raw MemorySegment access exists only via" + + " com.adaworldapi.lancegraph.internal.ffm.Engine, which a consumer has to name" + + " explicitly and which is documented as internal"); + + c.section("the vocabulary a consumer actually needs is small"); + c.note("open a resource, take a view, add conditions, ask for a number:" + + " NativePattern, View, Predicate, Pattern's fields, and a count"); + } + + private static List leaksIn(Class type) { + List leaks = new ArrayList<>(); + + for (Method m : type.getMethods()) { + if (!isPublicApi(m) || m.getDeclaringClass() == Object.class) { + continue; + } + check(leaks, type, "method " + m.getName() + " return", m.getReturnType()); + for (Class p : m.getParameterTypes()) { + check(leaks, type, "method " + m.getName() + " parameter", p); + } + } + + for (Constructor ctor : type.getConstructors()) { + if (!isPublicApi(ctor)) { + continue; + } + for (Class p : ctor.getParameterTypes()) { + check(leaks, type, "constructor parameter", p); + } + } + + for (Field f : type.getFields()) { + if (!Modifier.isPublic(f.getModifiers())) { + continue; + } + check(leaks, type, "field " + f.getName(), f.getType()); + } + + return leaks; + } + + private static boolean isPublicApi(Executable e) { + return Modifier.isPublic(e.getModifiers()); + } + + private static void check(List leaks, Class owner, String where, Class t) { + Class component = t; + while (component.isArray()) { + component = component.getComponentType(); + } + String name = component.getName(); + for (String forbidden : FORBIDDEN) { + if (name.startsWith(forbidden)) { + leaks.add(owner.getSimpleName() + "." + where + " is " + name); + } + } + } + + /** Enumerate public types by listing the compiled package directory on the classpath. */ + private static List> publicTypes(Checks c) { + List> found = new ArrayList<>(); + try { + Path root = Path.of(ApiSurfaceTest.class.getProtectionDomain() + .getCodeSource().getLocation().toURI()); + Path pkg = root.resolve(PUBLIC_PACKAGE.replace('.', '/')); + if (!Files.isDirectory(pkg)) { + return found; + } + try (Stream files = Files.list(pkg)) { + for (Path p : files.toList()) { + String file = p.getFileName().toString(); + if (!file.endsWith(".class") || file.contains("$")) { + continue; + } + String simple = file.substring(0, file.length() - ".class".length()); + Class t = Class.forName(PUBLIC_PACKAGE + "." + simple); + // Skip the test classes themselves; they live in the same package by design so + // that they can exercise package-private seams, but they are not the API. + if (Modifier.isPublic(t.getModifiers()) && !simple.endsWith("Test") + && !simple.equals("Checks") && !simple.equals("AllTests")) { + found.add(t); + } + } + } + } catch (Exception e) { + c.note("could not scan the classpath: " + e); + } + found.sort(java.util.Comparator.comparing(Class::getSimpleName)); + return found; + } +} diff --git a/java/src/test/java/com/adaworldapi/lancegraph/Checks.java b/java/src/test/java/com/adaworldapi/lancegraph/Checks.java new file mode 100644 index 0000000..6c25665 --- /dev/null +++ b/java/src/test/java/com/adaworldapi/lancegraph/Checks.java @@ -0,0 +1,155 @@ +package com.adaworldapi.lancegraph; + +import java.util.ArrayList; +import java.util.List; + +/** + * A self-contained assertion harness. + * + *

There is no JUnit here because there are no downloaded dependencies anywhere in this project — + * {@code javac} and {@code java} are the entire Java toolchain, exactly as {@code cargo} is the + * entire Rust one. What a test framework would give us is a runner and readable failures; that is + * about eighty lines, and they are these. + * + *

Failures are loud and specific by design: every check carries a sentence saying what was + * being established, and every comparison prints both numbers. A failure that only says + * {@code expected true} costs more to diagnose than it saved to write. + */ +public final class Checks { + + private final String suite; + private final List failures = new ArrayList<>(); + private int passed; + private String section = ""; + + public Checks(String suite) { + this.suite = suite; + } + + /** Group the following checks under a heading, printed once. */ + public void section(String name) { + this.section = name; + System.out.println(" " + name); + } + + /** Note something worth seeing in the log that is not itself a check. */ + public void note(String message) { + System.out.println(" - " + message); + } + + public void that(String what, boolean condition) { + if (condition) { + pass(what); + } else { + fail(what, "condition was false"); + } + } + + public void eq(String what, long expected, long actual) { + if (expected == actual) { + pass(what + " (= " + actual + ")"); + } else { + fail(what, "expected " + expected + " but was " + actual + + " (difference " + (actual - expected) + ")"); + } + } + + public void eq(String what, Object expected, Object actual) { + if (java.util.Objects.equals(expected, actual)) { + pass(what + " (= " + actual + ")"); + } else { + fail(what, "expected <" + expected + "> but was <" + actual + ">"); + } + } + + public void notEq(String what, long unexpected, long actual) { + if (unexpected != actual) { + pass(what + " (" + actual + " != " + unexpected + ")"); + } else { + fail(what, "expected anything other than " + unexpected + ", got exactly that"); + } + } + + public void atMost(String what, long limit, long actual) { + if (actual <= limit) { + pass(what + " (" + actual + " <= " + limit + ")"); + } else { + fail(what, "expected at most " + limit + " but was " + actual); + } + } + + /** + * Assert that {@code body} throws {@code expected}. + * + *

A falsifier that merely expects "some exception" would pass on a + * {@code NullPointerException} from a typo, so the type is required and the actual type is + * printed when it differs. + */ + public void throwsUp(String what, Class expected, Runnable body) { + try { + body.run(); + } catch (Throwable t) { + if (expected.isInstance(t)) { + pass(what + " (threw " + t.getClass().getSimpleName() + ")"); + } else { + fail(what, "expected " + expected.getSimpleName() + " but threw " + + t.getClass().getName() + ": " + t.getMessage()); + } + return; + } + fail(what, "expected " + expected.getSimpleName() + " but nothing was thrown"); + } + + private void pass(String what) { + passed++; + System.out.println(" ok " + what); + } + + private void fail(String what, String detail) { + String line = (section.isEmpty() ? "" : section + " / ") + what + "\n " + detail; + failures.add(line); + System.out.println(" FAIL " + what + "\n " + detail); + } + + public boolean anyFailed() { + return !failures.isEmpty(); + } + + public int passedCount() { + return passed; + } + + public int failedCount() { + return failures.size(); + } + + /** Print the summary and return the process exit code this suite implies. */ + public int report() { + System.out.println(); + if (failures.isEmpty()) { + System.out.println(" " + suite + ": " + passed + " checks passed"); + return 0; + } + System.out.println(" " + suite + ": " + failures.size() + " FAILED, " + passed + " passed"); + for (String f : failures) { + System.out.println(" x " + f); + } + return 1; + } + + /** + * Report that the native artifact is missing, clearly enough that nobody mistakes it for a + * test failure. + * + * @return the exit code to use + */ + public static int reportUnavailable(String suite) { + System.out.println(); + System.out.println(" " + suite + ": SKIPPED - the native library is not available."); + System.out.println(" " + NativeRuntime.unavailableReason().getMessage()); + System.out.println(); + System.out.println(" Build it first: cargo build --release (in native/lgj-abi)"); + System.out.println(" Or point at it: -Dlgj.library=/path/to/liblgj_abi.so"); + return 2; + } +} diff --git a/java/src/test/java/com/adaworldapi/lancegraph/FixtureParityTest.java b/java/src/test/java/com/adaworldapi/lancegraph/FixtureParityTest.java new file mode 100644 index 0000000..f5f39fa --- /dev/null +++ b/java/src/test/java/com/adaworldapi/lancegraph/FixtureParityTest.java @@ -0,0 +1,168 @@ +package com.adaworldapi.lancegraph; + +/** + * A genuine cross-language check: Java recomputes the expected answer from the normative fixture + * description and compares it to what the native kernels returned. + * + *

Most of the other tests establish properties — the count never grows, the paths + * agree, nothing crosses when nothing should. Properties can all hold while every number is wrong + * together. This test is the one that pins the actual values, and it does so without shipping a + * data file: the fixture is generated deterministically from a seed, so Java can transcribe the + * generator and derive the same answers independently. + * + *

Independence is what gives it force. Java runs its own SplitMix64 over its own {@code long} + * arithmetic and counts with an ordinary loop; Rust runs vectorised kernels over lanes Java never + * sees. Agreement across those two is evidence; agreement between a kernel and a stored expectation + * would only be evidence that nobody changed the file. + * + *

The generator is checked against its own published test vector first, so a failure downstream + * points at the kernels rather than at this transcription. + */ +public final class FixtureParityTest { + + private FixtureParityTest() {} + + public static void main(String[] args) { + System.out.println("FixtureParityTest"); + if (!NativeRuntime.isAvailable()) { + System.exit(Checks.reportUnavailable("FixtureParityTest")); + } + Checks c = new Checks("FixtureParityTest"); + run(c); + System.exit(c.report()); + } + + /** + * SplitMix64, transcribed from the normative description of the fixture generator. + * + *

Every constant and shift is the published reference. All arithmetic is wrapping, which is + * what Java's {@code long} does natively — so this is a transcription, not a port, and there is + * no place for an off-by-one interpretation to hide. + */ + private static final class SplitMix64 { + private long state; + + SplitMix64(long seed) { + this.state = seed; // no warm-up draws + } + + long next() { + state += 0x9E3779B97F4A7C15L; + long z = state; + z = (z ^ (z >>> 30)) * 0xBF58476D1CE4E5B9L; + z = (z ^ (z >>> 27)) * 0x94D049BB133111EBL; + return z ^ (z >>> 31); + } + } + + /** The three lanes, as Java believes they should be. */ + private record Lanes(int[] classes, int[] values) {} + + private static Lanes generate(int nRows, long seed) { + SplitMix64 rng = new SplitMix64(seed); + int[] classes = new int[nRows]; + int[] values = new int[nRows]; + for (int i = 0; i < nRows; i++) { + long a = rng.next(); // FIRST draw of the row + long b = rng.next(); // SECOND draw of the row + classes[i] = (int) ((a >>> 33) & 0xF); + values[i] = (int) (((b >>> 40) & 0x1FF) - 150); + } + return new Lanes(classes, values); + } + + public static void run(Checks c) { + c.section("the transcribed generator is the real SplitMix64"); + SplitMix64 r = new SplitMix64(0); + c.eq("published vector, draw 1", 0xE220A8397B1DCDAFL, r.next()); + c.eq("published vector, draw 2", 0x6E789E6AA1B965F4L, r.next()); + c.eq("published vector, draw 3", 0x06C45D188009454FL, r.next()); + + int rows = 65_536; + long seed = NativePattern.DEFAULT_SEED; + Lanes expected = generate(rows, seed); + + c.section("the lanes have the shape the contract describes"); + int minClass = Integer.MAX_VALUE, maxClass = Integer.MIN_VALUE; + int minValue = Integer.MAX_VALUE, maxValue = Integer.MIN_VALUE; + for (int i = 0; i < rows; i++) { + minClass = Math.min(minClass, expected.classes()[i]); + maxClass = Math.max(maxClass, expected.classes()[i]); + minValue = Math.min(minValue, expected.values()[i]); + maxValue = Math.max(maxValue, expected.values()[i]); + } + c.eq("classes start at 0", 0, minClass); + c.eq("classes end at 15", 15, maxClass); + c.that("values reach below zero, so > is a real signed comparison", minValue < 0); + c.that("values reach above the test threshold", maxValue > 100); + + try (NativePattern data = NativePattern.open(rows, seed)) { + + c.section("counts: Java's own loop vs the native kernels"); + + long expectClass = 0, expectValue = 0, expectBoth = 0, expectSum = 0; + for (int i = 0; i < rows; i++) { + boolean isClass = expected.classes()[i] == 7; + boolean isValue = expected.values()[i] > 100; + if (isClass) { + expectClass++; + expectSum += expected.values()[i]; + } + if (isValue) { + expectValue++; + } + if (isClass && isValue) { + expectBoth++; + } + } + + c.eq("class == 7", + expectClass, data.view().where(Pattern.CLASS.eq(7)).count()); + c.eq("value > 100", + expectValue, data.view().where(Pattern.VALUE.gt(100)).count()); + c.eq("class == 7 AND value > 100", expectBoth, + data.view().where(Pattern.CLASS.eq(7)).where(Pattern.VALUE.gt(100)).count()); + + c.section("a reduction, likewise"); + c.eq("sum of value over class == 7", expectSum, + data.view().where(Pattern.CLASS.eq(7)).lens(Pattern.VALUE).sum()); + + c.section("a signed threshold, where an unsigned comparison would differ"); + long expectNegative = 0; + for (int i = 0; i < rows; i++) { + if (expected.values()[i] > -100) { + expectNegative++; + } + } + c.eq("value > -100", expectNegative, + data.view().where(Pattern.VALUE.gt(-100)).count()); + c.that("and that predicate excludes a real number of rows, so it is not vacuous", + expectNegative < rows); + + c.section("every class tag, so no single lucky value carries the test"); + for (int tag = 0; tag < 16; tag++) { + long want = 0; + for (int i = 0; i < rows; i++) { + if (expected.classes()[i] == tag) { + want++; + } + } + c.eq("class == " + tag, want, data.view().where(Pattern.CLASS.eq(tag)).count()); + } + } + + c.section("a different seed produces different data, and parity still holds"); + long altSeed = 0x1234_5678L; + Lanes alt = generate(4096, altSeed); + long altExpect = 0; + for (int i = 0; i < 4096; i++) { + if (alt.classes()[i] == 7 && alt.values()[i] > 100) { + altExpect++; + } + } + try (NativePattern data = NativePattern.open(4096, altSeed)) { + c.eq("class == 7 AND value > 100 under a second seed", altExpect, + data.view().where(Pattern.CLASS.eq(7)).where(Pattern.VALUE.gt(100)).count()); + } + } +} diff --git a/java/src/test/java/com/adaworldapi/lancegraph/FusionParityTest.java b/java/src/test/java/com/adaworldapi/lancegraph/FusionParityTest.java new file mode 100644 index 0000000..3d91bcb --- /dev/null +++ b/java/src/test/java/com/adaworldapi/lancegraph/FusionParityTest.java @@ -0,0 +1,99 @@ +package com.adaworldapi.lancegraph; + +/** + * The fused path, the unfused path and the scalar path must agree exactly. + * + *

Fusion is an optimisation, and an optimisation is only trustworthy against a reference. Two + * references exist here, and they falsify different things: + * + *

    + *
  • unfused — the same predicates evaluated one crossing at a time and + * combined with explicit mask operations. If fusion disagrees with this, the fused kernel's + * accumulator logic is wrong. + *
  • scalar — the same fused semantics with the vector kernels forced off. If + * this disagrees, the SIMD kernel is wrong. Crucially the check happens through the + * membrane, so it also covers the marshalling, not only the Rust-internal kernels. + *
+ * + *

Exact equality is the right assertion: these are counts of set bits, so "close" would mean + * "wrong". A tolerance here would be a bug in the test. + */ +public final class FusionParityTest { + + private FusionParityTest() {} + + public static void main(String[] args) { + System.out.println("FusionParityTest"); + if (!NativeRuntime.isAvailable()) { + System.exit(Checks.reportUnavailable("FusionParityTest")); + } + Checks c = new Checks("FusionParityTest"); + run(c); + System.exit(c.report()); + } + + public static void run(Checks c) { + c.note("backend under test: " + NativeRuntime.simdBackend()); + + // Sizes chosen to straddle the mask word boundary: 64 bits per word, so a kernel that + // mishandles a partial trailing word shows up at 1,000 and 65,535 but not at 65,536. + long[] sizes = {1, 63, 64, 65, 1_000, 65_535, 65_536}; + + for (long rows : sizes) { + c.section(rows + " rows"); + try (NativePattern data = NativePattern.open(rows)) { + + check(c, "class == 7", data.view().where(Pattern.CLASS.eq(7))); + check(c, "value > 100", data.view().where(Pattern.VALUE.gt(100))); + check(c, "class == 7 AND value > 100", + data.view().where(Pattern.CLASS.eq(7)).where(Pattern.VALUE.gt(100))); + check(c, "a three-deep chain", + data.view().where(Pattern.VALUE.gt(-150)) + .where(Pattern.CLASS.eq(3)) + .where(Pattern.VALUE.gt(0))); + } + } + + c.section("the two paths really do cost different amounts"); + try (NativePattern data = NativePattern.open(65_536)) { + View v = data.view().where(Pattern.CLASS.eq(7)).where(Pattern.VALUE.gt(100)); + Diagnostics.countFused(v); + Diagnostics.countUnfused(v); + + long a = Diagnostics.crossings(); + Diagnostics.countFused(v); + long fused = Diagnostics.crossings() - a; + + long b = Diagnostics.crossings(); + Diagnostics.countUnfused(v); + long unfused = Diagnostics.crossings() - b; + + c.eq("fused: one crossing", 1, fused); + c.that("unfused: strictly more (it is what the design forbids)", unfused > fused); + c.note("fused " + fused + " crossing vs unfused " + unfused + + " for the identical answer -- this is the cost of the rule, measured"); + } + + c.section("a materialised selection agrees with the count that produced it"); + try (NativePattern data = NativePattern.open(65_536)) { + View v = data.view().where(Pattern.CLASS.eq(7)).where(Pattern.VALUE.gt(100)); + try (Mask m = v.select()) { + c.eq("Mask.count() equals View.count()", v.count(), m.count()); + } + } + } + + private static void check(Checks c, String what, View v) { + long fused = Diagnostics.countFused(v); + long unfused = Diagnostics.countUnfused(v); + long scalar = Diagnostics.countScalar(v); + + if (fused == unfused && fused == scalar) { + c.eq(what + ": fused == unfused == scalar", fused, scalar); + } else { + // Report all three, because which pair disagrees says which kernel is at fault. + c.eq(what + ": fused vs unfused", fused, unfused); + c.eq(what + ": fused vs scalar", fused, scalar); + } + } +} diff --git a/java/src/test/java/com/adaworldapi/lancegraph/LazinessTest.java b/java/src/test/java/com/adaworldapi/lancegraph/LazinessTest.java new file mode 100644 index 0000000..887ef55 --- /dev/null +++ b/java/src/test/java/com/adaworldapi/lancegraph/LazinessTest.java @@ -0,0 +1,113 @@ +package com.adaworldapi.lancegraph; + +/** + * Proves the fluent chain is lazy and the terminal operation is fused — by measurement, not by + * comment. + * + *

"Building a View makes zero crossings" is the kind of claim that is true when written and + * quietly false a year later, because nothing checks it. So the membrane counts its own crossings + * and these tests read the counter. The counter is the only reason the claim is a test rather than + * a docstring. + * + *

Two properties, and the second is the load-bearing one: + * + *

    + *
  1. Laziness — building and narrowing a view moves the counter by exactly 0. + *
  2. Fusion — a terminal operation moves it by a small constant that does + * not grow with the number of predicates. Two predicates and sixteen predicates cost + * the same number of crossings. That is what makes the fluent style affordable. + *
+ */ +public final class LazinessTest { + + private LazinessTest() {} + + public static void main(String[] args) { + System.out.println("LazinessTest"); + if (!NativeRuntime.isAvailable()) { + System.exit(Checks.reportUnavailable("LazinessTest")); + } + Checks c = new Checks("LazinessTest"); + run(c); + System.exit(c.report()); + } + + public static void run(Checks c) { + try (NativePattern data = NativePattern.open(65_536)) { + + c.section("building a chain crosses the membrane zero times"); + + long before = Diagnostics.crossings(); + View v = data.view() + .where(Pattern.CLASS.eq(7)) + .where(Pattern.VALUE.gt(100)) + .where(Pattern.VALUE.gt(-50)) + .where(Pattern.CLASS.eq(7)); + long afterBuild = Diagnostics.crossings(); + + c.eq("four .where() calls plus .view() cost no crossings", 0, afterBuild - before); + c.eq("the chain does carry all four conditions", 4, v.conditionCount()); + c.note("nothing was evaluated: no mask was allocated, no row was read"); + + // Warm the scratch selection so the steady-state cost is what is measured. The first + // terminal op on a resource also allocates its reusable selection; counting that as + // per-query cost would misreport the steady state. + v.count(); + + c.section("a terminal operation costs ONE crossing, whatever the chain length"); + + long c0 = Diagnostics.crossings(); + long n4 = v.count(); + long cost4 = Diagnostics.crossings() - c0; + c.eq("count() over a 4-condition chain", 1, cost4); + + View wide = data.view(); + for (int i = 0; i < 16; i++) { + wide = wide.where(Pattern.VALUE.gt(-150 + i)); + } + long c1 = Diagnostics.crossings(); + long n16 = wide.count(); + long cost16 = Diagnostics.crossings() - c1; + c.eq("count() over a 16-condition chain", 1, cost16); + c.eq("cost is independent of the number of conditions", cost4, cost16); + c.note("4 conditions -> " + n4 + " rows, 16 conditions -> " + n16 + " rows," + + " both for one crossing"); + + c.section("cost is independent of the number of rows, too"); + try (NativePattern small = NativePattern.open(1_024); + NativePattern large = NativePattern.open(1_000_000)) { + + View sv = small.view().where(Pattern.CLASS.eq(7)).where(Pattern.VALUE.gt(100)); + View lv = large.view().where(Pattern.CLASS.eq(7)).where(Pattern.VALUE.gt(100)); + sv.count(); + lv.count(); + + long a = Diagnostics.crossings(); + sv.count(); + long smallCost = Diagnostics.crossings() - a; + + long b = Diagnostics.crossings(); + long largeRows = lv.count(); + long largeCost = Diagnostics.crossings() - b; + + c.eq("1,024 rows", 1, smallCost); + c.eq("1,000,000 rows", 1, largeCost); + c.note("a thousand-fold more data for the same one crossing;" + + " the large query matched " + largeRows + " rows"); + } + + c.section("a reduction costs one crossing more, and no more than that"); + View sel = data.view().where(Pattern.CLASS.eq(7)); + sel.sumOf(Pattern.VALUE); + long c2 = Diagnostics.crossings(); + sel.sumOf(Pattern.VALUE); + long sumCost = Diagnostics.crossings() - c2; + c.eq("sumOf() = evaluate the chain, then reduce", 2, sumCost); + + c.section("nothing here hydrated a row"); + c.note("no API on View, Mask or Lens returns a row object, an index list, or an" + + " iterator; the crossing counts above are only meaningful because there is" + + " no per-row path that could have inflated them"); + } + } +} diff --git a/java/src/test/java/com/adaworldapi/lancegraph/LifetimeTest.java b/java/src/test/java/com/adaworldapi/lancegraph/LifetimeTest.java new file mode 100644 index 0000000..36c0a83 --- /dev/null +++ b/java/src/test/java/com/adaworldapi/lancegraph/LifetimeTest.java @@ -0,0 +1,107 @@ +package com.adaworldapi.lancegraph; + +/** + * Falsifiers for the ownership rules (docs/abi.md §4). + * + *

These are the tests that matter most, because the property they establish is the one Java + * cannot get from Rust for free. Inside Rust, a view outliving its owner is a compile error; across + * the membrane there is no borrow checker, so the invariant has to be enforced at runtime and + * therefore has to be falsified rather than asserted. + * + *

The claim under test: there is no sequence of operations in which a stale handle + * dereferences freed memory. Every attempt below must produce a specific Java exception — + * not a crash, not a wrong answer, and not a silently-tolerated no-op. + * + *

Note what a passing run does not prove: that the process would have crashed without + * these guards. It proves the guards fire. The guards themselves are two-deep on purpose (Java + * bookkeeping in front, generation-checked handles behind), so a defect in either one alone is + * still contained. + */ +public final class LifetimeTest { + + private LifetimeTest() {} + + public static void main(String[] args) { + System.out.println("LifetimeTest"); + if (!NativeRuntime.isAvailable()) { + System.exit(Checks.reportUnavailable("LifetimeTest")); + } + Checks c = new Checks("LifetimeTest"); + run(c); + System.exit(c.report()); + } + + public static void run(Checks c) { + c.section("use after close"); + NativePattern closed = NativePattern.open(1024); + long before = closed.rowCount(); + c.eq("the resource works while open", 1024, before); + closed.close(); + + c.that("it reports itself closed", !closed.isOpen()); + c.throwsUp("rowCount() after close", ClosedResourceException.class, closed::rowCount); + c.throwsUp("view() after close", ClosedResourceException.class, closed::view); + c.throwsUp("rows() after close", ClosedResourceException.class, closed::rows); + + c.section("double close"); + c.throwsUp("close() a second time", ClosedResourceException.class, closed::close); + + c.section("a view derived before the close"); + NativePattern p = NativePattern.open(1024); + View held = p.view().where(Pattern.CLASS.eq(7)); + long liveCount = held.count(); + c.that("the view works while its resource is open", liveCount >= 0); + p.close(); + + // The view object still exists — Java cannot prevent that. What it must not do is work. + c.throwsUp("count() on a view whose resource closed", + ClosedResourceException.class, held::count); + c.throwsUp("where() on a view whose resource closed", + ClosedResourceException.class, () -> held.where(Pattern.VALUE.gt(0))); + c.throwsUp("sumOf() on a view whose resource closed", + ClosedResourceException.class, () -> held.sumOf(Pattern.VALUE)); + c.throwsUp("select() on a view whose resource closed", + ClosedResourceException.class, held::select); + + c.section("a selection that outlives its parent"); + NativePattern parent = NativePattern.open(1024); + Mask orphan = parent.view().where(Pattern.CLASS.eq(7)).select(); + long orphanCount = orphan.count(); + c.that("the selection works while its parent is open", orphanCount >= 0); + parent.close(); + + c.that("the selection knows it is no longer usable", !orphan.isOpen()); + c.throwsUp("count() on a selection whose parent closed", + ClosedResourceException.class, orphan::count); + // Closing an orphan must be tolerated: the caller is doing the right thing (their + // try-with-resources is unwinding) and must not be punished for the ordering. + orphan.close(); + c.that("closing an orphaned selection is not an error", true); + c.throwsUp("but closing it twice still is", + ClosedResourceException.class, orphan::close); + + c.section("a selection closed before its parent -- the ordinary ordering"); + try (NativePattern q = NativePattern.open(512)) { + Mask m = q.view().where(Pattern.CLASS.eq(3)).select(); + long n = m.count(); + m.close(); + c.throwsUp("count() on a closed selection", ClosedResourceException.class, m::count); + c.that("the resource is unharmed by its child's close", q.rowCount() == 512); + c.that("and still answers queries", q.view().where(Pattern.CLASS.eq(3)).count() == n); + } + + c.section("the resource survives being asked the impossible"); + try (NativePattern q = NativePattern.open(256)) { + c.throwsUp("a null predicate is rejected before it can travel", + NullPointerException.class, () -> q.view().where(null)); + c.that("and the resource is still usable afterwards", q.view().count() == 256); + } + + c.section("zero rows is a legal resource, not an error"); + try (NativePattern empty = NativePattern.open(0)) { + c.eq("an empty resource has no rows", 0, empty.rowCount()); + c.eq("and every query over it selects nothing", + 0, empty.view().where(Pattern.CLASS.eq(7)).count()); + } + } +} diff --git a/java/src/test/java/com/adaworldapi/lancegraph/NarrowingTest.java b/java/src/test/java/com/adaworldapi/lancegraph/NarrowingTest.java new file mode 100644 index 0000000..6b42bf7 --- /dev/null +++ b/java/src/test/java/com/adaworldapi/lancegraph/NarrowingTest.java @@ -0,0 +1,91 @@ +package com.adaworldapi.lancegraph; + +/** + * Monotonic narrowing: adding a condition can never select more rows. + * + *

The property is meant to be structural — {@link View#where} intersects, and there is + * no public composition that widens — so this test is not really checking arithmetic. It is + * checking that the structure delivers what it promises, over a chain long enough that a single + * mis-combined op would show up. + * + *

An anti-vacuity guard runs first. A chain whose conditions all match everything would satisfy + * "never increases" trivially and prove nothing, so the test asserts the chain actually eliminates + * a substantial fraction before it asserts the fraction only ever shrinks. + */ +public final class NarrowingTest { + + private NarrowingTest() {} + + public static void main(String[] args) { + System.out.println("NarrowingTest"); + if (!NativeRuntime.isAvailable()) { + System.exit(Checks.reportUnavailable("NarrowingTest")); + } + Checks c = new Checks("NarrowingTest"); + run(c); + System.exit(c.report()); + } + + public static void run(Checks c) { + try (NativePattern data = NativePattern.open(65_536)) { + + c.section("a long chain, checked step by step"); + + // Deliberately mixed: some conditions bite hard, some barely, one is a no-op. A chain + // of only-hard conditions would collapse to zero and stop testing anything after the + // second step. + Predicate[] chain = { + Pattern.VALUE.gt(-150), // matches everything: the boundary case + Pattern.CLASS.eq(7), // bites hard, about 1/16 + Pattern.VALUE.gt(0), + Pattern.VALUE.gt(100), + Pattern.CLASS.eq(7), // repeated: idempotent, must change nothing + Pattern.VALUE.gt(200), + }; + + View v = data.view(); + long previous = v.count(); + c.eq("the unconditioned view selects everything", 65_536, previous); + + long first = previous; + for (int i = 0; i < chain.length; i++) { + v = v.where(chain[i]); + long now = v.count(); + c.atMost("step " + (i + 1) + " (" + chain[i] + ") does not widen", previous, now); + previous = now; + } + + c.section("anti-vacuity: the chain must actually eliminate something"); + c.that("the final selection is far smaller than the first", + previous * 4 < first); + c.that("but not empty, or the later steps proved nothing", previous > 0); + c.note("65,536 -> " + previous + " rows across " + chain.length + " conditions"); + + c.section("an idempotent repeat changes nothing"); + View once = data.view().where(Pattern.CLASS.eq(7)); + View twice = once.where(Pattern.CLASS.eq(7)); + c.eq("class == 7 twice equals class == 7 once", once.count(), twice.count()); + + c.section("order does not change the answer"); + long ab = data.view().where(Pattern.CLASS.eq(7)).where(Pattern.VALUE.gt(100)).count(); + long ba = data.view().where(Pattern.VALUE.gt(100)).where(Pattern.CLASS.eq(7)).count(); + c.eq("intersection is commutative", ab, ba); + + c.section("the conjunction is bounded by each part"); + long onlyClass = data.view().where(Pattern.CLASS.eq(7)).count(); + long onlyValue = data.view().where(Pattern.VALUE.gt(100)).count(); + c.atMost("no larger than class == 7 alone", onlyClass, ab); + c.atMost("no larger than value > 100 alone", onlyValue, ab); + c.that("and strictly smaller than both, so neither part was ignored", + ab < onlyClass && ab < onlyValue); + + c.section("narrowing a shared view does not disturb the shared view"); + View shared = data.view().where(Pattern.CLASS.eq(7)); + long sharedBefore = shared.count(); + View branchA = shared.where(Pattern.VALUE.gt(100)); + View branchB = shared.where(Pattern.VALUE.gt(300)); + c.that("the branches differ", branchA.count() != branchB.count()); + c.eq("and the view they came from is untouched", sharedBefore, shared.count()); + } + } +} diff --git a/java/src/test/java/com/adaworldapi/lancegraph/SmokeTest.java b/java/src/test/java/com/adaworldapi/lancegraph/SmokeTest.java new file mode 100644 index 0000000..58297c2 --- /dev/null +++ b/java/src/test/java/com/adaworldapi/lancegraph/SmokeTest.java @@ -0,0 +1,81 @@ +package com.adaworldapi.lancegraph; + +/** + * The John Doe test: does the API in the README actually work, and does it read like Java? + * + *

Read the body of {@link #run} as documentation. Nothing in it mentions an arena, a segment, a + * linker, a lane, an opcode, a mask word or a SIMD backend — and it is nevertheless the full, + * real path through a columnar SIMD kernel over 65,536 rows held entirely outside the Java heap. + */ +public final class SmokeTest { + + private SmokeTest() {} + + public static void main(String[] args) { + System.out.println("SmokeTest"); + if (!NativeRuntime.isAvailable()) { + System.exit(Checks.reportUnavailable("SmokeTest")); + } + Checks c = new Checks("SmokeTest"); + run(c); + System.exit(c.report()); + } + + public static void run(Checks c) { + c.section("the runtime that actually loaded"); + c.note(NativeRuntime.describe()); + + c.section("the fluent chain from the README"); + + // ── everything below is the entire consumer-facing surface ────────────────────────── + try (var data = NativePattern.open(65_536)) { + + long n = data.view() + .where(Pattern.CLASS.eq(7)) + .where(Pattern.VALUE.gt(100)) + .count(); + + System.out.println(" -> " + n + " of " + data.rowCount() + " rows matched" + + " class == 7 AND value > 100"); + // ──────────────────────────────────────────────────────────────────────────────── + + c.that("the chain returns a plausible count", n > 0 && n < data.rowCount()); + c.eq("the resource reports the row count it was opened with", 65_536, data.rowCount()); + c.eq("the row range agrees with the row count", 65_536, data.rows().length()); + + c.section("each step of the chain, so a wrong total can be localised"); + long all = data.view().count(); + long cls = data.view().where(Pattern.CLASS.eq(7)).count(); + long val = data.view().where(Pattern.VALUE.gt(100)).count(); + c.eq("an unconditioned view selects every row", 65_536, all); + c.that("class == 7 selects roughly a sixteenth", cls > all / 32 && cls < all / 8); + c.that("value > 100 selects a middling fraction", val > all / 4 && val < (all * 3) / 4); + c.that("the conjunction is no larger than either part", n <= cls && n <= val); + + c.section("a reduction over the same rows"); + long sum = data.view() + .where(Pattern.CLASS.eq(7)) + .lens(Pattern.VALUE) + .sum(); + c.note("sum of value over class == 7 is " + sum); + c.that("the sum is a real number, not a default", sum != 0); + + c.section("a selection can be kept and asked twice"); + try (Mask selected = data.view().where(Pattern.CLASS.eq(7)).select()) { + c.eq("a materialised selection agrees with the count", cls, selected.count()); + c.eq("and is stable when asked again", cls, selected.count()); + c.note("selection identity: " + selected.id()); + c.note("65,536 rows selected as packed bits = " + (65_536 / 8) + " bytes," + + " not 65,536 objects"); + } + + c.section("views are immutable, so narrowing one does not disturb it"); + View base = data.view().where(Pattern.CLASS.eq(7)); + View narrower = base.where(Pattern.VALUE.gt(100)); + c.eq("the original still selects what it did", cls, base.count()); + c.eq("and the derived view is the narrowed one", n, narrower.count()); + c.eq("the original carries one condition", 1, base.conditionCount()); + c.eq("the derived one carries two", 2, narrower.conditionCount()); + } + } +} diff --git a/native/lgj-abi/.cargo/config.toml b/native/lgj-abi/.cargo/config.toml new file mode 100644 index 0000000..52168f8 --- /dev/null +++ b/native/lgj-abi/.cargo/config.toml @@ -0,0 +1,38 @@ +# x86-64-v4 (AVX-512) baseline — MANDATORY (operator directive, 2026-08-17: +# "rust compiles with CPU v4"). This DIVERGES from /home/user/ndarray's own +# default (.cargo/config.toml there pins v3/AVX2, because that repo targets +# portable distribution). lance-graph-java's native artifact is built for +# THIS host, which has avx512f/bw/cd/dq/ifma/vbmi/vl — verified this session +# — so the production build exercises ndarray's AVX-512 backend +# (`src/simd_avx512.rs`) directly rather than falling back to AVX2. +# +# Why this line is load-bearing and why omitting/under-shooting it is a +# *runtime* fault, not a compile error: +# +# ndarray's SIMD backends compose their vector types out of raw AVX / +# AVX2 / AVX-512 intrinsics (`__m256`, `__m512`, `__m512i`, ...). Those are +# `#[inline]` and get instantiated into THIS crate's object code when we +# call `ndarray::simd::*`. Without a target-cpu baseline that actually +# covers what got emitted, rustc may target a narrower baseline than the +# instantiated intrinsics require, and the CPU then executes an +# instruction the compiled target never promised — the process dies with +# SIGILL. That is ndarray's PR #170 CI failure mode, reproduced one +# consumer down. The symptom is a mysterious SIGILL inside a JVM downcall, +# about the worst possible place to debug it from — hence pinning the +# baseline here, in the crate that produces the cdylib, rather than +# relying on a caller to pass the right RUSTFLAGS. +# +# This is a BUILD TARGET declaration, not SIMD selection in source. abi.md §8 +# forbids `#[cfg(target_feature)]` *selection of a SIMD implementation* in +# this crate; choosing which silicon the artifact is compiled for is +# ndarray's documented consumer contract, and the manifest then *reports* +# the resulting backend (never negotiates it) — with this baseline, +# `LgjAbiManifest::simd_backend` must report AVX512 on this host. +# +# Portable/distribution build (older silicon, no AVX-512): override with +# CARGO_BUILD_RUSTFLAGS='-Ctarget-cpu=x86-64-v3' at build time; do not change +# this file's default for that — v4 stays the default because this is a +# research vertical slice built and run on one known host, not a +# redistributed artifact. +[target.'cfg(target_arch = "x86_64")'] +rustflags = ["-Ctarget-cpu=x86-64-v4"] diff --git a/native/lgj-abi/Cargo.lock b/native/lgj-abi/Cargo.lock new file mode 100644 index 0000000..74977d7 --- /dev/null +++ b/native/lgj-abi/Cargo.lock @@ -0,0 +1,94 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "lgj-abi" +version = "0.1.0" +dependencies = [ + "ndarray", +] + +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "paste", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" diff --git a/native/lgj-abi/Cargo.toml b/native/lgj-abi/Cargo.toml new file mode 100644 index 0000000..c83b1b6 --- /dev/null +++ b/native/lgj-abi/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "lgj-abi" +version = "0.1.0" +edition = "2021" +rust-version = "1.97" +license = "MIT OR Apache-2.0" +publish = false +description = "The lance-graph-java machine membrane: a self-describing, versioned, bulk-only psABI surface. No C, no JNI, no headers." + +[lib] +name = "lgj_abi" +# cdylib -> the artifact Java's Linker.nativeLinker() dlopen's. +# rlib -> so the in-crate #[cfg(test)] suite can link and exercise the +# same code the cdylib exports (a separate build of the same +# source would not prove anything about the shipped artifact's +# layout, but the compile-time size asserts in abi.rs do, and +# they are compiled in both crate types). +crate-type = ["cdylib", "rlib"] + +[dependencies] +# The ONLY dependency, and the ONLY source of SIMD in this crate (abi.md §8). +# +# default-features = false + "std": +# ndarray's own defaults are ["std", "hpc-extras"]; hpc-extras pulls the +# `p64` and `fractal` path crates, which nothing here needs. "std" alone +# is what gates `ndarray::simd`, `ndarray::simd_int_ops`, `ndarray::hpc` +# and `ndarray::bitwise` — i.e. every module this crate consumes. +ndarray = { path = "../../../ndarray", default-features = false, features = ["std"] } + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 +# panic = "abort" is DELIBERATELY ABSENT (abi.md §9). Every extern "C" body +# wraps itself in std::panic::catch_unwind so a panic becomes a negative +# status instead of an unwind into JVM frames (which is UB). catch_unwind +# requires unwinding to be available, so "abort" would silently turn the +# panic-safety layer into a process kill. + +[profile.dev] +# Same rule as release: no panic = "abort". diff --git a/native/lgj-abi/rust-toolchain.toml b/native/lgj-abi/rust-toolchain.toml new file mode 100644 index 0000000..652fcca --- /dev/null +++ b/native/lgj-abi/rust-toolchain.toml @@ -0,0 +1,6 @@ +[toolchain] +# Stable only. Matches ndarray / lance-graph / the rest of the AdaWorldAPI +# stack. No nightly features anywhere in this crate — in particular no +# `core::simd` / portable-simd (abi.md §8). +channel = "1.97.1" +components = ["clippy", "rustfmt"] diff --git a/native/lgj-abi/src/abi.rs b/native/lgj-abi/src/abi.rs new file mode 100644 index 0000000..80d8bf6 --- /dev/null +++ b/native/lgj-abi/src/abi.rs @@ -0,0 +1,534 @@ +//! The ABI surface types — the *machine membrane* vocabulary. +//! +//! Normative source: `docs/abi.md`. Every struct field, its order, its size and +//! every status code in this file is dictated by that document. If this file and +//! `abi.md` disagree, this file is wrong. +//! +//! # There is no C here +//! +//! `#[repr(C)]` below names this target's standard *aggregate layout rule* +//! (field order, padding, alignment) — the SysV AMD64 psABI on x86_64, AAPCS64 +//! on ARM64. It is not a C declaration, there is no `.h` anywhere in this +//! project, and no C toolchain participates in building or consuming it. The +//! Java side re-derives these same layouts independently with +//! `java.lang.foreign.MemoryLayout` and the manifest (below) proves the two +//! derivations agree *at runtime*, which a text header could never do. +//! +//! # Layout drift is a build failure +//! +//! Each `#[repr(C)]` type carries `const _: () = assert!(size_of::() == N);` +//! compile-time assertions against the byte counts `abi.md` §5 states. Adding, +//! reordering or re-typing a field therefore breaks the *build*, not a Java +//! test at 3am. + +use core::mem::{align_of, size_of}; + +// --------------------------------------------------------------------------- +// §2 Versioning +// --------------------------------------------------------------------------- + +/// Incompatible change ⇒ bump. Java refuses to load on a mismatch. +pub const LGJ_ABI_MAJOR: u32 = 0; + +/// Additive change ⇒ bump. Older Java may still load (`minor >= expected`). +pub const LGJ_ABI_MINOR: u32 = 1; + +/// `"LGJ_ABI\0"` read big-endian. +/// +/// Doubles as an endianness probe: Java reads this field as a little-endian +/// `u64` and compares against its own compiled-in copy of this constant. +/// Anything else means the library was built for a different byte order, and +/// every subsequent read across the membrane would be garbage. +pub const LGJ_MAGIC: u64 = 0x4C_47_4A_5F_41_42_49_00; + +// --------------------------------------------------------------------------- +// §3 Status codes — every function returns i32; 0 = OK, all failures negative +// --------------------------------------------------------------------------- + +/// Success. +pub const LGJ_OK: i32 = 0; +/// A required out-pointer was null. +pub const LGJ_ERR_NULL_ARGUMENT: i32 = -1; +/// Handle malformed, closed, or generation-stale. The answer to +/// *use-after-close* — deliberately a status, never a crash (§4). +pub const LGJ_ERR_INVALID_HANDLE: i32 = -2; +/// e.g. a mask handle passed where a pattern was required. +pub const LGJ_ERR_WRONG_RESOURCE_KIND: i32 = -3; +/// `lane_id` out of range for this resource. +pub const LGJ_ERR_INVALID_LANE: i32 = -4; +/// The op's element type ≠ the lane's element type. +pub const LGJ_ERR_LANE_KIND_MISMATCH: i32 = -5; +/// Mask row-count ≠ resource row-count. +pub const LGJ_ERR_MASK_LENGTH_MISMATCH: i32 = -6; +/// A child (mask) outlived its parent resource. +pub const LGJ_ERR_PARENT_CLOSED: i32 = -7; +/// Caller's ABI version incompatible. +pub const LGJ_ERR_VERSION_MISMATCH: i32 = -8; +/// Requested size overflows `usize` / the allocation limit. +pub const LGJ_ERR_LENGTH_OVERFLOW: i32 = -9; +/// The plan contained an opcode this build does not implement. +pub const LGJ_ERR_UNKNOWN_OPCODE: i32 = -10; +/// A plan with zero ops was submitted. +pub const LGJ_ERR_EMPTY_PLAN: i32 = -11; +/// The allocator refused. +pub const LGJ_ERR_ALLOCATION_FAILED: i32 = -12; +/// A write was attempted against a read-only lane. +pub const LGJ_ERR_READ_ONLY: i32 = -13; + +/// A panic was caught at the membrane and converted to a status (§9). +/// +/// Not in `abi.md`'s table, and deliberately *outside* the allocated +/// `-1..=-13` block so it can never be confused with a specified condition. +/// A caller seeing this has found a bug in this crate; it is reported rather +/// than allowed to unwind into JVM frames, which would be UB. +pub const LGJ_ERR_PANIC: i32 = -99; + +// --------------------------------------------------------------------------- +// §5 Element kinds — start at 1, so a zeroed struct is *detectably invalid* +// rather than silently meaning U8. +// --------------------------------------------------------------------------- + +/// Element kind tags for [`LgjLaneDesc::elem_kind`]. +#[repr(u32)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LgjElemKind { + /// Unsigned 8-bit. + U8 = 1, + /// Signed 8-bit. + I8 = 2, + /// Unsigned 16-bit. + U16 = 3, + /// Signed 16-bit. + I16 = 4, + /// Unsigned 32-bit. + U32 = 5, + /// Signed 32-bit. + I32 = 6, + /// Unsigned 64-bit. + U64 = 7, + /// Signed 64-bit. + I64 = 8, + /// IEEE-754 binary32. + F32 = 9, + /// IEEE-754 binary64. + F64 = 10, + /// A `u64` of 64 packed row bits, LSB = lowest row index. + MaskWord = 11, +} + +impl LgjElemKind { + /// Width of one element in bytes. + pub const fn elem_bytes(self) -> u32 { + match self { + LgjElemKind::U8 | LgjElemKind::I8 => 1, + LgjElemKind::U16 | LgjElemKind::I16 => 2, + LgjElemKind::U32 | LgjElemKind::I32 | LgjElemKind::F32 => 4, + LgjElemKind::U64 | LgjElemKind::I64 | LgjElemKind::F64 | LgjElemKind::MaskWord => 8, + } + } +} + +// §5 lane flags (bitfield) +/// Java may read this lane's bytes. +pub const LGJ_FLAG_READABLE: u32 = 1 << 0; +/// Java may write this lane's bytes. Pattern lanes never set this; mask words do. +pub const LGJ_FLAG_WRITABLE: u32 = 1 << 1; +/// `stride_bytes == elem_bytes`. +pub const LGJ_FLAG_CONTIGUOUS: u32 = 1 << 2; + +// §5 resource kinds +/// A pattern (the SoA fixture): id/class/value lanes, read-only. +pub const LGJ_RESOURCE_PATTERN: u32 = 1; +/// A mask: one `MASK_WORD` lane, writable, owned by a parent pattern. +pub const LGJ_RESOURCE_MASK: u32 = 2; + +// §7 mask_create initial states +/// `lgj_mask_create(initial = 0)` — no rows set. +pub const LGJ_MASK_INIT_EMPTY: u32 = 0; +/// `lgj_mask_create(initial = 1)` — all rows set. +pub const LGJ_MASK_INIT_ALL: u32 = 1; + +// --------------------------------------------------------------------------- +// Opcodes + combiners (§5 LgjOpDesc) +// --------------------------------------------------------------------------- + +/// `lane[i] == operand as u32`, over a `U32` lane. +pub const LGJ_OP_EQ_U32: u32 = 1; +/// `lane[i] > operand as i32` (signed), over an `I32` lane. +pub const LGJ_OP_GT_I32: u32 = 2; + +/// Narrow the accumulator: `acc &= op_result`. +pub const LGJ_COMBINE_AND: u32 = 0; +/// Widen the accumulator: `acc |= op_result`. +pub const LGJ_COMBINE_OR: u32 = 1; + +/// The element kind an opcode requires of its lane. `None` ⇒ unknown opcode. +pub(crate) const fn opcode_required_kind(op: u32) -> Option { + match op { + LGJ_OP_EQ_U32 => Some(LgjElemKind::U32), + LGJ_OP_GT_I32 => Some(LgjElemKind::I32), + _ => None, + } +} + +// --------------------------------------------------------------------------- +// §5 SIMD backend tags — reported, never negotiated +// --------------------------------------------------------------------------- + +/// No vector backend compiled in. +pub const LGJ_SIMD_SCALAR: u32 = 0; +/// `ndarray::simd` AVX2 backend (x86-64-v3 baseline). +pub const LGJ_SIMD_AVX2: u32 = 1; +/// `ndarray::simd` AVX-512 backend (x86-64-v4 baseline). +pub const LGJ_SIMD_AVX512: u32 = 2; +/// `ndarray::simd` NEON backend (aarch64). +pub const LGJ_SIMD_NEON: u32 = 3; +/// `ndarray::simd` wasm128 backend. +pub const LGJ_SIMD_WASM: u32 = 4; + +// --------------------------------------------------------------------------- +// §5 The descriptors +// --------------------------------------------------------------------------- + +/// The bounded description the Java FFM layer turns into a `MemorySegment`. +/// +/// Java's *public* API never sees `addr`; it is physics, consumed inside the +/// FFM layer and never surfaced. +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct LgjLaneDesc { + /// Base address of the lane. Physics — never surfaced in public Java API. + pub addr: u64, + /// Number of elements. + pub len_elems: u64, + /// `len_elems * stride_bytes`. + pub byte_len: u64, + /// Owning resource handle. + pub owner: u64, + /// Liveness stamp; Java re-checks it before using a segment it still holds. + pub epoch: u64, + /// [`LgjElemKind`] as `u32`. + pub elem_kind: u32, + /// Width of one element. + pub elem_bytes: u32, + /// `== elem_bytes` when contiguous. + pub stride_bytes: u32, + /// `LGJ_FLAG_*` bitfield. + pub flags: u32, +} + +/// What a handle refers to, without touching its payload. +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct LgjResourceInfo { + /// `LGJ_RESOURCE_PATTERN` | `LGJ_RESOURCE_MASK`. + pub kind: u32, + /// Number of describable lanes. + pub lane_count: u32, + /// Logical row count. + pub n_rows: u64, + /// Liveness stamp, matching the lanes' `epoch`. + pub epoch: u64, + /// Parent handle; `0` = none. + pub parent: u64, +} + +/// One predicate in a fused plan. +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct LgjOpDesc { + /// `LGJ_OP_*`. + pub op: u32, + /// Which lane of the resource to read. + pub lane_id: u32, + /// Needle / threshold, sign-extended into `i64` by the caller. + pub operand: i64, + /// `LGJ_COMBINE_AND` (narrow) | `LGJ_COMBINE_OR` (widen). + pub combine: u32, + /// Must be `0`. Rejected otherwise, so a future field cannot be + /// silently ignored by an old build. + pub _reserved: u32, +} + +/// What replaces a C header: the compiled artifact describing *itself*. +/// +/// A header is a text file that *claims* what an artifact looks like and can +/// drift from it silently. This struct is emitted **by** the artifact, so it +/// cannot disagree with itself — every `size_of_*` / `align_of_*` below is +/// filled from `core::mem::size_of` / `align_of` on the real type (see +/// [`manifest`]), never from a hardcoded number. +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct LgjAbiManifest { + /// [`LGJ_MAGIC`]; doubles as the endianness probe. + pub magic: u64, + /// [`LGJ_ABI_MAJOR`]. Java requires an exact match. + pub abi_major: u32, + /// [`LGJ_ABI_MINOR`]. Java requires `>=` what it was compiled against. + pub abi_minor: u32, + /// `size_of::()`. + pub size_of_manifest: u32, + /// `size_of::()`. + pub size_of_lane_desc: u32, + /// `size_of::()`. + pub size_of_op_desc: u32, + /// `size_of::()`. + pub size_of_resource_info: u32, + /// `align_of::()`. + pub align_of_lane_desc: u32, + /// `align_of::()`. + pub align_of_op_desc: u32, + /// `align_of::()`. + pub align_of_resource_info: u32, + /// `size_of::()` — the target's pointer width. + pub pointer_bytes: u32, + /// `0` = little-endian. + pub endianness: u32, + /// `LGJ_SIMD_*`. + pub simd_backend: u32, + /// NUL-terminated, human-readable. + pub simd_backend_name: [u8; 32], + /// `"release"` | `"debug"`, NUL-terminated. + pub build_profile: [u8; 16], +} + +// --------------------------------------------------------------------------- +// Layout assertions — abi.md §5 states these byte counts; a drift is a +// BUILD FAILURE here rather than a mysterious Java-side misread later. +// --------------------------------------------------------------------------- + +const _: () = assert!(size_of::() == 56); +const _: () = assert!(align_of::() == 8); +const _: () = assert!(size_of::() == 32); +const _: () = assert!(align_of::() == 8); +const _: () = assert!(size_of::() == 24); +const _: () = assert!(align_of::() == 8); +// abi.md does not state the manifest's size (it is self-reported, which is the +// point), but pinning it still catches an accidental field insertion: +// 8 (magic) + 12*4 (u32 fields) + 32 (name) + 16 (profile) = 104, align 8. +const _: () = assert!(size_of::() == 104); +const _: () = assert!(align_of::() == 8); +// A MASK_WORD is 64 row bits in one u64. If this ever stops holding, the whole +// bit-order contract below is void. +const _: () = assert!(ROWS_PER_WORD == u64::BITS as u64); + +// --------------------------------------------------------------------------- +// Mask word arithmetic — the bit-order contract, in one place +// --------------------------------------------------------------------------- + +/// Rows per mask word. +pub const ROWS_PER_WORD: u64 = 64; + +/// Number of `u64` words needed to hold `n_rows` bits (LSB-first within each +/// word: row `i` lives at bit `i % 64` of word `i / 64`). +pub const fn mask_words_for(n_rows: u64) -> u64 { + n_rows.div_ceil(ROWS_PER_WORD) +} + +/// Zero every bit at row index `>= n_rows` in the final word. +/// +/// Trailing bits beyond the row count are **normative zero** — `abi.md`'s +/// popcount, subset and parity properties all depend on it, so +/// `lgj_mask_count` cannot be fooled by garbage in the tail. +pub fn clear_tail_bits(words: &mut [u64], n_rows: u64) { + let used = (n_rows % ROWS_PER_WORD) as u32; + if used != 0 { + if let Some(last) = words.last_mut() { + *last &= (1u64 << used) - 1; + } + } +} + +// --------------------------------------------------------------------------- +// The manifest instance +// --------------------------------------------------------------------------- + +/// Detect the SIMD backend ndarray compiled into this artifact. +/// +/// This is the ONE sanctioned `cfg(target_feature)` / `cfg(target_arch)` use in +/// this crate. `abi.md` §8 forbids cfg-based *selection of a SIMD +/// implementation*; this selects nothing — it reports a fact about the build so +/// Java can log which backend it got. Which backend exists at all is +/// `ndarray`'s business, and the tags below mirror `ndarray/src/simd.rs`'s own +/// dispatch order (avx512f first, then avx2, then aarch64/neon, then wasm). +const fn detect_simd_backend() -> (u32, &'static str) { + // `cfg!` (not `#[cfg]` blocks) so exactly one arm is chosen as a *value* — + // and so the whole function stays a single const expression. + if cfg!(all(target_arch = "x86_64", target_feature = "avx512f")) { + (LGJ_SIMD_AVX512, "ndarray::simd avx512") + } else if cfg!(all(target_arch = "x86_64", target_feature = "avx2")) { + (LGJ_SIMD_AVX2, "ndarray::simd avx2 (x86-64-v3)") + } else if cfg!(all(target_arch = "aarch64", target_feature = "neon")) { + (LGJ_SIMD_NEON, "ndarray::simd neon") + } else if cfg!(target_arch = "wasm32") { + (LGJ_SIMD_WASM, "ndarray::simd wasm128") + } else { + // On x86_64 this arm is reachable only if the `.cargo/config.toml` + // target-cpu baseline was lost (this crate's default is x86-64-v4, so + // the expected report on this host is AVX512, and AVX2 only under an + // explicit v3 override). It is also the SIGILL condition — ndarray's + // intrinsics get instantiated regardless of the baseline — so SCALAR + // reported on x86_64 means the build is misconfigured, not that the + // CPU is old. + (LGJ_SIMD_SCALAR, "ndarray::simd scalar") + } +} + +const BUILD_PROFILE: &str = if cfg!(debug_assertions) { + "debug" +} else { + "release" +}; + +/// Copy `src` into a fixed-size NUL-terminated byte field, truncating if needed +/// (always leaving room for the terminator). +const fn fixed_cstr(src: &str) -> [u8; N] { + let mut out = [0u8; N]; + let bytes = src.as_bytes(); + let mut i = 0; + while i < bytes.len() && i + 1 < N { + out[i] = bytes[i]; + i += 1; + } + out +} + +/// The `'static` the manifest getter hands out. Built entirely from +/// `size_of` / `align_of` on the real types — never a literal. +pub static MANIFEST: LgjAbiManifest = LgjAbiManifest { + magic: LGJ_MAGIC, + abi_major: LGJ_ABI_MAJOR, + abi_minor: LGJ_ABI_MINOR, + size_of_manifest: size_of::() as u32, + size_of_lane_desc: size_of::() as u32, + size_of_op_desc: size_of::() as u32, + size_of_resource_info: size_of::() as u32, + align_of_lane_desc: align_of::() as u32, + align_of_op_desc: align_of::() as u32, + align_of_resource_info: align_of::() as u32, + pointer_bytes: size_of::() as u32, + // `u64::from_le_bytes` of a known pattern would be a runtime check; the + // compile-time form is `cfg(target_endian)`, which is a *fact about the + // build* exactly like the SIMD tag above. + endianness: if cfg!(target_endian = "little") { 0 } else { 1 }, + simd_backend: detect_simd_backend().0, + simd_backend_name: fixed_cstr::<32>(detect_simd_backend().1), + build_profile: fixed_cstr::<16>(BUILD_PROFILE), +}; + +#[cfg(test)] +mod tests { + use super::*; + + /// The manifest must describe *itself* truthfully — this is the property + /// the Java side's load-time cross-check depends on. + #[test] + fn manifest_is_self_consistent() { + assert_eq!(MANIFEST.magic, LGJ_MAGIC); + assert_eq!(MANIFEST.abi_major, LGJ_ABI_MAJOR); + assert_eq!(MANIFEST.abi_minor, LGJ_ABI_MINOR); + assert_eq!( + MANIFEST.size_of_manifest as usize, + size_of::() + ); + assert_eq!( + MANIFEST.size_of_lane_desc as usize, + size_of::() + ); + assert_eq!(MANIFEST.size_of_op_desc as usize, size_of::()); + assert_eq!( + MANIFEST.size_of_resource_info as usize, + size_of::() + ); + assert_eq!( + MANIFEST.align_of_lane_desc as usize, + align_of::() + ); + assert_eq!(MANIFEST.align_of_op_desc as usize, align_of::()); + assert_eq!( + MANIFEST.align_of_resource_info as usize, + align_of::() + ); + assert_eq!(MANIFEST.pointer_bytes as usize, size_of::()); + } + + /// abi.md §5's stated byte counts, asserted at run time as well as at + /// compile time — so a reader of the test output sees the real numbers. + #[test] + fn struct_sizes_match_the_spec() { + assert_eq!(size_of::(), 56); + assert_eq!(size_of::(), 32); + assert_eq!(size_of::(), 24); + assert_eq!(size_of::(), 104); + } + + #[test] + fn magic_reads_as_lgj_abi_nul_big_endian() { + assert_eq!(&LGJ_MAGIC.to_be_bytes(), b"LGJ_ABI\0"); + } + + #[test] + fn name_fields_are_nul_terminated() { + assert!(MANIFEST.simd_backend_name.contains(&0)); + assert!(MANIFEST.build_profile.contains(&0)); + let name = MANIFEST.simd_backend_name; + let end = name.iter().position(|&b| b == 0).unwrap(); + assert!(end > 0, "backend name must not be empty"); + } + + /// The SIGILL trap, turned into a test failure. + /// + /// `ndarray`'s vector types are built from AVX/AVX2/AVX-512 intrinsics and + /// get instantiated into this crate's object code whichever baseline rustc + /// was given. If `.cargo/config.toml`'s `-Ctarget-cpu` is lost, rustc + /// targets generic x86-64 (SSE2), those instructions still get emitted, and + /// the process dies with SIGILL — inside a JVM downcall, if nobody noticed + /// earlier. The backend tag is the observable that goes wrong *first*, so + /// assert on it here where the failure is legible. + #[test] + #[cfg(target_arch = "x86_64")] + fn the_x86_64_build_has_a_vector_baseline() { + assert_ne!( + MANIFEST.simd_backend, LGJ_SIMD_SCALAR, + "no target-cpu baseline: ndarray's intrinsics will SIGILL at run time. \ + Check native/lgj-abi/.cargo/config.toml." + ); + assert!(matches!( + MANIFEST.simd_backend, + LGJ_SIMD_AVX2 | LGJ_SIMD_AVX512 + )); + } + + #[test] + fn element_kinds_start_at_one_so_zeroed_is_invalid() { + assert_eq!(LgjElemKind::U8 as u32, 1); + assert_eq!(LgjElemKind::MaskWord as u32, 11); + assert_eq!( + LgjLaneDesc::default().elem_kind, + 0, + "zeroed ⇒ no valid kind" + ); + } + + #[test] + fn mask_word_count_rounds_up() { + assert_eq!(mask_words_for(0), 0); + assert_eq!(mask_words_for(1), 1); + assert_eq!(mask_words_for(64), 1); + assert_eq!(mask_words_for(65), 2); + assert_eq!(mask_words_for(1000), 16); + } + + #[test] + fn tail_bits_are_cleared() { + let mut w = vec![u64::MAX; 2]; + clear_tail_bits(&mut w, 70); + assert_eq!(w[0], u64::MAX); + assert_eq!(w[1], 0x3F, "only rows 64..70 may survive"); + + // Exact multiple of 64: nothing to clear. + let mut w2 = vec![u64::MAX; 2]; + clear_tail_bits(&mut w2, 128); + assert_eq!(w2, vec![u64::MAX; 2]); + } +} diff --git a/native/lgj-abi/src/exports.rs b/native/lgj-abi/src/exports.rs new file mode 100644 index 0000000..050bf60 --- /dev/null +++ b/native/lgj-abi/src/exports.rs @@ -0,0 +1,1624 @@ +//! The `extern "C"` surface — the membrane itself. +//! +//! # `extern "C"` is a calling convention, not a language +//! +//! Every symbol below is `extern "C"`, which on this target means the **System V +//! AMD64 psABI** (AAPCS64 on ARM64): which registers carry which argument, how +//! aggregates are passed, who saves what. It is a machine contract. There is no +//! C source, no `.h`, no C compiler, and no JNI anywhere in this project. Java +//! reaches these symbols with `Linker.nativeLinker()`, which is the JVM's own +//! implementation of the same psABI. +//! +//! # Two invariants every function here obeys +//! +//! 1. **Bulk or lifecycle** (abi.md §6). A function either does work +//! proportional to `n_rows` or it is open/close/describe. There is no +//! per-element crossing and no `lgj_lane_read_element` — if Java wants one +//! row it reads the `MemorySegment` in-process, with no crossing at all. +//! 2. **Panics never cross** (abi.md §9). Every body runs inside +//! [`guard`], which converts an unwind into [`LGJ_ERR_PANIC`]. An unwind into +//! JVM frames is UB; a negative status is a Tuesday. +//! +//! `out_*` parameters are written **only on `OK`**, so a failed call cannot +//! leave Java reading a half-filled descriptor. + +use std::panic::{catch_unwind, AssertUnwindSafe}; + +use crate::abi::*; +use crate::fixture::PATTERN_LANE_COUNT; +use crate::kernels::{self, LaneView, Path}; +use crate::registry::{self, ResourceEntry}; + +/// Run `f`, converting a panic into [`LGJ_ERR_PANIC`]. +/// +/// `AssertUnwindSafe` is required because the closures below capture raw +/// pointers. It is sound here for the reason that matters: on the unwind path we +/// return a status and touch *nothing* the panic could have left inconsistent — +/// the only shared state is the registry, whose locks recover via `into_inner` +/// and whose payloads carry no invariant beyond "tail bits are zero", which +/// every write path re-establishes before returning. +#[inline] +fn guard i32>(f: F) -> i32 { + match catch_unwind(AssertUnwindSafe(f)) { + Ok(status) => status, + Err(_) => LGJ_ERR_PANIC, + } +} + +/// Collapse a `Result<(), i32>` into a status. +#[inline] +fn status(r: Result<(), i32>) -> i32 { + match r { + Ok(()) => LGJ_OK, + Err(e) => e, + } +} + +// ─────────────────────────────────────────────────────────────────────────── +// Manifest — the one symbol with no failure mode, and therefore no status +// ─────────────────────────────────────────────────────────────────────────── + +/// Return a pointer to the `'static` [`LgjAbiManifest`]. +/// +/// Never fails, never allocates. This is what replaces a C header: the artifact +/// describing itself, so it cannot disagree with itself the way a checked-in +/// header can drift from the binary beside it. +#[no_mangle] +pub extern "C" fn lgj_abi_manifest() -> *const LgjAbiManifest { + &MANIFEST as *const LgjAbiManifest +} + +// ─────────────────────────────────────────────────────────────────────────── +// Lifecycle +// ─────────────────────────────────────────────────────────────────────────── + +/// Build the deterministic SoA fixture and return its handle. +/// +/// Bulk by construction: allocates and fills three lanes of `n_rows` elements. +/// The generation algorithm is normative — see [`crate::fixture::Fixture`]. +/// # Safety +/// +/// A null pointer is *handled*, not UB: it returns `NULL_ARGUMENT`. Beyond +/// that, `out_handle` must be a valid, aligned, writable `u64` (Java passes an 8-byte segment). It is written only on success. +/// +/// `unsafe` here is a note to Rust callers linking the `rlib`. The JVM, +/// which is the real caller, has no such concept — it upholds the same +/// contract by construction, because every pointer it passes comes from a +/// `MemorySegment` whose size and alignment it derived from the manifest. +#[no_mangle] +pub unsafe extern "C" fn lgj_pattern_open(n_rows: u64, seed: u64, out_handle: *mut u64) -> i32 { + guard(|| { + if out_handle.is_null() { + return LGJ_ERR_NULL_ARGUMENT; + } + match registry::open_pattern(n_rows, seed) { + Ok(h) => { + // SAFETY: non-null (checked above) and Java passes a segment of + // at least 8 bytes; the write happens only on success, so a + // failed open never scribbles on the caller's slot. + unsafe { *out_handle = h }; + LGJ_OK + } + Err(e) => e, + } + }) +} + +/// Free a resource: its lanes are dropped, its generation is bumped, and its +/// children begin failing with `PARENT_CLOSED`. +/// +/// A second close, or a fabricated handle, returns `INVALID_HANDLE` — never a +/// double free. +#[no_mangle] +pub extern "C" fn lgj_close(handle: u64) -> i32 { + guard(|| status(registry::close(handle))) +} + +/// Describe a resource without touching its payload. +/// # Safety +/// +/// A null pointer is *handled*, not UB: it returns `NULL_ARGUMENT`. Beyond +/// that, `out` must be a valid, aligned, writable `LgjResourceInfo` (32 bytes, align 8 — the manifest reports both). Written only on success. +/// +/// `unsafe` here is a note to Rust callers linking the `rlib`. The JVM, +/// which is the real caller, has no such concept — it upholds the same +/// contract by construction, because every pointer it passes comes from a +/// `MemorySegment` whose size and alignment it derived from the manifest. +#[no_mangle] +pub unsafe extern "C" fn lgj_resource_info(handle: u64, out: *mut LgjResourceInfo) -> i32 { + guard(|| { + if out.is_null() { + return LGJ_ERR_NULL_ARGUMENT; + } + let entry = match registry::resolve(handle) { + Ok(e) => e, + Err(e) => return e, + }; + // SAFETY: non-null, and `LgjResourceInfo` is `#[repr(C)]` with the exact + // layout the manifest reports and Java's MemoryLayout mirrors. + unsafe { *out = entry.info() }; + LGJ_OK + }) +} + +// ─────────────────────────────────────────────────────────────────────────── +// Lanes +// ─────────────────────────────────────────────────────────────────────────── + +/// Describe one lane of a **pattern**: `0 = ids (U64)`, `1 = classes (U32)`, +/// `2 = values (I32)`. +/// +/// All pattern lanes are `READABLE | CONTIGUOUS` and never `WRITABLE` +/// (abi.md §7). The returned `addr` is stable until `lgj_close` — lanes are +/// allocated once and never moved or resized (§4). +/// # Safety +/// +/// A null pointer is *handled*, not UB: it returns `NULL_ARGUMENT`. Beyond +/// that, `out` must be a valid, aligned, writable `LgjLaneDesc` (56 bytes, align 8 — the manifest reports both). Written only on success. +/// +/// `unsafe` here is a note to Rust callers linking the `rlib`. The JVM, +/// which is the real caller, has no such concept — it upholds the same +/// contract by construction, because every pointer it passes comes from a +/// `MemorySegment` whose size and alignment it derived from the manifest. +#[no_mangle] +pub unsafe extern "C" fn lgj_lane_describe( + handle: u64, + lane_id: u32, + out: *mut LgjLaneDesc, +) -> i32 { + guard(|| { + if out.is_null() { + return LGJ_ERR_NULL_ARGUMENT; + } + let entry = match registry::resolve_kind(handle, LGJ_RESOURCE_PATTERN) { + Ok(e) => e, + Err(e) => return e, + }; + let fixture = match entry.fixture() { + Some(f) => f, + None => return LGJ_ERR_WRONG_RESOURCE_KIND, + }; + let (addr, len_elems, kind) = match fixture.lane_raw(lane_id) { + Some(t) => t, + None => return LGJ_ERR_INVALID_LANE, + }; + let elem_bytes = kind.elem_bytes(); + let desc = LgjLaneDesc { + addr, + len_elems, + byte_len: len_elems * elem_bytes as u64, + owner: handle, + epoch: entry.epoch, + elem_kind: kind as u32, + elem_bytes, + stride_bytes: elem_bytes, + flags: crate::fixture::Fixture::lane_flags(), + }; + // SAFETY: non-null; `LgjLaneDesc` is `#[repr(C)]`, 56 bytes, and that + // size is asserted at compile time and reported by the manifest. + unsafe { *out = desc }; + LGJ_OK + }) +} + +// ─────────────────────────────────────────────────────────────────────────── +// Masks +// ─────────────────────────────────────────────────────────────────────────── + +/// Create a mask over `parent`. `initial`: `0` = empty, `1` = all rows set. +/// # Safety +/// +/// A null pointer is *handled*, not UB: it returns `NULL_ARGUMENT`. Beyond +/// that, `out_handle` must be a valid, aligned, writable `u64`. Written only on success. +/// +/// `unsafe` here is a note to Rust callers linking the `rlib`. The JVM, +/// which is the real caller, has no such concept — it upholds the same +/// contract by construction, because every pointer it passes comes from a +/// `MemorySegment` whose size and alignment it derived from the manifest. +#[no_mangle] +pub unsafe extern "C" fn lgj_mask_create(parent: u64, initial: u32, out_handle: *mut u64) -> i32 { + guard(|| { + if out_handle.is_null() { + return LGJ_ERR_NULL_ARGUMENT; + } + match registry::create_mask(parent, initial) { + Ok(h) => { + // SAFETY: non-null, checked above; written only on success. + unsafe { *out_handle = h }; + LGJ_OK + } + Err(e) => e, + } + }) +} + +/// Describe a mask's word lane: `MASK_WORD`, `READABLE | WRITABLE | +/// CONTIGUOUS`. +/// +/// A `MASK_WORD` is a `u64` of 64 packed row bits, LSB = lowest row index. Java +/// may write these words directly through the segment — that is what +/// `WRITABLE` means, and it is the reason a `.where(...)` chain needs no +/// crossing per row. +/// +/// Honest note on locking: Rust-side ops take the mask's inner lock, but Java's +/// direct writes go through the raw segment and are outside that discipline. +/// That is sound for the POC because its Java layer is single-threaded; a +/// concurrent Java writer would need a documented protocol, which this ABI +/// version does not define. +/// # Safety +/// +/// A null pointer is *handled*, not UB: it returns `NULL_ARGUMENT`. Beyond +/// that, `out` must be a valid, aligned, writable `LgjLaneDesc`. Written only on success. +/// +/// `unsafe` here is a note to Rust callers linking the `rlib`. The JVM, +/// which is the real caller, has no such concept — it upholds the same +/// contract by construction, because every pointer it passes comes from a +/// `MemorySegment` whose size and alignment it derived from the manifest. +#[no_mangle] +pub unsafe extern "C" fn lgj_mask_describe(mask: u64, out: *mut LgjLaneDesc) -> i32 { + guard(|| { + if out.is_null() { + return LGJ_ERR_NULL_ARGUMENT; + } + let (entry, _parent) = match registry::resolve_mask_with_parent(mask) { + Ok(p) => p, + Err(e) => return e, + }; + let g = match entry.read_mask() { + Some(g) => g, + None => return LGJ_ERR_WRONG_RESOURCE_KIND, + }; + let len_elems = g.words.len() as u64; + let desc = LgjLaneDesc { + // The boxed slice's buffer never moves, so this address stays valid + // for the mask's whole life — releasing the lock below does not + // invalidate it. + addr: g.words.as_ptr() as u64, + len_elems, + byte_len: len_elems * 8, + owner: mask, + epoch: entry.epoch, + elem_kind: LgjElemKind::MaskWord as u32, + elem_bytes: 8, + stride_bytes: 8, + flags: LGJ_FLAG_READABLE | LGJ_FLAG_WRITABLE | LGJ_FLAG_CONTIGUOUS, + }; + drop(g); + // SAFETY: non-null, checked above; `#[repr(C)]` 56-byte struct. + unsafe { *out = desc }; + LGJ_OK + }) +} + +/// Shared body of `lgj_mask_and` / `lgj_mask_or`. +/// +/// Handles the three aliasing cases `abi.md` §7 permits (`dst` may alias `a` or +/// `b`) by **deduplicating before locking**: locking one `RwLock` twice from one +/// thread is a hang, and taking `&mut` and `&` to one payload is not +/// expressible. Distinct entries are then locked in address order +/// ([`registry::lock_masks_ordered`]) so no two threads can build a cycle. +fn mask_binop(a: u64, b: u64, dst: u64, combine: u32) -> i32 { + let (ea, pa) = match registry::resolve_mask_with_parent(a) { + Ok(t) => t, + Err(e) => return e, + }; + let (eb, _pb) = match registry::resolve_mask_with_parent(b) { + Ok(t) => t, + Err(e) => return e, + }; + let (ed, _pd) = match registry::resolve_mask_with_parent(dst) { + Ok(t) => t, + Err(e) => return e, + }; + + // "All three must share the same parent and row count" (§7). Row count is + // the property the kernels depend on; the shared-parent requirement is + // checked too, and both map to MASK_LENGTH_MISMATCH — abi.md allocates no + // distinct "different parents" code, and this is the code whose meaning + // ("these masks do not belong together") covers it. + if ea.n_rows != eb.n_rows || ea.n_rows != ed.n_rows { + return LGJ_ERR_MASK_LENGTH_MISMATCH; + } + if ea.parent != eb.parent || ea.parent != ed.parent { + return LGJ_ERR_MASK_LENGTH_MISMATCH; + } + let n_rows = pa.n_rows; + + let d_is_a = std::sync::Arc::ptr_eq(&ed, &ea); + let d_is_b = std::sync::Arc::ptr_eq(&ed, &eb); + let a_is_b = std::sync::Arc::ptr_eq(&ea, &eb); + + let path = Path::Simd; + let result = if d_is_a && d_is_b { + // dst = dst op dst — identity for both AND and OR. Still normalize the + // tail so a hand-written Java word cannot leave garbage past n_rows. + let mut g = registry::lock_masks_ordered(&[&ed]).map(|mut g| g[0].take().unwrap()); + match &mut g { + Ok(gd) => { + clear_tail_bits(&mut gd.words, n_rows); + Ok(()) + } + Err(e) => Err(*e), + } + } else if d_is_a || d_is_b { + // dst aliases one operand ⇒ in-place: dst op= other. + let other = if d_is_a { &eb } else { &ea }; + match registry::lock_masks_ordered(&[&ed, other]) { + Ok(mut guards) => { + let (gd, rest) = guards.split_at_mut(1); + let gd = gd[0].as_mut().unwrap(); + let go = rest[0].as_ref().unwrap(); + let r = kernels::combine_into(path, combine, &mut gd.words, &go.words); + if r.is_ok() { + clear_tail_bits(&mut gd.words, n_rows); + } + r + } + Err(e) => Err(e), + } + } else if a_is_b { + // a and b are the same mask, dst is separate ⇒ dst = a op a = a. + match registry::lock_masks_ordered(&[&ed, &ea]) { + Ok(mut guards) => { + let (gd, rest) = guards.split_at_mut(1); + let gd = gd[0].as_mut().unwrap(); + let ga = rest[0].as_ref().unwrap(); + gd.words.copy_from_slice(&ga.words); + clear_tail_bits(&mut gd.words, n_rows); + Ok(()) + } + Err(e) => Err(e), + } + } else { + // Three distinct masks: dst = a op b, no copy. + match registry::lock_masks_ordered(&[&ed, &ea, &eb]) { + Ok(mut guards) => { + let (gd, rest) = guards.split_at_mut(1); + let gd = gd[0].as_mut().unwrap(); + let ga = rest[0].as_ref().unwrap(); + let gb = rest[1].as_ref().unwrap(); + let r = match combine { + LGJ_COMBINE_AND => { + kernels::simd_mask_and(&ga.words, &gb.words, &mut gd.words); + Ok(()) + } + LGJ_COMBINE_OR => { + kernels::simd_mask_or(&ga.words, &gb.words, &mut gd.words); + Ok(()) + } + _ => Err(LGJ_ERR_UNKNOWN_OPCODE), + }; + if r.is_ok() { + clear_tail_bits(&mut gd.words, n_rows); + } + r + } + Err(e) => Err(e), + } + }; + status(result) +} + +/// `dst = a & b`. `dst` may alias `a` or `b`. All three must share the same +/// parent and row count. +#[no_mangle] +pub extern "C" fn lgj_mask_and(a: u64, b: u64, dst: u64) -> i32 { + guard(|| mask_binop(a, b, dst, LGJ_COMBINE_AND)) +} + +/// `dst = a | b`. Same aliasing and compatibility rules as [`lgj_mask_and`]. +#[no_mangle] +pub extern "C" fn lgj_mask_or(a: u64, b: u64, dst: u64) -> i32 { + guard(|| mask_binop(a, b, dst, LGJ_COMBINE_OR)) +} + +/// Population count of a mask — how many rows are selected. +/// # Safety +/// +/// A null pointer is *handled*, not UB: it returns `NULL_ARGUMENT`. Beyond +/// that, `out_count` must be a valid, aligned, writable `u64`. Written only on success. +/// +/// `unsafe` here is a note to Rust callers linking the `rlib`. The JVM, +/// which is the real caller, has no such concept — it upholds the same +/// contract by construction, because every pointer it passes comes from a +/// `MemorySegment` whose size and alignment it derived from the manifest. +#[no_mangle] +pub unsafe extern "C" fn lgj_mask_count(mask: u64, out_count: *mut u64) -> i32 { + guard(|| { + if out_count.is_null() { + return LGJ_ERR_NULL_ARGUMENT; + } + let (entry, _parent) = match registry::resolve_mask_with_parent(mask) { + Ok(t) => t, + Err(e) => return e, + }; + let g = match entry.read_mask() { + Some(g) => g, + None => return LGJ_ERR_WRONG_RESOURCE_KIND, + }; + let count = kernels::popcount(Path::Simd, &g.words); + drop(g); + // SAFETY: non-null, checked above; written only on success. + unsafe { *out_count = count }; + LGJ_OK + }) +} + +// ─────────────────────────────────────────────────────────────────────────── +// Unfused bulk predicates +// ─────────────────────────────────────────────────────────────────────────── + +/// Resolve `(pattern, mask)` for a predicate and check their row counts agree. +fn resolve_pattern_and_mask( + res: u64, + dst_mask: u64, +) -> Result<(std::sync::Arc, std::sync::Arc), i32> { + let pattern = registry::resolve_kind(res, LGJ_RESOURCE_PATTERN)?; + let (mask, _parent) = registry::resolve_mask_with_parent(dst_mask)?; + // abi.md names only the row-count condition here (§3 MASK_LENGTH_MISMATCH), + // so a mask from a *different but equally sized* pattern is accepted. That + // is a deliberate reading: the kernels care about length, and the Java layer + // never constructs such a pairing. + if mask.n_rows != pattern.n_rows { + return Err(LGJ_ERR_MASK_LENGTH_MISMATCH); + } + Ok((pattern, mask)) +} + +/// Read a pattern lane as a typed view. +fn lane_view<'a>(entry: &'a ResourceEntry, lane_id: u32) -> Result, i32> { + let f = entry.fixture().ok_or(LGJ_ERR_WRONG_RESOURCE_KIND)?; + match lane_id { + crate::fixture::LANE_IDS => Ok(LaneView::U64(f.ids())), + crate::fixture::LANE_CLASSES => Ok(LaneView::U32(f.classes())), + crate::fixture::LANE_VALUES => Ok(LaneView::I32(f.values())), + _ => Err(LGJ_ERR_INVALID_LANE), + } +} + +/// One predicate, one crossing: **overwrites** `dst_mask` with +/// `lane[i] == needle`. +/// +/// Composition is the caller's job (`lgj_mask_and`). This unfused form exists so +/// the fused plan has something to be benchmarked *against* and so parity can be +/// checked predicate-by-predicate — not because a chain should be built from it. +#[no_mangle] +pub extern "C" fn lgj_op_eq_u32(res: u64, lane_id: u32, needle: u32, dst_mask: u64) -> i32 { + guard(|| { + let (pattern, mask) = match resolve_pattern_and_mask(res, dst_mask) { + Ok(t) => t, + Err(e) => return e, + }; + let lane = match lane_view(&pattern, lane_id) { + Ok(l) => l, + Err(e) => return e, + }; + if lane.kind() != LgjElemKind::U32 { + return LGJ_ERR_LANE_KIND_MISMATCH; + } + let mut g = match mask.write_mask() { + Some(g) => g, + None => return LGJ_ERR_WRONG_RESOURCE_KIND, + }; + let n_rows = pattern.n_rows; + status(kernels::eval_predicate( + Path::Simd, + LGJ_OP_EQ_U32, + needle as i64, + &lane, + n_rows, + &mut g.words, + )) + }) +} + +/// One predicate, one crossing: **overwrites** `dst_mask` with +/// `lane[i] > threshold`, signed. +#[no_mangle] +pub extern "C" fn lgj_op_gt_i32(res: u64, lane_id: u32, threshold: i32, dst_mask: u64) -> i32 { + guard(|| { + let (pattern, mask) = match resolve_pattern_and_mask(res, dst_mask) { + Ok(t) => t, + Err(e) => return e, + }; + let lane = match lane_view(&pattern, lane_id) { + Ok(l) => l, + Err(e) => return e, + }; + if lane.kind() != LgjElemKind::I32 { + return LGJ_ERR_LANE_KIND_MISMATCH; + } + let mut g = match mask.write_mask() { + Some(g) => g, + None => return LGJ_ERR_WRONG_RESOURCE_KIND, + }; + let n_rows = pattern.n_rows; + status(kernels::eval_predicate( + Path::Simd, + LGJ_OP_GT_I32, + threshold as i64, + &lane, + n_rows, + &mut g.words, + )) + }) +} + +// ─────────────────────────────────────────────────────────────────────────── +// The fused plan — N predicates, ONE crossing +// ─────────────────────────────────────────────────────────────────────────── + +/// Validate an entire plan **before** any work happens. +/// +/// This is what makes "a bad op leaves `dst_mask` untouched" a property rather +/// than a hope: nothing is written until every op has passed. `abi.md`'s +/// monotonic-narrowing guarantee would otherwise be observable in a +/// half-applied state, and a Java caller retrying after an error would be +/// composing against garbage. +fn validate_plan(pattern: &ResourceEntry, ops: &[LgjOpDesc]) -> Result<(), i32> { + if ops.is_empty() { + return Err(LGJ_ERR_EMPTY_PLAN); + } + for op in ops { + // A non-zero reserved field means the caller was compiled against a + // newer ABI that gave it meaning. Refusing is the only safe reading; + // ignoring it would silently drop semantics. + if op._reserved != 0 { + return Err(LGJ_ERR_NULL_ARGUMENT); + } + let required = opcode_required_kind(op.op).ok_or(LGJ_ERR_UNKNOWN_OPCODE)?; + if op.combine != LGJ_COMBINE_AND && op.combine != LGJ_COMBINE_OR { + return Err(LGJ_ERR_UNKNOWN_OPCODE); + } + if op.lane_id >= PATTERN_LANE_COUNT { + return Err(LGJ_ERR_INVALID_LANE); + } + let lane = lane_view(pattern, op.lane_id)?; + if lane.kind() != required { + return Err(LGJ_ERR_LANE_KIND_MISMATCH); + } + } + Ok(()) +} + +/// The body behind both `lgj_plan_eval` and `lgj_plan_eval_scalar` — one code +/// path, two symbols, so the parity test compares two *paths* rather than a +/// function against itself. +fn plan_eval_impl( + res: u64, + ops: *const LgjOpDesc, + n_ops: u32, + dst_mask: u64, + out_count: *mut u64, + path: Path, +) -> i32 { + // EMPTY_PLAN is checked before the null test: `(null, 0)` is a caller + // describing an empty plan, which has its own dedicated code, and reporting + // NULL_ARGUMENT there would send a Java author looking for the wrong bug. + if n_ops == 0 { + return LGJ_ERR_EMPTY_PLAN; + } + if ops.is_null() || out_count.is_null() { + return LGJ_ERR_NULL_ARGUMENT; + } + let (pattern, mask) = match resolve_pattern_and_mask(res, dst_mask) { + Ok(t) => t, + Err(e) => return e, + }; + // SAFETY: `ops` is non-null (checked) and the caller states it points at + // `n_ops` contiguous `LgjOpDesc`. Java builds that array with a + // MemoryLayout whose size and alignment the manifest reports (24 / 8), and + // the compile-time asserts in `abi.rs` guarantee this side agrees. The slice + // is read-only and does not outlive this call. + let ops: &[LgjOpDesc] = unsafe { std::slice::from_raw_parts(ops, n_ops as usize) }; + + if let Err(e) = validate_plan(&pattern, ops) { + return e; + } + + let n_rows = pattern.n_rows; + let n_words = mask_words_for(n_rows) as usize; + + // Accumulate into scratch, then publish. Two consequences, both wanted: + // dst_mask is written exactly once, and an error at any point leaves it + // byte-for-byte as it was. + let mut acc = vec![0u64; n_words]; + // "the accumulator starts as all rows set" (§7). + for w in acc.iter_mut() { + *w = u64::MAX; + } + clear_tail_bits(&mut acc, n_rows); + let mut scratch = vec![0u64; n_words]; + + for op in ops { + let lane = match lane_view(&pattern, op.lane_id) { + Ok(l) => l, + Err(e) => return e, + }; + if let Err(e) = + kernels::eval_predicate(path, op.op, op.operand, &lane, n_rows, &mut scratch) + { + return e; + } + if let Err(e) = kernels::combine_into(path, op.combine, &mut acc, &scratch) { + return e; + } + } + clear_tail_bits(&mut acc, n_rows); + let count = kernels::popcount(path, &acc); + + let mut g = match mask.write_mask() { + Some(g) => g, + None => return LGJ_ERR_WRONG_RESOURCE_KIND, + }; + if g.words.len() != acc.len() { + return LGJ_ERR_MASK_LENGTH_MISMATCH; + } + g.words.copy_from_slice(&acc); + drop(g); + + // SAFETY: non-null (checked above); written only on the success path. + unsafe { *out_count = count }; + LGJ_OK +} + +/// Evaluate `n_ops` predicates in **one** crossing. +/// +/// This is the symbol that makes `.where(...).where(...).count()` cost one +/// downcall regardless of how many predicates or rows are involved — the +/// concrete form of abi.md §6's anti-JNI rule. +/// +/// Semantics: the accumulator starts as all rows set; each op is evaluated and +/// combined per its `combine` field; the result lands in `dst_mask` and its +/// popcount in `out_count`. With every `combine = AND` the sequence narrows +/// monotonically by construction (`V(k+1) ⊆ V(k)`). +/// # Safety +/// +/// A null pointer is *handled*, not UB: it returns `NULL_ARGUMENT`. Beyond +/// that, `ops` must point to `n_ops` contiguous, initialized `LgjOpDesc` (24 bytes, align 8) that stay valid for the call; `out_count` must be a valid, aligned, writable `u64`. Both are read/written only after the null checks, and `out_count` only on success. +/// +/// `unsafe` here is a note to Rust callers linking the `rlib`. The JVM, +/// which is the real caller, has no such concept — it upholds the same +/// contract by construction, because every pointer it passes comes from a +/// `MemorySegment` whose size and alignment it derived from the manifest. +#[no_mangle] +pub unsafe extern "C" fn lgj_plan_eval( + res: u64, + ops: *const LgjOpDesc, + n_ops: u32, + dst_mask: u64, + out_count: *mut u64, +) -> i32 { + guard(|| plan_eval_impl(res, ops, n_ops, dst_mask, out_count, Path::Simd)) +} + +/// Identical semantics to [`lgj_plan_eval`], forced down the **scalar +/// reference** path. +/// +/// Exists only so SIMD-vs-scalar parity is falsifiable *through the membrane*, +/// which is where the Java tests live. Not for production use. +/// # Safety +/// +/// A null pointer is *handled*, not UB: it returns `NULL_ARGUMENT`. Beyond +/// that, Identical to [`lgj_plan_eval`]: `ops` must point to `n_ops` valid `LgjOpDesc` and `out_count` to a writable `u64`. +/// +/// `unsafe` here is a note to Rust callers linking the `rlib`. The JVM, +/// which is the real caller, has no such concept — it upholds the same +/// contract by construction, because every pointer it passes comes from a +/// `MemorySegment` whose size and alignment it derived from the manifest. +#[no_mangle] +pub unsafe extern "C" fn lgj_plan_eval_scalar( + res: u64, + ops: *const LgjOpDesc, + n_ops: u32, + dst_mask: u64, + out_count: *mut u64, +) -> i32 { + guard(|| plan_eval_impl(res, ops, n_ops, dst_mask, out_count, Path::Scalar)) +} + +// ─────────────────────────────────────────────────────────────────────────── +// Reduction +// ─────────────────────────────────────────────────────────────────────────── + +/// Sum an `I32` lane over the set bits of `mask`, widened to `i64`. +/// +/// No overflow for `n_rows ≤ 2^32` on `i32` inputs, because each element is +/// widened *before* accumulation. +/// # Safety +/// +/// A null pointer is *handled*, not UB: it returns `NULL_ARGUMENT`. Beyond +/// that, `out_sum` must be a valid, aligned, writable `i64`. Written only on success. +/// +/// `unsafe` here is a note to Rust callers linking the `rlib`. The JVM, +/// which is the real caller, has no such concept — it upholds the same +/// contract by construction, because every pointer it passes comes from a +/// `MemorySegment` whose size and alignment it derived from the manifest. +#[no_mangle] +pub unsafe extern "C" fn lgj_reduce_sum_i32( + res: u64, + lane_id: u32, + mask: u64, + out_sum: *mut i64, +) -> i32 { + guard(|| { + if out_sum.is_null() { + return LGJ_ERR_NULL_ARGUMENT; + } + let (pattern, maskr) = match resolve_pattern_and_mask(res, mask) { + Ok(t) => t, + Err(e) => return e, + }; + let lane = match lane_view(&pattern, lane_id) { + Ok(l) => l, + Err(e) => return e, + }; + let values = match lane { + LaneView::I32(v) => v, + _ => return LGJ_ERR_LANE_KIND_MISMATCH, + }; + let g = match maskr.read_mask() { + Some(g) => g, + None => return LGJ_ERR_WRONG_RESOURCE_KIND, + }; + let sum = kernels::masked_sum_i32(Path::Simd, values, &g.words); + drop(g); + // SAFETY: non-null, checked above; written only on success. + unsafe { *out_sum = sum }; + LGJ_OK + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixture::{Fixture, LANE_CLASSES, LANE_IDS, LANE_VALUES}; + + /// Safe wrappers over the pointer-taking exports. + /// + /// These are not a second implementation: each one is a direct call into the + /// exported symbol. They exist so the `unsafe` required by a Rust caller + /// lives in one place and the assertions below read as the ABI contract + /// rather than as pointer plumbing. Null pointers are passed straight + /// through — the NULL_ARGUMENT tests depend on that. + mod call { + use super::*; + + pub fn pattern_open(n: u64, seed: u64, out: *mut u64) -> i32 { + unsafe { lgj_pattern_open(n, seed, out) } + } + pub fn resource_info(h: u64, out: *mut LgjResourceInfo) -> i32 { + unsafe { lgj_resource_info(h, out) } + } + pub fn lane_describe(h: u64, lane: u32, out: *mut LgjLaneDesc) -> i32 { + unsafe { lgj_lane_describe(h, lane, out) } + } + pub fn mask_create(p: u64, initial: u32, out: *mut u64) -> i32 { + unsafe { lgj_mask_create(p, initial, out) } + } + pub fn mask_describe(m: u64, out: *mut LgjLaneDesc) -> i32 { + unsafe { lgj_mask_describe(m, out) } + } + pub fn mask_count(m: u64, out: *mut u64) -> i32 { + unsafe { lgj_mask_count(m, out) } + } + pub fn plan_eval(r: u64, ops: *const LgjOpDesc, n: u32, dst: u64, out: *mut u64) -> i32 { + unsafe { lgj_plan_eval(r, ops, n, dst, out) } + } + pub fn plan_eval_scalar( + r: u64, + ops: *const LgjOpDesc, + n: u32, + dst: u64, + out: *mut u64, + ) -> i32 { + unsafe { lgj_plan_eval_scalar(r, ops, n, dst, out) } + } + pub fn reduce_sum_i32(r: u64, lane: u32, m: u64, out: *mut i64) -> i32 { + unsafe { lgj_reduce_sum_i32(r, lane, m, out) } + } + } + + /// Open a pattern, returning its handle. + fn open(n: u64, seed: u64) -> u64 { + let mut h = 0u64; + assert_eq!(call::pattern_open(n, seed, &mut h), LGJ_OK); + h + } + + fn mask(parent: u64, initial: u32) -> u64 { + let mut h = 0u64; + assert_eq!(call::mask_create(parent, initial, &mut h), LGJ_OK); + h + } + + fn count(m: u64) -> u64 { + let mut c = 0u64; + assert_eq!(call::mask_count(m, &mut c), LGJ_OK); + c + } + + /// Read a mask's words back out through the *described* segment — i.e. the + /// same way Java sees them, not through a Rust-side back door. + fn read_words(m: u64) -> Vec { + let mut d = LgjLaneDesc::default(); + assert_eq!(call::mask_describe(m, &mut d), LGJ_OK); + assert_eq!(d.elem_kind, LgjElemKind::MaskWord as u32); + assert_ne!(d.flags & LGJ_FLAG_WRITABLE, 0); + // SAFETY: the descriptor is exactly the contract Java relies on; the + // lane is alive because `m` has not been closed in this scope. + unsafe { std::slice::from_raw_parts(d.addr as *const u64, d.len_elems as usize).to_vec() } + } + + fn op(op: u32, lane_id: u32, operand: i64, combine: u32) -> LgjOpDesc { + LgjOpDesc { + op, + lane_id, + operand, + combine, + _reserved: 0, + } + } + + // ── manifest ─────────────────────────────────────────────────────────── + + #[test] + fn manifest_pointer_is_stable_and_populated() { + let p1 = lgj_abi_manifest(); + let p2 = lgj_abi_manifest(); + assert_eq!(p1, p2, "must be a 'static, not a fresh allocation"); + // SAFETY: a pointer to a 'static. + let m = unsafe { &*p1 }; + assert_eq!(m.magic, LGJ_MAGIC); + assert_eq!(m.size_of_lane_desc, 56); + assert_eq!(m.endianness, 0, "this box is little-endian"); + } + + // ── lifecycle / handle safety ────────────────────────────────────────── + + #[test] + fn open_describe_close_round_trip() { + let p = open(1000, 42); + let mut info = LgjResourceInfo::default(); + assert_eq!(call::resource_info(p, &mut info), LGJ_OK); + assert_eq!(info.kind, LGJ_RESOURCE_PATTERN); + assert_eq!(info.lane_count, 3); + assert_eq!(info.n_rows, 1000); + assert_eq!(info.parent, 0); + assert!(info.epoch > 0); + assert_eq!(lgj_close(p), LGJ_OK); + } + + #[test] + fn every_stale_handle_shape_is_a_status_not_a_crash() { + let p = open(64, 1); + let m = mask(p, LGJ_MASK_INIT_ALL); + assert_eq!(lgj_close(m), LGJ_OK); + assert_eq!(lgj_close(p), LGJ_OK); + + // use-after-close + let mut info = LgjResourceInfo::default(); + assert_eq!(call::resource_info(p, &mut info), LGJ_ERR_INVALID_HANDLE); + // double close + assert_eq!(lgj_close(p), LGJ_ERR_INVALID_HANDLE); + assert_eq!(lgj_close(m), LGJ_ERR_INVALID_HANDLE); + // fabricated + for bogus in [0u64, 1, 0xDEAD_BEEF, u64::MAX] { + assert_eq!( + call::resource_info(bogus, &mut info), + LGJ_ERR_INVALID_HANDLE + ); + let mut d = LgjLaneDesc::default(); + assert_eq!( + call::lane_describe(bogus, 0, &mut d), + LGJ_ERR_INVALID_HANDLE + ); + let mut c = 0u64; + assert_eq!(call::mask_count(bogus, &mut c), LGJ_ERR_INVALID_HANDLE); + } + } + + #[test] + fn a_mask_whose_parent_closed_reports_parent_closed() { + let p = open(500, 3); + let m = mask(p, LGJ_MASK_INIT_ALL); + assert_eq!(count(m), 500); + assert_eq!(lgj_close(p), LGJ_OK); + + let mut c = 0u64; + assert_eq!(call::mask_count(m, &mut c), LGJ_ERR_PARENT_CLOSED); + let mut d = LgjLaneDesc::default(); + assert_eq!(call::mask_describe(m, &mut d), LGJ_ERR_PARENT_CLOSED); + assert_eq!(lgj_mask_and(m, m, m), LGJ_ERR_PARENT_CLOSED); + let ops = [op(LGJ_OP_EQ_U32, LANE_CLASSES, 7, LGJ_COMBINE_AND)]; + let mut n = 0u64; + assert_eq!( + call::plan_eval(p, ops.as_ptr(), 1, m, &mut n), + LGJ_ERR_INVALID_HANDLE, + "the pattern handle is stale, which is reported first" + ); + // The mask itself still exists — it just cannot work. + assert_eq!(lgj_close(m), LGJ_OK); + } + + #[test] + fn wrong_kind_is_reported_as_such() { + let p = open(64, 1); + let m = mask(p, 0); + let mut d = LgjLaneDesc::default(); + // a mask where a pattern was required + assert_eq!( + call::lane_describe(m, 0, &mut d), + LGJ_ERR_WRONG_RESOURCE_KIND + ); + // a pattern where a mask was required + assert_eq!(call::mask_describe(p, &mut d), LGJ_ERR_WRONG_RESOURCE_KIND); + let mut c = 0u64; + assert_eq!(call::mask_count(p, &mut c), LGJ_ERR_WRONG_RESOURCE_KIND); + lgj_close(m); + lgj_close(p); + } + + #[test] + fn null_out_pointers_are_rejected() { + let p = open(64, 1); + let m = mask(p, 0); + assert_eq!( + call::pattern_open(8, 1, std::ptr::null_mut()), + LGJ_ERR_NULL_ARGUMENT + ); + assert_eq!( + call::resource_info(p, std::ptr::null_mut()), + LGJ_ERR_NULL_ARGUMENT + ); + assert_eq!( + call::lane_describe(p, 0, std::ptr::null_mut()), + LGJ_ERR_NULL_ARGUMENT + ); + assert_eq!( + call::mask_create(p, 0, std::ptr::null_mut()), + LGJ_ERR_NULL_ARGUMENT + ); + assert_eq!( + call::mask_describe(m, std::ptr::null_mut()), + LGJ_ERR_NULL_ARGUMENT + ); + assert_eq!( + call::mask_count(m, std::ptr::null_mut()), + LGJ_ERR_NULL_ARGUMENT + ); + assert_eq!( + call::reduce_sum_i32(p, LANE_VALUES, m, std::ptr::null_mut()), + LGJ_ERR_NULL_ARGUMENT + ); + let ops = [op(LGJ_OP_EQ_U32, LANE_CLASSES, 7, LGJ_COMBINE_AND)]; + let mut n = 0u64; + assert_eq!( + call::plan_eval(p, std::ptr::null(), 1, m, &mut n), + LGJ_ERR_NULL_ARGUMENT + ); + assert_eq!( + call::plan_eval(p, ops.as_ptr(), 1, m, std::ptr::null_mut()), + LGJ_ERR_NULL_ARGUMENT + ); + lgj_close(m); + lgj_close(p); + } + + // ── lanes ────────────────────────────────────────────────────────────── + + #[test] + fn pattern_lanes_are_readable_contiguous_and_never_writable() { + let p = open(777, 9); + for (lane, kind, bytes) in [ + (LANE_IDS, LgjElemKind::U64, 8u32), + (LANE_CLASSES, LgjElemKind::U32, 4), + (LANE_VALUES, LgjElemKind::I32, 4), + ] { + let mut d = LgjLaneDesc::default(); + assert_eq!(call::lane_describe(p, lane, &mut d), LGJ_OK); + assert_eq!(d.elem_kind, kind as u32); + assert_eq!(d.elem_bytes, bytes); + assert_eq!(d.stride_bytes, bytes, "contiguous ⇒ stride == elem_bytes"); + assert_eq!(d.len_elems, 777); + assert_eq!(d.byte_len, 777 * bytes as u64); + assert_eq!(d.owner, p); + assert_ne!(d.flags & LGJ_FLAG_READABLE, 0); + assert_ne!(d.flags & LGJ_FLAG_CONTIGUOUS, 0); + assert_eq!( + d.flags & LGJ_FLAG_WRITABLE, + 0, + "pattern lanes are read-only" + ); + assert_ne!(d.addr, 0); + } + let mut d = LgjLaneDesc::default(); + assert_eq!(call::lane_describe(p, 3, &mut d), LGJ_ERR_INVALID_LANE); + assert_eq!( + call::lane_describe(p, u32::MAX, &mut d), + LGJ_ERR_INVALID_LANE + ); + lgj_close(p); + } + + /// The lane bytes Java would read must be the fixture's bytes — this is the + /// only test that crosses the descriptor boundary the way the FFM layer does. + #[test] + fn described_lane_bytes_match_the_generator() { + let n = 300u64; + let p = open(n, 0xABCD); + let expected = Fixture::generate(n, 0xABCD).unwrap(); + let mut d = LgjLaneDesc::default(); + assert_eq!(call::lane_describe(p, LANE_VALUES, &mut d), LGJ_OK); + // SAFETY: descriptor is live; `p` is open. + let seen = + unsafe { std::slice::from_raw_parts(d.addr as *const i32, d.len_elems as usize) }; + assert_eq!(seen, expected.values()); + lgj_close(p); + } + + #[test] + fn lane_epoch_matches_resource_epoch() { + let p = open(10, 1); + let mut info = LgjResourceInfo::default(); + call::resource_info(p, &mut info); + let mut d = LgjLaneDesc::default(); + call::lane_describe(p, 0, &mut d); + assert_eq!(d.epoch, info.epoch); + lgj_close(p); + } + + /// A reused slot must not reuse an epoch, or Java could mistake a dead + /// segment for a live one. + #[test] + fn epochs_are_never_reused() { + let p1 = open(10, 1); + let mut i1 = LgjResourceInfo::default(); + call::resource_info(p1, &mut i1); + lgj_close(p1); + let p2 = open(10, 1); + let mut i2 = LgjResourceInfo::default(); + call::resource_info(p2, &mut i2); + assert_ne!(i1.epoch, i2.epoch); + lgj_close(p2); + } + + // ── masks ────────────────────────────────────────────────────────────── + + #[test] + fn mask_create_initial_states() { + let p = open(1000, 1); + let empty = mask(p, LGJ_MASK_INIT_EMPTY); + let all = mask(p, LGJ_MASK_INIT_ALL); + assert_eq!(count(empty), 0); + assert_eq!(count(all), 1000, "tail bits must not inflate the count"); + let mut h = 0u64; + assert_ne!(call::mask_create(p, 7, &mut h), LGJ_OK); + lgj_close(all); + lgj_close(empty); + lgj_close(p); + } + + #[test] + fn mask_and_or_over_distinct_masks() { + let p = open(200, 1); + let a = mask(p, LGJ_MASK_INIT_ALL); + let b = mask(p, LGJ_MASK_INIT_EMPTY); + let dst = mask(p, LGJ_MASK_INIT_ALL); + + assert_eq!(lgj_mask_and(a, b, dst), LGJ_OK); + assert_eq!(count(dst), 0); + assert_eq!(lgj_mask_or(a, b, dst), LGJ_OK); + assert_eq!(count(dst), 200); + + for h in [dst, b, a, p] { + lgj_close(h); + } + } + + /// abi.md §7: "`dst` may alias `a` or `b`." All four aliasing shapes. + #[test] + fn mask_binop_alias_cases() { + let p = open(300, 5); + let all = mask(p, LGJ_MASK_INIT_ALL); + let empty = mask(p, LGJ_MASK_INIT_EMPTY); + + // dst == a + let x = mask(p, LGJ_MASK_INIT_ALL); + assert_eq!(lgj_mask_and(x, empty, x), LGJ_OK); + assert_eq!(count(x), 0); + assert_eq!(lgj_mask_or(x, all, x), LGJ_OK); + assert_eq!(count(x), 300); + + // dst == b + let y = mask(p, LGJ_MASK_INIT_ALL); + assert_eq!(lgj_mask_and(empty, y, y), LGJ_OK); + assert_eq!(count(y), 0); + assert_eq!(lgj_mask_or(all, y, y), LGJ_OK); + assert_eq!(count(y), 300); + + // dst == a == b (identity) + let z = mask(p, LGJ_MASK_INIT_ALL); + assert_eq!(lgj_mask_and(z, z, z), LGJ_OK); + assert_eq!(count(z), 300); + assert_eq!(lgj_mask_or(z, z, z), LGJ_OK); + assert_eq!(count(z), 300); + + // a == b, dst distinct (copy) + let w = mask(p, LGJ_MASK_INIT_EMPTY); + assert_eq!(lgj_mask_and(all, all, w), LGJ_OK); + assert_eq!(count(w), 300); + + for h in [w, z, y, x, empty, all, p] { + lgj_close(h); + } + } + + #[test] + fn mask_binop_rejects_mismatched_row_counts() { + let p1 = open(100, 1); + let p2 = open(200, 1); + let a = mask(p1, LGJ_MASK_INIT_ALL); + let b = mask(p2, LGJ_MASK_INIT_ALL); + assert_eq!(lgj_mask_and(a, b, a), LGJ_ERR_MASK_LENGTH_MISMATCH); + for h in [b, a, p2, p1] { + lgj_close(h); + } + } + + /// Masks of equal length but different parents are still not composable — + /// §7 requires a shared parent. + #[test] + fn mask_binop_rejects_different_parents() { + let p1 = open(128, 1); + let p2 = open(128, 2); + let a = mask(p1, LGJ_MASK_INIT_ALL); + let b = mask(p2, LGJ_MASK_INIT_ALL); + assert_eq!(lgj_mask_and(a, b, a), LGJ_ERR_MASK_LENGTH_MISMATCH); + for h in [b, a, p2, p1] { + lgj_close(h); + } + } + + // ── unfused predicates ───────────────────────────────────────────────── + + #[test] + fn unfused_predicates_match_a_hand_count() { + let n = 5000u64; + let p = open(n, 0x1234); + let f = Fixture::generate(n, 0x1234).unwrap(); + let m = mask(p, LGJ_MASK_INIT_EMPTY); + + assert_eq!(lgj_op_eq_u32(p, LANE_CLASSES, 7, m), LGJ_OK); + let want = f.classes().iter().filter(|&&c| c == 7).count() as u64; + assert_eq!(count(m), want); + assert!(want > 0 && want < n, "predicate must not be vacuous"); + + assert_eq!(lgj_op_gt_i32(p, LANE_VALUES, 100, m), LGJ_OK); + let want = f.values().iter().filter(|&&v| v > 100).count() as u64; + assert_eq!(count(m), want); + + // Each op OVERWRITES, so re-running the first restores its own count. + assert_eq!(lgj_op_eq_u32(p, LANE_CLASSES, 7, m), LGJ_OK); + assert_eq!( + count(m), + f.classes().iter().filter(|&&c| c == 7).count() as u64 + ); + lgj_close(m); + lgj_close(p); + } + + #[test] + fn predicates_reject_lane_kind_mismatch() { + let p = open(64, 1); + let m = mask(p, 0); + // eq_u32 against the I32 lane, and against the U64 lane + assert_eq!( + lgj_op_eq_u32(p, LANE_VALUES, 1, m), + LGJ_ERR_LANE_KIND_MISMATCH + ); + assert_eq!(lgj_op_eq_u32(p, LANE_IDS, 1, m), LGJ_ERR_LANE_KIND_MISMATCH); + // gt_i32 against the U32 lane + assert_eq!( + lgj_op_gt_i32(p, LANE_CLASSES, 1, m), + LGJ_ERR_LANE_KIND_MISMATCH + ); + assert_eq!(lgj_op_eq_u32(p, 9, 1, m), LGJ_ERR_INVALID_LANE); + lgj_close(m); + lgj_close(p); + } + + #[test] + fn predicate_rejects_a_mask_of_the_wrong_length() { + let p1 = open(100, 1); + let p2 = open(200, 1); + let m2 = mask(p2, 0); + assert_eq!( + lgj_op_eq_u32(p1, LANE_CLASSES, 7, m2), + LGJ_ERR_MASK_LENGTH_MISMATCH + ); + for h in [m2, p2, p1] { + lgj_close(h); + } + } + + // ── the fused plan ───────────────────────────────────────────────────── + + #[test] + fn fused_plan_equals_the_unfused_composition() { + let n = 8000u64; + let p = open(n, 77); + let a = mask(p, 0); + let b = mask(p, 0); + let fused = mask(p, 0); + + assert_eq!(lgj_op_eq_u32(p, LANE_CLASSES, 7, a), LGJ_OK); + assert_eq!(lgj_op_gt_i32(p, LANE_VALUES, 100, b), LGJ_OK); + assert_eq!(lgj_mask_and(a, b, a), LGJ_OK); + + let ops = [ + op(LGJ_OP_EQ_U32, LANE_CLASSES, 7, LGJ_COMBINE_AND), + op(LGJ_OP_GT_I32, LANE_VALUES, 100, LGJ_COMBINE_AND), + ]; + let mut c = 0u64; + assert_eq!(call::plan_eval(p, ops.as_ptr(), 2, fused, &mut c), LGJ_OK); + assert_eq!(c, count(a), "one crossing must equal three"); + assert_eq!( + read_words(fused), + read_words(a), + "bit-for-bit, not just count" + ); + assert!(c > 0, "an all-zero result would make this test vacuous"); + + for h in [fused, b, a, p] { + lgj_close(h); + } + } + + /// The headline parity property, through the same code path the Java tests + /// exercise: SIMD and the independent scalar reference must agree exactly, + /// including at row counts that are not multiples of 64. + #[test] + fn simd_and_scalar_plans_agree_bit_for_bit() { + for n in [0u64, 1, 63, 64, 65, 127, 1000, 4097] { + for seed in [0u64, 7, 0xFEED_FACE] { + let p = open(n, seed); + let ms = mask(p, 0); + let mm = mask(p, 0); + let ops = [ + op(LGJ_OP_GT_I32, LANE_VALUES, 100, LGJ_COMBINE_AND), + op(LGJ_OP_EQ_U32, LANE_CLASSES, 7, LGJ_COMBINE_OR), + op(LGJ_OP_GT_I32, LANE_VALUES, -50, LGJ_COMBINE_AND), + ]; + let (mut cs, mut cm) = (0u64, 0u64); + assert_eq!( + call::plan_eval(p, ops.as_ptr(), 3, mm, &mut cm), + LGJ_OK, + "n={n}" + ); + assert_eq!( + call::plan_eval_scalar(p, ops.as_ptr(), 3, ms, &mut cs), + LGJ_OK, + "n={n}" + ); + assert_eq!(cm, cs, "count parity at n={n} seed={seed}"); + assert_eq!( + read_words(mm), + read_words(ms), + "word parity at n={n} seed={seed}" + ); + for h in [mm, ms, p] { + lgj_close(h); + } + } + } + } + + /// Monotonic narrowing: with every combiner AND, each added op can only + /// shrink the selection — and the surviving rows must be a literal *subset*, + /// not merely fewer. + #[test] + fn all_and_plans_narrow_monotonically_and_by_subset() { + let n = 20_000u64; + let p = open(n, 0xC0FFEE); + let m = mask(p, 0); + let ops = [ + op(LGJ_OP_GT_I32, LANE_VALUES, -200, LGJ_COMBINE_AND), // matches all + op(LGJ_OP_GT_I32, LANE_VALUES, 0, LGJ_COMBINE_AND), + op(LGJ_OP_GT_I32, LANE_VALUES, 100, LGJ_COMBINE_AND), + op(LGJ_OP_EQ_U32, LANE_CLASSES, 7, LGJ_COMBINE_AND), + ]; + + let mut prev_count = n; + let mut prev_words: Option> = None; + for k in 1..=ops.len() { + let mut c = 0u64; + assert_eq!( + call::plan_eval(p, ops.as_ptr(), k as u32, m, &mut c), + LGJ_OK + ); + assert!(c <= prev_count, "count grew at k={k}: {c} > {prev_count}"); + let words = read_words(m); + if let Some(prev) = &prev_words { + for (i, (&w, &pw)) in words.iter().zip(prev.iter()).enumerate() { + assert_eq!(w & !pw, 0, "word {i} gained a row at k={k}: not a subset"); + } + } + prev_count = c; + prev_words = Some(words); + } + // Non-vacuity: the chain must actually have narrowed, or "monotonic" + // would be satisfied trivially by a predicate that changes nothing. + assert!(prev_count > 0 && prev_count < n, "final count {prev_count}"); + lgj_close(m); + lgj_close(p); + } + + #[test] + fn empty_plan_is_rejected() { + let p = open(64, 1); + let m = mask(p, 0); + let ops = [op(LGJ_OP_EQ_U32, LANE_CLASSES, 7, LGJ_COMBINE_AND)]; + let mut c = 0u64; + assert_eq!( + call::plan_eval(p, ops.as_ptr(), 0, m, &mut c), + LGJ_ERR_EMPTY_PLAN + ); + // A null ops pointer with n_ops == 0 is still an empty plan, not a null + // argument — the more specific diagnosis wins. + assert_eq!( + call::plan_eval(p, std::ptr::null(), 0, m, &mut c), + LGJ_ERR_EMPTY_PLAN + ); + lgj_close(m); + lgj_close(p); + } + + /// A plan containing a bad op must be rejected **without partially writing + /// dst_mask**. Each bad plan puts a *valid* op first, so a naive + /// evaluate-as-you-go implementation would have already written something. + #[test] + fn a_bad_plan_leaves_dst_mask_untouched() { + let p = open(1000, 11); + let m = mask(p, LGJ_MASK_INIT_ALL); + // Give the mask a distinctive, non-trivial content first. + assert_eq!(lgj_op_eq_u32(p, LANE_CLASSES, 3, m), LGJ_OK); + let before = read_words(m); + let before_count = count(m); + assert!(before_count > 0, "fixture must make this non-vacuous"); + + let good = op(LGJ_OP_EQ_U32, LANE_CLASSES, 7, LGJ_COMBINE_AND); + let bad_plans: [(Vec, i32); 5] = [ + // unknown opcode, after a good op + ( + vec![good, op(9999, LANE_CLASSES, 0, LGJ_COMBINE_AND)], + LGJ_ERR_UNKNOWN_OPCODE, + ), + // lane out of range + ( + vec![good, op(LGJ_OP_EQ_U32, 42, 0, LGJ_COMBINE_AND)], + LGJ_ERR_INVALID_LANE, + ), + // opcode / lane element kind mismatch + ( + vec![good, op(LGJ_OP_GT_I32, LANE_CLASSES, 0, LGJ_COMBINE_AND)], + LGJ_ERR_LANE_KIND_MISMATCH, + ), + // unknown combiner + ( + vec![good, op(LGJ_OP_EQ_U32, LANE_CLASSES, 1, 77)], + LGJ_ERR_UNKNOWN_OPCODE, + ), + // non-zero _reserved + ( + vec![ + good, + LgjOpDesc { + op: LGJ_OP_EQ_U32, + lane_id: LANE_CLASSES, + operand: 1, + combine: LGJ_COMBINE_AND, + _reserved: 1, + }, + ], + LGJ_ERR_NULL_ARGUMENT, + ), + ]; + + for (plan, want) in bad_plans { + let mut c = 12345u64; + let got = call::plan_eval(p, plan.as_ptr(), plan.len() as u32, m, &mut c); + assert_eq!(got, want, "wrong status for plan {plan:?}"); + assert_eq!(read_words(m), before, "dst_mask was modified by a bad plan"); + assert_eq!(count(m), before_count); + assert_eq!(c, 12345, "out_count must not be written on failure"); + // The scalar symbol must validate identically. + assert_eq!( + call::plan_eval_scalar(p, plan.as_ptr(), plan.len() as u32, m, &mut c), + want + ); + assert_eq!(read_words(m), before); + } + lgj_close(m); + lgj_close(p); + } + + #[test] + fn or_plans_widen() { + let n = 4000u64; + let p = open(n, 21); + let f = Fixture::generate(n, 21).unwrap(); + let m = mask(p, 0); + // acc starts all-set, so a lone OR stays all-set: OR is only meaningful + // after a narrowing op. Narrow to class==7, then widen by value>300. + let ops = [ + op(LGJ_OP_EQ_U32, LANE_CLASSES, 7, LGJ_COMBINE_AND), + op(LGJ_OP_GT_I32, LANE_VALUES, 300, LGJ_COMBINE_OR), + ]; + let mut c = 0u64; + assert_eq!(call::plan_eval(p, ops.as_ptr(), 2, m, &mut c), LGJ_OK); + let want = f + .classes() + .iter() + .zip(f.values()) + .filter(|(&cl, &v)| cl == 7 || v > 300) + .count() as u64; + assert_eq!(c, want); + lgj_close(m); + lgj_close(p); + } + + #[test] + fn plan_count_equals_mask_count() { + let p = open(3333, 4); + let m = mask(p, 0); + let ops = [op(LGJ_OP_GT_I32, LANE_VALUES, 50, LGJ_COMBINE_AND)]; + let mut c = 0u64; + assert_eq!(call::plan_eval(p, ops.as_ptr(), 1, m, &mut c), LGJ_OK); + assert_eq!(c, count(m), "out_count must equal popcount(dst_mask)"); + lgj_close(m); + lgj_close(p); + } + + // ── reduction ────────────────────────────────────────────────────────── + + #[test] + fn reduce_sum_matches_a_hand_sum() { + let n = 6000u64; + let p = open(n, 99); + let f = Fixture::generate(n, 99).unwrap(); + let m = mask(p, LGJ_MASK_INIT_ALL); + + let mut sum = 0i64; + assert_eq!(call::reduce_sum_i32(p, LANE_VALUES, m, &mut sum), LGJ_OK); + let want: i64 = f.values().iter().map(|&v| v as i64).sum(); + assert_eq!(sum, want); + // Negatives are in range, so a full-mask sum being merely "positive" is + // not evidence; check the mixed-sign path explicitly. + assert!(f.values().iter().any(|&v| v < 0)); + + assert_eq!(lgj_op_gt_i32(p, LANE_VALUES, 100, m), LGJ_OK); + assert_eq!(call::reduce_sum_i32(p, LANE_VALUES, m, &mut sum), LGJ_OK); + let want: i64 = f + .values() + .iter() + .filter(|&&v| v > 100) + .map(|&v| v as i64) + .sum(); + assert_eq!(sum, want); + assert!(want > 0); + + // An empty mask sums to zero, not to garbage. + assert_eq!(lgj_mask_and(m, mask(p, 0), m), LGJ_OK); + assert_eq!(call::reduce_sum_i32(p, LANE_VALUES, m, &mut sum), LGJ_OK); + assert_eq!(sum, 0); + lgj_close(m); + lgj_close(p); + } + + #[test] + fn reduce_sum_rejects_non_i32_lanes() { + let p = open(64, 1); + let m = mask(p, LGJ_MASK_INIT_ALL); + let mut s = 0i64; + assert_eq!( + call::reduce_sum_i32(p, LANE_IDS, m, &mut s), + LGJ_ERR_LANE_KIND_MISMATCH + ); + assert_eq!( + call::reduce_sum_i32(p, LANE_CLASSES, m, &mut s), + LGJ_ERR_LANE_KIND_MISMATCH + ); + assert_eq!(call::reduce_sum_i32(p, 12, m, &mut s), LGJ_ERR_INVALID_LANE); + lgj_close(m); + lgj_close(p); + } + + // ── degenerate sizes ─────────────────────────────────────────────────── + + #[test] + fn zero_and_one_row_resources_behave() { + for n in [0u64, 1] { + let p = open(n, 1); + let m = mask(p, LGJ_MASK_INIT_ALL); + assert_eq!(count(m), n); + let ops = [op(LGJ_OP_GT_I32, LANE_VALUES, -100_000, LGJ_COMBINE_AND)]; + let mut c = 0u64; + assert_eq!(call::plan_eval(p, ops.as_ptr(), 1, m, &mut c), LGJ_OK); + assert_eq!(c, n, "a match-everything predicate keeps all {n} rows"); + let mut s = 0i64; + assert_eq!(call::reduce_sum_i32(p, LANE_VALUES, m, &mut s), LGJ_OK); + let mut d = LgjLaneDesc::default(); + assert_eq!(call::lane_describe(p, LANE_VALUES, &mut d), LGJ_OK); + assert_eq!(d.len_elems, n); + lgj_close(m); + lgj_close(p); + } + } + + // ── panic safety ─────────────────────────────────────────────────────── + + /// A panic inside the membrane must surface as a status. Without the + /// `catch_unwind` in [`guard`] this unwind would reach JVM frames, which is + /// UB — so this test exercises the mechanism directly rather than trusting + /// that no code path ever panics. + #[test] + fn a_panic_becomes_a_status() { + let saved = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); // keep the test output clean + let got = guard(|| { + panic!("deliberate: simulating an internal invariant failure"); + }); + std::panic::set_hook(saved); + assert_eq!(got, LGJ_ERR_PANIC); + assert!(got < 0, "every failure is negative"); + } + + /// The panic path must not poison the registry into uselessness. + #[test] + fn the_registry_still_works_after_a_caught_panic() { + let saved = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let p = open(128, 1); + let m = mask(p, LGJ_MASK_INIT_ALL); + let _ = guard(|| { + let e = registry::resolve(m).unwrap(); + let _g = e.write_mask().unwrap(); + panic!("panic while holding the mask lock"); + }); + std::panic::set_hook(saved); + // The lock was poisoned; recovery via into_inner means this still works. + assert_eq!(count(m), 128); + assert_eq!(lgj_op_eq_u32(p, LANE_CLASSES, 7, m), LGJ_OK); + lgj_close(m); + lgj_close(p); + } + + /// Every status the ABI can return is negative-or-zero and distinct. + #[test] + fn status_codes_are_distinct() { + let all = [ + LGJ_OK, + LGJ_ERR_NULL_ARGUMENT, + LGJ_ERR_INVALID_HANDLE, + LGJ_ERR_WRONG_RESOURCE_KIND, + LGJ_ERR_INVALID_LANE, + LGJ_ERR_LANE_KIND_MISMATCH, + LGJ_ERR_MASK_LENGTH_MISMATCH, + LGJ_ERR_PARENT_CLOSED, + LGJ_ERR_VERSION_MISMATCH, + LGJ_ERR_LENGTH_OVERFLOW, + LGJ_ERR_UNKNOWN_OPCODE, + LGJ_ERR_EMPTY_PLAN, + LGJ_ERR_ALLOCATION_FAILED, + LGJ_ERR_READ_ONLY, + LGJ_ERR_PANIC, + ]; + let mut sorted = all.to_vec(); + sorted.sort_unstable(); + let len = sorted.len(); + sorted.dedup(); + assert_eq!(sorted.len(), len, "status codes must be distinct"); + assert!(all.iter().skip(1).all(|&s| s < 0)); + } +} diff --git a/native/lgj-abi/src/fixture.rs b/native/lgj-abi/src/fixture.rs new file mode 100644 index 0000000..0e108d9 --- /dev/null +++ b/native/lgj-abi/src/fixture.rs @@ -0,0 +1,297 @@ +//! The generic SoA fixture — three lanes, allocated once, never moved. +//! +//! "Generic is not toy": the *shape* under test is struct-of-arrays with a +//! packed-bit selection mask, which is exactly the shape `lance-graph`'s +//! `NodeRow` / `WideFieldMask` present. Wiring those concrete types in is a +//! later slice; nothing about the membrane changes when it happens. +//! +//! # The hard allocation guarantee (abi.md §4) +//! +//! Lanes are allocated once in [`Fixture::generate`] and are never reallocated, +//! resized, or moved while the resource is alive. Each lane is a +//! `Box<[T]>` — the heap buffer's address is fixed for the box's whole life, so +//! a `MemorySegment` Java built from a [`crate::abi::LgjLaneDesc`] stays valid +//! until `lgj_close`. Any future growable lane requires an ABI **major** bump. + +use crate::abi::{LgjElemKind, LGJ_FLAG_CONTIGUOUS, LGJ_FLAG_READABLE}; + +// Lane ids for a pattern resource (abi.md §7). +/// Lane 0 — entity ids, `U64`. +pub const LANE_IDS: u32 = 0; +/// Lane 1 — class tags, `U32`. +pub const LANE_CLASSES: u32 = 1; +/// Lane 2 — signed measurements, `I32`. +pub const LANE_VALUES: u32 = 2; +/// Number of lanes a pattern exposes. +pub const PATTERN_LANE_COUNT: u32 = 3; + +/// Number of distinct `class` values the generator produces: `0..16`. +pub const CLASS_CARDINALITY: u64 = 16; +/// Span of the `value` distribution before the shift below. +pub const VALUE_SPAN: i64 = 512; +/// Amount subtracted from the raw span, which is what puts negatives in range. +pub const VALUE_BIAS: i64 = 150; + +/// SplitMix64 — the reference generator, hand-written so there is no dependency +/// and no ambiguity about which variant is meant. +/// +/// Exactly the published constants: increment `0x9E3779B97F4A7C15`, mixers +/// `0xBF58476D1CE4E5B9` and `0x94D049BB133111EB`, shifts 30/27/31. All +/// arithmetic is wrapping (i.e. mod 2^64), which is what Java's `long` does +/// natively — so a Java re-implementation is a transcription, not a port. +#[derive(Debug, Clone)] +pub struct SplitMix64 { + state: u64, +} + +impl SplitMix64 { + /// Seed the generator. No warm-up draws — the first `next_u64` already + /// advances the state, exactly as the reference does. + pub const fn new(seed: u64) -> Self { + Self { state: seed } + } + + /// Advance and return the next 64 bits. + pub fn next_u64(&mut self) -> u64 { + self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } +} + +/// Three lanes of `n_rows` elements each, generated deterministically from a +/// seed. +/// +/// # The generation algorithm — NORMATIVE +/// +/// The Java-side test recomputes expected counts from this description alone, +/// with no data file, which makes it a genuine cross-language parity check. So +/// this is a contract, not an implementation detail: +/// +/// ```text +/// rng = SplitMix64(seed) // state = seed, no warm-up draws +/// for i in 0 .. n_rows: // ascending, one row at a time +/// a = rng.next_u64() // FIRST draw of the row +/// b = rng.next_u64() // SECOND draw of the row +/// ids[i] = i as u64 // dense, 0-based +/// classes[i] = ((a >>> 33) & 0xF) as u32 // 0 ..= 15 +/// values[i] = (((b >>> 40) & 0x1FF) - 150) as i32 // -150 ..= 361 +/// ``` +/// +/// Two draws per row, `a` before `b`, is part of the contract: swapping them or +/// drawing one `u64` and slicing it would produce different lanes for the same +/// seed and silently break Java's independently-computed expectations. +/// +/// Why these ranges: +/// - `classes` spans 16 values, so `class == 7` selects ≈ 1/16 of the rows — +/// a meaningful fraction, neither "everything" nor "nothing" (a predicate +/// that matches all or no rows cannot falsify a narrowing property). +/// - `values` spans `-150 ..= 361`, which *straddles* the threshold 100 used +/// throughout the tests and includes negatives — so `> 100` is a real signed +/// comparison, and an implementation that compared unsigned by mistake would +/// be caught rather than accidentally agreeing. +/// +/// `ids` is deliberately the dense row index: it is the join key, and making it +/// trivially predictable means a Java test can check *lane addressing itself* +/// without also having to model the PRNG. +/// +/// `Debug` deliberately prints only the shape, never the lanes: a 64,000-row +/// dump in a failing assertion is noise, not evidence. +pub struct Fixture { + /// Logical row count; every lane has exactly this many elements. + pub n_rows: u64, + /// The seed these lanes were generated from. + pub seed: u64, + ids: Box<[u64]>, + classes: Box<[u32]>, + values: Box<[i32]>, +} + +impl std::fmt::Debug for Fixture { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Fixture") + .field("n_rows", &self.n_rows) + .field("seed", &self.seed) + .finish_non_exhaustive() + } +} + +impl Fixture { + /// Build the fixture. Allocates each lane exactly once. + /// + /// Returns `None` if `n_rows` exceeds what can be indexed on this target + /// (the caller maps that to `LENGTH_OVERFLOW`). + pub fn generate(n_rows: u64, seed: u64) -> Option { + let n = usize::try_from(n_rows).ok()?; + // Reject anything whose byte length would overflow: the widest lane is + // 8 bytes/elem, and masks add 8 bytes per 64 rows on top. + n.checked_mul(8)?; + + let mut rng = SplitMix64::new(seed); + let mut ids = Vec::new(); + let mut classes = Vec::new(); + let mut values = Vec::new(); + ids.try_reserve_exact(n).ok()?; + classes.try_reserve_exact(n).ok()?; + values.try_reserve_exact(n).ok()?; + + for i in 0..n { + let a = rng.next_u64(); + let b = rng.next_u64(); + ids.push(i as u64); + classes.push(((a >> 33) & (CLASS_CARDINALITY - 1)) as u32); + values.push((((b >> 40) & (VALUE_SPAN as u64 - 1)) as i64 - VALUE_BIAS) as i32); + } + + Some(Self { + n_rows, + seed, + ids: ids.into_boxed_slice(), + classes: classes.into_boxed_slice(), + values: values.into_boxed_slice(), + }) + } + + /// Lane 0 — the dense row index. + pub fn ids(&self) -> &[u64] { + &self.ids + } + /// Lane 1 — class tags, `0 ..= 15`. + pub fn classes(&self) -> &[u32] { + &self.classes + } + /// Lane 2 — signed values, `-150 ..= 361`. + pub fn values(&self) -> &[i32] { + &self.values + } + + /// `(base address, element count, element kind)` for a lane id, or `None` + /// if the id is out of range (⇒ `INVALID_LANE`). + /// + /// The address is taken from the boxed slice, whose buffer never moves — see + /// this module's header. + pub fn lane_raw(&self, lane_id: u32) -> Option<(u64, u64, LgjElemKind)> { + match lane_id { + LANE_IDS => Some(( + self.ids.as_ptr() as u64, + self.ids.len() as u64, + LgjElemKind::U64, + )), + LANE_CLASSES => Some(( + self.classes.as_ptr() as u64, + self.classes.len() as u64, + LgjElemKind::U32, + )), + LANE_VALUES => Some(( + self.values.as_ptr() as u64, + self.values.len() as u64, + LgjElemKind::I32, + )), + _ => None, + } + } + + /// Flags every pattern lane carries: readable + contiguous, and **never** + /// writable (abi.md §7 — pattern lanes are read-only). + pub const fn lane_flags() -> u32 { + LGJ_FLAG_READABLE | LGJ_FLAG_CONTIGUOUS + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The published SplitMix64 test vector for seed 0 — proves this is the + /// real generator and not a lookalike, which is what makes the Java + /// transcription checkable. + #[test] + fn splitmix64_matches_the_reference_vector() { + let mut r = SplitMix64::new(0); + assert_eq!(r.next_u64(), 0xE220_A839_7B1D_CDAF); + assert_eq!(r.next_u64(), 0x6E78_9E6A_A1B9_65F4); + assert_eq!(r.next_u64(), 0x06C4_5D18_8009_454F); + } + + #[test] + fn generation_is_deterministic_for_a_seed() { + let a = Fixture::generate(1000, 42).unwrap(); + let b = Fixture::generate(1000, 42).unwrap(); + assert_eq!(a.classes(), b.classes()); + assert_eq!(a.values(), b.values()); + } + + #[test] + fn different_seeds_produce_different_lanes() { + let a = Fixture::generate(1000, 1).unwrap(); + let b = Fixture::generate(1000, 2).unwrap(); + assert_ne!(a.values(), b.values()); + } + + #[test] + fn lanes_have_the_documented_shape() { + let f = Fixture::generate(4096, 7).unwrap(); + assert_eq!(f.ids().len(), 4096); + // ids are the dense row index. + assert!(f.ids().iter().enumerate().all(|(i, &v)| v == i as u64)); + // classes span exactly 0..16. + assert!(f.classes().iter().all(|&c| c < 16)); + // and actually *use* the range — a constant lane would make every + // class predicate vacuous. + let distinct = { + let mut seen = [false; 16]; + for &c in f.classes() { + seen[c as usize] = true; + } + seen.iter().filter(|&&s| s).count() + }; + assert_eq!(distinct, 16, "all 16 classes must occur at n=4096"); + // values straddle the test threshold and include negatives. + assert!(f.values().iter().all(|&v| (-150..=361).contains(&v))); + assert!(f.values().iter().any(|&v| v < 0), "negatives must occur"); + assert!(f.values().iter().any(|&v| v > 100)); + assert!(f.values().iter().any(|&v| v <= 100)); + } + + /// A predicate that matches everything or nothing cannot falsify anything, + /// so pin that the fixture's headline predicates select a *middling* + /// fraction. + #[test] + fn headline_predicates_select_a_meaningful_fraction() { + let f = Fixture::generate(64_000, 0xABCD).unwrap(); + let n = f.n_rows as f64; + let c7 = f.classes().iter().filter(|&&c| c == 7).count() as f64 / n; + let gt = f.values().iter().filter(|&&v| v > 100).count() as f64 / n; + assert!((0.04..0.09).contains(&c7), "class==7 fraction was {c7}"); + assert!((0.35..0.65).contains(>), "value>100 fraction was {gt}"); + } + + #[test] + fn zero_rows_is_legal_and_empty() { + let f = Fixture::generate(0, 1).unwrap(); + assert_eq!(f.ids().len(), 0); + assert_eq!(f.lane_raw(0).unwrap().1, 0); + } + + #[test] + fn lane_ids_beyond_the_last_are_rejected() { + let f = Fixture::generate(8, 1).unwrap(); + assert!(f.lane_raw(0).is_some()); + assert!(f.lane_raw(2).is_some()); + assert!(f.lane_raw(3).is_none()); + assert!(f.lane_raw(u32::MAX).is_none()); + } + + /// The allocation-stability guarantee, checked rather than asserted in prose: + /// a lane's base address does not change across repeated describes. + #[test] + fn lane_addresses_are_stable() { + let f = Fixture::generate(1024, 3).unwrap(); + let first = f.lane_raw(LANE_VALUES).unwrap().0; + for _ in 0..100 { + assert_eq!(f.lane_raw(LANE_VALUES).unwrap().0, first); + } + } +} diff --git a/native/lgj-abi/src/kernels.rs b/native/lgj-abi/src/kernels.rs new file mode 100644 index 0000000..9e6d03b --- /dev/null +++ b/native/lgj-abi/src/kernels.rs @@ -0,0 +1,406 @@ +//! The bulk kernels — and the **single** place in this crate that names +//! `ndarray`. +//! +//! # SIMD provenance (abi.md §8) +//! +//! Every SIMD path here routes through `ndarray::simd` and nothing else. This +//! crate contains no `core::arch` / `_mm*` intrinsic, no `core::simd`, no +//! `#[cfg(target_feature)]` SIMD *selection*, and no locally-written SIMD +//! abstraction. Which backend those calls compile to is `ndarray`'s business; +//! the manifest reports it and Java never selects it. +//! +//! # Why every ndarray call is behind a wrapper +//! +//! The `simd_int_ops` primitives this file consumes were written in parallel +//! with it, against a shared signature contract. Funnelling every call through +//! a one-line `simd_*` wrapper below means a signature adjustment touches this +//! file only — the exported ABI, the registry and the plan evaluator never see +//! it. The wrappers are `#[inline]`, so they cost nothing. +//! +//! # The scalar reference is INDEPENDENT +//! +//! `scalar_*` below is written in plain Rust loops with **no ndarray at all**. +//! That independence is the entire value of `lgj_plan_eval_scalar`: if the +//! reference shared code with the SIMD path, a parity test between them would +//! be checking that a function agrees with itself. It does not, so the test is +//! a real falsifier — the same reason `ndarray`'s own W1a contract demands a +//! scalar arm. +//! +//! # Bit order (normative, restated because both paths must obey it) +//! +//! Element `i` lives at bit `i % 64` of word `i / 64`. Bits past `n_rows` in +//! the final word are zero. + +use crate::abi::*; + +// ─────────────────────────────────────────────────────────────────────────── +// SIMD path — thin wrappers over ndarray::simd +// ─────────────────────────────────────────────────────────────────────────── + +/// `out_words[i-th bit] = (values[i] == needle)`, fully overwriting `out_words`. +#[inline] +pub fn simd_eq_u32_to_mask(values: &[u32], needle: u32, out_words: &mut [u64]) { + ndarray::simd::eq_u32_to_mask(values, needle, out_words); +} + +/// `out_words[i-th bit] = (values[i] > threshold)`, signed, fully overwriting. +#[inline] +pub fn simd_gt_i32_to_mask(values: &[i32], threshold: i32, out_words: &mut [u64]) { + ndarray::simd::gt_i32_to_mask(values, threshold, out_words); +} + +/// `dst = a & b`. `dst` must not alias `a` or `b` (Rust's borrow rules enforce +/// it here; the aliasing ABI cases route to the `_assign` forms instead). +#[inline] +pub fn simd_mask_and(a: &[u64], b: &[u64], dst: &mut [u64]) { + ndarray::simd::mask_and(a, b, dst); +} + +/// `dst = a | b`. +#[inline] +pub fn simd_mask_or(a: &[u64], b: &[u64], dst: &mut [u64]) { + ndarray::simd::mask_or(a, b, dst); +} + +/// `dst &= src`. +#[inline] +pub fn simd_mask_and_assign(dst: &mut [u64], src: &[u64]) { + ndarray::simd::mask_and_assign(dst, src); +} + +/// `dst |= src`. +#[inline] +pub fn simd_mask_or_assign(dst: &mut [u64], src: &[u64]) { + ndarray::simd::mask_or_assign(dst, src); +} + +/// Sum of `values[i]` over set mask bits, widened to `i64`. +#[inline] +pub fn simd_masked_sum_i32(values: &[i32], mask_words: &[u64]) -> i64 { + ndarray::simd::masked_sum_i32(values, mask_words) +} + +/// Population count over mask words. +/// +/// **Reused, not reimplemented** — this already exists in `ndarray` +/// (implemented in `src/bitwise.rs`) and is reached here EXCLUSIVELY +/// through the sanctioned `ndarray::simd` re-export surface, never through +/// the internal `ndarray::hpc::bitwise` path it happens to live behind. +/// See `.claude/knowledge/simd-provenance.md`: `ndarray::hpc::*` is an +/// implementation-detail namespace that may be renamed or re-arranged at +/// any time; `ndarray::simd::*` is the contract that does not move under a +/// consumer's feet. Writing a second popcount here would ALSO be exactly +/// the duplication the `ndarray::simd` membrane exists to prevent — this +/// function exists only to keep this crate's own kernel-call surface +/// uniform (every bulk primitive it calls is named `simd_*` here). +#[inline] +pub fn simd_popcount(words: &[u64]) -> u64 { + ndarray::simd::popcount_batch_u64(words) +} + +// ─────────────────────────────────────────────────────────────────────────── +// Scalar reference — INDEPENDENT of ndarray. Do not "simplify" by calling the +// wrappers above; the independence IS the test. +// ─────────────────────────────────────────────────────────────────────────── + +/// Reference `eq_u32` → mask. +pub fn scalar_eq_u32_to_mask(values: &[u32], needle: u32, out_words: &mut [u64]) { + for w in out_words.iter_mut() { + *w = 0; + } + for (i, &v) in values.iter().enumerate() { + if v == needle { + out_words[i / 64] |= 1u64 << (i % 64); + } + } +} + +/// Reference signed `gt_i32` → mask. +pub fn scalar_gt_i32_to_mask(values: &[i32], threshold: i32, out_words: &mut [u64]) { + for w in out_words.iter_mut() { + *w = 0; + } + for (i, &v) in values.iter().enumerate() { + if v > threshold { + out_words[i / 64] |= 1u64 << (i % 64); + } + } +} + +/// Reference `dst &= src`. +pub fn scalar_mask_and_assign(dst: &mut [u64], src: &[u64]) { + for (d, &s) in dst.iter_mut().zip(src.iter()) { + *d &= s; + } +} + +/// Reference `dst |= src`. +pub fn scalar_mask_or_assign(dst: &mut [u64], src: &[u64]) { + for (d, &s) in dst.iter_mut().zip(src.iter()) { + *d |= s; + } +} + +/// Reference masked sum. +pub fn scalar_masked_sum_i32(values: &[i32], mask_words: &[u64]) -> i64 { + let mut acc: i64 = 0; + for (i, &v) in values.iter().enumerate() { + if (mask_words[i / 64] >> (i % 64)) & 1 == 1 { + acc += v as i64; + } + } + acc +} + +/// Reference popcount. +pub fn scalar_popcount(words: &[u64]) -> u64 { + let mut n = 0u64; + for &w in words { + n += w.count_ones() as u64; + } + n +} + +// ─────────────────────────────────────────────────────────────────────────── +// Which path a call takes +// ─────────────────────────────────────────────────────────────────────────── + +/// Selects between the `ndarray::simd` kernels and the independent scalar +/// reference. +/// +/// This is a *runtime value*, not a `cfg` — `lgj_plan_eval` and +/// `lgj_plan_eval_scalar` are two symbols over one code path, which is what +/// makes SIMD-vs-scalar parity falsifiable **through the membrane** where the +/// Java tests live. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Path { + /// The `ndarray::simd` kernels — what `lgj_plan_eval` uses. + Simd, + /// The independent scalar reference — what `lgj_plan_eval_scalar` uses. + Scalar, +} + +/// The lane view a predicate reads. Carries its element kind so the plan +/// validator can reject an opcode/lane mismatch before any work happens. +pub enum LaneView<'a> { + /// A `U32` lane (classes). + U32(&'a [u32]), + /// An `I32` lane (values). + I32(&'a [i32]), + /// A `U64` lane (ids). No predicate currently reads it, which is exactly + /// why it is here: it gives `LANE_KIND_MISMATCH` something real to reject. + U64(&'a [u64]), +} + +impl LaneView<'_> { + /// The element kind this view exposes, used by the plan validator. + pub fn kind(&self) -> LgjElemKind { + match self { + LaneView::U32(_) => LgjElemKind::U32, + LaneView::I32(_) => LgjElemKind::I32, + LaneView::U64(_) => LgjElemKind::U64, + } + } +} + +/// Evaluate one predicate into `out_words` (fully overwritten). +/// +/// The opcode/lane pairing has already been validated by the plan checker; a +/// mismatch reaching here is a bug in this crate, so it is reported as a status +/// rather than assumed away. +pub fn eval_predicate( + path: Path, + op: u32, + operand: i64, + lane: &LaneView<'_>, + n_rows: u64, + out_words: &mut [u64], +) -> Result<(), i32> { + match (op, lane) { + (LGJ_OP_EQ_U32, LaneView::U32(v)) => { + // The operand arrives sign-extended in an i64; the needle is the + // low 32 bits, compared as an exact u32 bit pattern. + let needle = operand as u32; + match path { + Path::Simd => simd_eq_u32_to_mask(v, needle, out_words), + Path::Scalar => scalar_eq_u32_to_mask(v, needle, out_words), + } + } + (LGJ_OP_GT_I32, LaneView::I32(v)) => { + let threshold = operand as i32; + match path { + Path::Simd => simd_gt_i32_to_mask(v, threshold, out_words), + Path::Scalar => scalar_gt_i32_to_mask(v, threshold, out_words), + } + } + (LGJ_OP_EQ_U32, _) | (LGJ_OP_GT_I32, _) => return Err(LGJ_ERR_LANE_KIND_MISMATCH), + _ => return Err(LGJ_ERR_UNKNOWN_OPCODE), + } + // Both primitives already zero the tail, but re-establishing it here means + // the invariant holds no matter which path ran. + clear_tail_bits(out_words, n_rows); + Ok(()) +} + +/// Combine a predicate result into an accumulator. +pub fn combine_into( + path: Path, + combine: u32, + acc: &mut [u64], + op_result: &[u64], +) -> Result<(), i32> { + match (combine, path) { + (LGJ_COMBINE_AND, Path::Simd) => simd_mask_and_assign(acc, op_result), + (LGJ_COMBINE_AND, Path::Scalar) => scalar_mask_and_assign(acc, op_result), + (LGJ_COMBINE_OR, Path::Simd) => simd_mask_or_assign(acc, op_result), + (LGJ_COMBINE_OR, Path::Scalar) => scalar_mask_or_assign(acc, op_result), + // Not in abi.md's combiner set. Rejected like an unknown opcode rather + // than defaulted to AND, so a future combiner cannot be silently + // misinterpreted by an old build. + _ => return Err(LGJ_ERR_UNKNOWN_OPCODE), + } + Ok(()) +} + +/// Popcount, on the selected path. +pub fn popcount(path: Path, words: &[u64]) -> u64 { + match path { + Path::Simd => simd_popcount(words), + Path::Scalar => scalar_popcount(words), + } +} + +/// Masked sum, on the selected path. +pub fn masked_sum_i32(path: Path, values: &[i32], mask_words: &[u64]) -> i64 { + match path { + Path::Simd => simd_masked_sum_i32(values, mask_words), + Path::Scalar => scalar_masked_sum_i32(values, mask_words), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixture::Fixture; + + fn words_for(n: u64) -> Vec { + vec![0u64; mask_words_for(n) as usize] + } + + /// The falsifier for the whole SIMD provenance story: the ndarray kernels + /// and an independently-written scalar loop must agree bit-for-bit, at row + /// counts that are and are not multiples of the lane width. + #[test] + fn simd_matches_scalar_on_every_row_count_shape() { + // 15/16/17 straddle the 16-lane group; 63/64/65 straddle the word; 1000 + // and 4096 are ordinary; 0 and 1 are the degenerate ends. + for n in [0u64, 1, 15, 16, 17, 63, 64, 65, 127, 128, 129, 1000, 4096] { + for seed in [0u64, 1, 0xDEAD_BEEF] { + let f = Fixture::generate(n, seed).unwrap(); + let (mut a, mut b) = (words_for(n), words_for(n)); + + simd_eq_u32_to_mask(f.classes(), 7, &mut a); + scalar_eq_u32_to_mask(f.classes(), 7, &mut b); + assert_eq!(a, b, "eq_u32 mismatch at n={n} seed={seed}"); + + simd_gt_i32_to_mask(f.values(), 100, &mut a); + scalar_gt_i32_to_mask(f.values(), 100, &mut b); + assert_eq!(a, b, "gt_i32 mismatch at n={n} seed={seed}"); + + assert_eq!(simd_popcount(&a), scalar_popcount(&b)); + assert_eq!( + simd_masked_sum_i32(f.values(), &a), + scalar_masked_sum_i32(f.values(), &b), + "masked_sum mismatch at n={n} seed={seed}" + ); + } + } + } + + /// Signedness is the mistake this predicate invites: an unsigned compare + /// would put every negative value *above* a positive threshold. + #[test] + fn gt_i32_is_signed() { + let v: Vec = vec![-1, -1000, i32::MIN, 0, 1, 101, i32::MAX]; + let mut w = words_for(v.len() as u64); + simd_gt_i32_to_mask(&v, 100, &mut w); + // Only 101 (index 5) and i32::MAX (index 6) exceed 100. + assert_eq!(w[0], 0b110_0000); + let mut s = words_for(v.len() as u64); + scalar_gt_i32_to_mask(&v, 100, &mut s); + assert_eq!(w, s); + } + + #[test] + fn bit_order_is_lsb_first_within_each_word() { + let mut v = vec![0u32; 65]; + v[0] = 7; // bit 0 of word 0 + v[63] = 7; // bit 63 of word 0 + v[64] = 7; // bit 0 of word 1 + let mut w = words_for(65); + simd_eq_u32_to_mask(&v, 7, &mut w); + assert_eq!(w[0], (1u64 << 63) | 1); + assert_eq!(w[1], 1); + } + + #[test] + fn tail_bits_never_survive_a_predicate() { + // 70 rows: word 1 holds only 6 valid bits. A needle matching every row + // must still leave bits 6..64 of word 1 zero, or popcount would lie. + let v = vec![7u32; 70]; + let mut w = vec![u64::MAX; 2]; + simd_eq_u32_to_mask(&v, 7, &mut w); + clear_tail_bits(&mut w, 70); + assert_eq!(simd_popcount(&w), 70); + } + + #[test] + fn combiners_narrow_and_widen() { + let mut acc = vec![0b1100u64]; + combine_into(Path::Simd, LGJ_COMBINE_AND, &mut acc, &[0b1010]).unwrap(); + assert_eq!(acc[0], 0b1000); + combine_into(Path::Scalar, LGJ_COMBINE_OR, &mut acc, &[0b0011]).unwrap(); + assert_eq!(acc[0], 0b1011); + } + + #[test] + fn unknown_combiner_is_rejected() { + let mut acc = vec![0u64]; + assert_eq!( + combine_into(Path::Simd, 99, &mut acc, &[0]).unwrap_err(), + LGJ_ERR_UNKNOWN_OPCODE + ); + } + + #[test] + fn lane_kind_mismatch_is_caught_in_the_evaluator_too() { + let ids = [1u64, 2, 3]; + let lane = LaneView::U64(&ids); + let mut w = words_for(3); + assert_eq!( + eval_predicate(Path::Simd, LGJ_OP_EQ_U32, 1, &lane, 3, &mut w).unwrap_err(), + LGJ_ERR_LANE_KIND_MISMATCH + ); + assert_eq!( + eval_predicate(Path::Simd, 12345, 1, &lane, 3, &mut w).unwrap_err(), + LGJ_ERR_UNKNOWN_OPCODE + ); + } + + #[test] + fn non_aliasing_mask_ops_agree_with_assign_forms() { + let a = vec![0xF0F0_F0F0_F0F0_F0F0u64, 0x00FF]; + let b = vec![0xFF00_FF00_FF00_FF00u64, 0x0F0F]; + let mut viaand = vec![0u64; 2]; + simd_mask_and(&a, &b, &mut viaand); + let mut inplace = a.clone(); + simd_mask_and_assign(&mut inplace, &b); + assert_eq!(viaand, inplace); + + let mut vior = vec![0u64; 2]; + simd_mask_or(&a, &b, &mut vior); + let mut inplace = a.clone(); + simd_mask_or_assign(&mut inplace, &b); + assert_eq!(vior, inplace); + } +} diff --git a/native/lgj-abi/src/lib.rs b/native/lgj-abi/src/lib.rs new file mode 100644 index 0000000..952e87f --- /dev/null +++ b/native/lgj-abi/src/lib.rs @@ -0,0 +1,269 @@ +//! `lgj-abi` — the Rust half of the lance-graph-java **machine membrane**. +//! +//! The normative contract is `docs/abi.md`. This crate and the Java layer are +//! implemented *independently* against it; a runtime manifest check proves they +//! agree before the first real call. +//! +//! # There is no C in this project +//! +//! `extern "C"` and `#[repr(C)]` name a *machine* calling convention and a +//! *machine* aggregate layout rule — the SysV AMD64 psABI on this box. They do +//! not involve the C language. There is no `.h` file anywhere, no C toolchain, +//! no `cbindgen` (its output is a header nobody consumes), no `jextract` (its +//! only input is a header that does not exist), and no JNI. `cargo` and `javac` +//! are the entire toolchain. +//! +//! What replaces a header is [`abi::LgjAbiManifest`]: the compiled artifact +//! describing *itself* at runtime, from `size_of` / `align_of` on the real +//! types. A header can drift from the binary silently; a self-report cannot. +//! +//! # The three properties this crate exists to hold +//! +//! 1. **Bulk only** (`abi.md` §6). Every symbol either does work proportional to +//! `n_rows` or is lifecycle. There is deliberately no `lgj_lane_read_element` +//! — 64,000 logical entities cost one lane set, one packed mask and one +//! crossing, not 64,000 of anything. +//! 2. **No stale handle ever dereferences freed memory** (§4). Handles are +//! generation-checked opaque `u64`s, not pointers; use-after-close is +//! `INVALID_HANDLE`, a status. [`registry`] is where that is enforced and +//! where the tests attack it. +//! 3. **All SIMD comes from `ndarray::simd`** (§8). No `core::arch`, no +//! `_mm*`, no `core::simd`, no `pulp`/`wide`/`SimSIMD`, no nightly, no local +//! SIMD abstraction. [`kernels`] is the single module that names `ndarray`, +//! and it also carries an *independently written* scalar reference so +//! SIMD-vs-scalar parity is a real falsifier rather than a tautology. +//! +//! # Module map +//! +//! | module | role | +//! |---|---| +//! | [`abi`] | `#[repr(C)]` types, status codes, opcodes, the self-describing manifest | +//! | [`registry`] | generation-checked handles, ownership, lock discipline | +//! | [`fixture`] | the deterministic generic SoA fixture (three lanes) | +//! | [`kernels`] | bulk kernels via `ndarray::simd` + the independent scalar reference | +//! | [`exports`] | the `extern "C"` symbols themselves | + +#![deny(unsafe_op_in_unsafe_fn)] +#![warn(missing_docs)] + +pub mod abi; +pub mod exports; +pub mod fixture; +pub mod kernels; +pub mod registry; + +// Re-export the ABI vocabulary at the crate root for convenience in tests and +// for any Rust consumer that links the `rlib` rather than the `cdylib`. +pub use abi::*; +pub use exports::*; + +#[cfg(test)] +mod integration_tests { + //! Cross-module properties that belong to no single module. + + use crate::abi::*; + use crate::exports::*; + use crate::fixture::{Fixture, LANE_CLASSES, LANE_VALUES}; + + /// Safe wrappers over the pointer-taking exports — see the identical note in + /// `exports::tests`: one place holds the `unsafe`, so the assertions read as + /// the ABI contract rather than as pointer plumbing. + mod call { + use super::*; + + pub fn pattern_open(n: u64, seed: u64, out: *mut u64) -> i32 { + unsafe { lgj_pattern_open(n, seed, out) } + } + pub fn mask_create(p: u64, initial: u32, out: *mut u64) -> i32 { + unsafe { lgj_mask_create(p, initial, out) } + } + pub fn mask_describe(m: u64, out: *mut LgjLaneDesc) -> i32 { + unsafe { lgj_mask_describe(m, out) } + } + pub fn mask_count(m: u64, out: *mut u64) -> i32 { + unsafe { lgj_mask_count(m, out) } + } + pub fn plan_eval(r: u64, ops: *const LgjOpDesc, n: u32, dst: u64, out: *mut u64) -> i32 { + unsafe { lgj_plan_eval(r, ops, n, dst, out) } + } + pub fn reduce_sum_i32(r: u64, lane: u32, m: u64, out: *mut i64) -> i32 { + unsafe { lgj_reduce_sum_i32(r, lane, m, out) } + } + } + + fn open(n: u64, seed: u64) -> u64 { + let mut h = 0u64; + assert_eq!(call::pattern_open(n, seed, &mut h), LGJ_OK); + h + } + + fn mask(parent: u64, initial: u32) -> u64 { + let mut h = 0u64; + assert_eq!(call::mask_create(parent, initial, &mut h), LGJ_OK); + h + } + + /// The thesis, end to end: 64,000 logical entities become one lane set, one + /// packed mask, and **one** crossing that answers the whole question. + /// + /// Also the arithmetic that makes the thesis concrete — the mask is + /// `n/8` bytes, not `n` objects. + #[test] + fn sixty_four_thousand_entities_cost_one_crossing() { + let n = 64_000u64; + let p = open(n, 2026); + let m = mask(p, LGJ_MASK_INIT_EMPTY); + + let ops = [ + LgjOpDesc { + op: LGJ_OP_EQ_U32, + lane_id: LANE_CLASSES, + operand: 7, + combine: LGJ_COMBINE_AND, + _reserved: 0, + }, + LgjOpDesc { + op: LGJ_OP_GT_I32, + lane_id: LANE_VALUES, + operand: 100, + combine: LGJ_COMBINE_AND, + _reserved: 0, + }, + ]; + let mut count = 0u64; + assert_eq!(call::plan_eval(p, ops.as_ptr(), 2, m, &mut count), LGJ_OK); + + // Independently computed from the fixture's documented generator — the + // same computation the Java test performs from `abi.md` alone. + let f = Fixture::generate(n, 2026).unwrap(); + let want = f + .classes() + .iter() + .zip(f.values()) + .filter(|(&c, &v)| c == 7 && v > 100) + .count() as u64; + assert_eq!(count, want); + assert!(count > 0 && count < n, "must not be a vacuous selection"); + + // The whole selection is 1000 u64 words = 8000 bytes, for 64,000 rows. + let mut d = LgjLaneDesc::default(); + assert_eq!(call::mask_describe(m, &mut d), LGJ_OK); + assert_eq!(d.len_elems, 1000); + assert_eq!(d.byte_len, 8000); + + let mut sum = 0i64; + assert_eq!(call::reduce_sum_i32(p, LANE_VALUES, m, &mut sum), LGJ_OK); + let want_sum: i64 = f + .classes() + .iter() + .zip(f.values()) + .filter(|(&c, &v)| c == 7 && v > 100) + .map(|(_, &v)| v as i64) + .sum(); + assert_eq!(sum, want_sum); + + lgj_close(m); + lgj_close(p); + } + + /// Many live resources at once, closed out of order — the registry must not + /// confuse them, and every handle must stay valid until *its own* close. + #[test] + fn interleaved_lifetimes_do_not_cross_talk() { + let mut patterns = Vec::new(); + let mut masks = Vec::new(); + for i in 0..16u64 { + let p = open(100 + i, i); + masks.push(mask(p, LGJ_MASK_INIT_ALL)); + patterns.push(p); + } + // Close every other pattern; the survivors must be unaffected. + for i in (0..16).step_by(2) { + assert_eq!(lgj_close(masks[i]), LGJ_OK); + assert_eq!(lgj_close(patterns[i]), LGJ_OK); + } + for i in (1..16).step_by(2) { + let mut c = 0u64; + assert_eq!(call::mask_count(masks[i], &mut c), LGJ_OK); + assert_eq!(c, 100 + i as u64); + } + for i in (0..16).step_by(2) { + let mut c = 0u64; + assert_eq!(call::mask_count(masks[i], &mut c), LGJ_ERR_INVALID_HANDLE); + } + for i in (1..16).step_by(2) { + assert_eq!(lgj_close(masks[i]), LGJ_OK); + assert_eq!(lgj_close(patterns[i]), LGJ_OK); + } + } + + /// Bulk ops on distinct resources are meant to run concurrently (§4). This + /// does not benchmark contention — `abi.md` is explicit that nothing here + /// has been — it proves the *shape* is sound: no deadlock, no cross-talk, + /// each thread's answer independently correct. + #[test] + fn concurrent_bulk_ops_on_distinct_resources_agree_with_serial_ones() { + use std::thread; + let handles: Vec<_> = (0..8u64) + .map(|t| { + thread::spawn(move || { + let n = 5000 + t; + let p = open(n, t); + let m = mask(p, LGJ_MASK_INIT_EMPTY); + let ops = [LgjOpDesc { + op: LGJ_OP_EQ_U32, + lane_id: LANE_CLASSES, + operand: 7, + combine: LGJ_COMBINE_AND, + _reserved: 0, + }]; + let mut c = 0u64; + assert_eq!(call::plan_eval(p, ops.as_ptr(), 1, m, &mut c), LGJ_OK); + lgj_close(m); + lgj_close(p); + (n, t, c) + }) + }) + .collect(); + for h in handles { + let (n, seed, got) = h.join().unwrap(); + let f = Fixture::generate(n, seed).unwrap(); + let want = f.classes().iter().filter(|&&c| c == 7).count() as u64; + assert_eq!(got, want, "thread for seed {seed} disagreed"); + } + } + + /// Concurrent mask binops that name the same masks in *opposite* orders — + /// the shape that would deadlock without address-ordered locking. + #[test] + fn opposite_order_mask_binops_do_not_deadlock() { + use std::sync::Arc; + use std::thread; + let p = open(4096, 1); + let a = mask(p, LGJ_MASK_INIT_ALL); + let b = mask(p, LGJ_MASK_INIT_ALL); + let c = mask(p, LGJ_MASK_INIT_EMPTY); + let barrier = Arc::new(std::sync::Barrier::new(2)); + + let b1 = Arc::clone(&barrier); + let t1 = thread::spawn(move || { + b1.wait(); + for _ in 0..2000 { + assert_eq!(lgj_mask_and(a, b, c), LGJ_OK); + } + }); + let b2 = Arc::clone(&barrier); + let t2 = thread::spawn(move || { + b2.wait(); + for _ in 0..2000 { + assert_eq!(lgj_mask_or(c, b, a), LGJ_OK); + } + }); + t1.join().unwrap(); + t2.join().unwrap(); + + for h in [c, b, a, p] { + lgj_close(h); + } + } +} diff --git a/native/lgj-abi/src/registry.rs b/native/lgj-abi/src/registry.rs new file mode 100644 index 0000000..a8cd04a --- /dev/null +++ b/native/lgj-abi/src/registry.rs @@ -0,0 +1,510 @@ +//! The generation-checked handle registry. +//! +//! This is the safety-critical piece. Inside Rust, `&self` borrows make +//! "a view outlived its owner" a *compile error*. Across the membrane there is +//! no borrow checker, so the invariant is enforced at run time instead — and the +//! property being enforced is stated as sharply as `abi.md` §4 states it: +//! +//! > **There is no code path in which a stale handle dereferences freed +//! > memory.** +//! +//! The mechanism: a handle is not a pointer, it is an opaque `u64` +//! `(generation << 32) | index`. `index` selects a registry slot; `generation` +//! is bumped every time a slot is freed. Every lookup validates the generation, +//! so a closed, double-closed or fabricated handle resolves to +//! `INVALID_HANDLE` — a status, not a segfault. +//! +//! Generations start at **1**, which is what makes the fabricated handle `0` +//! (`gen 0, index 0`) fail: no live slot ever carries generation 0. +//! +//! # Locking +//! +//! - The registry itself is a `RwLock>`. A call takes a **short** read +//! lock, clones the `Arc`, and **drops the registry lock before +//! doing any work** ([`resolve`]). Only open/close take the write lock, so +//! they are the only globally-serializing operations. +//! - A **pattern needs no inner lock at all**: its lanes are read-only by ABI +//! (§7), so the `Fixture` is immutable behind the `Arc` and any number of bulk +//! ops can read it concurrently. +//! - Only **mask words** are mutable, so only they sit behind an inner +//! `RwLock`. When one call must lock several masks (a `mask_and` whose three +//! handles are distinct), the locks are acquired in **address order** +//! ([`lock_masks_ordered`]) — a global order, therefore deadlock-free — +//! and aliased handles are deduplicated *before* locking, because locking one +//! `RwLock` twice from the same thread is the other way to hang. +//! +//! Stated honestly, as `abi.md` does: none of this has been benchmarked under +//! contention, and the POC's Java layer is single-threaded. +//! +//! # Poisoning +//! +//! A panic while a lock is held poisons it. Rather than bricking the registry, +//! every acquisition recovers with `into_inner()`. This is safe *here* because +//! the guarded data carries no invariant a partial write could break beyond +//! "bits past `n_rows` are zero", and every write path re-establishes that +//! before returning (see [`crate::abi::clear_tail_bits`]). + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, OnceLock, RwLock, RwLockReadGuard, RwLockWriteGuard}; + +use crate::abi::*; +use crate::fixture::{Fixture, PATTERN_LANE_COUNT}; + +/// The mutable half of a mask: the packed row bits. +#[derive(Debug)] +pub struct MaskWords { + /// The packed row bits. Allocated once; the buffer never moves. + pub words: Box<[u64]>, +} + +/// What a handle refers to. +/// +/// `Pattern` is unlocked on purpose (read-only lanes); `Mask` carries its own +/// `RwLock` so bulk ops on distinct masks do not serialize. +#[derive(Debug)] +pub enum Payload { + /// A read-only SoA fixture — no lock needed, because no ABI path mutates it. + Pattern(Fixture), + /// Mutable mask words behind their own lock. + Mask(RwLock), +} + +/// One registry entry, shared by `Arc` so a call can drop the registry lock and +/// still hold its resource alive for the duration of the work. +#[derive(Debug)] +pub struct ResourceEntry { + /// `LGJ_RESOURCE_PATTERN` | `LGJ_RESOURCE_MASK`. + pub kind: u32, + /// Liveness stamp. Globally unique and monotonic, so a *reused slot* never + /// reuses an epoch — which is what lets Java notice a `MemorySegment` it + /// still holds belongs to a dead resource. + pub epoch: u64, + /// Logical rows. + pub n_rows: u64, + /// Parent handle, `0` for a pattern. + pub parent: u64, + /// The parent's generation at creation time. Re-checked on every mask + /// operation so a mask whose parent was closed reports `PARENT_CLOSED` + /// rather than operating against a dead pattern. + pub parent_gen: u32, + /// The resource's data. + pub payload: Payload, +} + +impl ResourceEntry { + /// `Some(&Fixture)` iff this is a pattern. + pub fn fixture(&self) -> Option<&Fixture> { + match &self.payload { + Payload::Pattern(f) => Some(f), + Payload::Mask(_) => None, + } + } + + /// `Some(&RwLock)` iff this is a mask. + pub fn mask(&self) -> Option<&RwLock> { + match &self.payload { + Payload::Mask(m) => Some(m), + Payload::Pattern(_) => None, + } + } + + /// Read-lock the mask words, recovering from poisoning (see module header). + pub fn read_mask(&self) -> Option> { + self.mask() + .map(|m| m.read().unwrap_or_else(|e| e.into_inner())) + } + + /// Write-lock the mask words, recovering from poisoning (see module header). + pub fn write_mask(&self) -> Option> { + self.mask() + .map(|m| m.write().unwrap_or_else(|e| e.into_inner())) + } + + /// Fill an [`LgjResourceInfo`]. Takes no handle: the description is about + /// the resource, and the caller already knows the handle it passed in. + pub fn info(&self) -> LgjResourceInfo { + LgjResourceInfo { + kind: self.kind, + lane_count: match self.kind { + LGJ_RESOURCE_PATTERN => PATTERN_LANE_COUNT, + // A mask exposes exactly one MASK_WORD lane. + _ => 1, + }, + n_rows: self.n_rows, + epoch: self.epoch, + parent: self.parent, + } + } +} + +/// A registry slot. `entry == None` means free; `generation` is bumped on every +/// free so handles pointing here go stale. +struct Slot { + generation: u32, + entry: Option>, +} + +static REGISTRY: OnceLock>> = OnceLock::new(); +static EPOCH: AtomicU64 = AtomicU64::new(1); + +fn registry() -> &'static RwLock> { + REGISTRY.get_or_init(|| RwLock::new(Vec::new())) +} + +fn next_epoch() -> u64 { + EPOCH.fetch_add(1, Ordering::Relaxed) +} + +/// `(generation << 32) | index`. +pub const fn encode_handle(generation: u32, index: u32) -> u64 { + ((generation as u64) << 32) | (index as u64) +} + +/// Split a handle into `(generation, index)`. Every value of `u64` decodes to +/// *something*; validity is decided by [`resolve`], never here. +pub const fn decode_handle(handle: u64) -> (u32, u32) { + ((handle >> 32) as u32, (handle & 0xFFFF_FFFF) as u32) +} + +/// Install an entry and return its handle. +pub fn insert(entry: ResourceEntry) -> Result { + let mut reg = registry().write().unwrap_or_else(|e| e.into_inner()); + let arc = Arc::new(entry); + + // Reuse a free slot if there is one; its generation is already ahead of any + // handle that used to point at it. + if let Some((idx, slot)) = reg.iter_mut().enumerate().find(|(_, s)| s.entry.is_none()) { + slot.entry = Some(arc); + return Ok(encode_handle(slot.generation, idx as u32)); + } + + let idx = reg.len(); + if idx > u32::MAX as usize { + return Err(LGJ_ERR_ALLOCATION_FAILED); + } + // Generation 1, never 0 — see this module's header. + reg.push(Slot { + generation: 1, + entry: Some(arc), + }); + Ok(encode_handle(1, idx as u32)) +} + +/// Resolve a handle to its entry, validating the generation. +/// +/// Takes a short read lock, clones the `Arc`, and drops the lock — so the +/// caller does its bulk work without holding the registry. +pub fn resolve(handle: u64) -> Result, i32> { + let (gen, idx) = decode_handle(handle); + let reg = registry().read().unwrap_or_else(|e| e.into_inner()); + let slot = reg.get(idx as usize).ok_or(LGJ_ERR_INVALID_HANDLE)?; + if slot.generation != gen { + // Closed and the slot was reused, or fabricated outright. + return Err(LGJ_ERR_INVALID_HANDLE); + } + let entry = slot.entry.clone().ok_or(LGJ_ERR_INVALID_HANDLE)?; + Ok(entry) + // `reg` drops here; nothing below this point holds the registry lock. +} + +/// Resolve and require a specific resource kind. +pub fn resolve_kind(handle: u64, kind: u32) -> Result, i32> { + let e = resolve(handle)?; + if e.kind != kind { + return Err(LGJ_ERR_WRONG_RESOURCE_KIND); + } + Ok(e) +} + +/// Resolve a mask **and** prove its parent is still alive. +/// +/// Returns `(mask, parent)`. A mask whose parent was closed yields +/// `PARENT_CLOSED` — it may still *exist*, it just cannot *work* (§4). +pub fn resolve_mask_with_parent( + handle: u64, +) -> Result<(Arc, Arc), i32> { + let mask = resolve_kind(handle, LGJ_RESOURCE_MASK)?; + let (pgen, pidx) = decode_handle(mask.parent); + let reg = registry().read().unwrap_or_else(|e| e.into_inner()); + let slot = reg.get(pidx as usize).ok_or(LGJ_ERR_PARENT_CLOSED)?; + if slot.generation != pgen || pgen != mask.parent_gen { + return Err(LGJ_ERR_PARENT_CLOSED); + } + let parent = slot.entry.clone().ok_or(LGJ_ERR_PARENT_CLOSED)?; + drop(reg); + Ok((mask, parent)) +} + +/// Free a slot: take the entry out and bump the generation, so every handle +/// that pointed here is now stale. +/// +/// Returns `INVALID_HANDLE` on a stale/fabricated handle — which is also what +/// makes a *double* close fail cleanly rather than free twice. +pub fn close(handle: u64) -> Result<(), i32> { + let mut reg = registry().write().unwrap_or_else(|e| e.into_inner()); + let (gen, idx) = decode_handle(handle); + let slot = reg.get_mut(idx as usize).ok_or(LGJ_ERR_INVALID_HANDLE)?; + if slot.generation != gen || slot.entry.is_none() { + return Err(LGJ_ERR_INVALID_HANDLE); + } + // Take the entry out FIRST, then bump: after this point no new resolve can + // succeed for `handle`. + let entry = slot.entry.take(); + // Wrapping is documented rather than ignored: after 2^32 closes of the same + // slot a generation repeats. `saturating` would be worse (it would freeze a + // generation and make every stale handle valid forever). + slot.generation = slot.generation.wrapping_add(1); + if slot.generation == 0 { + slot.generation = 1; // never hand out generation 0 + } + drop(reg); + // Drop the Arc outside the registry lock. Other in-flight calls may still + // hold clones; the storage is freed when the last one goes, which is why a + // concurrent bulk op cannot be reading freed lanes. + drop(entry); + Ok(()) +} + +/// Create a pattern resource from a deterministic fixture. +pub fn open_pattern(n_rows: u64, seed: u64) -> Result { + let fixture = Fixture::generate(n_rows, seed).ok_or(LGJ_ERR_LENGTH_OVERFLOW)?; + insert(ResourceEntry { + kind: LGJ_RESOURCE_PATTERN, + epoch: next_epoch(), + n_rows, + parent: 0, + parent_gen: 0, + payload: Payload::Pattern(fixture), + }) +} + +/// Create a mask over `parent`, all bits `0` or all bits `1`. +pub fn create_mask(parent_handle: u64, initial: u32) -> Result { + let parent = resolve_kind(parent_handle, LGJ_RESOURCE_PATTERN)?; + let n_rows = parent.n_rows; + let n_words = usize::try_from(mask_words_for(n_rows)).map_err(|_| LGJ_ERR_LENGTH_OVERFLOW)?; + + let fill = match initial { + LGJ_MASK_INIT_EMPTY => 0u64, + LGJ_MASK_INIT_ALL => u64::MAX, + // An out-of-range `initial` is a caller bug, not an alternative default. + _ => return Err(LGJ_ERR_NULL_ARGUMENT), + }; + let mut words = Vec::new(); + words + .try_reserve_exact(n_words) + .map_err(|_| LGJ_ERR_ALLOCATION_FAILED)?; + words.resize(n_words, fill); + let mut words = words.into_boxed_slice(); + // Bits past n_rows are normative zero even for an "all" mask. + clear_tail_bits(&mut words, n_rows); + + let (pgen, _) = decode_handle(parent_handle); + insert(ResourceEntry { + kind: LGJ_RESOURCE_MASK, + epoch: next_epoch(), + n_rows, + parent: parent_handle, + parent_gen: pgen, + payload: Payload::Mask(RwLock::new(MaskWords { words })), + }) +} + +/// Write-lock up to three **distinct** mask entries in address order. +/// +/// Address order is a global order, so no two threads can build a lock cycle. +/// The returned array is indexed **by role** (the caller's original argument +/// position), while acquisition happened in address order — that separation is +/// the whole point. +/// +/// # Panics / misuse +/// +/// The caller must pass pairwise-distinct entries (`Arc::ptr_eq` false). Passing +/// the same entry twice would deadlock on its own `RwLock`; callers dedup first +/// (see `exports::mask_binop`). +#[allow(clippy::type_complexity)] +pub fn lock_masks_ordered<'a>( + entries: &[&'a Arc], +) -> Result<[Option>; 3], i32> { + debug_assert!(entries.len() <= 3); + let mut order: Vec = (0..entries.len()).collect(); + order.sort_by_key(|&i| Arc::as_ptr(entries[i]) as usize); + + let mut guards: [Option>; 3] = [None, None, None]; + for &role in &order { + let lock = entries[role].mask().ok_or(LGJ_ERR_WRONG_RESOURCE_KIND)?; + guards[role] = Some(lock.write().unwrap_or_else(|e| e.into_inner())); + } + Ok(guards) +} + +/// Test-only: how many slots the registry has ever needed. Used to prove slot +/// reuse actually happens (and therefore that generation checking is load +/// bearing, not decorative). +#[cfg(test)] +pub fn slot_count() -> usize { + registry().read().unwrap_or_else(|e| e.into_inner()).len() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn handle_round_trips() { + for (g, i) in [(1u32, 0u32), (7, 3), (u32::MAX, u32::MAX), (2, 0)] { + assert_eq!(decode_handle(encode_handle(g, i)), (g, i)); + } + } + + #[test] + fn a_live_handle_resolves() { + let h = open_pattern(100, 1).unwrap(); + let e = resolve(h).unwrap(); + assert_eq!(e.kind, LGJ_RESOURCE_PATTERN); + assert_eq!(e.n_rows, 100); + close(h).unwrap(); + } + + #[test] + fn handles_are_never_zero() { + let h = open_pattern(8, 1).unwrap(); + assert_ne!(h, 0, "generation starts at 1 so handle 0 is unreachable"); + close(h).unwrap(); + } + + #[test] + fn fabricated_handles_are_rejected_not_dereferenced() { + for bogus in [0u64, 1, 0xDEAD_BEEF, u64::MAX, encode_handle(0, 0)] { + assert_eq!(resolve(bogus).unwrap_err(), LGJ_ERR_INVALID_HANDLE); + assert_eq!(close(bogus).unwrap_err(), LGJ_ERR_INVALID_HANDLE); + } + } + + #[test] + fn use_after_close_is_a_status_not_a_crash() { + let h = open_pattern(64, 1).unwrap(); + close(h).unwrap(); + assert_eq!(resolve(h).unwrap_err(), LGJ_ERR_INVALID_HANDLE); + } + + #[test] + fn double_close_fails_the_second_time() { + let h = open_pattern(64, 1).unwrap(); + assert!(close(h).is_ok()); + assert_eq!(close(h).unwrap_err(), LGJ_ERR_INVALID_HANDLE); + } + + /// The generation check earns its keep only if slots are actually reused. + /// Prove reuse happens *and* that the old handle still fails afterwards. + #[test] + fn a_reused_slot_invalidates_the_old_handle() { + let before = slot_count(); + let h1 = open_pattern(32, 1).unwrap(); + close(h1).unwrap(); + let h2 = open_pattern(32, 1).unwrap(); + let (g1, i1) = decode_handle(h1); + let (g2, i2) = decode_handle(h2); + if i1 == i2 { + // Slot really was reused: same index, higher generation. + assert!(g2 > g1); + assert_eq!(resolve(h1).unwrap_err(), LGJ_ERR_INVALID_HANDLE); + assert!(resolve(h2).is_ok()); + } + assert!(slot_count() >= before); + close(h2).unwrap(); + } + + #[test] + fn wrong_kind_is_distinguished_from_invalid() { + let p = open_pattern(64, 1).unwrap(); + let m = create_mask(p, LGJ_MASK_INIT_ALL).unwrap(); + assert_eq!( + resolve_kind(m, LGJ_RESOURCE_PATTERN).unwrap_err(), + LGJ_ERR_WRONG_RESOURCE_KIND + ); + assert_eq!( + resolve_kind(p, LGJ_RESOURCE_MASK).unwrap_err(), + LGJ_ERR_WRONG_RESOURCE_KIND + ); + close(m).unwrap(); + close(p).unwrap(); + } + + #[test] + fn mask_over_closed_parent_reports_parent_closed() { + let p = open_pattern(200, 5).unwrap(); + let m = create_mask(p, LGJ_MASK_INIT_ALL).unwrap(); + assert!(resolve_mask_with_parent(m).is_ok()); + close(p).unwrap(); + assert_eq!( + resolve_mask_with_parent(m).unwrap_err(), + LGJ_ERR_PARENT_CLOSED + ); + // The mask handle itself still resolves — it exists, it just cannot work. + assert!(resolve(m).is_ok()); + close(m).unwrap(); + } + + #[test] + fn mask_initial_states_are_exact() { + let p = open_pattern(70, 1).unwrap(); + let empty = create_mask(p, LGJ_MASK_INIT_EMPTY).unwrap(); + let all = create_mask(p, LGJ_MASK_INIT_ALL).unwrap(); + + let e = resolve(empty).unwrap(); + let g = e.read_mask().unwrap(); + assert_eq!(g.words.len(), 2); + assert!(g.words.iter().all(|&w| w == 0)); + drop(g); + + let a = resolve(all).unwrap(); + let g = a.read_mask().unwrap(); + assert_eq!(g.words[0], u64::MAX); + assert_eq!(g.words[1], 0x3F, "tail bits past row 70 must be zero"); + drop(g); + + close(all).unwrap(); + close(empty).unwrap(); + close(p).unwrap(); + } + + #[test] + fn bad_initial_value_is_rejected() { + let p = open_pattern(8, 1).unwrap(); + assert!(create_mask(p, 2).is_err()); + assert!(create_mask(p, u32::MAX).is_err()); + close(p).unwrap(); + } + + #[test] + fn mask_cannot_be_created_over_a_mask() { + let p = open_pattern(8, 1).unwrap(); + let m = create_mask(p, 0).unwrap(); + assert_eq!(create_mask(m, 0).unwrap_err(), LGJ_ERR_WRONG_RESOURCE_KIND); + close(m).unwrap(); + close(p).unwrap(); + } + + /// Locking distinct masks in address order must not deadlock regardless of + /// the order the caller names them in. + #[test] + fn ordered_locking_is_order_insensitive() { + let p = open_pattern(128, 1).unwrap(); + let a = create_mask(p, 0).unwrap(); + let b = create_mask(p, 0).unwrap(); + let c = create_mask(p, 0).unwrap(); + let (ea, eb, ec) = ( + resolve(a).unwrap(), + resolve(b).unwrap(), + resolve(c).unwrap(), + ); + for perm in [[&ea, &eb, &ec], [&ec, &eb, &ea], [&eb, &ec, &ea]] { + let g = lock_masks_ordered(&perm).unwrap(); + assert!(g[0].is_some() && g[1].is_some() && g[2].is_some()); + } + for h in [c, b, a, p] { + close(h).unwrap(); + } + } +} From bdd40b2c22f9b5e285681de2dc494b6b12633e19 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 21:30:04 +0000 Subject: [PATCH 02/30] Valhalla lab: three-truths method, causal isolation, 3 real reproducers Completes D-LGJ-F. One experiment source (src/shared/), compiled twice against real JDKs -- stable JDK 26 GA (record) and the official JEP 401 early-access binary (value record) -- via a self-verifying run.sh that mechanically diffs the two Vocab.java files modulo the 'value' keyword before trusting the A/B is honest. Experiments: IdentityExperiment (semantic truth -- is identity actually unobservable), FootprintExperiment (real per-object/array/field bytes via allocation-delta + JOL where available), FfmAddressingExperiment (is the wrapper free where it touches native memory), ThesisExperiment (the mandatory headline: 65,536 rows as one native lane vs hydrated Java objects, on both platforms). Causal isolation via three additional run.sh passes: escape analysis off, and UseArrayFlattening/UseFieldFlattening toggled independently -- isolates which flag actually drives the measured difference rather than inferring it. Three real Valhalla limitations reproduced and filed under reproducers/, none of which changed the production API: - R1: @NullRestricted field on an identity class is a VerifyError (javac's fault -- no source form expresses the required strict-field init order relative to super()) - R2: array flattening has a hard 8-byte payload cliff, confirmed via -XX:+PrintFlatArrayLayout. LaneId/Ordinal/MaskId (<=8B) flatten; RowRange/Row (16B) do not. This turns "Valhalla helps descriptors, not entities" from a hand-wave into a measured VM cutoff -- and RowRange landing on the wrong side is flagged as the one place the expectation was too optimistic. - R3: the densest null-restricted array form is jdk.internal-only and generics erase flattening entirely; Foo! null-restricted type syntax confirmed not to parse, matching the earlier archaeology finding. One real defect found and fixed before landing: IdentityExperiment and the stable Platform called Class::isValue() directly on four vocabulary types with a comment incorrectly claiming it was "final API on JDK 26" -- it does not exist there at all, confirmed by a real javac failure. Fixed by routing every query through Platform.isValueClass(Class), answered honestly per platform. Generated by [Claude Code](https://claude.ai/code) --- .claude/board/EPIPHANIES.md | 51 ++++++ .claude/board/STATUS_BOARD.md | 4 +- .gitignore | 4 + valhalla-lab/README.md | 100 +++++++++++ valhalla-lab/reproducers/R1-observed.txt | 4 + ...R1_NullRestrictedFieldInIdentityClass.java | 36 ++++ valhalla-lab/reproducers/R2-observed.txt | 8 + .../reproducers/R2_FlatteningCliff.java | 40 +++++ .../reproducers/R3-bang-syntax-observed.txt | 6 + valhalla-lab/reproducers/R3-observed.txt | 4 + .../R3_NoSupportedFlatSurface.java | 54 ++++++ valhalla-lab/reproducers/README.md | 159 ++++++++++++++++ valhalla-lab/results/AB-default.diff | 132 ++++++++++++++ valhalla-lab/results/stable-api-javac.log | 1 + valhalla-lab/results/stable-default.txt | 91 ++++++++++ valhalla-lab/results/stable-lab-javac.log | 1 + valhalla-lab/results/stable-noea.txt | 91 ++++++++++ valhalla-lab/results/valhalla-api-javac.log | 1 + valhalla-lab/results/valhalla-default.txt | 91 ++++++++++ valhalla-lab/results/valhalla-lab-javac.log | 3 + valhalla-lab/results/valhalla-noarrayflat.txt | 91 ++++++++++ valhalla-lab/results/valhalla-noea.txt | 91 ++++++++++ valhalla-lab/results/valhalla-nofieldflat.txt | 91 ++++++++++ valhalla-lab/results/valhalla-noflat.txt | 91 ++++++++++ valhalla-lab/results/vocab-diff.txt | 0 valhalla-lab/run.sh | 99 ++++++++++ .../adaworldapi/lancegraph/NativeAccess.java | 49 +++++ .../lab/FfmAddressingExperiment.java | 86 +++++++++ .../lab/FlatteningCliffExperiment.java | 46 +++++ .../lancegraph/lab/FootprintExperiment.java | 111 ++++++++++++ .../lancegraph/lab/IdentityExperiment.java | 66 +++++++ .../com/adaworldapi/lancegraph/lab/Lab.java | 144 +++++++++++++++ .../adaworldapi/lancegraph/lab/RunAll.java | 46 +++++ .../lancegraph/lab/ThesisExperiment.java | 170 ++++++++++++++++++ .../lancegraph/lab/Containers.java | 31 ++++ .../adaworldapi/lancegraph/lab/Platform.java | 68 +++++++ .../com/adaworldapi/lancegraph/lab/Vocab.java | 61 +++++++ .../lancegraph/lab/Containers.java | 46 +++++ .../adaworldapi/lancegraph/lab/Platform.java | 65 +++++++ .../com/adaworldapi/lancegraph/lab/Vocab.java | 61 +++++++ 40 files changed, 2392 insertions(+), 2 deletions(-) create mode 100644 valhalla-lab/README.md create mode 100644 valhalla-lab/reproducers/R1-observed.txt create mode 100644 valhalla-lab/reproducers/R1_NullRestrictedFieldInIdentityClass.java create mode 100644 valhalla-lab/reproducers/R2-observed.txt create mode 100644 valhalla-lab/reproducers/R2_FlatteningCliff.java create mode 100644 valhalla-lab/reproducers/R3-bang-syntax-observed.txt create mode 100644 valhalla-lab/reproducers/R3-observed.txt create mode 100644 valhalla-lab/reproducers/R3_NoSupportedFlatSurface.java create mode 100644 valhalla-lab/reproducers/README.md create mode 100644 valhalla-lab/results/AB-default.diff create mode 100644 valhalla-lab/results/stable-api-javac.log create mode 100644 valhalla-lab/results/stable-default.txt create mode 100644 valhalla-lab/results/stable-lab-javac.log create mode 100644 valhalla-lab/results/stable-noea.txt create mode 100644 valhalla-lab/results/valhalla-api-javac.log create mode 100644 valhalla-lab/results/valhalla-default.txt create mode 100644 valhalla-lab/results/valhalla-lab-javac.log create mode 100644 valhalla-lab/results/valhalla-noarrayflat.txt create mode 100644 valhalla-lab/results/valhalla-noea.txt create mode 100644 valhalla-lab/results/valhalla-nofieldflat.txt create mode 100644 valhalla-lab/results/valhalla-noflat.txt create mode 100644 valhalla-lab/results/vocab-diff.txt create mode 100755 valhalla-lab/run.sh create mode 100644 valhalla-lab/src/shared/com/adaworldapi/lancegraph/NativeAccess.java create mode 100644 valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/FfmAddressingExperiment.java create mode 100644 valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/FlatteningCliffExperiment.java create mode 100644 valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/FootprintExperiment.java create mode 100644 valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/IdentityExperiment.java create mode 100644 valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/Lab.java create mode 100644 valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/RunAll.java create mode 100644 valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/ThesisExperiment.java create mode 100644 valhalla-lab/src/stable/com/adaworldapi/lancegraph/lab/Containers.java create mode 100644 valhalla-lab/src/stable/com/adaworldapi/lancegraph/lab/Platform.java create mode 100644 valhalla-lab/src/stable/com/adaworldapi/lancegraph/lab/Vocab.java create mode 100644 valhalla-lab/src/valhalla/com/adaworldapi/lancegraph/lab/Containers.java create mode 100644 valhalla-lab/src/valhalla/com/adaworldapi/lancegraph/lab/Platform.java create mode 100644 valhalla-lab/src/valhalla/com/adaworldapi/lancegraph/lab/Vocab.java diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 05eeae2..d3cb4fb 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -4,6 +4,57 @@ > `**Status:**`/`**Confidence:**` line. A correction gets its own new, > dated entry that references the one it corrects — the storno rule. +## 2026-08-17 — E-LGJ-VALHALLA-MEASURED-NOT-ASSUMED-1 + +**Status:** FINDING. **Confidence:** High (real numbers, both JDKs actually +run, reproducible via `valhalla-lab/README.md`). + +The mandatory N-objects-vs-N-values-vs-1-lane experiment +(`.claude/knowledge/valhalla-three-truths-method.md`'s "one experiment that +must never be skipped") ran on both real JDKs. Headline, on 65,536 rows, +identical question, identical answer on every path: + +| | native, one crossing | hydrate 65,536 `Row`, then scan | +|---|---:|---:| +| stable JDK 26 | 19.5 µs, 289 KiB | 746 µs, 2.00 MiB | +| Valhalla JDK 27 EA | 15.7 µs, 289.5 KiB | 900 µs, 2.50 MiB | + +**The thesis's prediction held, and the reason why is itself a measured +finding, not an assumption:** `LaneId` (one field) measured `FLAT` under +Valhalla via the real VM query `ValueClass.isFlatArray` (2.90 B/element vs +16.00 B on stable — ~5.5× smaller), but `Row` (multiple fields) measured +**`NOT-FLAT`** even under Valhalla, and its per-row heap cost (40.01 B) was +*larger* than the stable JDK's own record-array cost (32.01 B). Valhalla +genuinely helps a single-field descriptor; it did not flatten the +multi-field materialization the thesis explicitly said to check rather +than assume away. + +**One real defect found and fixed before this landed** — a bug of the +falsifiability-discipline-caught-it, not the happy-path-hid-it kind. The +first version of `IdentityExperiment` and the stable-JDK `Platform` called +`Class::isValue()` directly on four vocabulary types, with a comment +incorrectly asserting *"Class::isValue is final API on JDK 26."* It does +not exist there at all — confirmed by a real `javac` compile failure, not +by re-reading documentation. Fixed by routing every identity query through +`Platform.isValueClass(Class)`: the stable half answers `false` +honestly (a JDK with no value-class concept can never produce one — the +answer is exact, not a guess, unlike the genuinely-unknowable +`arrayFlatness` case the same file already handles correctly), the +Valhalla half answers with the real `type.isValue()`. The correction +mirrors `E-LGJ-CORE-SLICE-GREEN-DISABLE-VERIFIED-1`'s finding about +`kernels.rs`: an agent's own doc comment stated the WRONG fact confidently +one line above the code that relied on it, and only compiling both +variants for real (not trusting the report that they "should" compile) +caught it. + +**Two javac usage facts worth keeping** (real dead ends this session hit +and resolved, recorded so a future session doesn't re-hit them): +`--release N` cannot be combined with `--add-exports` for a system module +(a hard javac restriction, not a bug) — use `-source N` instead when +compiling for the same JDK you'll run on; and `--enable-preview` requires +an explicit `-source`/`--release` to be present at all, it is not +self-sufficient. + ## 2026-08-17 — E-LGJ-CORE-SLICE-GREEN-DISABLE-VERIFIED-1 **Status:** FINDING. **Confidence:** High (measured, not asserted — every diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 112599f..f4be4bc 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -13,8 +13,8 @@ list. | D-LGJ-C | `native/lgj-abi` — manifest, generation-checked registry, generic SoA fixture, kernels, `extern "C"` surface | **DONE 2026-08-17** — `cargo test` **72/72**, `clippy -D warnings` clean, `fmt --check` clean, release build → 14/14 symbols verified via `nm -D`. **Disable-verified**: the registry's generation check was short-circuited and exactly the 2 tests that should catch it went red, 70 stayed green; restored, re-verified 72/72 | D, H | | D-LGJ-D | Java FFM membrane `internal/ffm` | **DONE 2026-08-17** — compiles clean with `-Xlint:all`; 7 `[restricted]` warnings, all in `internal/ffm/*` or a test deliberately exercising it; `AbiContractTest` 7/7 incl. proving the manifest cross-check genuinely rejects a wrong `.so` (`libz.so.1` loads but is refused for exporting no `lgj_abi_manifest`) | E | | D-LGJ-E | Java public facade (`NativePattern`/`View`/`Predicate`/`Pattern`/`Mask`) | **DONE 2026-08-17** — `AllTests` **132/132**: `ApiSurfaceTest` (reflection-enforced zero-FFM-leakage), `SmokeTest` 14/14, `FixtureParityTest` 30/30 (Java independently recomputes expected counts from the transcribed generator), `FusionParityTest` 31/31 (fused/unfused/scalar bit-identical across 6 row-count shapes), `LazinessTest` 8/8 (empirically: 0 crossings to build a 16-condition chain, exactly 1 to evaluate it, independent of rows up to 1,000,000 — the thesis's central claim, measured), `NarrowingTest` 16/16, `LifetimeTest` 23/23 | F, G | -| D-LGJ-F | Valhalla lab — three-truths method on the small semantic value vocabulary | **In flight** — sequenced after E, now reading the real Java types; deferred to a follow-up PR, not blocking PR #1 | I | -| D-LGJ-G | Java Vector API comparative bench vs Panama→`ndarray::simd` | **In flight** — real JMH jars fetched (`jmh-core`/`jmh-generator-annprocess`/`jopt-simple`/`commons-math3`) to `bench/lib/` (gitignored); no bench source written yet; deferred to the same follow-up PR as F | I | +| D-LGJ-F | Valhalla lab — three-truths method on the small semantic value vocabulary | **DONE 2026-08-17** — `valhalla-lab/`: 4 experiments + a self-verifying `run.sh` (mechanically diffs the two `Vocab.java`s modulo the `value` keyword before trusting the A/B) + 3 causal-isolation runs (escape-analysis off; `UseArrayFlattening`/`UseFieldFlattening` toggled independently). 3 real Valhalla limitations reproduced and filed under `reproducers/` (R1: `@NullRestricted` field on an identity class is a `VerifyError`, javac's fault — no source form expresses required strict-field order; **R2: array flattening has a hard 8-byte payload cliff, VM-confirmed via `-XX:+PrintFlatArrayLayout`** — `LaneId`/`Ordinal`/`MaskId` (≤8B) flatten, `RowRange`/`Row` (16B) do not, so "Valhalla helps descriptors not entities" is a measured VM cutoff, not a hand-wave, and `RowRange` landing on the wrong side is flagged as the one place the expectation was too optimistic; R3: the densest null-restricted array form is `jdk.internal`-only and generics erase flattening entirely — `Foo!` type syntax confirmed NOT to parse, matching the archaeology finding). 1 real defect found + fixed before landing (see `EPIPHANIES.md`). None of the three limitations changed the production API — the migration path stays exactly `record` → `value record` | I | +| D-LGJ-G | Java Vector API comparative bench vs Panama→`ndarray::simd` | **In flight** — real JMH + JOL jars fetched (`jmh-core`/`jmh-generator-annprocess`/`jopt-simple`/`commons-math3`/`jol-core`) to `bench/lib/` (gitignored); no bench source written yet; the ONLY remaining open row | I | | D-LGJ-H | Falsification: handle lifecycle (adversarial), SIMD/scalar parity, Java/native parity | **DONE 2026-08-17 for the Rust+Java core** — see D-LGJ-C's disable-verification and D-LGJ-E's `FusionParityTest`/`LifetimeTest`. Re-opens for F/G once the Lab lands | I | | D-LGJ-I | Docs: `architecture.md`, `panama.md`, `valhalla-lab.md`, `execution-boundary.md` | **Queued** — gated on F/G landing (the docs synthesize Lab results, not just the core) | — | | D-LGJ-AUDIT | Mechanical post-fan-out audit: `grep` for `ndarray::hpc` imports, any `.h`/`cbindgen`/`jextract` artifact, any FFM type leaking into public Java API | **DONE 2026-08-17** — 1 real violation found (`kernels.rs::simd_popcount` used the internal `ndarray::hpc::bitwise` path), fixed in place; everything else confirmed to be the one sanctioned exception or explanatory prose | closed D-LGJ-C/D/E for the core | diff --git a/.gitignore b/.gitignore index 0d6f235..8c7e5b4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,10 @@ Cargo.lock.bak /bench/out/ /bench/lib/*.jar /valhalla-lab/out/ +# run.sh's own compiled output (results/*-api, results/*-lab class trees) — the +# .txt/.diff/.log evidence files alongside them ARE committed, the compiled +# classes are pure build residue, regenerated by ./run.sh on demand. +/valhalla-lab/results/*/ # Downloaded JDKs and artifacts (never committed — see docs/abi.md and # .claude/knowledge/jdk-toolchain-facts.md for how to obtain them) diff --git a/valhalla-lab/README.md b/valhalla-lab/README.md new file mode 100644 index 0000000..8255c41 --- /dev/null +++ b/valhalla-lab/README.md @@ -0,0 +1,100 @@ +# The Valhalla laboratory + +The three-truths method (`.claude/knowledge/valhalla-three-truths-method.md`) applied to this +project's small semantic value vocabulary — `LaneId`, `Ordinal`, `MaskId`, `RowRange`, `Row` — and +to the mission's mandatory headline experiment: does Valhalla rescue per-entity materialization at +65,536-row scale, or only the tiny descriptor vocabulary around it? + +**Same experiment source, compiled twice** — once against a stable JDK where the vocabulary types +are plain `record`s, once against the JEP 401 early-access JDK where they are `value record`s — so +the comparison is genuinely apples-to-apples, not two different programs. + +## Layout + +``` +src/shared/ experiment logic, byte-identical on both compiles +src/stable/ Vocab.java (record), Containers.java, Platform.java — the stable-JDK half of the A/B +src/valhalla/ Vocab.java (value record), Containers.java, Platform.java — the Valhalla half +``` + +`Platform` is the one seam between them: same signatures on both sides, so `src/shared/` never +branches on which platform it's running on except by asking `Platform` — never by calling a +Valhalla-only API (like `Class::isValue` or `jdk.internal.value.ValueClass`) directly. That is a +real rule, not a style preference: `Class::isValue` does not exist at all on a stable JDK, so a +direct call would fail to *compile* the stable half, not just report the wrong answer. + +`NativeAccess` (in `src/shared/`, package `com.adaworldapi.lancegraph`) is a read-only, split-package +escape hatch into the shipped library's package-private handle — documented in the file itself. It +exists because the lab has to build the very thing the thesis says you should not build (65,536 Java +objects) from the *same bytes* the native kernel reads, or the comparison proves nothing. Nothing +under `java/` changes to support this. + +## Build and run + +### Stable half (JDK 26 GA, plain `record`s) + +```sh +javac -d out-stable $(find ../java/src/main/java src/shared src/stable -name '*.java') + +java --enable-native-access=ALL-UNNAMED \ + -Dlgj.library=../target/release/liblgj_abi.so \ + -cp out-stable com.adaworldapi.lancegraph.lab.RunAll +``` + +### Valhalla half (the JEP 401 EA build, `value record`s) + +`--release` cannot be combined with `--add-exports` (a real javac restriction — `--release` uses a +stricter cross-compilation module model). Use `-source` instead when compiling *for* the JDK you are +also running on, which is the case here. + +```sh +javac --enable-preview -source 27 \ + --add-exports java.base/jdk.internal.value=ALL-UNNAMED \ + --add-exports java.base/jdk.internal.vm.annotation=ALL-UNNAMED \ + -d out-valhalla $(find ../java/src/main/java src/shared src/valhalla -name '*.java') + +java --enable-preview --enable-native-access=ALL-UNNAMED \ + --add-exports java.base/jdk.internal.value=ALL-UNNAMED \ + --add-exports java.base/jdk.internal.vm.annotation=ALL-UNNAMED \ + -Dlgj.library=../target/release/liblgj_abi.so \ + -cp out-valhalla com.adaworldapi.lancegraph.lab.RunAll +``` + +Both need the JDK paths from `.claude/knowledge/jdk-toolchain-facts.md` — do not use `/usr/bin/java` +(JDK 21, no value classes at all) for either. + +## What each experiment measures + +| Class | Question | +|---|---| +| `IdentityExperiment` | Truth (a), semantic: is identity actually unobservable? `Class::isValue`, reference equality, array flatness, `synchronized` legality — measured on both platforms, asked to agree everywhere except reference equality (which no caller in the production API uses). | +| `FootprintExperiment` | Truth (b)/(c), representation: per-object bytes, array layout, field flattening, call-argument passing — via `jol-core`'s real VM instrumentation where available, allocation-delta measurement elsewhere. | +| `FfmAddressingExperiment` | Is the wrapper free where it actually touches native memory — a `RowRange`/`Ordinal` around an FFM offset vs a bare `long`? | +| `ThesisExperiment` | The mandatory headline: 65,536 rows as (1) one native lane + one packed mask + one crossing, vs (2)/(3) hydrated Java objects, on the SAME question and the SAME answer. Heap cost and wall time, both platforms. | + +## Measured headline (2026-08-17, this environment) + +Real numbers from a real run — reproduce with the commands above before citing a different number. + +| | native, one crossing | hydrate 65,536 `Row`, then scan | +|---|---:|---:| +| stable JDK 26 | 19.5 µs, 289 KiB Java-side | 746 µs, 2.00 MiB | +| Valhalla (JDK 27 EA) | 15.7 µs, 289.5 KiB Java-side | 900 µs, 2.50 MiB | + +The native path wins by roughly **38–57×** on time and **7–9×** on Java heap, on **both** platforms — +Valhalla does not close this gap, because `Row` (multiple fields) measured `NOT-FLAT` even under +Valhalla, while the single-field `LaneId` measured `FLAT` (2.90 B/element vs 16.00 B on stable, ~5.5× +smaller). This is the mission thesis's prediction, confirmed rather than assumed: **Valhalla helps +the tiny descriptor vocabulary; it does not rescue per-entity materialization at this scale.** See +`IdentityExperiment`'s and `FootprintExperiment`'s full output for the field-by-field evidence. + +## A defect found and fixed while wiring this up + +The first version of `IdentityExperiment`/stable `Platform` called `Class::isValue()` directly for +four of the five vocabulary types (`Ordinal`/`MaskId`/`RowRange`/`Row`), with a comment incorrectly +claiming it was "final API on JDK 26." It is not — `javac` on JDK 26 GA does not have that method at +all, confirmed by a real compile failure, not by reading documentation. Fixed by routing every +identity query through `Platform.isValueClass(Class)`, which the stable half answers `false` (a +JDK with no value-class concept can never produce one, so the answer is exact, not a guess) and the +Valhalla half answers with the real `type.isValue()`. See `EPIPHANIES.md` +`E-LGJ-CORE-SLICE-GREEN-DISABLE-VERIFIED-1` for the audit discipline this caught it under. diff --git a/valhalla-lab/reproducers/R1-observed.txt b/valhalla-lab/reproducers/R1-observed.txt new file mode 100644 index 0000000..f03f8ba --- /dev/null +++ b/valhalla-lab/reproducers/R1-observed.txt @@ -0,0 +1,4 @@ +Picked up JAVA_TOOL_OPTIONS: +identity container FAILED: java.lang.VerifyError + All strict final fields must be initialized before super(): 1 field(s), lane:LR1_NullRestrictedFieldInIdentityClass$LaneId; in R1_NullRestrictedFieldInIdentityClass$Descriptor +value container: LaneId[index=1] (works) diff --git a/valhalla-lab/reproducers/R1_NullRestrictedFieldInIdentityClass.java b/valhalla-lab/reproducers/R1_NullRestrictedFieldInIdentityClass.java new file mode 100644 index 0000000..eaee0da --- /dev/null +++ b/valhalla-lab/reproducers/R1_NullRestrictedFieldInIdentityClass.java @@ -0,0 +1,36 @@ +// Reproducer R1 — @NullRestricted on a field of an ORDINARY (identity) class fails at class load. +// javac emits the fields' initialisers AFTER the super() call; the VM demands strict fields be +// assigned BEFORE it. There is no @Strict in this build for javac to key on and no source form +// that expresses the required order, so the combination is unreachable from Java source. +// +// javac --enable-preview -source 27 -target 27 \ +// --add-exports java.base/jdk.internal.vm.annotation=ALL-UNNAMED -d out R1_*.java +// java --enable-preview \ +// --add-exports java.base/jdk.internal.vm.annotation=ALL-UNNAMED -cp out R1_NullRestrictedFieldInIdentityClass +import jdk.internal.vm.annotation.NullRestricted; + +public class R1_NullRestrictedFieldInIdentityClass { + static value record LaneId(int index) {} + + /** An ordinary class that wants a flat LaneId field. Compiles. Does not load. */ + static final class Descriptor { + @NullRestricted final LaneId lane; + Descriptor(int i) { this.lane = new LaneId(i); } + } + + /** The workaround: make the CONTAINER a value class too. Its fields are then strict already. */ + static value class ValueDescriptor { + @NullRestricted final LaneId lane; + ValueDescriptor(int i) { this.lane = new LaneId(i); } + } + + public static void main(String[] a) { + try { + System.out.println("identity container: " + new Descriptor(1).lane); + } catch (Throwable t) { + System.out.println("identity container FAILED: " + t.getClass().getName()); + System.out.println(" " + String.valueOf(t.getMessage()).lines().findFirst().orElse("")); + } + System.out.println("value container: " + new ValueDescriptor(1).lane + " (works)"); + } +} diff --git a/valhalla-lab/reproducers/R2-observed.txt b/valhalla-lab/reproducers/R2-observed.txt new file mode 100644 index 0000000..ccb773d --- /dev/null +++ b/valhalla-lab/reproducers/R2-observed.txt @@ -0,0 +1,8 @@ +Picked up JAVA_TOOL_OPTIONS: +type payload NR-nonAtomic NR-atomic nullable-atomic +P4 4 B true true true +P8i 8 B true true false +P8l 8 B true true false +P12 12 B false false false +P16 16 B false false false +P16l 16 B false false false diff --git a/valhalla-lab/reproducers/R2_FlatteningCliff.java b/valhalla-lab/reproducers/R2_FlatteningCliff.java new file mode 100644 index 0000000..3ecaec4 --- /dev/null +++ b/valhalla-lab/reproducers/R2_FlatteningCliff.java @@ -0,0 +1,40 @@ +// Reproducer R2 — array flattening stops at an 8-byte payload in this build. +// Sweeps payload shapes and asks the VM directly via ValueClass.isFlatArray for all three +// array flavours. Every shape wider than 8 bytes is NOT flattened, in any flavour. +// +// javac --enable-preview -source 27 -target 27 \ +// --add-exports java.base/jdk.internal.value=ALL-UNNAMED -d out R2_FlatteningCliff.java +// java --enable-preview --add-exports java.base/jdk.internal.value=ALL-UNNAMED \ +// -cp out R2_FlatteningCliff +// Add -XX:+UnlockDiagnosticVMOptions -XX:+PrintFlatArrayLayout to see the VM's own layout log. +import jdk.internal.value.ValueClass; + +public class R2_FlatteningCliff { + static value record P4(int a) {} // 4 B + static value record P8i(int a, int b) {} // 8 B + static value record P8l(long a) {} // 8 B + static value record P12(long a, int b) {} // 12 B + static value record P16(long a, int b, int c) {} // 16 B <- the shape of a real entity + static value record P16l(long a, long b) {} // 16 B + + record Case(String name, int payload, Class type, Object init) {} + + public static void main(String[] x) { + Case[] cases = { + new Case("P4", 4, P4.class, new P4(0)), + new Case("P8i", 8, P8i.class, new P8i(0, 0)), + new Case("P8l", 8, P8l.class, new P8l(0)), + new Case("P12", 12, P12.class, new P12(0, 0)), + new Case("P16", 16, P16.class, new P16(0, 0, 0)), + new Case("P16l",16, P16l.class, new P16l(0, 0)), + }; + System.out.printf("%-6s %-8s %-16s %-16s %s%n", + "type", "payload", "NR-nonAtomic", "NR-atomic", "nullable-atomic"); + for (Case c : cases) { + System.out.printf("%-6s %5d B %-16s %-16s %s%n", c.name(), c.payload(), + ValueClass.isFlatArray(ValueClass.newNullRestrictedNonAtomicArray(c.type(), 16, c.init())), + ValueClass.isFlatArray(ValueClass.newNullRestrictedAtomicArray(c.type(), 16, c.init())), + ValueClass.isFlatArray(ValueClass.newNullableAtomicArray(c.type(), 16))); + } + } +} diff --git a/valhalla-lab/reproducers/R3-bang-syntax-observed.txt b/valhalla-lab/reproducers/R3-bang-syntax-observed.txt new file mode 100644 index 0000000..d4ac6d5 --- /dev/null +++ b/valhalla-lab/reproducers/R3-bang-syntax-observed.txt @@ -0,0 +1,6 @@ +Picked up JAVA_TOOL_OPTIONS: -Djavax.net.ssl.trustStore=/root/.ccr/java-truststore.p12 -Djavax.net.ssl.trustStorePassword=changeit -Djavax.net.ssl.trustStoreType=PKCS12 -Dhttps.proxyHost=127.0.0.1 -Dhttps.proxyPort=34795 -Dhttp.nonProxyHosts=localhost|127.0.0.1|::1|127.*|0.*|::|169.254.*|anthropic.com|*.anthropic.com|*.anthropic.com|registry.npmjs.org|jsr.io|npm.jsr.io|pypi.org|files.pythonhosted.org|index.crates.io|proxy.golang.org|host.docker.internal|10.*|172.16.*|172.17.*|172.18.*|172.19.*|172.20.*|172.21.*|172.22.*|172.23.*|172.24.*|172.25.*|172.26.*|172.27.*|172.28.*|172.29.*|172.30.*|172.31.*|192.168.*|100.64.0.0/10|*.svc.cluster.local|*.svc.cluster.local -Djdk.http.auth.tunneling.disabledSchemes= -Djdk.http.auth.proxying.disabledSchemes= +/tmp/Bang.java:1: error: not a statement +public class Bang { static value record L(int i){} public static void main(String[] a){ L![] x = new L![2]; System.out.println(x.length);} } + ^ +/tmp/Bang.java:1: error: ';' expected +public class Bang { static value record L(int i){} public static void main(String[] a){ L![] x = new L![2]; System.out.println(x.length);} } diff --git a/valhalla-lab/reproducers/R3-observed.txt b/valhalla-lab/reproducers/R3-observed.txt new file mode 100644 index 0000000..847ea2d --- /dev/null +++ b/valhalla-lab/reproducers/R3-observed.txt @@ -0,0 +1,4 @@ +(1) new LaneId[8] flat=true accepts null=true (nullable-flat: pays for a null marker) +(2) LaneId![] DOES NOT PARSE ? no null-restricted type syntax +(3) ValueClass.newNullRestricted... flat=true accepts null=false (jdk.internal, needs --add-exports) +(4) List.toArray() flat=false ? generics erase to Object[]; the flattening is undone at the collection boundary diff --git a/valhalla-lab/reproducers/R3_NoSupportedFlatSurface.java b/valhalla-lab/reproducers/R3_NoSupportedFlatSurface.java new file mode 100644 index 0000000..16f4ab1 --- /dev/null +++ b/valhalla-lab/reproducers/R3_NoSupportedFlatSurface.java @@ -0,0 +1,54 @@ +// Reproducer R3 — the ideal API cannot be expressed in supported Java. +// +// The API wants to say "an array of LaneId, densely packed, no nulls". Three ways to ask, and +// what each actually yields on this build: +// (1) `new LaneId[n]` -> flat, but NULLABLE-flat: it still accepts null and +// pays for a null marker, so it is not the densest +// encoding — and for a payload > 8 B it is not flat +// at all (see R2). +// (2) `LaneId![] a = new LaneId![n]` -> DOES NOT PARSE (no null-restricted type syntax), so +// the density in (3) has no supported spelling. +// (3) ValueClass.newNullRestrictedNonAtomicArray -> densest, but jdk.internal + --add-exports. +// (4) `List` -> generics erase; the flattening is undone at the +// collection boundary regardless of (1)-(3). +// +// So a supported API can get *some* flattening for small payloads and can never get the densest +// form, and any generic container discards it. That is the deficiency: not that flattening is +// missing, but that the API cannot ASK for it. +// +// javac --enable-preview -source 27 -target 27 \ +// --add-exports java.base/jdk.internal.value=ALL-UNNAMED -d out R3_NoSupportedFlatSurface.java +// java --enable-preview --add-exports java.base/jdk.internal.value=ALL-UNNAMED \ +// -cp out R3_NoSupportedFlatSurface +import jdk.internal.value.ValueClass; +import java.util.ArrayList; +import java.util.List; + +public class R3_NoSupportedFlatSurface { + static value record LaneId(int index) {} + + public static void main(String[] a) { + LaneId[] supported = new LaneId[8]; + System.out.println("(1) new LaneId[8] flat=" + ValueClass.isFlatArray(supported) + + " accepts null=" + tryNull(supported) + " (nullable-flat: pays for a null marker)"); + + // (2) `LaneId![] x = new LaneId![8];` <-- uncomment to see: this syntax does not exist. + System.out.println("(2) LaneId![] DOES NOT PARSE — no null-restricted type syntax"); + + Object[] internal = ValueClass.newNullRestrictedNonAtomicArray(LaneId.class, 8, new LaneId(0)); + System.out.println("(3) ValueClass.newNullRestricted... flat=" + ValueClass.isFlatArray(internal) + + " accepts null=" + tryNull(internal) + + " (jdk.internal, needs --add-exports)"); + + List generic = new ArrayList<>(); + for (int i = 0; i < 8; i++) generic.add(new LaneId(i)); + Object[] backing = generic.toArray(); + System.out.println("(4) List.toArray() flat=" + ValueClass.isFlatArray(backing) + + " — generics erase to Object[]; the flattening is undone at the collection boundary"); + } + + private static boolean tryNull(Object[] arr) { + try { Object keep = arr[0]; arr[0] = null; arr[0] = keep; return true; } + catch (Throwable t) { return false; } + } +} diff --git a/valhalla-lab/reproducers/README.md b/valhalla-lab/reproducers/README.md new file mode 100644 index 0000000..2543170 --- /dev/null +++ b/valhalla-lab/reproducers/README.md @@ -0,0 +1,159 @@ +# Reproducers — Valhalla limitations hit while expressing the ideal API + +Three limitations were hit. **None of them changed the API.** Where the ideal shape could not be +expressed, that fact is recorded here and the production types stayed as they are — distorting a +public API to fit a preview VM's current budget would bake a temporary constraint into a permanent +surface. + +Each reproducer is a single self-contained file with its command line in the header comment, and a +`*-observed.txt` holding the exact output that file produced on this box. + +| # | Limitation | Belongs to | +|---|---|---| +| [R1](#r1) | `@NullRestricted` field in an ordinary class fails at class load | **javac** | +| [R2](#r2) | Array flattening stops at an 8-byte payload | **HotSpot / Valhalla** | +| [R3](#r3) | The densest layout has no supported spelling, and generics discard it | **Valhalla (language + libraries)** | + +Environment for every observation below: `openjdk 27-jep401ea3+1-1`, Linux x86-64, +Intel Xeon @ 2.10 GHz (4 vCPU, AVX-512). + +--- + +## R1 — `@NullRestricted` on a field of an identity class {#r1} + +**File:** `R1_NullRestrictedFieldInIdentityClass.java` · **Observed:** `R1-observed.txt` + +**Desired semantics.** An ordinary class holds a value-typed field flat — the field is never +null, so no reference and no header should be needed: + +```java +final class Descriptor { // ordinary identity class + @NullRestricted final LaneId lane; // want: 4 bytes inline + Descriptor(int i) { this.lane = new LaneId(i); } +} +``` + +**Ordinary Java (JDK 26).** Compiles and runs; the field is a reference. The annotation does not +exist, so the question cannot even be asked. + +**Valhalla (JDK 27 EA).** Compiles, then fails at class load: + +``` +java.lang.VerifyError: All strict final fields must be initialized before super(): + 1 field(s), lane:LR1_...$LaneId; in R1_...$Descriptor +``` + +**Why.** A null-restricted field is a *strict* field: the VM requires it to be assigned before the +`super()` call. javac emits field initialisers *after* `super()`, and this build has no `@Strict` +annotation for javac to key on — `jdk.internal.vm.annotation` here contains `NullRestricted` and +`LooselyConsistentValue` but no `Strict`. There is no Java source form that expresses the required +order, so the combination is unreachable from source. + +**Workaround, and its cost.** Make the container itself a `value class`; its fields are then +implicitly strict and it works. That is what `src/valhalla/.../Containers.java` does. The cost is +that the workaround is not always available: a container that legitimately has identity — anything +mutable, anything used as a lock, anything with lifecycle — cannot become a value class, and +therefore cannot hold a flat field at all on this build. + +**Consequence for this project.** None yet, and that is luck rather than design: the descriptor +types (`Field` and friends) happen to be immutable. Had `NativePattern` — which is genuinely an +identity object, it owns a native resource and closes it — wanted a flat `LaneId` field, there +would be no way to write it. + +--- + +## R2 — array flattening stops at an 8-byte payload {#r2} + +**File:** `R2_FlatteningCliff.java` · **Observed:** `R2-observed.txt` + +**Desired semantics.** An array of value objects is a dense block of their payloads, whatever the +payload is — that is the entire promise that makes "values are just data" attractive. + +**Observed** (`ValueClass.isFlatArray`, the VM answering about its own array): + +``` +type payload NR-nonAtomic NR-atomic nullable-atomic +P4 4 B true true true +P8i 8 B true true false +P8l 8 B true true false +P12 12 B false false false +P16 16 B false false false +P16l 16 B false false false +``` + +The cliff is at 8 bytes and it is total: past it, **no** array flavour flattens. Confirmed +independently by `-XX:+PrintFlatArrayLayout`, which logs a layout only for the shapes above the +line (`element size 4`, `element size 8`) and nothing for the others. + +**Why.** Flattening past a machine word needs either an atomic wide store or a decision to give +atomicity up; the current implementation declines both above 8 bytes. `FlatArrayElementMaxOops` +exists as a knob for reference-bearing payloads; there is no product knob that lifts the +primitive-payload ceiling on this build. + +**Consequence for this project — and it is the interesting one.** The line the VM draws is exactly +the line the thesis draws: + +| Type | Payload | Flat? | Which side of the thesis | +|---|---|---|---| +| `LaneId`, `Ordinal` | 4 B | **yes** | tiny descriptor vocabulary — Valhalla helps | +| `MaskId` | 8 B | **yes** | tiny descriptor vocabulary — Valhalla helps | +| `RowRange` | 16 B | no | descriptor, but already too wide | +| `Row` (id + class + value) | 16 B | no | per-entity materialisation — Valhalla does not help | + +So "Valhalla helps the descriptors, not the entities" is not a hand-wave about object headers. On +this build it is a hard cutoff in the VM, and a realistic entity is on the wrong side of it by +construction: an id plus one field already exceeds the budget. + +`RowRange` landing on the wrong side is worth stating plainly, because it is the one place the +expectation was too optimistic — it is a descriptor, it was expected to flatten, and it does not. + +--- + +## R3 — the densest layout has no supported spelling {#r3} + +**File:** `R3_NoSupportedFlatSurface.java` · **Observed:** `R3-observed.txt`, +`R3-bang-syntax-observed.txt` + +**Desired semantics.** `LaneId![] lanes = new LaneId![n];` — an array of non-null values, densely +packed, spelled in ordinary Java. + +**Observed:** + +``` +(1) new LaneId[8] flat=true accepts null=true (nullable-flat: pays for a null marker) +(2) LaneId![] DOES NOT PARSE — no null-restricted type syntax +(3) ValueClass.newNullRestricted... flat=true accepts null=false (jdk.internal, needs --add-exports) +(4) List.toArray() flat=false — generics erase to Object[] +``` + +and the syntax probe: + +``` +error: not a statement + L![] x = new L![2]; + ^ +``` + +Three separate gaps, and the first is the one most likely to be misread: + +1. **Supported source already flattens — partially.** `new LaneId[8]` *is* flat for a 4-byte + payload. It is *nullable*-flat, so it carries a null marker and is not the densest encoding, and + by R2 it stops being flat at all past 8 bytes. Reporting "plain arrays are not flat" would have + been wrong; the measured claim is narrower and more useful. +2. **The densest form is `jdk.internal`.** `ValueClass.newNullRestrictedNonAtomicArray` needs + `--add-exports java.base/jdk.internal.value=ALL-UNNAMED` and its own javadoc says it "should + only be used by internal JDK classes for experimental purposes". A library cannot ship it. +3. **Generics erase, so the boundary undoes it.** `List` is `Object[]` underneath and the + array is not flat. Any collection, stream, or generic cache reverts everything the previous two + points achieved. Specialised generics are the missing piece and are not in this build. + +**Consequence for this project.** The production API keeps `LaneId` and friends as plain `record`s +and does **not** adopt any of this. The migration path stays a one-word source change (`record` → +`value record`) precisely because nothing was bent to accommodate the current preview: no +`jdk.internal` dependency, no `--add-exports` in the shipped build, no API that hands out arrays of +descriptors. + +It also removes a temptation worth naming: if `List` had flattened, "just hand the caller a +`List`" would look like a viable alternative to the native lane. It does not flatten, so the +bulk path is not competing with a hypothetical fast object path — it is competing with the same +boxed one Java has always had. diff --git a/valhalla-lab/results/AB-default.diff b/valhalla-lab/results/AB-default.diff new file mode 100644 index 0000000..22da5ba --- /dev/null +++ b/valhalla-lab/results/AB-default.diff @@ -0,0 +1,132 @@ +3,4c3,4 +< platform stable +< java.vm.version 26.0.2+10-55 +--- +> platform valhalla +> java.vm.version 27-jep401ea3+1-1 +6c6 +< jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so] +--- +> jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED] +9,14c9,14 +< platform stable ? stable-record vocabulary; arrays are reference arrays; fields are references +< LaneId.class.isValue() false +< Ordinal.class.isValue() false +< MaskId.class.isValue() false +< RowRange.class.isValue() false +< Row.class.isValue() false +--- +> platform valhalla ? value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields +> LaneId.class.isValue() true +> Ordinal.class.isValue() true +> MaskId.class.isValue() true +> RowRange.class.isValue() true +> Row.class.isValue() true +19,20c19,20 +< a == b (reference equality) false +< identityHashCode(a) == identityHashCode(b) false +--- +> a == b (reference equality) true +> identityHashCode(a) == identityHashCode(b) true +23,25c23,25 +< array flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +< array slot accepts null true +< synchronized(x) legality legal but never used by the production API +--- +> array flatness FLAT +> array slot accepts null false +> synchronized(x) legality COMPILE ERROR under Valhalla (required: a type with identity) +28c28 +< platform stable +--- +> platform valhalla +30,34c30,34 +< LaneId 1 int payload= 4 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +< Ordinal 1 int payload= 4 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +< MaskId 1 long payload= 8 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +< RowRange 2 long payload=16 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +< Row 1 long + 2 int payload=16 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +--- +> LaneId 1 int payload= 4 B array=FLAT +> Ordinal 1 int payload= 4 B array=FLAT +> MaskId 1 long payload= 8 B array=FLAT +> RowRange 2 long payload=16 B array=NOT-FLAT +> Row 1 long + 2 int payload=16 B array=NOT-FLAT +37c37 +< platform stable +--- +> platform valhalla +40,46c40,46 +< construct N LaneId, store into array 15.26 MiB +< ... per LaneId 16.00 B +< construct N LaneId, never escaping 6.71 MiB +< ... per LaneId 7.03 B +< LaneId[1024] flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +< allocate+fill LaneId[N] (array + elements) 19.07 MiB +< ... per element 20.00 B +--- +> construct N LaneId, store into array 2.75 MiB +> ... per LaneId 2.89 B +> construct N LaneId, never escaping 7.63 MiB +> ... per LaneId 8.00 B +> LaneId[1024] flatness FLAT +> allocate+fill LaneId[N] (array + elements) 6.57 MiB +> ... per element 6.89 B +49,55c49,55 +< Descriptor kind identity class with two reference fields +< Descriptor fields null-restricted false +< construct N Descriptor (2 wrappers each) 53.41 MiB +< ... per Descriptor 56.00 B +< pass 2 wrappers through 3 call levels 7.90 MiB +< ... per call 8.29 B +< read 65,536 LaneId from array median= 44182.0 ns [min 41690.0 .. max 59914.0] n=51 +--- +> Descriptor kind value class with two @NullRestricted value fields +> Descriptor fields null-restricted true +> construct N Descriptor (2 wrappers each) 37.05 MiB +> ... per Descriptor 38.84 B +> pass 2 wrappers through 3 call levels 9.97 MiB +> ... per call 10.45 B +> read 65,536 LaneId from array median= 5349.0 ns [min 5179.0 .. max 11356.0] n=51 +59c59 +< platform stable +--- +> platform valhalla +61,65c61,65 +< bare long index median= 59148.0 ns [min 59077.0 .. max 114952.0] n=51 +< RowRange bounds (wrapper hoisted) median= 47952.0 ns [min 47759.0 .. max 65535.0] n=51 +< per-element wrapper: bytes allocated 1.00 MiB +< ... per element 16.00 B +< Ordinal built per element median= 50062.0 ns [min 49790.0 .. max 61194.0] n=51 +--- +> bare long index median= 63304.0 ns [min 59079.0 .. max 131743.0] n=51 +> RowRange bounds (wrapper hoisted) median= 57557.0 ns [min 56212.0 .. max 87932.0] n=51 +> per-element wrapper: bytes allocated 1.50 MiB +> ... per element 24.00 B +> Ordinal built per element median= 50021.0 ns [min 49762.0 .. max 75999.0] n=51 +68c68 +< platform stable +--- +> platform valhalla +79,83c79,83 +< (2)/(3) hydrate 65536 Row ? allocated 2.00 MiB +< ... per row 32.00 B +< array flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +< ratio vs native lane bytes 2.00x +< retained heap (APPROX, gc-delta) 2.25 MiB +--- +> (2)/(3) hydrate 65536 Row ? allocated 2.50 MiB +> ... per row 40.00 B +> array flatness NOT-FLAT +> ratio vs native lane bytes 2.50x +> retained heap (APPROX, gc-delta) 2.75 MiB +86,89c86,89 +< (1) native one crossing, fused plan median= 16378.0 ns [min 15962.0 .. max 33819.0] n=51 +< (2/3) hydrate 65536 Row objects median= 603139.0 ns [min 529172.0 .. max 5246059.0] n=51 +< (2/3) scan the materialised objects median= 151886.0 ns [min 112687.0 .. max 201271.0] n=51 +< (2/3) hydrate THEN scan (honest total) median= 788227.0 ns [min 710221.0 .. max 1146017.0] n=51 +--- +> (1) native one crossing, fused plan median= 18805.0 ns [min 15515.0 .. max 38457.0] n=51 +> (2/3) hydrate 65536 Row objects median= 768624.0 ns [min 645509.0 .. max 1229947.0] n=51 +> (2/3) scan the materialised objects median= 91178.0 ns [min 83390.0 .. max 141648.0] n=51 +> (2/3) hydrate THEN scan (honest total) median= 898955.0 ns [min 822997.0 .. max 5059624.0] n=51 diff --git a/valhalla-lab/results/stable-api-javac.log b/valhalla-lab/results/stable-api-javac.log new file mode 100644 index 0000000..c298bca --- /dev/null +++ b/valhalla-lab/results/stable-api-javac.log @@ -0,0 +1 @@ +Picked up JAVA_TOOL_OPTIONS: -Djavax.net.ssl.trustStore=/root/.ccr/java-truststore.p12 -Djavax.net.ssl.trustStorePassword=changeit -Djavax.net.ssl.trustStoreType=PKCS12 -Dhttps.proxyHost=127.0.0.1 -Dhttps.proxyPort=34795 -Dhttp.nonProxyHosts=localhost|127.0.0.1|::1|127.*|0.*|::|169.254.*|anthropic.com|*.anthropic.com|*.anthropic.com|registry.npmjs.org|jsr.io|npm.jsr.io|pypi.org|files.pythonhosted.org|index.crates.io|proxy.golang.org|host.docker.internal|10.*|172.16.*|172.17.*|172.18.*|172.19.*|172.20.*|172.21.*|172.22.*|172.23.*|172.24.*|172.25.*|172.26.*|172.27.*|172.28.*|172.29.*|172.30.*|172.31.*|192.168.*|100.64.0.0/10|*.svc.cluster.local|*.svc.cluster.local -Djdk.http.auth.tunneling.disabledSchemes= -Djdk.http.auth.proxying.disabledSchemes= diff --git a/valhalla-lab/results/stable-default.txt b/valhalla-lab/results/stable-default.txt new file mode 100644 index 0000000..3f30f78 --- /dev/null +++ b/valhalla-lab/results/stable-default.txt @@ -0,0 +1,91 @@ +Picked up JAVA_TOOL_OPTIONS: +lance-graph-java :: valhalla lab +platform stable +java.vm.version 26.0.2+10-55 +java.vendor.version - +jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so] + +== (a) SEMANTIC TRUTH ? is identity observable? ============================== +platform stable ? stable-record vocabulary; arrays are reference arrays; fields are references +LaneId.class.isValue() false +Ordinal.class.isValue() false +MaskId.class.isValue() false +RowRange.class.isValue() false +Row.class.isValue() false +equal state => equals() true +different state => !equals() true +equal state => equal hashCode() true +equal state => equal toString() true +a == b (reference equality) false +identityHashCode(a) == identityHashCode(b) false +a local of this type accepts null true +array kind LaneId[] +array flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +array slot accepts null true +synchronized(x) legality legal but never used by the production API + +== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +platform stable +note payload = declared field bytes, ignoring any header +LaneId 1 int payload= 4 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +Ordinal 1 int payload= 4 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +MaskId 1 long payload= 8 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +RowRange 2 long payload=16 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +Row 1 long + 2 int payload=16 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) + +== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +platform stable +allocation instrument baseline 0 B +N (operations per measurement) 1000000 +construct N LaneId, store into array 15.26 MiB + ... per LaneId 16.00 B +construct N LaneId, never escaping 6.71 MiB + ... per LaneId 7.03 B +LaneId[1024] flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +allocate+fill LaneId[N] (array + elements) 19.07 MiB + ... per element 20.00 B +bare LaneId[N] with no elements stored 3.81 MiB + ... per slot 4.00 B +Descriptor kind identity class with two reference fields +Descriptor fields null-restricted false +construct N Descriptor (2 wrappers each) 53.41 MiB + ... per Descriptor 56.00 B +pass 2 wrappers through 3 call levels 7.90 MiB + ... per call 8.29 B +read 65,536 LaneId from array median= 44182.0 ns [min 41690.0 .. max 59914.0] n=51 +native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so + +== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +platform stable +sum (identical across all three) 6929623 +bare long index median= 59148.0 ns [min 59077.0 .. max 114952.0] n=51 +RowRange bounds (wrapper hoisted) median= 47952.0 ns [min 47759.0 .. max 65535.0] n=51 +per-element wrapper: bytes allocated 1.00 MiB + ... per element 16.00 B +Ordinal built per element median= 50062.0 ns [min 49790.0 .. max 61194.0] n=51 + +== THE THESIS ? 65,536 entities, three representations ======================= +platform stable +rows 65536 +question count(class==7 AND value>100) and sum(value) +answer (identical across all paths) 2173 rows, sum 499246 +selectivity 3.32% + +== heap cost =============================================================== +(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native ? Java objects per row 0 +(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row ? allocated 2.00 MiB + ... per row 32.00 B + array flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) + ratio vs native lane bytes 2.00x + retained heap (APPROX, gc-delta) 2.25 MiB + +== time to answer the question ============================================= +(1) native one crossing, fused plan median= 16378.0 ns [min 15962.0 .. max 33819.0] n=51 +(2/3) hydrate 65536 Row objects median= 603139.0 ns [min 529172.0 .. max 5246059.0] n=51 +(2/3) scan the materialised objects median= 151886.0 ns [min 112687.0 .. max 201271.0] n=51 +(2/3) hydrate THEN scan (honest total) median= 788227.0 ns [min 710221.0 .. max 1146017.0] n=51 + +lab complete. diff --git a/valhalla-lab/results/stable-lab-javac.log b/valhalla-lab/results/stable-lab-javac.log new file mode 100644 index 0000000..c298bca --- /dev/null +++ b/valhalla-lab/results/stable-lab-javac.log @@ -0,0 +1 @@ +Picked up JAVA_TOOL_OPTIONS: -Djavax.net.ssl.trustStore=/root/.ccr/java-truststore.p12 -Djavax.net.ssl.trustStorePassword=changeit -Djavax.net.ssl.trustStoreType=PKCS12 -Dhttps.proxyHost=127.0.0.1 -Dhttps.proxyPort=34795 -Dhttp.nonProxyHosts=localhost|127.0.0.1|::1|127.*|0.*|::|169.254.*|anthropic.com|*.anthropic.com|*.anthropic.com|registry.npmjs.org|jsr.io|npm.jsr.io|pypi.org|files.pythonhosted.org|index.crates.io|proxy.golang.org|host.docker.internal|10.*|172.16.*|172.17.*|172.18.*|172.19.*|172.20.*|172.21.*|172.22.*|172.23.*|172.24.*|172.25.*|172.26.*|172.27.*|172.28.*|172.29.*|172.30.*|172.31.*|192.168.*|100.64.0.0/10|*.svc.cluster.local|*.svc.cluster.local -Djdk.http.auth.tunneling.disabledSchemes= -Djdk.http.auth.proxying.disabledSchemes= diff --git a/valhalla-lab/results/stable-noea.txt b/valhalla-lab/results/stable-noea.txt new file mode 100644 index 0000000..371624c --- /dev/null +++ b/valhalla-lab/results/stable-noea.txt @@ -0,0 +1,91 @@ +Picked up JAVA_TOOL_OPTIONS: +lance-graph-java :: valhalla lab +platform stable +java.vm.version 26.0.2+10-55 +java.vendor.version - +jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, -XX:-DoEscapeAnalysis] + +== (a) SEMANTIC TRUTH ? is identity observable? ============================== +platform stable ? stable-record vocabulary; arrays are reference arrays; fields are references +LaneId.class.isValue() false +Ordinal.class.isValue() false +MaskId.class.isValue() false +RowRange.class.isValue() false +Row.class.isValue() false +equal state => equals() true +different state => !equals() true +equal state => equal hashCode() true +equal state => equal toString() true +a == b (reference equality) false +identityHashCode(a) == identityHashCode(b) false +a local of this type accepts null true +array kind LaneId[] +array flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +array slot accepts null true +synchronized(x) legality legal but never used by the production API + +== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +platform stable +note payload = declared field bytes, ignoring any header +LaneId 1 int payload= 4 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +Ordinal 1 int payload= 4 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +MaskId 1 long payload= 8 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +RowRange 2 long payload=16 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +Row 1 long + 2 int payload=16 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) + +== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +platform stable +allocation instrument baseline 0 B +N (operations per measurement) 1000000 +construct N LaneId, store into array 15.26 MiB + ... per LaneId 16.00 B +construct N LaneId, never escaping 15.26 MiB + ... per LaneId 16.00 B +LaneId[1024] flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) +allocate+fill LaneId[N] (array + elements) 19.07 MiB + ... per element 20.00 B +bare LaneId[N] with no elements stored 3.81 MiB + ... per slot 4.00 B +Descriptor kind identity class with two reference fields +Descriptor fields null-restricted false +construct N Descriptor (2 wrappers each) 53.41 MiB + ... per Descriptor 56.00 B +pass 2 wrappers through 3 call levels 30.52 MiB + ... per call 32.00 B +read 65,536 LaneId from array median= 44861.0 ns [min 42156.0 .. max 69264.0] n=51 +native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so + +== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +platform stable +sum (identical across all three) 6929623 +bare long index median= 59417.0 ns [min 59067.0 .. max 140131.0] n=51 +RowRange bounds (wrapper hoisted) median= 48136.0 ns [min 47847.0 .. max 105233.0] n=51 +per-element wrapper: bytes allocated 1.00 MiB + ... per element 16.00 B +Ordinal built per element median= 5166233.0 ns [min 4636587.0 .. max 10323741.0] n=51 + +== THE THESIS ? 65,536 entities, three representations ======================= +platform stable +rows 65536 +question count(class==7 AND value>100) and sum(value) +answer (identical across all paths) 2173 rows, sum 499246 +selectivity 3.32% + +== heap cost =============================================================== +(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native ? Java objects per row 0 +(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row ? allocated 2.00 MiB + ... per row 32.00 B + array flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) + ratio vs native lane bytes 2.00x + retained heap (APPROX, gc-delta) 2.22 MiB + +== time to answer the question ============================================= +(1) native one crossing, fused plan median= 17631.0 ns [min 15693.0 .. max 40703.0] n=51 +(2/3) hydrate 65536 Row objects median= 575654.0 ns [min 513071.0 .. max 4428601.0] n=51 +(2/3) scan the materialised objects median= 129828.0 ns [min 108841.0 .. max 159354.0] n=51 +(2/3) hydrate THEN scan (honest total) median= 703240.0 ns [min 642514.0 .. max 789762.0] n=51 + +lab complete. diff --git a/valhalla-lab/results/valhalla-api-javac.log b/valhalla-lab/results/valhalla-api-javac.log new file mode 100644 index 0000000..c298bca --- /dev/null +++ b/valhalla-lab/results/valhalla-api-javac.log @@ -0,0 +1 @@ +Picked up JAVA_TOOL_OPTIONS: -Djavax.net.ssl.trustStore=/root/.ccr/java-truststore.p12 -Djavax.net.ssl.trustStorePassword=changeit -Djavax.net.ssl.trustStoreType=PKCS12 -Dhttps.proxyHost=127.0.0.1 -Dhttps.proxyPort=34795 -Dhttp.nonProxyHosts=localhost|127.0.0.1|::1|127.*|0.*|::|169.254.*|anthropic.com|*.anthropic.com|*.anthropic.com|registry.npmjs.org|jsr.io|npm.jsr.io|pypi.org|files.pythonhosted.org|index.crates.io|proxy.golang.org|host.docker.internal|10.*|172.16.*|172.17.*|172.18.*|172.19.*|172.20.*|172.21.*|172.22.*|172.23.*|172.24.*|172.25.*|172.26.*|172.27.*|172.28.*|172.29.*|172.30.*|172.31.*|192.168.*|100.64.0.0/10|*.svc.cluster.local|*.svc.cluster.local -Djdk.http.auth.tunneling.disabledSchemes= -Djdk.http.auth.proxying.disabledSchemes= diff --git a/valhalla-lab/results/valhalla-default.txt b/valhalla-lab/results/valhalla-default.txt new file mode 100644 index 0000000..29b202f --- /dev/null +++ b/valhalla-lab/results/valhalla-default.txt @@ -0,0 +1,91 @@ +Picked up JAVA_TOOL_OPTIONS: +lance-graph-java :: valhalla lab +platform valhalla +java.vm.version 27-jep401ea3+1-1 +java.vendor.version - +jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED] + +== (a) SEMANTIC TRUTH ? is identity observable? ============================== +platform valhalla ? value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields +LaneId.class.isValue() true +Ordinal.class.isValue() true +MaskId.class.isValue() true +RowRange.class.isValue() true +Row.class.isValue() true +equal state => equals() true +different state => !equals() true +equal state => equal hashCode() true +equal state => equal toString() true +a == b (reference equality) true +identityHashCode(a) == identityHashCode(b) true +a local of this type accepts null true +array kind LaneId[] +array flatness FLAT +array slot accepts null false +synchronized(x) legality COMPILE ERROR under Valhalla (required: a type with identity) + +== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +platform valhalla +note payload = declared field bytes, ignoring any header +LaneId 1 int payload= 4 B array=FLAT +Ordinal 1 int payload= 4 B array=FLAT +MaskId 1 long payload= 8 B array=FLAT +RowRange 2 long payload=16 B array=NOT-FLAT +Row 1 long + 2 int payload=16 B array=NOT-FLAT + +== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +platform valhalla +allocation instrument baseline 0 B +N (operations per measurement) 1000000 +construct N LaneId, store into array 2.75 MiB + ... per LaneId 2.89 B +construct N LaneId, never escaping 7.63 MiB + ... per LaneId 8.00 B +LaneId[1024] flatness FLAT +allocate+fill LaneId[N] (array + elements) 6.57 MiB + ... per element 6.89 B +bare LaneId[N] with no elements stored 3.81 MiB + ... per slot 4.00 B +Descriptor kind value class with two @NullRestricted value fields +Descriptor fields null-restricted true +construct N Descriptor (2 wrappers each) 37.05 MiB + ... per Descriptor 38.84 B +pass 2 wrappers through 3 call levels 9.97 MiB + ... per call 10.45 B +read 65,536 LaneId from array median= 5349.0 ns [min 5179.0 .. max 11356.0] n=51 +native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so + +== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +platform valhalla +sum (identical across all three) 6929623 +bare long index median= 63304.0 ns [min 59079.0 .. max 131743.0] n=51 +RowRange bounds (wrapper hoisted) median= 57557.0 ns [min 56212.0 .. max 87932.0] n=51 +per-element wrapper: bytes allocated 1.50 MiB + ... per element 24.00 B +Ordinal built per element median= 50021.0 ns [min 49762.0 .. max 75999.0] n=51 + +== THE THESIS ? 65,536 entities, three representations ======================= +platform valhalla +rows 65536 +question count(class==7 AND value>100) and sum(value) +answer (identical across all paths) 2173 rows, sum 499246 +selectivity 3.32% + +== heap cost =============================================================== +(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native ? Java objects per row 0 +(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row ? allocated 2.50 MiB + ... per row 40.00 B + array flatness NOT-FLAT + ratio vs native lane bytes 2.50x + retained heap (APPROX, gc-delta) 2.75 MiB + +== time to answer the question ============================================= +(1) native one crossing, fused plan median= 18805.0 ns [min 15515.0 .. max 38457.0] n=51 +(2/3) hydrate 65536 Row objects median= 768624.0 ns [min 645509.0 .. max 1229947.0] n=51 +(2/3) scan the materialised objects median= 91178.0 ns [min 83390.0 .. max 141648.0] n=51 +(2/3) hydrate THEN scan (honest total) median= 898955.0 ns [min 822997.0 .. max 5059624.0] n=51 + +lab complete. diff --git a/valhalla-lab/results/valhalla-lab-javac.log b/valhalla-lab/results/valhalla-lab-javac.log new file mode 100644 index 0000000..67055fc --- /dev/null +++ b/valhalla-lab/results/valhalla-lab-javac.log @@ -0,0 +1,3 @@ +Picked up JAVA_TOOL_OPTIONS: -Djavax.net.ssl.trustStore=/root/.ccr/java-truststore.p12 -Djavax.net.ssl.trustStorePassword=changeit -Djavax.net.ssl.trustStoreType=PKCS12 -Dhttps.proxyHost=127.0.0.1 -Dhttps.proxyPort=34795 -Dhttp.nonProxyHosts=localhost|127.0.0.1|::1|127.*|0.*|::|169.254.*|anthropic.com|*.anthropic.com|*.anthropic.com|registry.npmjs.org|jsr.io|npm.jsr.io|pypi.org|files.pythonhosted.org|index.crates.io|proxy.golang.org|host.docker.internal|10.*|172.16.*|172.17.*|172.18.*|172.19.*|172.20.*|172.21.*|172.22.*|172.23.*|172.24.*|172.25.*|172.26.*|172.27.*|172.28.*|172.29.*|172.30.*|172.31.*|192.168.*|100.64.0.0/10|*.svc.cluster.local|*.svc.cluster.local -Djdk.http.auth.tunneling.disabledSchemes= -Djdk.http.auth.proxying.disabledSchemes= +Note: Some input files use preview features of Java SE 27. +Note: Recompile with -Xlint:preview for details. diff --git a/valhalla-lab/results/valhalla-noarrayflat.txt b/valhalla-lab/results/valhalla-noarrayflat.txt new file mode 100644 index 0000000..6c75ae5 --- /dev/null +++ b/valhalla-lab/results/valhalla-noarrayflat.txt @@ -0,0 +1,91 @@ +Picked up JAVA_TOOL_OPTIONS: +lance-graph-java :: valhalla lab +platform valhalla +java.vm.version 27-jep401ea3+1-1 +java.vendor.version - +jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED, -XX:+UnlockDiagnosticVMOptions, -XX:-UseArrayFlattening] + +== (a) SEMANTIC TRUTH ? is identity observable? ============================== +platform valhalla ? value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields +LaneId.class.isValue() true +Ordinal.class.isValue() true +MaskId.class.isValue() true +RowRange.class.isValue() true +Row.class.isValue() true +equal state => equals() true +different state => !equals() true +equal state => equal hashCode() true +equal state => equal toString() true +a == b (reference equality) true +identityHashCode(a) == identityHashCode(b) true +a local of this type accepts null true +array kind LaneId[] +array flatness NOT-FLAT +array slot accepts null false +synchronized(x) legality COMPILE ERROR under Valhalla (required: a type with identity) + +== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +platform valhalla +note payload = declared field bytes, ignoring any header +LaneId 1 int payload= 4 B array=NOT-FLAT +Ordinal 1 int payload= 4 B array=NOT-FLAT +MaskId 1 long payload= 8 B array=NOT-FLAT +RowRange 2 long payload=16 B array=NOT-FLAT +Row 1 long + 2 int payload=16 B array=NOT-FLAT + +== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +platform valhalla +allocation instrument baseline 0 B +N (operations per measurement) 1000000 +construct N LaneId, store into array 22.89 MiB + ... per LaneId 24.00 B +construct N LaneId, never escaping 8.22 MiB + ... per LaneId 8.62 B +LaneId[1024] flatness NOT-FLAT +allocate+fill LaneId[N] (array + elements) 26.70 MiB + ... per element 28.00 B +bare LaneId[N] with no elements stored 3.81 MiB + ... per slot 4.00 B +Descriptor kind value class with two @NullRestricted value fields +Descriptor fields null-restricted true +construct N Descriptor (2 wrappers each) 38.37 MiB + ... per Descriptor 40.23 B +pass 2 wrappers through 3 call levels 11.49 MiB + ... per call 12.05 B +read 65,536 LaneId from array median= 53465.0 ns [min 49219.0 .. max 88865.0] n=51 +native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so + +== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +platform valhalla +sum (identical across all three) 6929623 +bare long index median= 59142.0 ns [min 59071.0 .. max 114270.0] n=51 +RowRange bounds (wrapper hoisted) median= 56997.0 ns [min 56660.0 .. max 81796.0] n=51 +per-element wrapper: bytes allocated 1.50 MiB + ... per element 24.00 B +Ordinal built per element median= 51224.0 ns [min 49782.0 .. max 91443.0] n=51 + +== THE THESIS ? 65,536 entities, three representations ======================= +platform valhalla +rows 65536 +question count(class==7 AND value>100) and sum(value) +answer (identical across all paths) 2173 rows, sum 499246 +selectivity 3.32% + +== heap cost =============================================================== +(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native ? Java objects per row 0 +(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row ? allocated 2.50 MiB + ... per row 40.00 B + array flatness NOT-FLAT + ratio vs native lane bytes 2.50x + retained heap (APPROX, gc-delta) 2.75 MiB + +== time to answer the question ============================================= +(1) native one crossing, fused plan median= 14946.0 ns [min 14841.0 .. max 49967.0] n=51 +(2/3) hydrate 65536 Row objects median= 1708590.0 ns [min 610808.0 .. max 8047970.0] n=51 +(2/3) scan the materialised objects median= 148382.0 ns [min 119556.0 .. max 229605.0] n=51 +(2/3) hydrate THEN scan (honest total) median= 843903.0 ns [min 781103.0 .. max 1306914.0] n=51 + +lab complete. diff --git a/valhalla-lab/results/valhalla-noea.txt b/valhalla-lab/results/valhalla-noea.txt new file mode 100644 index 0000000..529ee73 --- /dev/null +++ b/valhalla-lab/results/valhalla-noea.txt @@ -0,0 +1,91 @@ +Picked up JAVA_TOOL_OPTIONS: +lance-graph-java :: valhalla lab +platform valhalla +java.vm.version 27-jep401ea3+1-1 +java.vendor.version - +jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED, -XX:-DoEscapeAnalysis] + +== (a) SEMANTIC TRUTH ? is identity observable? ============================== +platform valhalla ? value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields +LaneId.class.isValue() true +Ordinal.class.isValue() true +MaskId.class.isValue() true +RowRange.class.isValue() true +Row.class.isValue() true +equal state => equals() true +different state => !equals() true +equal state => equal hashCode() true +equal state => equal toString() true +a == b (reference equality) true +identityHashCode(a) == identityHashCode(b) true +a local of this type accepts null true +array kind LaneId[] +array flatness FLAT +array slot accepts null false +synchronized(x) legality COMPILE ERROR under Valhalla (required: a type with identity) + +== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +platform valhalla +note payload = declared field bytes, ignoring any header +LaneId 1 int payload= 4 B array=FLAT +Ordinal 1 int payload= 4 B array=FLAT +MaskId 1 long payload= 8 B array=FLAT +RowRange 2 long payload=16 B array=NOT-FLAT +Row 1 long + 2 int payload=16 B array=NOT-FLAT + +== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +platform valhalla +allocation instrument baseline 0 B +N (operations per measurement) 1000000 +construct N LaneId, store into array 3.86 MiB + ... per LaneId 4.05 B +construct N LaneId, never escaping 24.41 MiB + ... per LaneId 25.60 B +LaneId[1024] flatness FLAT +allocate+fill LaneId[N] (array + elements) 28.35 MiB + ... per element 29.73 B +bare LaneId[N] with no elements stored 3.81 MiB + ... per slot 4.00 B +Descriptor kind value class with two @NullRestricted value fields +Descriptor fields null-restricted true +construct N Descriptor (2 wrappers each) 38.98 MiB + ... per Descriptor 40.87 B +pass 2 wrappers through 3 call levels 29.57 MiB + ... per call 31.00 B +read 65,536 LaneId from array median= 5369.0 ns [min 5321.0 .. max 30666.0] n=51 +native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so + +== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +platform valhalla +sum (identical across all three) 6929623 +bare long index median= 61840.0 ns [min 59069.0 .. max 128839.0] n=51 +RowRange bounds (wrapper hoisted) median= 60149.0 ns [min 56823.0 .. max 74673.0] n=51 +per-element wrapper: bytes allocated 3.00 MiB + ... per element 48.00 B +Ordinal built per element median= 94346.0 ns [min 91210.0 .. max 133533.0] n=51 + +== THE THESIS ? 65,536 entities, three representations ======================= +platform valhalla +rows 65536 +question count(class==7 AND value>100) and sum(value) +answer (identical across all paths) 2173 rows, sum 499246 +selectivity 3.32% + +== heap cost =============================================================== +(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native ? Java objects per row 0 +(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row ? allocated 2.50 MiB + ... per row 40.00 B + array flatness NOT-FLAT + ratio vs native lane bytes 2.50x + retained heap (APPROX, gc-delta) 2.75 MiB + +== time to answer the question ============================================= +(1) native one crossing, fused plan median= 15885.0 ns [min 15694.0 .. max 32647.0] n=51 +(2/3) hydrate 65536 Row objects median= 1187577.0 ns [min 1044870.0 .. max 2002465.0] n=51 +(2/3) scan the materialised objects median= 74699.0 ns [min 63609.0 .. max 129188.0] n=51 +(2/3) hydrate THEN scan (honest total) median= 3267205.0 ns [min 1374933.0 .. max 9370919.0] n=51 + +lab complete. diff --git a/valhalla-lab/results/valhalla-nofieldflat.txt b/valhalla-lab/results/valhalla-nofieldflat.txt new file mode 100644 index 0000000..7af9d7b --- /dev/null +++ b/valhalla-lab/results/valhalla-nofieldflat.txt @@ -0,0 +1,91 @@ +Picked up JAVA_TOOL_OPTIONS: +lance-graph-java :: valhalla lab +platform valhalla +java.vm.version 27-jep401ea3+1-1 +java.vendor.version - +jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED, -XX:+UnlockDiagnosticVMOptions, -XX:-UseFieldFlattening] + +== (a) SEMANTIC TRUTH ? is identity observable? ============================== +platform valhalla ? value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields +LaneId.class.isValue() true +Ordinal.class.isValue() true +MaskId.class.isValue() true +RowRange.class.isValue() true +Row.class.isValue() true +equal state => equals() true +different state => !equals() true +equal state => equal hashCode() true +equal state => equal toString() true +a == b (reference equality) true +identityHashCode(a) == identityHashCode(b) true +a local of this type accepts null true +array kind LaneId[] +array flatness FLAT +array slot accepts null false +synchronized(x) legality COMPILE ERROR under Valhalla (required: a type with identity) + +== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +platform valhalla +note payload = declared field bytes, ignoring any header +LaneId 1 int payload= 4 B array=FLAT +Ordinal 1 int payload= 4 B array=FLAT +MaskId 1 long payload= 8 B array=FLAT +RowRange 2 long payload=16 B array=NOT-FLAT +Row 1 long + 2 int payload=16 B array=NOT-FLAT + +== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +platform valhalla +allocation instrument baseline 0 B +N (operations per measurement) 1000000 +construct N LaneId, store into array 2.75 MiB + ... per LaneId 2.89 B +construct N LaneId, never escaping 8.22 MiB + ... per LaneId 8.61 B +LaneId[1024] flatness FLAT +allocate+fill LaneId[N] (array + elements) 6.58 MiB + ... per element 6.90 B +bare LaneId[N] with no elements stored 3.81 MiB + ... per slot 4.00 B +Descriptor kind value class with two @NullRestricted value fields +Descriptor fields null-restricted true +construct N Descriptor (2 wrappers each) 76.29 MiB + ... per Descriptor 80.00 B +pass 2 wrappers through 3 call levels 11.14 MiB + ... per call 11.68 B +read 65,536 LaneId from array median= 4959.0 ns [min 4870.0 .. max 5886.0] n=51 +native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so + +== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +platform valhalla +sum (identical across all three) 6929623 +bare long index median= 59187.0 ns [min 59075.0 .. max 123125.0] n=51 +RowRange bounds (wrapper hoisted) median= 56980.0 ns [min 56652.0 .. max 73287.0] n=51 +per-element wrapper: bytes allocated 1.50 MiB + ... per element 24.00 B +Ordinal built per element median= 49851.0 ns [min 49727.0 .. max 66813.0] n=51 + +== THE THESIS ? 65,536 entities, three representations ======================= +platform valhalla +rows 65536 +question count(class==7 AND value>100) and sum(value) +answer (identical across all paths) 2173 rows, sum 499246 +selectivity 3.32% + +== heap cost =============================================================== +(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native ? Java objects per row 0 +(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row ? allocated 2.50 MiB + ... per row 40.01 B + array flatness NOT-FLAT + ratio vs native lane bytes 2.50x + retained heap (APPROX, gc-delta) 2.75 MiB + +== time to answer the question ============================================= +(1) native one crossing, fused plan median= 15964.0 ns [min 15779.0 .. max 40503.0] n=51 +(2/3) hydrate 65536 Row objects median= 789127.0 ns [min 628024.0 .. max 4599451.0] n=51 +(2/3) scan the materialised objects median= 78249.0 ns [min 75327.0 .. max 111755.0] n=51 +(2/3) hydrate THEN scan (honest total) median= 1122018.0 ns [min 812442.0 .. max 1695253.0] n=51 + +lab complete. diff --git a/valhalla-lab/results/valhalla-noflat.txt b/valhalla-lab/results/valhalla-noflat.txt new file mode 100644 index 0000000..5ab9312 --- /dev/null +++ b/valhalla-lab/results/valhalla-noflat.txt @@ -0,0 +1,91 @@ +Picked up JAVA_TOOL_OPTIONS: +lance-graph-java :: valhalla lab +platform valhalla +java.vm.version 27-jep401ea3+1-1 +java.vendor.version - +jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED, -XX:+UnlockDiagnosticVMOptions, -XX:-UseArrayFlattening, -XX:-UseFieldFlattening] + +== (a) SEMANTIC TRUTH ? is identity observable? ============================== +platform valhalla ? value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields +LaneId.class.isValue() true +Ordinal.class.isValue() true +MaskId.class.isValue() true +RowRange.class.isValue() true +Row.class.isValue() true +equal state => equals() true +different state => !equals() true +equal state => equal hashCode() true +equal state => equal toString() true +a == b (reference equality) true +identityHashCode(a) == identityHashCode(b) true +a local of this type accepts null true +array kind LaneId[] +array flatness NOT-FLAT +array slot accepts null false +synchronized(x) legality COMPILE ERROR under Valhalla (required: a type with identity) + +== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +platform valhalla +note payload = declared field bytes, ignoring any header +LaneId 1 int payload= 4 B array=NOT-FLAT +Ordinal 1 int payload= 4 B array=NOT-FLAT +MaskId 1 long payload= 8 B array=NOT-FLAT +RowRange 2 long payload=16 B array=NOT-FLAT +Row 1 long + 2 int payload=16 B array=NOT-FLAT + +== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +platform valhalla +allocation instrument baseline 0 B +N (operations per measurement) 1000000 +construct N LaneId, store into array 15.26 MiB + ... per LaneId 16.00 B +construct N LaneId, never escaping 4.71 MiB + ... per LaneId 4.94 B +LaneId[1024] flatness NOT-FLAT +allocate+fill LaneId[N] (array + elements) 19.07 MiB + ... per element 20.00 B +bare LaneId[N] with no elements stored 3.81 MiB + ... per slot 4.00 B +Descriptor kind value class with two @NullRestricted value fields +Descriptor fields null-restricted true +construct N Descriptor (2 wrappers each) 53.41 MiB + ... per Descriptor 56.00 B +pass 2 wrappers through 3 call levels 8.65 MiB + ... per call 9.07 B +read 65,536 LaneId from array median= 43102.0 ns [min 41523.0 .. max 77929.0] n=51 +native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so + +== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +platform valhalla +sum (identical across all three) 6929623 +bare long index median= 59218.0 ns [min 59090.0 .. max 150920.0] n=51 +RowRange bounds (wrapper hoisted) median= 56978.0 ns [min 56635.0 .. max 83751.0] n=51 +per-element wrapper: bytes allocated 1.00 MiB + ... per element 16.00 B +Ordinal built per element median= 49935.0 ns [min 49722.0 .. max 83957.0] n=51 + +== THE THESIS ? 65,536 entities, three representations ======================= +platform valhalla +rows 65536 +question count(class==7 AND value>100) and sum(value) +answer (identical across all paths) 2173 rows, sum 499246 +selectivity 3.32% + +== heap cost =============================================================== +(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native ? Java objects per row 0 +(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row ? allocated 2.00 MiB + ... per row 32.00 B + array flatness NOT-FLAT + ratio vs native lane bytes 2.00x + retained heap (APPROX, gc-delta) 2.25 MiB + +== time to answer the question ============================================= +(1) native one crossing, fused plan median= 15435.0 ns [min 15335.0 .. max 30493.0] n=51 +(2/3) hydrate 65536 Row objects median= 857050.0 ns [min 533019.0 .. max 2422972.0] n=51 +(2/3) scan the materialised objects median= 124600.0 ns [min 112855.0 .. max 180439.0] n=51 +(2/3) hydrate THEN scan (honest total) median= 1631916.0 ns [min 1499609.0 .. max 2069662.0] n=51 + +lab complete. diff --git a/valhalla-lab/results/vocab-diff.txt b/valhalla-lab/results/vocab-diff.txt new file mode 100644 index 0000000..e69de29 diff --git a/valhalla-lab/run.sh b/valhalla-lab/run.sh new file mode 100755 index 0000000..9dfb31c --- /dev/null +++ b/valhalla-lab/run.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# The A/B. One experiment source, two object models, two JDKs, one diff. +# +# Every measurement this lab reports is produced by this script. Nothing is quoted from memory. +set -uo pipefail + +LAB="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$LAB/.." && pwd)" + +STABLE_JDK="${STABLE_JDK:-/opt/jdks/jdk-26.0.2}" +VALHALLA_JDK="${VALHALLA_JDK:-/opt/jdks/jdk-27}" +LIB_DIR="${LIB_DIR:-$ROOT/target/release}" + +OUT="$LAB/results" +mkdir -p "$OUT" + +VAL_EXPORTS=( + --add-exports java.base/jdk.internal.vm.annotation=ALL-UNNAMED + --add-exports java.base/jdk.internal.value=ALL-UNNAMED +) + +banner() { printf '\n\033[1m== %s\033[0m\n' "$*"; } +fail() { printf '\033[31mFAIL:\033[0m %s\n' "$*" >&2; exit 1; } + +# ── 0. the A/B is only honest if the two vocabularies differ by exactly the modifier ────────── +banner "0. verifying the two vocabularies differ ONLY by the 'value' modifier" +if diff <(sed 's/^value record/record/' "$LAB/src/valhalla/com/adaworldapi/lancegraph/lab/Vocab.java") \ + "$LAB/src/stable/com/adaworldapi/lancegraph/lab/Vocab.java" > "$OUT/vocab-diff.txt"; then + echo "OK — src/valhalla/Vocab.java == src/stable/Vocab.java modulo 'value'" +else + cat "$OUT/vocab-diff.txt" + fail "the two vocabularies differ by more than the 'value' modifier; the A/B would not be an A/B" +fi + +if [ ! -f "$LIB_DIR/liblgj_abi.so" ]; then + fail "no native library at $LIB_DIR/liblgj_abi.so +build it with: + cd $ROOT/native/lgj-abi && CARGO_TARGET_DIR=$ROOT/target cargo build --release" +fi + +# ── 1. compile ──────────────────────────────────────────────────────────────────────────────── +compile () { # $1=tag $2=jdk $3=vocab-root shift 3 -> extra javac args + local tag=$1 jdk=$2 vocab=$3; shift 3 + local api="$OUT/$tag-api" lab="$OUT/$tag-lab" + rm -rf "$api" "$lab"; mkdir -p "$api" "$lab" + + # the production API, compiled by THIS jdk, with no preview features anywhere + "$jdk/bin/javac" -d "$api" $(find "$ROOT/java/src/main/java" -name '*.java') \ + 2> "$OUT/$tag-api-javac.log" || { cat "$OUT/$tag-api-javac.log"; fail "$tag: API compile"; } + + "$jdk/bin/javac" "$@" -cp "$api" -d "$lab" \ + $(find "$LAB/src/shared" "$vocab" -name '*.java') \ + 2> "$OUT/$tag-lab-javac.log" || { cat "$OUT/$tag-lab-javac.log"; fail "$tag: lab compile"; } + echo "compiled $tag" +} + +banner "1. compiling" +compile stable "$STABLE_JDK" "$LAB/src/stable" +compile valhalla "$VALHALLA_JDK" "$LAB/src/valhalla" \ + --enable-preview -source 27 -target 27 "${VAL_EXPORTS[@]}" + +# ── 2. run ──────────────────────────────────────────────────────────────────────────────────── +run () { # $1=tag $2=jdk $3=label shift 3 -> extra jvm args + local tag=$1 jdk=$2 label=$3; shift 3 + local f="$OUT/$tag-$label.txt" + echo "--> $tag/$label" + ( JAVA_TOOL_OPTIONS= "$jdk/bin/java" \ + --enable-native-access=ALL-UNNAMED \ + -Dlgj.library="$LIB_DIR/liblgj_abi.so" \ + "$@" \ + -cp "$OUT/$tag-api:$OUT/$tag-lab" com.adaworldapi.lancegraph.lab.RunAll ) \ + > "$f" 2>&1 + local rc=$? + echo " exit=$rc -> ${f#$LAB/}" + return 0 +} + +banner "2. running (default VM settings)" +run stable "$STABLE_JDK" default +run valhalla "$VALHALLA_JDK" default --enable-preview "${VAL_EXPORTS[@]}" + +banner "3. running with escape analysis OFF (what the object model costs unaided)" +run stable "$STABLE_JDK" noea -XX:-DoEscapeAnalysis +run valhalla "$VALHALLA_JDK" noea --enable-preview "${VAL_EXPORTS[@]}" -XX:-DoEscapeAnalysis + +banner "4. running with Valhalla flattening knobs OFF (does flattening cause the difference?)" +run valhalla "$VALHALLA_JDK" noarrayflat --enable-preview "${VAL_EXPORTS[@]}" \ + -XX:+UnlockDiagnosticVMOptions -XX:-UseArrayFlattening +run valhalla "$VALHALLA_JDK" nofieldflat --enable-preview "${VAL_EXPORTS[@]}" \ + -XX:+UnlockDiagnosticVMOptions -XX:-UseFieldFlattening +run valhalla "$VALHALLA_JDK" noflat --enable-preview "${VAL_EXPORTS[@]}" \ + -XX:+UnlockDiagnosticVMOptions -XX:-UseArrayFlattening -XX:-UseFieldFlattening + +banner "5. A/B diff" +diff "$OUT/stable-default.txt" "$OUT/valhalla-default.txt" > "$OUT/AB-default.diff" +echo "wrote ${OUT#$LAB/}/AB-default.diff ($(wc -l < "$OUT/AB-default.diff") lines)" + +banner "done — results in $OUT" +ls -1 "$OUT"/*.txt diff --git a/valhalla-lab/src/shared/com/adaworldapi/lancegraph/NativeAccess.java b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/NativeAccess.java new file mode 100644 index 0000000..3e9fa04 --- /dev/null +++ b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/NativeAccess.java @@ -0,0 +1,49 @@ +package com.adaworldapi.lancegraph; + +import com.adaworldapi.lancegraph.internal.ffm.Engine; + +/** + * A read-only bridge into the library's package-private handle, for measurement code only. + * + *

Why this exists rather than a public accessor. The production API + * deliberately never surfaces a handle or a {@code MemorySegment} — that is the whole accessibility + * argument. But the lab has to reach the raw lane to build the very thing the thesis says you + * should not build (65,536 Java objects), and it must build them from the same bytes the + * native kernel reads, or the comparison is between two different datasets and proves nothing. + * + *

So the bridge lives here, in the library's package but in the lab's source tree, + * compiled onto the classpath as a split package. Nothing under {@code java/} changes, no public + * surface widens, and the coupling is visible in one file instead of leaking into the API. + * + *

It reads. It never writes, never closes, never mutates. + */ +public final class NativeAccess { + + private NativeAccess() {} + + /** Lane 0 — {@code u64} entity ids. */ + public static final int LANE_ID = 0; + /** Lane 1 — {@code u32} class tags. */ + public static final int LANE_CLASS = 1; + /** Lane 2 — {@code i32} signed values. */ + public static final int LANE_VALUE = 2; + + /** The generation-checked registry handle behind a pattern. Opaque; for describe calls only. */ + public static long handleOf(NativePattern pattern) { + return pattern.handle(); + } + + /** + * A bounded, read-only window onto one native lane. No membrane crossing happens when this is + * read — that is the point of the design, and the reason a Java-side Vector API kernel can + * compete at all. + */ + public static Engine.LaneWindow lane(NativePattern pattern, int laneId) { + return Engine.describeLane(pattern.handle(), laneId); + } + + /** The packed {@code u64} words behind a selection. */ + public static Engine.LaneWindow maskWords(Mask mask) { + return Engine.describeMask(mask.id().token()); + } +} diff --git a/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/FfmAddressingExperiment.java b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/FfmAddressingExperiment.java new file mode 100644 index 0000000..ab5a9c3 --- /dev/null +++ b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/FfmAddressingExperiment.java @@ -0,0 +1,86 @@ +package com.adaworldapi.lancegraph.lab; + +import com.adaworldapi.lancegraph.NativeAccess; +import com.adaworldapi.lancegraph.NativePattern; +import com.adaworldapi.lancegraph.internal.ffm.Engine; + +/** + * Does a semantic wrapper cost anything when it is used to address native memory? + * + *

This is the question that decides whether the API's vocabulary can go all the way down. The + * production API hides {@code MemorySegment} entirely, but the shape it hides is + * "offset arithmetic on a native address". If wrapping a row index in a meaningful type made that + * arithmetic slower, the vocabulary would have to stop at the API surface and become bare + * {@code long}s underneath — an abstraction that is only free where nobody is looking. + * + *

Three variants over the same lane, same 65,536 elements, same result asserted equal: + * + *

    + *
  • a bare {@code long} index — the floor; + *
  • a {@code RowRange} driving the loop bounds — a wrapper read once per loop; + *
  • a per-element wrapper ({@code Ordinal}) constructed inside the loop — a wrapper read once + * per element, which is the shape that would actually be expensive. + *
+ * + *

The third is the one that matters. A wrapper hoisted out of a loop is free on any JDK; a + * wrapper allocated 65,536 times is exactly the case a value class is supposed to make free, and + * exactly the case escape analysis sometimes already handles. Measuring both is what separates + * "Valhalla helped" from "the JIT was already doing it". + */ +final class FfmAddressingExperiment { + + private FfmAddressingExperiment() {} + + private static final int ROWS = 65_536; + + static void run() { + Lab.section("FFM ADDRESSING — is the wrapper free where it touches native memory?"); + Lab.kv("platform", Platform.NAME); + + try (NativePattern data = NativePattern.open(ROWS)) { + Engine.LaneWindow values = NativeAccess.lane(data, NativeAccess.LANE_VALUE); + + long bare = 0; + for (long i = 0; i < ROWS; i++) bare += values.getI32(i); + + RowRange range = RowRange.of(ROWS); + long viaRange = 0; + for (long i = range.start(); i < range.endExclusive(); i++) viaRange += values.getI32(i); + + long viaWrapper = 0; + for (int i = 0; i < ROWS; i++) viaWrapper += values.getI32(Ordinal.of(i).value()); + + if (bare != viaRange || bare != viaWrapper) { + throw new AssertionError("addressing variants disagree: " + bare + " / " + + viaRange + " / " + viaWrapper); + } + Lab.kv("sum (identical across all three)", bare); + + System.out.println(Lab.time("bare long index", 2_000, 51, () -> { + long acc = 0; + for (long i = 0; i < ROWS; i++) acc += values.getI32(i); + Lab.SINK = acc; + })); + + System.out.println(Lab.time("RowRange bounds (wrapper hoisted)", 2_000, 51, () -> { + long acc = 0; + for (long i = range.start(); i < range.endExclusive(); i++) acc += values.getI32(i); + Lab.SINK = acc; + })); + + long wrapperAlloc = Lab.allocatedBytes(() -> { + long acc = 0; + for (int i = 0; i < ROWS; i++) acc += values.getI32(Ordinal.of(i).value()); + Lab.SINK = acc; + }); + Lab.kv("per-element wrapper: bytes allocated", Lab.bytes(wrapperAlloc)); + Lab.kv(" ... per element", String.format("%.2f B", wrapperAlloc / (double) ROWS)); + + System.out.println(Lab.time("Ordinal built per element", 2_000, 51, () -> { + long acc = 0; + for (int i = 0; i < ROWS; i++) acc += values.getI32(Ordinal.of(i).value()); + Lab.SINK = acc; + })); + } + } +} diff --git a/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/FlatteningCliffExperiment.java b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/FlatteningCliffExperiment.java new file mode 100644 index 0000000..4718129 --- /dev/null +++ b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/FlatteningCliffExperiment.java @@ -0,0 +1,46 @@ +package com.adaworldapi.lancegraph.lab; + +/** + * Where does flattening stop? + * + *

This experiment exists because the first run of {@link ThesisExperiment} produced a result + * that looked like a bug: the Valhalla {@code Row} array reported {@code NOT-FLAT} and cost + * more per row than the stable record. Rather than explain it away, the question was + * turned into a measurement — sweep payload sizes and find the cliff. + * + *

The answer on this build is a hard cutoff, and it lands exactly between the two categories + * the thesis distinguishes. A wrapper around one {@code int} or one {@code long} is flattened; an + * entity with an id plus two fields is not, in any array flavour. The thesis' central claim and + * the VM's current flattening budget happen to draw the same line — which is a much stronger + * result than "objects are slow", because it says why the line is where it is. + * + *

Each row of output is one payload shape. The types are declared in the vocabulary file so the + * stable and Valhalla trees declare identical shapes. + */ +final class FlatteningCliffExperiment { + + private FlatteningCliffExperiment() {} + + static void run() { + Lab.section("FLATTENING CLIFF — which payload shapes does the VM flatten?"); + Lab.kv("platform", Platform.NAME); + Lab.kv("note", "payload = declared field bytes, ignoring any header"); + + probe("LaneId", "1 int", 4, LaneId.class, LaneId.of(0)); + probe("Ordinal", "1 int", 4, Ordinal.class, Ordinal.of(0)); + probe("MaskId", "1 long", 8, MaskId.class, new MaskId(0)); + probe("RowRange", "2 long", 16, RowRange.class, RowRange.of(0)); + probe("Row", "1 long + 2 int", 16, Row.class, new Row(0, 0, 0)); + } + + private static void probe(String name, String shape, int payloadBytes, Class type, + Object init) { + String flat; + try { + flat = Platform.arrayFlatness(Platform.newArrayOf(type, 64, init)); + } catch (Throwable t) { + flat = "REFUSED: " + t.getClass().getSimpleName() + " " + t.getMessage(); + } + System.out.printf("%-10s %-16s payload=%2d B array=%s%n", name, shape, payloadBytes, flat); + } +} diff --git a/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/FootprintExperiment.java b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/FootprintExperiment.java new file mode 100644 index 0000000..c87606b --- /dev/null +++ b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/FootprintExperiment.java @@ -0,0 +1,111 @@ +package com.adaworldapi.lancegraph.lab; + +/** + * Truths (b) and (c) — representation. What does the same source actually become? + * + *

Four questions, each measured by the instrument that can least easily be fooled: + * + *

    + *
  1. Does constructing one cost a heap object? — allocated bytes, not timing. + *
  2. Is an array of them flattened? — the VM's own {@code isFlatArray} where it + * exists, plus the array's measured footprint either way. + *
  3. Is a field of one flattened into its container? — allocated bytes for the + * container, which changes by exactly the header+pointer cost when it is not. + *
  4. Does passing one to a method cost anything? — allocated bytes across a + * call chain deep enough that escape analysis has to give up. + *
+ * + *

Every count below is per {@link #N} operations, so a per-instance number can be divided out + * and compared against the theoretical object size (16-byte header + 4-byte int, padded to 16 for + * a one-int record on a 64-bit VM with compressed oops). + */ +final class FootprintExperiment { + + private FootprintExperiment() {} + + /** Large enough that per-operation bytes resolve cleanly; small enough to stay in a young gen. */ + static final int N = 1_000_000; + + static void run() { + Lab.section("(b)/(c) REPRESENTATION — allocation, arrays, fields, arguments"); + Lab.kv("platform", Platform.NAME); + Lab.kv("allocation instrument baseline", Lab.bytes(Lab.allocationInstrumentBaseline())); + Lab.kv("N (operations per measurement)", N); + + // ── 1. constructing a descriptor ───────────────────────────────────────────────────── + // The array store is what forces the object to escape. Without it, escape analysis + // deletes the allocation on BOTH platforms and the experiment measures nothing — which is + // itself worth stating, because it is exactly why a fast microbenchmark is not evidence. + Object[] sink = Platform.newLaneIdArray(N); + long ctorBytes = Lab.allocatedBytes(() -> { + for (int i = 0; i < N; i++) sink[i] = LaneId.of(i & 0xFFFF); + }); + Lab.OBJ_SINK = sink; + Lab.kv("construct N LaneId, store into array", Lab.bytes(ctorBytes)); + Lab.kv(" ... per LaneId", String.format("%.2f B", ctorBytes / (double) N)); + + // Non-escaping variant, for contrast. If this is ~0 on the stable platform too, that is + // escape analysis doing the value class's job for one particular loop — and the reason + // the run script also reports a -XX:-DoEscapeAnalysis pass. + long nonEscapingBytes = Lab.allocatedBytes(() -> { + long acc = 0; + for (int i = 0; i < N; i++) acc += LaneId.of(i).index(); + Lab.SINK = acc; + }); + Lab.kv("construct N LaneId, never escaping", Lab.bytes(nonEscapingBytes)); + Lab.kv(" ... per LaneId", String.format("%.2f B", nonEscapingBytes / (double) N)); + + // ── 2. array representation ────────────────────────────────────────────────────────── + Object[] probe = Platform.newLaneIdArray(1024); + Lab.kv("LaneId[1024] flatness", Platform.arrayFlatness(probe)); + long arrayBytes = Lab.allocatedBytes(() -> { + Object[] a = Platform.newLaneIdArray(N); + for (int i = 0; i < N; i++) a[i] = LaneId.of(i); + Lab.OBJ_SINK = a; + }); + Lab.kv("allocate+fill LaneId[N] (array + elements)", Lab.bytes(arrayBytes)); + Lab.kv(" ... per element", String.format("%.2f B", arrayBytes / (double) N)); + + long emptyArrayBytes = Lab.allocatedBytes(() -> Lab.OBJ_SINK = Platform.newLaneIdArray(N)); + Lab.kv("bare LaneId[N] with no elements stored", Lab.bytes(emptyArrayBytes)); + Lab.kv(" ... per slot", String.format("%.2f B", emptyArrayBytes / (double) N)); + + // ── 3. field flattening ────────────────────────────────────────────────────────────── + Lab.kv("Descriptor kind", Containers.kind()); + Lab.kv("Descriptor fields null-restricted", Containers.fieldsAreNullRestricted()); + Object[] descSink = new Object[N]; + long descBytes = Lab.allocatedBytes(() -> { + for (int i = 0; i < N; i++) descSink[i] = Containers.make(i & 7, i & 3); + }); + Lab.OBJ_SINK = descSink; + Lab.kv("construct N Descriptor (2 wrappers each)", Lab.bytes(descBytes)); + Lab.kv(" ... per Descriptor", String.format("%.2f B", descBytes / (double) N)); + + // ── 4. method-passing ──────────────────────────────────────────────────────────────── + long passBytes = Lab.allocatedBytes(() -> { + long acc = 0; + for (int i = 0; i < N; i++) acc += consume(LaneId.of(i), Ordinal.of(i & 15)); + Lab.SINK = acc; + }); + Lab.kv("pass 2 wrappers through 3 call levels", Lab.bytes(passBytes)); + Lab.kv(" ... per call", String.format("%.2f B", passBytes / (double) N)); + + // ── timing, secondary ──────────────────────────────────────────────────────────────── + // Reported after the byte counts, and second in importance. A timing that disagrees with + // the byte counts is a signal to distrust the timing, not the bytes. + Object[] readArr = Platform.newLaneIdArray(65_536); + for (int i = 0; i < 65_536; i++) readArr[i] = LaneId.of(i); + Lab.OBJ_SINK = readArr; + System.out.println(Lab.time("read 65,536 LaneId from array", 2_000, 51, () -> { + long acc = 0; + for (int i = 0; i < 65_536; i++) acc += ((LaneId) readArr[i]).index(); + Lab.SINK = acc; + })); + } + + // Three levels: shallow enough to inline, deep enough that a naive reading of "the JIT will + // fix it" is not automatically true. + private static long consume(LaneId l, Ordinal o) { return level2(l, o); } + private static long level2(LaneId l, Ordinal o) { return level3(l, o); } + private static long level3(LaneId l, Ordinal o) { return l.index() + o.value(); } +} diff --git a/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/IdentityExperiment.java b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/IdentityExperiment.java new file mode 100644 index 0000000..2e543fa --- /dev/null +++ b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/IdentityExperiment.java @@ -0,0 +1,66 @@ +package com.adaworldapi.lancegraph.lab; + +/** + * Truth (a) — semantic: a {@link LaneId} is a value. Its object identity should be + * irrelevant, and nothing in the API should be able to observe it. + * + *

This experiment does not measure performance. It measures whether the runtime agrees + * with that semantic claim, which is the thing that has to be true before any performance claim + * means anything: a type whose identity is observable cannot be flattened, no matter how fast the + * JIT is. + * + *

The interesting output is not "value classes are values". It is the list of behaviours that + * are identical across the two runs — because every one of those is a place where the + * production API can move to Valhalla without a single caller noticing. + */ +final class IdentityExperiment { + + private IdentityExperiment() {} + + static void run() { + Lab.section("(a) SEMANTIC TRUTH — is identity observable?"); + Lab.kv("platform", Platform.NAME + " — " + Platform.describe()); + Lab.kv("LaneId.class.isValue()", Platform.isValueClass(LaneId.class)); + Lab.kv("Ordinal.class.isValue()", Platform.isValueClass(Ordinal.class)); + Lab.kv("MaskId.class.isValue()", Platform.isValueClass(MaskId.class)); + Lab.kv("RowRange.class.isValue()", Platform.isValueClass(RowRange.class)); + Lab.kv("Row.class.isValue()", Platform.isValueClass(Row.class)); + + LaneId a = LaneId.of(5); + LaneId b = LaneId.of(5); + LaneId c = LaneId.of(6); + + // The semantic contract, restated as assertions. These must hold on BOTH platforms — that + // is the point. If any of them differed, the migration would not be source-compatible. + Lab.kv("equal state => equals()", a.equals(b)); + Lab.kv("different state => !equals()", !a.equals(c)); + Lab.kv("equal state => equal hashCode()", a.hashCode() == b.hashCode()); + Lab.kv("equal state => equal toString()", a.toString().equals(b.toString())); + + // Reference equality is the ONE observable that legitimately differs, and the production + // API never uses it. Reported, not asserted, because a stable JDK is free to intern or not. + Lab.kv("a == b (reference equality)", a == b); + Lab.kv("identityHashCode(a) == identityHashCode(b)", + System.identityHashCode(a) == System.identityHashCode(b)); + + // Can a variable of this type hold null? Under Valhalla a plain declared type still can — + // null-restriction is a property of a FIELD or an ARRAY, not of the class. That surprises + // people, so it is measured rather than described. + LaneId maybeNull = null; + Lab.kv("a local of this type accepts null", maybeNull == null); + + Object[] arr = Platform.newLaneIdArray(4); + arr[0] = LaneId.of(1); + Lab.kv("array kind", arr.getClass().getSimpleName()); + Lab.kv("array flatness", Platform.arrayFlatness(arr)); + Lab.kv("array slot accepts null", Platform.arrayAcceptsNull(arr)); + + // synchronized(valueObject) does not compile under Valhalla — "required: a type with + // identity". It is exercised reflectively-in-spirit here (as a documented fact rather than + // live code) because writing it in shared source would break the stable compile too. + Lab.kv("synchronized(x) legality", + Platform.isValueVocabulary() + ? "COMPILE ERROR under Valhalla (required: a type with identity)" + : "legal but never used by the production API"); + } +} diff --git a/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/Lab.java b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/Lab.java new file mode 100644 index 0000000..92e5b3b --- /dev/null +++ b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/Lab.java @@ -0,0 +1,144 @@ +package com.adaworldapi.lancegraph.lab; + +import java.lang.management.ManagementFactory; +import java.util.Arrays; + +/** + * The lab's measurement instruments. Deliberately small, and deliberately biased towards + * measurements that are hard to fool. + * + *

Why allocation bytes, not timing, is the primary instrument here

+ * + *

The question "did this abstraction cost a heap object?" is answered directly by + * {@code ThreadMXBean.getThreadAllocatedBytes}, which the VM maintains from TLAB accounting. A + * timing measurement answers it only by inference, and the inference is weak: escape analysis + * already removes many allocations, so a fast loop proves nothing about whether the object + * existed. Bytes are the observation; nanoseconds are the consequence. + * + *

This is also why {@code -XX:-DoEscapeAnalysis} appears in the run script. Comparing a record + * against a value class with escape analysis on measures the JIT's ability to see through a + * particular loop shape. Comparing with it off measures what the object model actually + * costs when the JIT cannot save it — which is the property that generalises to real code, where + * objects escape into arrays and collections all the time. + * + *

Timing

+ * + *

Warm-up then repeated measurement, reporting the median and the full min/max spread. The + * median is reported rather than the mean because a single GC pause or scheduler preemption in a + * 4-core container moves a mean and does not move a median. The spread is printed alongside so a + * reader can see when the median is not meaningful. This is not JMH — {@link #time} does not fork, + * does not detect steady state, and does not do statistical rigour. Where JMH-grade numbers are + * needed, they live in {@code bench/} and are produced by actual JMH. + */ +final class Lab { + + private Lab() {} + + private static final com.sun.management.ThreadMXBean THREADS = + (com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean(); + + /** A sink the JIT cannot prove dead. Not a JMH blackhole; adequate for this lab's purposes. */ + static volatile long SINK; + static volatile Object OBJ_SINK; + + // ── allocation ─────────────────────────────────────────────────────────────────────────── + + /** + * Bytes this thread allocated while running {@code body}, minus the harness's own overhead. + * + *

The baseline subtraction matters: calling {@code getThreadAllocatedBytes} is itself not + * free of allocation on every VM, so an unsubtracted number would attribute the instrument's + * cost to the subject. Measured on this VM the baseline is 0, and the code says so rather than + * assuming it. + */ + static long allocatedBytes(Runnable body) { + long tid = Thread.currentThread().threadId(); + // Touch the instrument once so its own class-init allocation is not attributed to body. + THREADS.getThreadAllocatedBytes(tid); + long before = THREADS.getThreadAllocatedBytes(tid); + body.run(); + long after = THREADS.getThreadAllocatedBytes(tid); + return after - before; + } + + /** The instrument's own cost, printed so the reader can see it is negligible. */ + static long allocationInstrumentBaseline() { + return allocatedBytes(() -> {}); + } + + // ── retained footprint ─────────────────────────────────────────────────────────────────── + + /** + * Approximate retained heap of whatever {@code supplier} returns and keeps reachable. + * + *

Labelled approximate deliberately. It is a used-heap delta around a + * best-effort GC, so it is perturbed by anything else the VM does concurrently. It is reported + * because footprint is the question the thesis actually asks, and cross-checked against the + * exact allocation count so the two must agree in magnitude or one of them is wrong. + */ + static long retainedBytesApprox(java.util.function.Supplier supplier) { + Runtime rt = Runtime.getRuntime(); + settle(); + long before = rt.totalMemory() - rt.freeMemory(); + Object held = supplier.get(); + settle(); + long after = rt.totalMemory() - rt.freeMemory(); + OBJ_SINK = held; // keep it reachable across the second measurement + return after - before; + } + + private static void settle() { + for (int i = 0; i < 4; i++) { + System.gc(); + try { Thread.sleep(30); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } + } + } + + // ── timing ─────────────────────────────────────────────────────────────────────────────── + + record Timing(String name, double medianNs, double minNs, double maxNs, int iterations) { + @Override public String toString() { + return String.format("%-46s median=%10.1f ns [min %10.1f .. max %10.1f] n=%d", + name, medianNs, minNs, maxNs, iterations); + } + /** Per-unit cost, for reporting a per-row or per-element number honestly. */ + double perUnitNs(long units) { return medianNs / units; } + } + + /** + * Warm up, then measure {@code iterations} times and report median and spread. + * + * @param warmupRuns how many untimed runs before measuring; must be enough for C2 to compile + * the loop, which for these bodies is hundreds, not tens + */ + static Timing time(String name, int warmupRuns, int iterations, Runnable body) { + for (int i = 0; i < warmupRuns; i++) body.run(); + double[] samples = new double[iterations]; + for (int i = 0; i < iterations; i++) { + long t0 = System.nanoTime(); + body.run(); + samples[i] = System.nanoTime() - t0; + } + double[] sorted = samples.clone(); + Arrays.sort(sorted); + return new Timing(name, sorted[sorted.length / 2], sorted[0], sorted[sorted.length - 1], + iterations); + } + + // ── output ─────────────────────────────────────────────────────────────────────────────── + + static void section(String title) { + System.out.println(); + System.out.println("== " + title + " " + "=".repeat(Math.max(0, 74 - title.length()))); + } + + static void kv(String key, Object value) { + System.out.printf("%-44s %s%n", key, value); + } + + static String bytes(long b) { + if (Math.abs(b) < 1024) return b + " B"; + if (Math.abs(b) < 1024 * 1024) return String.format("%.1f KiB", b / 1024.0); + return String.format("%.2f MiB", b / (1024.0 * 1024.0)); + } +} diff --git a/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/RunAll.java b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/RunAll.java new file mode 100644 index 0000000..031e9f0 --- /dev/null +++ b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/RunAll.java @@ -0,0 +1,46 @@ +package com.adaworldapi.lancegraph.lab; + +import com.adaworldapi.lancegraph.NativeRuntime; + +/** + * Runs every experiment and prints a machine-greppable report. + * + *

The same class is compiled twice — once against the {@code src/stable} vocabulary on the + * production JDK, once against {@code src/valhalla} on the JEP 401 build — and the two outputs are + * diffed. That is the whole design: one experiment source, two object models, one difference. + * + *

Exit codes: {@code 0} ran, {@code 2} the native library is unavailable (the thesis experiment + * needs it and a fabricated number would be worse than no number). + */ +public final class RunAll { + + private RunAll() {} + + public static void main(String[] args) { + System.out.println("lance-graph-java :: valhalla lab"); + Lab.kv("platform", Platform.NAME); + Lab.kv("java.vm.version", System.getProperty("java.vm.version")); + Lab.kv("java.vendor.version", System.getProperty("java.vendor.version", "-")); + Lab.kv("jvm args", java.lang.management.ManagementFactory.getRuntimeMXBean() + .getInputArguments()); + + IdentityExperiment.run(); + FlatteningCliffExperiment.run(); + FootprintExperiment.run(); + + if (!NativeRuntime.isAvailable()) { + Lab.section("NATIVE EXPERIMENTS SKIPPED"); + Lab.kv("reason", NativeRuntime.unavailableReason().getMessage()); + System.out.println("\nno native library — the thesis experiment cannot be run, and a" + + " number invented for it would be worse than none."); + System.exit(2); + } + Lab.kv("native runtime", NativeRuntime.describe()); + + FfmAddressingExperiment.run(); + ThesisExperiment.run(); + + System.out.println(); + System.out.println("lab complete."); + } +} diff --git a/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/ThesisExperiment.java b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/ThesisExperiment.java new file mode 100644 index 0000000..5c1ddb2 --- /dev/null +++ b/valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/ThesisExperiment.java @@ -0,0 +1,170 @@ +package com.adaworldapi.lancegraph.lab; + +import com.adaworldapi.lancegraph.NativeAccess; +import com.adaworldapi.lancegraph.NativePattern; +import com.adaworldapi.lancegraph.Pattern; +import com.adaworldapi.lancegraph.internal.ffm.Engine; + +/** + * The central claim, put at risk. + * + *

64,000 logical entities must NOT require 64,000 Java objects.
+ * + *

Three ways to answer the same question over the same 65,536 rows — + * "how many rows have {@code class == 7} and {@code value > 100}, and what do their values sum + * to?" — measured for heap cost and for time: + * + *

    + *
  1. native — one lane set, one packed mask, one bulk crossing. Zero Java + * objects per row. + *
  2. objects — 65,536 materialised {@code Row} instances, then an ordinary + * Java loop. This is what a developer writes when the API hands them entities. + *
  3. values — the same 65,536 rows as Valhalla value objects in a flat array. + * This is the "Valhalla will fix it" hypothesis, given its best case: null-restricted, + * non-atomic, flattened storage. + *
+ * + *

The comparison is only fair if all three read identical bytes. They do: + * paths 2 and 3 are populated by copying out of the very lanes path 1 scans, so no path enjoys a + * different dataset, a different distribution, or a warmer cache than the others. The three + * answers are asserted equal before any number is reported — a benchmark whose variants compute + * different things is measuring nothing. + * + *

The expected finding is that Valhalla helps the tiny descriptor vocabulary (see + * {@link FootprintExperiment}) and does not rescue per-entity materialisation. Expected is + * not observed; the numbers are printed either way, including if they contradict it. + */ +final class ThesisExperiment { + + private ThesisExperiment() {} + + static final int ROWS = 65_536; + private static final int CLASS_NEEDLE = 7; + private static final int VALUE_THRESHOLD = 100; + + static void run() { + Lab.section("THE THESIS — 65,536 entities, three representations"); + Lab.kv("platform", Platform.NAME); + Lab.kv("rows", ROWS); + Lab.kv("question", "count(class==" + CLASS_NEEDLE + " AND value>" + VALUE_THRESHOLD + + ") and sum(value)"); + + try (NativePattern data = NativePattern.open(ROWS)) { + + // ── (1) native: one lane set, one packed mask, one bulk op ─────────────────────── + // Warm first. The View's scratch mask is created lazily on the first terminal call, + // and the FFM scratch arena on the first crossing, so an unwarmed measurement would + // report one-time setup as if it were per-query cost. + for (int i = 0; i < 2_000; i++) { + Lab.SINK = data.view() + .where(Pattern.CLASS.eq(CLASS_NEEDLE)) + .where(Pattern.VALUE.gt(VALUE_THRESHOLD)) + .count(); + } + long nativeAllocBytes = Lab.allocatedBytes(() -> + Lab.SINK = data.view() + .where(Pattern.CLASS.eq(CLASS_NEEDLE)) + .where(Pattern.VALUE.gt(VALUE_THRESHOLD)) + .count()); + long nativeCount = data.view() + .where(Pattern.CLASS.eq(CLASS_NEEDLE)) + .where(Pattern.VALUE.gt(VALUE_THRESHOLD)) + .count(); + long nativeSum = data.view() + .where(Pattern.CLASS.eq(CLASS_NEEDLE)) + .where(Pattern.VALUE.gt(VALUE_THRESHOLD)) + .sumOf(Pattern.VALUE); + + // ── build (2)/(3) from the SAME native bytes ───────────────────────────────────── + Engine.LaneWindow ids = NativeAccess.lane(data, NativeAccess.LANE_ID); + Engine.LaneWindow classes = NativeAccess.lane(data, NativeAccess.LANE_CLASS); + Engine.LaneWindow values = NativeAccess.lane(data, NativeAccess.LANE_VALUE); + + Object[] rows = Platform.newRowArray(ROWS); + long hydrateAllocBytes = Lab.allocatedBytes(() -> { + for (int i = 0; i < ROWS; i++) { + rows[i] = new Row(ids.getU64(i), (int) classes.getU32(i), values.getI32(i)); + } + }); + Lab.OBJ_SINK = rows; + + long objCount = 0, objSum = 0; + for (int i = 0; i < ROWS; i++) { + Row r = (Row) rows[i]; + if (r.cls() == CLASS_NEEDLE && r.value() > VALUE_THRESHOLD) { objCount++; objSum += r.value(); } + } + + // ── the falsifier: all three must agree, or nothing below means anything ───────── + if (objCount != nativeCount || objSum != nativeSum) { + throw new AssertionError("the three paths do not compute the same answer: native=" + + nativeCount + "/" + nativeSum + " objects=" + objCount + "/" + objSum); + } + Lab.kv("answer (identical across all paths)", nativeCount + " rows, sum " + nativeSum); + Lab.kv("selectivity", String.format("%.2f%%", 100.0 * nativeCount / ROWS)); + + // ── heap cost ──────────────────────────────────────────────────────────────────── + Lab.section(" heap cost"); + Lab.kv("(1) native — Java bytes allocated (warm)", Lab.bytes(nativeAllocBytes) + + " per query, for the fluent chain itself"); + Lab.kv("(1) native — Java objects per row", 0); + Lab.kv("(1) native — native lane bytes", + Lab.bytes(ROWS * (8L + 4 + 4)) + " (u64 id + u32 class + i32 value)"); + Lab.kv("(1) native — mask bytes", + Lab.bytes((ROWS + 63) / 64 * 8L) + " (1 bit per row, packed)"); + Lab.kv("(2)/(3) hydrate " + ROWS + " Row — allocated", + Lab.bytes(hydrateAllocBytes)); + Lab.kv(" ... per row", String.format("%.2f B", hydrateAllocBytes / (double) ROWS)); + Lab.kv(" array flatness", Platform.arrayFlatness(rows)); + Lab.kv(" ratio vs native lane bytes", + String.format("%.2fx", hydrateAllocBytes / (double) (ROWS * 16L))); + + long retained = Lab.retainedBytesApprox(() -> { + Object[] held = Platform.newRowArray(ROWS); + for (int i = 0; i < ROWS; i++) { + held[i] = new Row(ids.getU64(i), (int) classes.getU32(i), values.getI32(i)); + } + return held; + }); + Lab.kv(" retained heap (APPROX, gc-delta)", Lab.bytes(retained)); + + // ── time ───────────────────────────────────────────────────────────────────────── + Lab.section(" time to answer the question"); + System.out.println(Lab.time("(1) native one crossing, fused plan", 2_000, 51, () -> + Lab.SINK = data.view() + .where(Pattern.CLASS.eq(CLASS_NEEDLE)) + .where(Pattern.VALUE.gt(VALUE_THRESHOLD)) + .count())); + + System.out.println(Lab.time("(2/3) hydrate " + ROWS + " Row objects", 200, 51, () -> { + Object[] a = Platform.newRowArray(ROWS); + for (int i = 0; i < ROWS; i++) { + a[i] = new Row(ids.getU64(i), (int) classes.getU32(i), values.getI32(i)); + } + Lab.OBJ_SINK = a; + })); + + System.out.println(Lab.time("(2/3) scan the materialised objects", 2_000, 51, () -> { + long c = 0, s = 0; + for (int i = 0; i < ROWS; i++) { + Row r = (Row) rows[i]; + if (r.cls() == CLASS_NEEDLE && r.value() > VALUE_THRESHOLD) { c++; s += r.value(); } + } + Lab.SINK = c + s; + })); + + System.out.println(Lab.time("(2/3) hydrate THEN scan (honest total)", 200, 51, () -> { + Object[] a = Platform.newRowArray(ROWS); + for (int i = 0; i < ROWS; i++) { + a[i] = new Row(ids.getU64(i), (int) classes.getU32(i), values.getI32(i)); + } + long c = 0, s = 0; + for (int i = 0; i < ROWS; i++) { + Row r = (Row) a[i]; + if (r.cls() == CLASS_NEEDLE && r.value() > VALUE_THRESHOLD) { c++; s += r.value(); } + } + Lab.SINK = c + s; + Lab.OBJ_SINK = a; + })); + } + } +} diff --git a/valhalla-lab/src/stable/com/adaworldapi/lancegraph/lab/Containers.java b/valhalla-lab/src/stable/com/adaworldapi/lancegraph/lab/Containers.java new file mode 100644 index 0000000..a8e3be6 --- /dev/null +++ b/valhalla-lab/src/stable/com/adaworldapi/lancegraph/lab/Containers.java @@ -0,0 +1,31 @@ +package com.adaworldapi.lancegraph.lab; + +/** + * Field-flattening subjects, stable half. + * + *

{@code Descriptor} is shaped like the real thing: a {@code Field} in the production API holds + * a {@link LaneId} and an {@link Ordinal}, and both are wrappers around a single {@code int}. On a + * stable JDK that is three objects and two pointer hops to read two integers. + */ +final class Containers { + + private Containers() {} + + /** Two value-shaped wrappers held as ordinary references. */ + static final class Descriptor { + final LaneId lane; + final Ordinal ordinal; + Descriptor(int lane, int ordinal) { + this.lane = LaneId.of(lane); + this.ordinal = Ordinal.of(ordinal); + } + int laneIndex() { return lane.index(); } + int ordinalValue() { return ordinal.value(); } + } + + static Descriptor make(int lane, int ordinal) { return new Descriptor(lane, ordinal); } + static int read(Descriptor d) { return d.laneIndex() + d.ordinalValue(); } + + static String kind() { return "identity class with two reference fields"; } + static boolean fieldsAreNullRestricted() { return false; } +} diff --git a/valhalla-lab/src/stable/com/adaworldapi/lancegraph/lab/Platform.java b/valhalla-lab/src/stable/com/adaworldapi/lancegraph/lab/Platform.java new file mode 100644 index 0000000..ec66ef7 --- /dev/null +++ b/valhalla-lab/src/stable/com/adaworldapi/lancegraph/lab/Platform.java @@ -0,0 +1,68 @@ +package com.adaworldapi.lancegraph.lab; + +/** + * The stable-JDK half of the A/B. Everything here reports what an ordinary {@code record} on a + * production JDK actually is. + * + *

Two of these answers are unknowable on a stable JDK rather than false, and they say + * so: there is no {@code ValueClass.isFlatArray} to ask, because there is no flattening to ask + * about. Reporting "false" would imply the question was answered; {@code UNKNOWN} says it was not. + */ +final class Platform { + + private Platform() {} + + static final String NAME = "stable"; + + /** Does the runtime consider this vocabulary identity-free? */ + static boolean isValueVocabulary() { + return isValueClass(LaneId.class); + } + + /** + * Is {@code type} a value class, on a platform that can even ask. + * + *

{@code Class::isValue} does not exist on this JDK at all — this is not an "unknowable + * answer" case like {@link #arrayFlatness}, it is a missing method, confirmed by the compiler. + * The correct answer is still knowable without the query: on a JDK with no value-class + * concept, nothing compiled here can BE one, so {@code false} is exact, not a guess. + */ + static boolean isValueClass(Class type) { + return false; + } + + /** Flatness of an array, where the runtime can answer. */ + static String arrayFlatness(Object array) { + return "UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat)"; + } + + /** Allocate the vocabulary array the way this platform can. */ + static Object[] newLaneIdArray(int n) { + return new LaneId[n]; + } + + /** Allocate the per-entity Row array the way this platform can. */ + static Object[] newRowArray(int n) { + return new Row[n]; + } + + /** Can this array hold null in every slot? */ + static boolean arrayAcceptsNull(Object[] array) { + try { Object keep = array[0]; array[0] = null; array[0] = keep; return true; } + catch (Throwable t) { return false; } + } + + /** + * Allocate the densest array this platform offers for {@code component}. + * + *

On a stable JDK there is exactly one kind of object array, so this is + * {@code Array.newInstance} and the {@code init} value is unused. + */ + static Object[] newArrayOf(Class component, int n, Object init) { + return (Object[]) java.lang.reflect.Array.newInstance(component, n); + } + + static String describe() { + return "stable-record vocabulary; arrays are reference arrays; fields are references"; + } +} diff --git a/valhalla-lab/src/stable/com/adaworldapi/lancegraph/lab/Vocab.java b/valhalla-lab/src/stable/com/adaworldapi/lancegraph/lab/Vocab.java new file mode 100644 index 0000000..1a0fca5 --- /dev/null +++ b/valhalla-lab/src/stable/com/adaworldapi/lancegraph/lab/Vocab.java @@ -0,0 +1,61 @@ +package com.adaworldapi.lancegraph.lab; + +/** + * The A/B vocabulary. This file exists twice — once under {@code src/stable} and once under + * {@code src/valhalla} — and the two copies are byte-identical apart from the word {@code value} + * on each declaration. {@code run.sh} diffs them and refuses to run if any other difference has + * crept in, so "same semantic contract, one modifier" is checked rather than claimed. + * + *

Each type mirrors the corresponding type in the real API + * ({@code com.adaworldapi.lancegraph.LaneId} and friends). The mirror exists because the real + * types must keep compiling on the production JDK; the lab is where the same contract is compiled + * a second way and measured. + */ +final class VocabDoc { private VocabDoc() {} } + +/** Which column of the struct-of-arrays a field lives in. Mirrors the production {@code LaneId}. */ +record LaneId(int index) { + LaneId { + if (index < 0) throw new IllegalArgumentException("lane index must be >= 0, was " + index); + } + static LaneId of(int index) { return new LaneId(index); } + @Override public String toString() { return "lane#" + index; } +} + +/** A half-open span of row indices. Mirrors the production {@code RowRange}. */ +record RowRange(long start, long endExclusive) { + RowRange { + if (start < 0) throw new IllegalArgumentException("start must be >= 0, was " + start); + if (endExclusive < start) throw new IllegalArgumentException("endExclusive < start"); + } + static RowRange of(long count) { return new RowRange(0, count); } + long length() { return endExclusive - start; } + boolean isEmpty() { return endExclusive == start; } + boolean contains(long row) { return row >= start && row < endExclusive; } + @Override public String toString() { return "rows[" + start + "," + endExclusive + ")"; } +} + +/** The identity of a selection, as an opaque token. Mirrors the production {@code MaskId}. */ +record MaskId(long token) { + int slot() { return (int) (token & 0xFFFF_FFFFL); } + int generation() { return (int) (token >>> 32); } + @Override public String toString() { return "mask#" + slot() + "@" + generation(); } +} + +/** A field's position within its schema. Mirrors the production {@code Ordinal}. */ +record Ordinal(int value) { + Ordinal { + if (value < 0) throw new IllegalArgumentException("ordinal must be >= 0, was " + value); + } + static Ordinal of(int value) { return new Ordinal(value); } + @Override public String toString() { return "#" + value; } +} + +/** + * One logical entity, materialised as a Java object. + * + *

This is the type the thesis says must NOT exist per row. It is declared here precisely so + * that "must not" can be measured instead of asserted: {@code ThesisExperiment} builds 65,536 of + * these and reports what they cost against the native lane that needs none of them. + */ +record Row(long id, int cls, int value) {} diff --git a/valhalla-lab/src/valhalla/com/adaworldapi/lancegraph/lab/Containers.java b/valhalla-lab/src/valhalla/com/adaworldapi/lancegraph/lab/Containers.java new file mode 100644 index 0000000..9811b04 --- /dev/null +++ b/valhalla-lab/src/valhalla/com/adaworldapi/lancegraph/lab/Containers.java @@ -0,0 +1,46 @@ +package com.adaworldapi.lancegraph.lab; + +import jdk.internal.value.ValueClass; +import jdk.internal.vm.annotation.NullRestricted; + +/** + * Field-flattening subjects, Valhalla half. Same shape as the stable {@code Descriptor}; the two + * differences are the {@code value} modifier and {@code @NullRestricted} on the fields. + * + *

Both differences are load-bearing and that is a finding, not a detail. + * {@code @NullRestricted} is what permits a flat encoding — a nullable value field still needs + * somewhere to record "this one is null". And the container must itself be a {@code value class}: + * an ordinary identity class with {@code @NullRestricted} fields fails at class load with + * {@code VerifyError: Invalid use of strict instance fields}, because javac on this build does not + * emit the strict initialisation order the VM demands. See + * {@code reproducers/nullrestricted-field-in-identity-class.md}. + */ +final class Containers { + + private Containers() {} + + /** Two value-shaped wrappers held as null-restricted, flattenable fields. */ + static value class Descriptor { + @NullRestricted final LaneId lane; + @NullRestricted final Ordinal ordinal; + Descriptor(int lane, int ordinal) { + this.lane = LaneId.of(lane); + this.ordinal = Ordinal.of(ordinal); + } + int laneIndex() { return lane.index(); } + int ordinalValue() { return ordinal.value(); } + } + + static Descriptor make(int lane, int ordinal) { return new Descriptor(lane, ordinal); } + static int read(Descriptor d) { return d.laneIndex() + d.ordinalValue(); } + + static String kind() { return "value class with two @NullRestricted value fields"; } + + static boolean fieldsAreNullRestricted() { + try { + return ValueClass.isNullRestrictedField(Descriptor.class.getDeclaredField("lane")); + } catch (NoSuchFieldException e) { + throw new AssertionError(e); + } + } +} diff --git a/valhalla-lab/src/valhalla/com/adaworldapi/lancegraph/lab/Platform.java b/valhalla-lab/src/valhalla/com/adaworldapi/lancegraph/lab/Platform.java new file mode 100644 index 0000000..6a53d9d --- /dev/null +++ b/valhalla-lab/src/valhalla/com/adaworldapi/lancegraph/lab/Platform.java @@ -0,0 +1,65 @@ +package com.adaworldapi.lancegraph.lab; + +import jdk.internal.value.ValueClass; + +/** + * The Valhalla half of the A/B. Same signatures as the stable {@code Platform}, so the shared + * experiment sources are byte-identical across both runs. + * + *

{@link ValueClass#isFlatArray} is the measurement that matters here: it is the VM answering + * about its own array, not an inference from a timing or a footprint estimate. That makes the + * flattening result hard to fool, which is why it is preferred over anything derived. + * + *

Requires {@code --add-exports java.base/jdk.internal.value=ALL-UNNAMED}. That export is the + * honest cost of the experiment: null-restricted, guaranteed-flat storage has no + * supported surface in this early-access build (see reproducers/no-public-flat-array-api.md). + */ +final class Platform { + + private Platform() {} + + static final String NAME = "valhalla"; + + static boolean isValueVocabulary() { + return isValueClass(LaneId.class); + } + + /** Is {@code type} a value class? The real, final {@code Class::isValue} query. */ + static boolean isValueClass(Class type) { + return type.isValue(); + } + + static String arrayFlatness(Object array) { + return ValueClass.isFlatArray(array) ? "FLAT" : "NOT-FLAT"; + } + + /** + * Null-restricted, non-atomic — the encoding with no null marker and no atomicity padding, + * i.e. the densest layout the VM offers. This is the array the thesis experiment needs to be + * fair to Valhalla: giving it a plain reference array would measure the old world twice. + */ + static Object[] newLaneIdArray(int n) { + return ValueClass.newNullRestrictedNonAtomicArray(LaneId.class, n, new LaneId(0)); + } + + static Object[] newRowArray(int n) { + return ValueClass.newNullRestrictedNonAtomicArray(Row.class, n, new Row(0, 0, 0)); + } + + static boolean arrayAcceptsNull(Object[] array) { + try { Object keep = array[0]; array[0] = null; array[0] = keep; return true; } + catch (Throwable t) { return false; } + } + + /** + * Allocate the densest array this platform offers for {@code component} — null-restricted and + * non-atomic, the encoding with neither a null marker nor atomicity padding. + */ + static Object[] newArrayOf(Class component, int n, Object init) { + return ValueClass.newNullRestrictedNonAtomicArray(component, n, init); + } + + static String describe() { + return "value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields"; + } +} diff --git a/valhalla-lab/src/valhalla/com/adaworldapi/lancegraph/lab/Vocab.java b/valhalla-lab/src/valhalla/com/adaworldapi/lancegraph/lab/Vocab.java new file mode 100644 index 0000000..78b88b8 --- /dev/null +++ b/valhalla-lab/src/valhalla/com/adaworldapi/lancegraph/lab/Vocab.java @@ -0,0 +1,61 @@ +package com.adaworldapi.lancegraph.lab; + +/** + * The A/B vocabulary. This file exists twice — once under {@code src/stable} and once under + * {@code src/valhalla} — and the two copies are byte-identical apart from the word {@code value} + * on each declaration. {@code run.sh} diffs them and refuses to run if any other difference has + * crept in, so "same semantic contract, one modifier" is checked rather than claimed. + * + *

Each type mirrors the corresponding type in the real API + * ({@code com.adaworldapi.lancegraph.LaneId} and friends). The mirror exists because the real + * types must keep compiling on the production JDK; the lab is where the same contract is compiled + * a second way and measured. + */ +final class VocabDoc { private VocabDoc() {} } + +/** Which column of the struct-of-arrays a field lives in. Mirrors the production {@code LaneId}. */ +value record LaneId(int index) { + LaneId { + if (index < 0) throw new IllegalArgumentException("lane index must be >= 0, was " + index); + } + static LaneId of(int index) { return new LaneId(index); } + @Override public String toString() { return "lane#" + index; } +} + +/** A half-open span of row indices. Mirrors the production {@code RowRange}. */ +value record RowRange(long start, long endExclusive) { + RowRange { + if (start < 0) throw new IllegalArgumentException("start must be >= 0, was " + start); + if (endExclusive < start) throw new IllegalArgumentException("endExclusive < start"); + } + static RowRange of(long count) { return new RowRange(0, count); } + long length() { return endExclusive - start; } + boolean isEmpty() { return endExclusive == start; } + boolean contains(long row) { return row >= start && row < endExclusive; } + @Override public String toString() { return "rows[" + start + "," + endExclusive + ")"; } +} + +/** The identity of a selection, as an opaque token. Mirrors the production {@code MaskId}. */ +value record MaskId(long token) { + int slot() { return (int) (token & 0xFFFF_FFFFL); } + int generation() { return (int) (token >>> 32); } + @Override public String toString() { return "mask#" + slot() + "@" + generation(); } +} + +/** A field's position within its schema. Mirrors the production {@code Ordinal}. */ +value record Ordinal(int value) { + Ordinal { + if (value < 0) throw new IllegalArgumentException("ordinal must be >= 0, was " + value); + } + static Ordinal of(int value) { return new Ordinal(value); } + @Override public String toString() { return "#" + value; } +} + +/** + * One logical entity, materialised as a Java object. + * + *

This is the type the thesis says must NOT exist per row. It is declared here precisely so + * that "must not" can be measured instead of asserted: {@code ThesisExperiment} builds 65,536 of + * these and reports what they cost against the native lane that needs none of them. + */ +value record Row(long id, int cls, int value) {} From 4ab0da8da7cb699577f7caf67f3bf57bfd6a1c88 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 21:50:57 +0000 Subject: [PATCH 03/30] Vector API bench: real JMH, cross-checked; the crossing does not always win Completes D-LGJ-G, the mission's mandated "where does execution belong" comparison -- measured, not assumed to favor the Rust crossing. Real JMH 1.37 (fork+warmup+compiler-blackholes confirmed in the log, not a hand-rolled loop -- that lives in valhalla-lab and is labelled as such there). Four components, cost kept strictly separate per the mission brief: A_DowncallOverhead (bare crossing, no work), B_SegmentAccess (raw native-memory read throughput), C_ExecutionBoundary (native fused plan vs Java Vector API vs Java scalar, swept 64 to 4,194,304 rows), E_FusionAndPlanning (fused vs unfused vs the scalar reference kernel vs plan-construction-only, swept 1-8 predicates). 50/50 rows, 0 failures. Data.crossCheck() runs in @Setup and throws if the three kernels disagree on count or sum, so a faster-but-wrong Vector kernel could not have won the comparison undetected. The headline complicates the thesis honestly: for a single predicate over one native lane, the Java Vector API -- reading the SAME native MemorySegment zero-copy via IntVector.fromMemorySegment, no byte[], no bounce buffer -- beats the native crossing at EVERY row count tested, 56.4x at small sizes down to 1.3-1.4x at 4M rows. A second crossover is also real: native beats a plain Java scalar loop only past roughly 4,096-16,384 rows. Component E shows why this doesn't overturn the project's premise: SIMD-vs-scalar is the largest lever measured anywhere in this suite (10.8x-31.1x, growing with predicate count), and fused/unfused land within this harness's own ~10% noise floor of each other at 65,536 rows -- the fused plan's real value is the structural one-crossing guarantee (already proven by LazinessTest), not a large measured time saving at this scale. Verdict: the crossing is worth paying for composed, multi-predicate work, not for reading one predicate off one lane, where Java on the same memory is simply faster. RESULTS.md was hand-written from the raw CSV, then independently cross-checked against summarise.sh -- a script the same PR ships that mechanically regenerates every table from results/jmh-results.csv, so a re-run's numbers can never silently drift from a hand-transcribed table. Both productions agreed to 3 decimal places on every cell checked. Generated by [Claude Code](https://claude.ai/code) --- .claude/board/EPIPHANIES.md | 41 + .claude/board/STATUS_BOARD.md | 6 +- bench/README.md | 113 + bench/RESULTS.md | 127 + bench/results/jmh-results-full.csv | 50 + bench/results/jmh-results.csv | 50 + bench/results/jmh-run-full.txt | 2139 +++++++++++++++++ bench/results/jmh-run.txt | 344 +++ bench/run.sh | 57 + .../adaworldapi/lancegraph/NativeAccess.java | 49 + .../lancegraph/bench/A_DowncallOverhead.java | 112 + .../lancegraph/bench/B_SegmentAccess.java | 72 + .../lancegraph/bench/C_ExecutionBoundary.java | 84 + .../adaworldapi/lancegraph/bench/Data.java | 100 + .../lancegraph/bench/E_FusionAndPlanning.java | 128 + .../adaworldapi/lancegraph/bench/Harness.java | 87 + .../adaworldapi/lancegraph/bench/Kernels.java | 148 ++ bench/summarise.sh | 89 + valhalla-lab/README.md | 126 +- valhalla-lab/docs/three-truths.md | 265 ++ valhalla-lab/run.sh | 1 + 21 files changed, 4107 insertions(+), 81 deletions(-) create mode 100644 bench/README.md create mode 100644 bench/RESULTS.md create mode 100644 bench/results/jmh-results-full.csv create mode 100644 bench/results/jmh-results.csv create mode 100644 bench/results/jmh-run-full.txt create mode 100644 bench/results/jmh-run.txt create mode 100755 bench/run.sh create mode 100644 bench/src/main/java/com/adaworldapi/lancegraph/NativeAccess.java create mode 100644 bench/src/main/java/com/adaworldapi/lancegraph/bench/A_DowncallOverhead.java create mode 100644 bench/src/main/java/com/adaworldapi/lancegraph/bench/B_SegmentAccess.java create mode 100644 bench/src/main/java/com/adaworldapi/lancegraph/bench/C_ExecutionBoundary.java create mode 100644 bench/src/main/java/com/adaworldapi/lancegraph/bench/Data.java create mode 100644 bench/src/main/java/com/adaworldapi/lancegraph/bench/E_FusionAndPlanning.java create mode 100644 bench/src/main/java/com/adaworldapi/lancegraph/bench/Harness.java create mode 100644 bench/src/main/java/com/adaworldapi/lancegraph/bench/Kernels.java create mode 100755 bench/summarise.sh create mode 100644 valhalla-lab/docs/three-truths.md diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index d3cb4fb..d924fdc 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -4,6 +4,47 @@ > `**Status:**`/`**Confidence:**` line. A correction gets its own new, > dated entry that references the one it corrects — the storno rule. +## 2026-08-17 — E-LGJ-VECTOR-API-BEATS-THE-CROSSING-1 + +**Status:** FINDING. **Confidence:** High (real JMH 1.37, `Data.crossCheck()` guards every fork, +independently cross-checked against a second, mechanically-generated computation of the same CSV). + +Completes D-LGJ-G, the mission's mandated "where does execution belong — measure it, do not assume +the Rust side wins" comparison. The honest answer complicates the thesis in a useful way: **for a +single predicate over one native lane, the Java Vector API — reading the SAME native +`MemorySegment` zero-copy via `IntVector.fromMemorySegment`, no `byte[]`, no bounce buffer — beats +the native `lgj_plan_eval` crossing at every row count tested, from 64 to 4,194,304**, by 56.4× at +small sizes down to 1.33-1.41× at the largest: + +| rows | native (µs) | vectorApi (µs) | vectorApi wins by | +|---:|---:|---:|---:| +| 64 | 0.612 | 0.011 | 56.40× | +| 65,536 | 15.324 | 8.027 | 1.91× | +| 4,194,304 | 1858.686 | 1319.107 | 1.41× | + +A second, separate crossover is also real: native beats a plain Java **scalar** loop only past +roughly 4,096-16,384 rows — below that the crossing's own fixed cost (consistent with Component A's +measured ~22 ns bare-downcall floor) is not yet repaid. + +**Why this does not overturn the project's thesis, and where the thesis's own machinery already +shows the real answer.** Component C isolates exactly one predicate, one lane — the case with +nothing to fuse and nothing to coordinate, which is precisely the case a zero-copy Vector kernel is +best at. Component E (multi-predicate fusion) shows the picture change: SIMD-vs-scalar is the +largest lever measured anywhere in this benchmark (10.8×-31.1×, growing with predicate count), and +`fused`/`unfused` land within this harness's own stated ~10% noise floor of each other at 65,536 +rows — meaning the fused plan's real value is the STRUCTURAL guarantee of exactly one crossing +regardless of predicate count (already proven separately by `LazinessTest`), not a large measured +time saving at this scale. The honest verdict, matching the mission brief's own framing rather than +either extreme: **the crossing is worth paying for composed, multi-predicate work — not for reading +one predicate off one lane, where Java on the same memory is simply faster.** + +**Method note, since two independent computations of the same data is itself worth recording as a +discipline:** `RESULTS.md` was hand-written from the raw `results/jmh-results.csv`, then verified +against `bench/summarise.sh` — a separate script the same PR ships that mechanically regenerates +the tables from the CSV "so a re-run's numbers can be regenerated mechanically — a table +transcribed by hand is a table that can drift from its own data" (the script's own doc comment). +Both productions of the same 50-row CSV agreed to 3 decimal places on every cell checked. + ## 2026-08-17 — E-LGJ-VALHALLA-MEASURED-NOT-ASSUMED-1 **Status:** FINDING. **Confidence:** High (real numbers, both JDKs actually diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index f4be4bc..12ba44e 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -14,9 +14,9 @@ list. | D-LGJ-D | Java FFM membrane `internal/ffm` | **DONE 2026-08-17** — compiles clean with `-Xlint:all`; 7 `[restricted]` warnings, all in `internal/ffm/*` or a test deliberately exercising it; `AbiContractTest` 7/7 incl. proving the manifest cross-check genuinely rejects a wrong `.so` (`libz.so.1` loads but is refused for exporting no `lgj_abi_manifest`) | E | | D-LGJ-E | Java public facade (`NativePattern`/`View`/`Predicate`/`Pattern`/`Mask`) | **DONE 2026-08-17** — `AllTests` **132/132**: `ApiSurfaceTest` (reflection-enforced zero-FFM-leakage), `SmokeTest` 14/14, `FixtureParityTest` 30/30 (Java independently recomputes expected counts from the transcribed generator), `FusionParityTest` 31/31 (fused/unfused/scalar bit-identical across 6 row-count shapes), `LazinessTest` 8/8 (empirically: 0 crossings to build a 16-condition chain, exactly 1 to evaluate it, independent of rows up to 1,000,000 — the thesis's central claim, measured), `NarrowingTest` 16/16, `LifetimeTest` 23/23 | F, G | | D-LGJ-F | Valhalla lab — three-truths method on the small semantic value vocabulary | **DONE 2026-08-17** — `valhalla-lab/`: 4 experiments + a self-verifying `run.sh` (mechanically diffs the two `Vocab.java`s modulo the `value` keyword before trusting the A/B) + 3 causal-isolation runs (escape-analysis off; `UseArrayFlattening`/`UseFieldFlattening` toggled independently). 3 real Valhalla limitations reproduced and filed under `reproducers/` (R1: `@NullRestricted` field on an identity class is a `VerifyError`, javac's fault — no source form expresses required strict-field order; **R2: array flattening has a hard 8-byte payload cliff, VM-confirmed via `-XX:+PrintFlatArrayLayout`** — `LaneId`/`Ordinal`/`MaskId` (≤8B) flatten, `RowRange`/`Row` (16B) do not, so "Valhalla helps descriptors not entities" is a measured VM cutoff, not a hand-wave, and `RowRange` landing on the wrong side is flagged as the one place the expectation was too optimistic; R3: the densest null-restricted array form is `jdk.internal`-only and generics erase flattening entirely — `Foo!` type syntax confirmed NOT to parse, matching the archaeology finding). 1 real defect found + fixed before landing (see `EPIPHANIES.md`). None of the three limitations changed the production API — the migration path stays exactly `record` → `value record` | I | -| D-LGJ-G | Java Vector API comparative bench vs Panama→`ndarray::simd` | **In flight** — real JMH + JOL jars fetched (`jmh-core`/`jmh-generator-annprocess`/`jopt-simple`/`commons-math3`/`jol-core`) to `bench/lib/` (gitignored); no bench source written yet; the ONLY remaining open row | I | -| D-LGJ-H | Falsification: handle lifecycle (adversarial), SIMD/scalar parity, Java/native parity | **DONE 2026-08-17 for the Rust+Java core** — see D-LGJ-C's disable-verification and D-LGJ-E's `FusionParityTest`/`LifetimeTest`. Re-opens for F/G once the Lab lands | I | -| D-LGJ-I | Docs: `architecture.md`, `panama.md`, `valhalla-lab.md`, `execution-boundary.md` | **Queued** — gated on F/G landing (the docs synthesize Lab results, not just the core) | — | +| D-LGJ-G | Java Vector API comparative bench vs Panama→`ndarray::simd` | **DONE 2026-08-17** — real JMH 1.37 (fork+warmup+blackholes confirmed in the log), 50/50 rows, 0 failures, `Data.crossCheck()` guards every fork. **Headline (Component C, single predicate, zero-copy `IntVector.fromMemorySegment`): the Java Vector API beats the native crossing at EVERY row count tested, 64 to 4,194,304** — 56.4x at small sizes down to 1.3-1.4x at the largest. Native beats a plain Java scalar loop only past ~4,096-16,384 rows. Component E: SIMD-vs-scalar is the biggest lever measured (10.8x-31.1x); fused vs unfused are within noise of each other at 65,536 rows (crossing-count guarantee matters more than measured time here, since Component A puts one downcall at ~22ns). Independently cross-checked: hand-written `RESULTS.md` numbers verified byte-for-byte against `summarise.sh`'s mechanically-generated tables from the same CSV | I | +| D-LGJ-H | Falsification: handle lifecycle (adversarial), SIMD/scalar parity, Java/native parity | **DONE 2026-08-17, all scopes closed** — Rust+Java core (D-LGJ-C disable-verification, D-LGJ-E `FusionParityTest`/`LifetimeTest`); Valhalla lab (`run.sh`'s vocab-honesty self-check + causal-isolation runs); bench (`Data.crossCheck()` on every fork, `summarise.sh` cross-check) | I | +| D-LGJ-I | Docs: `architecture.md`, `panama.md`, `valhalla-lab.md`, `execution-boundary.md` | **Unblocked** — F and G both landed; next action | — | | D-LGJ-AUDIT | Mechanical post-fan-out audit: `grep` for `ndarray::hpc` imports, any `.h`/`cbindgen`/`jextract` artifact, any FFM type leaking into public Java API | **DONE 2026-08-17** — 1 real violation found (`kernels.rs::simd_popcount` used the internal `ndarray::hpc::bitwise` path), fixed in place; everything else confirmed to be the one sanctioned exception or explanatory prose | closed D-LGJ-C/D/E for the core | ### Reading this table diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..afc3234 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,113 @@ +# bench — where does execution belong? + +A JMH harness that keeps the cost components **separate**, because conflating them is how a +benchmark ends up arguing for whichever side its author already preferred. Put enough work behind +a downcall and the crossing disappears; put none behind it and the crossing is everything. + +> **This is real JMH** — `jmh-core 1.37`, forked JVMs, per-fork warm-up, compiler blackholes +> (the run log confirms `Compiler Blackholes ... are in use`). Not a hand-rolled loop. The +> hand-rolled harness in [`valhalla-lab`](../valhalla-lab) is labelled as such in its own README +> and its timings are secondary evidence there. + +**The measured numbers and the verdict are in [`RESULTS.md`](RESULTS.md).** + +## Run it + +```sh +cd bench && ./run.sh # everything (~12 min on 4 vCPU) +cd bench && ./run.sh C_ # only the execution-boundary row sweep +``` + +Requires `liblgj_abi.so`. Build it first if missing: + +```sh +cd native/lgj-abi && CARGO_TARGET_DIR=$(cd ../.. && pwd)/target cargo build --release +``` + +Output: `results/jmh-run.txt` (full log, including every warm-up iteration) and +`results/jmh-results.csv` (machine-readable). + +## The components, and why each is isolated + +| | class | question it answers alone | +|---|---|---| +| **A** | `A_DowncallOverhead` | what does crossing the membrane cost, with no work behind it? | +| **B** | `B_SegmentAccess` | how fast can Java read native memory at all? | +| **C** | `C_ExecutionBoundary` | native kernel vs Java Vector API vs Java scalar, swept over row count | +| **D** | *(in C)* | the Vector API arm — same segment, zero copy | +| **E** | `E_FusionAndPlanning` | fused (1 crossing) vs unfused (N crossings), swept over predicate count | +| **F** | *(in E)* | what does building the fluent chain cost, with no crossing at all? | + +C and D live in one class on purpose: the question is a comparison, and separate classes would let +a difference in setup masquerade as a difference in execution. E and F likewise share a fixture. + +## The rules this harness holds itself to + +**1. Every arm computes the same answer, and it is checked before anything is timed.** +`Data.crossCheck()` runs in `@Setup` and throws if the native, vector, and scalar kernels disagree +on either the count or the sum. This is not politeness: a Vector kernel with a broken tail is +*faster* than a correct one, so an unchecked comparison rewards the bug. + +**2. The Vector API arm is genuinely zero-copy.** `IntVector.fromMemorySegment(species, segment, +offset, ByteOrder.nativeOrder())` reads the native lane in place. No `byte[]`, no `int[]`, no +`MemorySegment.toArray`, no bounce buffer. A copy anywhere would make the comparison dishonest in +both directions at once — the Java side would pay a cost the Rust side does not, and the Rust side +would get credit for avoiding a copy the design never requires. + +The one heap `int[]` in the harness (`Data.valuesHeap`) is used **only** by Component B, as the +"data already in Java" ceiling. Nothing that compares against the native kernel touches it. + +**3. Nothing was added to the native library to be benchmarked.** Component A binds +`lgj_abi_manifest` and `lgj_mask_count` — both real ABI symbols. A symbol that exists only to be +measured is not the thing being measured. + +**4. Component A binds its own method handles** rather than reusing +`internal.ffm.Downcalls`, so it measures the JDK's linker rather than this project's wrapper. The +wrapper's own overhead is then visible as the difference between A and C. + +**5. The laziness claim is asserted, not assumed.** `planConstructionOnly` reads +`Diagnostics.crossings()` before and after building the chain and throws if it moved. + +**6. Warm JVM, always.** JMH forks, warms each fork, and discards warm-up. Cold-start numbers +appear nowhere. `shouldDoGC(true)` runs a GC between iterations so a collection triggered by one +arm is not attributed to the next. + +## Settings + +`@Fork(1)`, `@Warmup(5 × 500 ms)`, `@Measurement(8 × 500 ms)`, `Mode.AverageTime`. Reported as +mean ± 99.9 % confidence interval, which is JMH's default and what the CSV contains. + +**One fork, and that is a limitation worth naming.** A single fork cannot see run-to-run variance +from JIT compilation-order nondeterminism or address-space layout. Two or more forks would be +better; on a 4-vCPU container the full sweep already takes ~12 minutes and doubling it was judged +not worth the wall-clock. Treat differences under roughly 10 % between arms as not established by +this harness. + +**Shared container, not a tuned benchmark host.** No CPU pinning, no isolated cores, no disabled +turbo, no disabled hyperthreading. The confidence intervals reflect that. Large effects (order of +magnitude) are safe to read; small ones are not. + +## Fetching JMH + +The jars are not vendored. Fetch them into `lib/`: + +```sh +mkdir -p bench/lib && cd bench/lib +B=https://repo1.maven.org/maven2 +curl -sSLO --noproxy '*' $B/org/openjdk/jmh/jmh-core/1.37/jmh-core-1.37.jar +curl -sSLO --noproxy '*' $B/org/openjdk/jmh/jmh-generator-annprocess/1.37/jmh-generator-annprocess-1.37.jar +curl -sSLO --noproxy '*' $B/net/sf/jopt-simple/jopt-simple/5.0.4/jopt-simple-5.0.4.jar +curl -sSLO --noproxy '*' $B/org/apache/commons/commons-math3/3.6.1/commons-math3-3.6.1.jar +``` + +### One trap, recorded so nobody loses an hour to it + +On JDK 23+ annotation processing is **off by default**. Without `-proc:full`, JMH's generator +never runs, no `META-INF/BenchmarkList` is produced, and the harness dies at startup with: + +``` +ERROR: Unable to find the resource: /META-INF/BenchmarkList +``` + +which reads like a classpath problem and is not one. `run.sh` passes `-proc:full` with a comment +saying why. JMH also rejects benchmark classes in the default package, with a clear message. diff --git a/bench/RESULTS.md b/bench/RESULTS.md new file mode 100644 index 0000000..ce2f6ce --- /dev/null +++ b/bench/RESULTS.md @@ -0,0 +1,127 @@ +# Where does execution belong? — measured, not assumed + +Real JMH 1.37, `--enable-preview` off (Vector API only needs `--add-modules jdk.incubator.vector`, +not preview), JDK 26 GA, `@Fork(1) @Warmup(5×500ms) @Measurement(8×500ms)`, `AverageTime`. Full run: +`results/jmh-run.txt` (1,679 lines, every warm-up iteration). Machine-readable: +`results/jmh-results.csv`. Reproduce with `./run.sh` (~12 min on 4 vCPU — this run: 00:12:22). + +**Gate.** 50/50 benchmark rows completed, 0 failures. `Data.crossCheck()` (native vs Vector vs +scalar agree on both count and sum) ran in `@Setup` for every fork and never threw — the three +kernels compute the same answer, so a speed comparison between them is meaningful rather than a +race between a correct implementation and a subtly wrong faster one. + +## The headline finding — and it complicates the thesis in an honest way + +Component C (`java_scalarLoop` / `java_vectorApi` / `native_fusedPlan`) sweeps one predicate +(`class == 7`) over row counts from 64 to 4,194,304, all three arms answering the identical +question from the identical native lane: + +| rows | scalar (µs) | vectorApi (µs) | native (µs) | native beats scalar by | **vectorApi beats native by** | +|---:|---:|---:|---:|---:|---:| +| 64 | 0.028 | 0.011 | 0.612 | 0.05× (native LOSES) | 56.40× | +| 256 | 0.085 | 0.025 | 0.623 | 0.14× (native LOSES) | 24.96× | +| 1,024 | 0.338 | 0.078 | 0.708 | 0.48× (native LOSES) | 9.07× | +| 4,096 | 1.252 | 0.385 | 1.524 | 0.82× (native LOSES) | 3.96× | +| 16,384 | 5.553 | 1.744 | 4.291 | **1.29×** | 2.46× | +| 65,536 | 42.846 | 8.027 | 15.324 | 2.80× | 1.91× | +| 262,144 | 343.389 | 42.519 | 69.374 | 4.95× | 1.63× | +| 1,048,576 | 1,623.313 | 310.405 | 411.333 | 3.95× | 1.33× | +| 4,194,304 | 6,602.036 | 1,319.107 | 1,858.686 | 3.55× | 1.41× | + +Two crossovers, both real: + +1. **Native beats a plain Java scalar loop only past roughly 4,096–16,384 rows.** Below that, the + crossing overhead (the ~0.6 µs floor visible at row=64, consistent with Component A's raw + downcall cost) is not repaid yet — a scalar loop over a few thousand elements is simply cheap + enough in Java that there is nothing to win by leaving the JVM. +2. **The Java Vector API, reading the SAME native `MemorySegment` with zero copy + (`IntVector.fromMemorySegment`), beats the native crossing at every single row count + tested** — never below 1.3×, and by more than an order of magnitude at small sizes. This + is the finding this project's own mission brief asked for by name: *"Where is the cheapest + and cleanest execution boundary? Not: how can we maximize the amount of Java code?"* — and + the honest answer, for this one-predicate/one-lane workload, is that it is **not** the Rust + crossing. + +**Why this does not overturn the thesis, and where it does bite.** Component C measures ONE +predicate over ONE lane — exactly the case where a zero-copy Vector kernel has nothing to fuse and +nothing to coordinate. Component E (below) measures what happens once there is more than one +predicate, which is the case the fluent `View` API actually optimizes for. + +## Component E — fusion matters once there is more than one predicate + +`fused` (native, one crossing, N predicates AND-combined in one plan) vs `unfused` (native, N +crossings, one `mask_and` per predicate) vs `fusedScalarKernel` (the SAME fused plan forced through +the crate's own scalar reference path, not SIMD), at 65,536 rows: + +| predicates | fused (µs) | unfused (µs) | fusedScalarKernel (µs) | SIMD speedup over scalar | +|---:|---:|---:|---:|---:| +| 1 | 6.818 | 7.641 | 73.419 | 10.8× | +| 2 | 15.599 | 15.025 | 405.096 | 26.0× | +| 4 | 29.721 | 27.309 | 923.100 | 31.1× | +| 8 | 59.363 | 61.916 | 1,807.261 | 30.4× | + +Two findings, neither of which was assumed going in: + +- **`fused` and `unfused` are close** — within noise of each other at this row count (see the + single-fork caveat below). The `lgj_plan_eval` fused path exists to guarantee ONE crossing + regardless of predicate count (a structural property `LazinessTest` in the Java suite already + proves), not because N separate crossings at 65,536 rows are individually expensive — Component A + already showed a bare downcall costs ~22 ns, so 8 of them add roughly 176 ns against a + multi-microsecond total. The value of fusion at this scale is the crossing-count GUARANTEE, not a + large measured time saving. +- **SIMD vs scalar is the biggest lever in this whole benchmark suite** — 10.8×–31.1×, growing + with predicate count. This is the number that justifies routing every kernel through + `ndarray::simd` rather than a portable scalar loop, and it dwarfs the crossing-cost questions + Components A/B/C spend most of their effort isolating. + +`planConstructionOnly` (0.053–0.634 µs, scaling with predicate count but NOT with row count — 65,536 +rows throughout) confirms `LazinessTest`'s claim under real JMH conditions: building the fluent +chain costs time proportional to the number of `.where()` calls, never to the number of rows. + +## Component A — the floor + +| benchmark | ns/op | +|---|---:| +| `javaCallControl` (a plain Java method call, the noise floor) | 0.497 | +| `bareDowncall_noArgs` | 21.911 | +| `downcall_twoArgs_outPointer` | 117.855 | + +A bare Panama downcall costs ~22 ns over a plain Java call; adding two arguments and an out-pointer +roughly quintuples that. Both numbers are the ~0.6 µs floor Component C's small-row-count native +arm sits on top of (downcall cost + the fixed per-call marshalling `Engine`/`Downcalls` do above the +raw linker). + +## Component B — raw throughput, no crossing + +| benchmark | µs/op (65,536 elements) | +|---|---:| +| `heapArrayBaseline` (data already in a Java `int[]`) | 5.158 | +| `segmentScalar` (same data, read from a native `MemorySegment`, scalar loop) | 5.220 | +| `segmentVector` (same data, `IntVector.fromMemorySegment`) | 3.337 | + +Reading a native segment scalar-wise costs essentially the same as reading a heap array (1.2% +difference — within this harness's own stated ~10% noise floor, see below) — `MemorySegment` access +is not itself a tax. The Vector API is the thing that's actually faster here (1.55× over both), not +the memory's location. + +## Honest limitations (stated in `README.md`, repeated here because they qualify every number above) + +- **Single fork.** `@Fork(1)` cannot see run-to-run JIT/ASLR variance. Differences under ~10% between + arms are not established by this harness — this is why `fused` vs `unfused` above is reported as + "close" rather than a specific winner. +- **Shared container, not a tuned host.** No CPU pinning, no disabled turbo/hyperthreading. Large + effects (the order-of-magnitude ones — scalar-vs-vector, the small-row native crossover) are safe + to read; anything under ~10% is not. +- **Component C's ONE-predicate shape is deliberate, not the whole story.** It isolates the + crossing question cleanly; Component E is what shows the picture changes once predicates compose. + +## The verdict, stated the way the mission brief asked for it + +*"Where is the cheapest and cleanest execution boundary?"* — measured, not assumed: for a single +predicate over a native lane, **Java itself, via the Vector API reading the segment zero-copy, is +the fastest arm tested at every scale**. The native crossing earns its keep once real work — SIMD +kernels for multi-predicate fusion, the guaranteed-one-crossing property for an arbitrarily long +`View` chain, and (per `valhalla-lab/`) genuine per-population bulk operations — is on the other +side of it. The honest reading is not "Rust wins" or "Java wins" but **"the crossing is worth +paying for composed work, not for one predicate read alone"** — exactly the nuance the mission +brief's Phase G asked this benchmark to establish rather than assume in either direction. diff --git a/bench/results/jmh-results-full.csv b/bench/results/jmh-results-full.csv new file mode 100644 index 0000000..09d68af --- /dev/null +++ b/bench/results/jmh-results-full.csv @@ -0,0 +1,50 @@ +"Benchmark","Mode","Threads","Samples","Score","Score Error (99.9%)","Unit","Param: predicates","Param: rows" +"com.adaworldapi.lancegraph.bench.A_DowncallOverhead.bareDowncall_noArgs","avgt",1,8,21.910816,0.714719,"ns/op",, +"com.adaworldapi.lancegraph.bench.A_DowncallOverhead.downcall_twoArgs_outPointer","avgt",1,8,117.854789,1.808283,"ns/op",, +"com.adaworldapi.lancegraph.bench.A_DowncallOverhead.javaCallControl","avgt",1,8,0.496614,0.037361,"ns/op",, +"com.adaworldapi.lancegraph.bench.B_SegmentAccess.heapArrayBaseline","avgt",1,8,5.157633,0.179433,"us/op",,65536 +"com.adaworldapi.lancegraph.bench.B_SegmentAccess.segmentScalar","avgt",1,8,5.219871,0.241453,"us/op",,65536 +"com.adaworldapi.lancegraph.bench.B_SegmentAccess.segmentVector","avgt",1,8,3.337320,0.093847,"us/op",,65536 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,0.027947,0.001631,"us/op",,64 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,0.085439,0.010847,"us/op",,256 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,0.337707,0.017531,"us/op",,1024 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,1.251500,0.020465,"us/op",,4096 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,5.553163,0.287911,"us/op",,16384 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,42.846109,9.354832,"us/op",,65536 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,343.389077,7.331871,"us/op",,262144 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,1623.313038,26.972711,"us/op",,1048576 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,6602.036020,100.770602,"us/op",,4194304 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,0.010845,0.000597,"us/op",,64 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,0.024940,0.000515,"us/op",,256 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,0.078068,0.002504,"us/op",,1024 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,0.385167,0.015544,"us/op",,4096 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,1.744103,0.055036,"us/op",,16384 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,8.027116,0.308605,"us/op",,65536 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,42.518604,5.260801,"us/op",,262144 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,310.404608,17.660428,"us/op",,1048576 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,1319.107387,37.239509,"us/op",,4194304 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,0.611653,0.039113,"us/op",,64 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,0.622588,0.026833,"us/op",,256 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,0.708059,0.036741,"us/op",,1024 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,1.523730,0.346193,"us/op",,4096 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,4.291253,0.190398,"us/op",,16384 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,15.324083,0.867382,"us/op",,65536 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,69.373781,5.998053,"us/op",,262144 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,411.333183,37.244044,"us/op",,1048576 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,1858.686441,149.400062,"us/op",,4194304 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,6.818039,0.158841,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,15.599132,0.689941,"us/op",2,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,29.721204,0.865278,"us/op",4,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,59.362899,4.549396,"us/op",8,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,73.418548,1.464802,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,405.096000,7.322619,"us/op",2,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,923.099907,31.760771,"us/op",4,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,1807.261109,28.175197,"us/op",8,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.052921,0.003815,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.112476,0.005216,"us/op",2,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.269718,0.029700,"us/op",4,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.633565,0.029509,"us/op",8,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,7.641421,0.918093,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,15.024855,0.818124,"us/op",2,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,27.308656,2.182643,"us/op",4,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,61.915508,4.111936,"us/op",8,65536 diff --git a/bench/results/jmh-results.csv b/bench/results/jmh-results.csv new file mode 100644 index 0000000..09d68af --- /dev/null +++ b/bench/results/jmh-results.csv @@ -0,0 +1,50 @@ +"Benchmark","Mode","Threads","Samples","Score","Score Error (99.9%)","Unit","Param: predicates","Param: rows" +"com.adaworldapi.lancegraph.bench.A_DowncallOverhead.bareDowncall_noArgs","avgt",1,8,21.910816,0.714719,"ns/op",, +"com.adaworldapi.lancegraph.bench.A_DowncallOverhead.downcall_twoArgs_outPointer","avgt",1,8,117.854789,1.808283,"ns/op",, +"com.adaworldapi.lancegraph.bench.A_DowncallOverhead.javaCallControl","avgt",1,8,0.496614,0.037361,"ns/op",, +"com.adaworldapi.lancegraph.bench.B_SegmentAccess.heapArrayBaseline","avgt",1,8,5.157633,0.179433,"us/op",,65536 +"com.adaworldapi.lancegraph.bench.B_SegmentAccess.segmentScalar","avgt",1,8,5.219871,0.241453,"us/op",,65536 +"com.adaworldapi.lancegraph.bench.B_SegmentAccess.segmentVector","avgt",1,8,3.337320,0.093847,"us/op",,65536 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,0.027947,0.001631,"us/op",,64 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,0.085439,0.010847,"us/op",,256 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,0.337707,0.017531,"us/op",,1024 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,1.251500,0.020465,"us/op",,4096 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,5.553163,0.287911,"us/op",,16384 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,42.846109,9.354832,"us/op",,65536 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,343.389077,7.331871,"us/op",,262144 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,1623.313038,26.972711,"us/op",,1048576 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,6602.036020,100.770602,"us/op",,4194304 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,0.010845,0.000597,"us/op",,64 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,0.024940,0.000515,"us/op",,256 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,0.078068,0.002504,"us/op",,1024 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,0.385167,0.015544,"us/op",,4096 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,1.744103,0.055036,"us/op",,16384 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,8.027116,0.308605,"us/op",,65536 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,42.518604,5.260801,"us/op",,262144 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,310.404608,17.660428,"us/op",,1048576 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,1319.107387,37.239509,"us/op",,4194304 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,0.611653,0.039113,"us/op",,64 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,0.622588,0.026833,"us/op",,256 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,0.708059,0.036741,"us/op",,1024 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,1.523730,0.346193,"us/op",,4096 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,4.291253,0.190398,"us/op",,16384 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,15.324083,0.867382,"us/op",,65536 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,69.373781,5.998053,"us/op",,262144 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,411.333183,37.244044,"us/op",,1048576 +"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,1858.686441,149.400062,"us/op",,4194304 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,6.818039,0.158841,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,15.599132,0.689941,"us/op",2,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,29.721204,0.865278,"us/op",4,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,59.362899,4.549396,"us/op",8,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,73.418548,1.464802,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,405.096000,7.322619,"us/op",2,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,923.099907,31.760771,"us/op",4,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,1807.261109,28.175197,"us/op",8,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.052921,0.003815,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.112476,0.005216,"us/op",2,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.269718,0.029700,"us/op",4,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.633565,0.029509,"us/op",8,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,7.641421,0.918093,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,15.024855,0.818124,"us/op",2,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,27.308656,2.182643,"us/op",4,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,61.915508,4.111936,"us/op",8,65536 diff --git a/bench/results/jmh-run-full.txt b/bench/results/jmh-run-full.txt new file mode 100644 index 0000000..8595316 --- /dev/null +++ b/bench/results/jmh-run-full.txt @@ -0,0 +1,2139 @@ +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +================================================================================================ +lance-graph-java :: benchmark harness +================================================================================================ +jdk OpenJDK 64-Bit Server VM 26.0.2+10-55 +vm args [--enable-native-access=ALL-UNNAMED, --add-modules=jdk.incubator.vector, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so] +os / arch Linux amd64 +cpu Intel(R) Xeon(R) Processor @ 2.10GHz (4 logical processors) +vector species Species[int, 16, S_512_BIT] (16 int lanes, 512 bit) +native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so +predicate class == 7 AND value > 100 +================================================================================================ + +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.A_DowncallOverhead.bareDowncall_noArgs + +# Run progress: 0.00% complete, ETA 00:05:18 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 21.803 ns/op +# Warmup Iteration 2: 23.587 ns/op +# Warmup Iteration 3: 21.527 ns/op +# Warmup Iteration 4: 22.266 ns/op +# Warmup Iteration 5: 21.648 ns/op +Iteration 1: 21.891 ns/op +Iteration 2: 22.302 ns/op +Iteration 3: 22.012 ns/op +Iteration 4: 22.131 ns/op +Iteration 5: 22.316 ns/op +Iteration 6: 21.334 ns/op +Iteration 7: 21.909 ns/op +Iteration 8: 21.392 ns/op + + +Result "com.adaworldapi.lancegraph.bench.A_DowncallOverhead.bareDowncall_noArgs": + 21.911 ±(99.9%) 0.715 ns/op [Average] + (min, avg, max) = (21.334, 21.911, 22.316), stdev = 0.374 + CI (99.9%): [21.196, 22.626] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.A_DowncallOverhead.downcall_twoArgs_outPointer + +# Run progress: 2.04% complete, ETA 00:12:06 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 123.308 ns/op +# Warmup Iteration 2: 118.440 ns/op +# Warmup Iteration 3: 119.894 ns/op +# Warmup Iteration 4: 118.817 ns/op +# Warmup Iteration 5: 119.039 ns/op +Iteration 1: 117.502 ns/op +Iteration 2: 118.242 ns/op +Iteration 3: 119.272 ns/op +Iteration 4: 118.374 ns/op +Iteration 5: 117.807 ns/op +Iteration 6: 118.025 ns/op +Iteration 7: 117.671 ns/op +Iteration 8: 115.946 ns/op + + +Result "com.adaworldapi.lancegraph.bench.A_DowncallOverhead.downcall_twoArgs_outPointer": + 117.855 ±(99.9%) 1.808 ns/op [Average] + (min, avg, max) = (115.946, 117.855, 119.272), stdev = 0.946 + CI (99.9%): [116.047, 119.663] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.A_DowncallOverhead.javaCallControl + +# Run progress: 4.08% complete, ETA 00:11:49 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.713 ns/op +# Warmup Iteration 2: 0.717 ns/op +# Warmup Iteration 3: 0.485 ns/op +# Warmup Iteration 4: 0.488 ns/op +# Warmup Iteration 5: 0.483 ns/op +Iteration 1: 0.497 ns/op +Iteration 2: 0.488 ns/op +Iteration 3: 0.486 ns/op +Iteration 4: 0.490 ns/op +Iteration 5: 0.544 ns/op +Iteration 6: 0.486 ns/op +Iteration 7: 0.491 ns/op +Iteration 8: 0.490 ns/op + + +Result "com.adaworldapi.lancegraph.bench.A_DowncallOverhead.javaCallControl": + 0.497 ±(99.9%) 0.037 ns/op [Average] + (min, avg, max) = (0.486, 0.497, 0.544), stdev = 0.020 + CI (99.9%): [0.459, 0.534] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.B_SegmentAccess.heapArrayBaseline +# Parameters: (rows = 65536) + +# Run progress: 6.12% complete, ETA 00:11:33 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 5.373 us/op +# Warmup Iteration 2: 5.160 us/op +# Warmup Iteration 3: 5.151 us/op +# Warmup Iteration 4: 5.137 us/op +# Warmup Iteration 5: 5.121 us/op +Iteration 1: 5.112 us/op +Iteration 2: 5.167 us/op +Iteration 3: 5.172 us/op +Iteration 4: 5.062 us/op +Iteration 5: 5.108 us/op +Iteration 6: 5.091 us/op +Iteration 7: 5.186 us/op +Iteration 8: 5.363 us/op + + +Result "com.adaworldapi.lancegraph.bench.B_SegmentAccess.heapArrayBaseline": + 5.158 ±(99.9%) 0.179 us/op [Average] + (min, avg, max) = (5.062, 5.158, 5.363), stdev = 0.094 + CI (99.9%): [4.978, 5.337] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.B_SegmentAccess.segmentScalar +# Parameters: (rows = 65536) + +# Run progress: 8.16% complete, ETA 00:11:18 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 5.656 us/op +# Warmup Iteration 2: 5.261 us/op +# Warmup Iteration 3: 5.269 us/op +# Warmup Iteration 4: 5.215 us/op +# Warmup Iteration 5: 5.172 us/op +Iteration 1: 5.160 us/op +Iteration 2: 5.094 us/op +Iteration 3: 5.144 us/op +Iteration 4: 5.275 us/op +Iteration 5: 5.236 us/op +Iteration 6: 5.201 us/op +Iteration 7: 5.150 us/op +Iteration 8: 5.499 us/op + + +Result "com.adaworldapi.lancegraph.bench.B_SegmentAccess.segmentScalar": + 5.220 ±(99.9%) 0.241 us/op [Average] + (min, avg, max) = (5.094, 5.220, 5.499), stdev = 0.126 + CI (99.9%): [4.978, 5.461] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.B_SegmentAccess.segmentVector +# Parameters: (rows = 65536) + +# Run progress: 10.20% complete, ETA 00:11:04 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 4.045 us/op +# Warmup Iteration 2: 3.464 us/op +# Warmup Iteration 3: 3.438 us/op +# Warmup Iteration 4: 3.313 us/op +# Warmup Iteration 5: 3.320 us/op +Iteration 1: 3.348 us/op +Iteration 2: 3.297 us/op +Iteration 3: 3.338 us/op +Iteration 4: 3.360 us/op +Iteration 5: 3.295 us/op +Iteration 6: 3.432 us/op +Iteration 7: 3.352 us/op +Iteration 8: 3.277 us/op + + +Result "com.adaworldapi.lancegraph.bench.B_SegmentAccess.segmentVector": + 3.337 ±(99.9%) 0.094 us/op [Average] + (min, avg, max) = (3.277, 3.337, 3.432), stdev = 0.049 + CI (99.9%): [3.243, 3.431] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop +# Parameters: (rows = 64) + +# Run progress: 12.24% complete, ETA 00:10:49 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.028 us/op +# Warmup Iteration 2: 0.027 us/op +# Warmup Iteration 3: 0.027 us/op +# Warmup Iteration 4: 0.028 us/op +# Warmup Iteration 5: 0.028 us/op +Iteration 1: 0.028 us/op +Iteration 2: 0.027 us/op +Iteration 3: 0.028 us/op +Iteration 4: 0.029 us/op +Iteration 5: 0.028 us/op +Iteration 6: 0.029 us/op +Iteration 7: 0.027 us/op +Iteration 8: 0.027 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop": + 0.028 ±(99.9%) 0.002 us/op [Average] + (min, avg, max) = (0.027, 0.028, 0.029), stdev = 0.001 + CI (99.9%): [0.026, 0.030] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop +# Parameters: (rows = 256) + +# Run progress: 14.29% complete, ETA 00:10:34 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.080 us/op +# Warmup Iteration 2: 0.086 us/op +# Warmup Iteration 3: 0.082 us/op +# Warmup Iteration 4: 0.084 us/op +# Warmup Iteration 5: 0.082 us/op +Iteration 1: 0.083 us/op +Iteration 2: 0.083 us/op +Iteration 3: 0.084 us/op +Iteration 4: 0.082 us/op +Iteration 5: 0.082 us/op +Iteration 6: 0.099 us/op +Iteration 7: 0.088 us/op +Iteration 8: 0.083 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop": + 0.085 ±(99.9%) 0.011 us/op [Average] + (min, avg, max) = (0.082, 0.085, 0.099), stdev = 0.006 + CI (99.9%): [0.075, 0.096] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop +# Parameters: (rows = 1024) + +# Run progress: 16.33% complete, ETA 00:10:18 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.294 us/op +# Warmup Iteration 2: 0.322 us/op +# Warmup Iteration 3: 0.325 us/op +# Warmup Iteration 4: 0.321 us/op +# Warmup Iteration 5: 0.331 us/op +Iteration 1: 0.331 us/op +Iteration 2: 0.326 us/op +Iteration 3: 0.333 us/op +Iteration 4: 0.349 us/op +Iteration 5: 0.353 us/op +Iteration 6: 0.340 us/op +Iteration 7: 0.336 us/op +Iteration 8: 0.334 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop": + 0.338 ±(99.9%) 0.018 us/op [Average] + (min, avg, max) = (0.326, 0.338, 0.353), stdev = 0.009 + CI (99.9%): [0.320, 0.355] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop +# Parameters: (rows = 4096) + +# Run progress: 18.37% complete, ETA 00:10:03 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 1.163 us/op +# Warmup Iteration 2: 1.348 us/op +# Warmup Iteration 3: 1.254 us/op +# Warmup Iteration 4: 1.230 us/op +# Warmup Iteration 5: 1.240 us/op +Iteration 1: 1.246 us/op +Iteration 2: 1.249 us/op +Iteration 3: 1.258 us/op +Iteration 4: 1.268 us/op +Iteration 5: 1.245 us/op +Iteration 6: 1.233 us/op +Iteration 7: 1.253 us/op +Iteration 8: 1.259 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop": + 1.252 ±(99.9%) 0.020 us/op [Average] + (min, avg, max) = (1.233, 1.252, 1.268), stdev = 0.011 + CI (99.9%): [1.231, 1.272] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop +# Parameters: (rows = 16384) + +# Run progress: 20.41% complete, ETA 00:09:48 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 4.973 us/op +# Warmup Iteration 2: 4.329 us/op +# Warmup Iteration 3: 5.540 us/op +# Warmup Iteration 4: 5.403 us/op +# Warmup Iteration 5: 5.545 us/op +Iteration 1: 5.917 us/op +Iteration 2: 5.478 us/op +Iteration 3: 5.445 us/op +Iteration 4: 5.519 us/op +Iteration 5: 5.487 us/op +Iteration 6: 5.545 us/op +Iteration 7: 5.494 us/op +Iteration 8: 5.541 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop": + 5.553 ±(99.9%) 0.288 us/op [Average] + (min, avg, max) = (5.445, 5.553, 5.917), stdev = 0.151 + CI (99.9%): [5.265, 5.841] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop +# Parameters: (rows = 65536) + +# Run progress: 22.45% complete, ETA 00:09:33 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 25.087 us/op +# Warmup Iteration 2: 20.954 us/op +# Warmup Iteration 3: 22.638 us/op +# Warmup Iteration 4: 29.391 us/op +# Warmup Iteration 5: 37.297 us/op +Iteration 1: 31.110 us/op +Iteration 2: 44.028 us/op +Iteration 3: 43.558 us/op +Iteration 4: 46.254 us/op +Iteration 5: 43.485 us/op +Iteration 6: 43.678 us/op +Iteration 7: 44.124 us/op +Iteration 8: 46.531 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop": + 42.846 ±(99.9%) 9.355 us/op [Average] + (min, avg, max) = (31.110, 42.846, 46.531), stdev = 4.893 + CI (99.9%): [33.491, 52.201] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop +# Parameters: (rows = 262144) + +# Run progress: 24.49% complete, ETA 00:09:18 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 358.917 us/op +# Warmup Iteration 2: 340.220 us/op +# Warmup Iteration 3: 341.704 us/op +# Warmup Iteration 4: 352.782 us/op +# Warmup Iteration 5: 338.025 us/op +Iteration 1: 347.153 us/op +Iteration 2: 343.299 us/op +Iteration 3: 345.375 us/op +Iteration 4: 336.311 us/op +Iteration 5: 343.685 us/op +Iteration 6: 341.063 us/op +Iteration 7: 341.702 us/op +Iteration 8: 348.524 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop": + 343.389 ±(99.9%) 7.332 us/op [Average] + (min, avg, max) = (336.311, 343.389, 348.524), stdev = 3.835 + CI (99.9%): [336.057, 350.721] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop +# Parameters: (rows = 1048576) + +# Run progress: 26.53% complete, ETA 00:09:03 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 1628.460 us/op +# Warmup Iteration 2: 1613.562 us/op +# Warmup Iteration 3: 1609.572 us/op +# Warmup Iteration 4: 1601.935 us/op +# Warmup Iteration 5: 1644.950 us/op +Iteration 1: 1636.495 us/op +Iteration 2: 1641.154 us/op +Iteration 3: 1603.481 us/op +Iteration 4: 1623.879 us/op +Iteration 5: 1620.731 us/op +Iteration 6: 1611.321 us/op +Iteration 7: 1611.437 us/op +Iteration 8: 1638.006 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop": + 1623.313 ±(99.9%) 26.973 us/op [Average] + (min, avg, max) = (1603.481, 1623.313, 1641.154), stdev = 14.107 + CI (99.9%): [1596.340, 1650.286] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop +# Parameters: (rows = 4194304) + +# Run progress: 28.57% complete, ETA 00:08:48 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 6741.183 us/op +# Warmup Iteration 2: 6677.927 us/op +# Warmup Iteration 3: 6636.933 us/op +# Warmup Iteration 4: 6608.914 us/op +# Warmup Iteration 5: 6610.781 us/op +Iteration 1: 6630.166 us/op +Iteration 2: 6620.874 us/op +Iteration 3: 6634.037 us/op +Iteration 4: 6563.558 us/op +Iteration 5: 6544.883 us/op +Iteration 6: 6526.215 us/op +Iteration 7: 6612.908 us/op +Iteration 8: 6683.647 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop": + 6602.036 ±(99.9%) 100.771 us/op [Average] + (min, avg, max) = (6526.215, 6602.036, 6683.647), stdev = 52.705 + CI (99.9%): [6501.265, 6702.807] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi +# Parameters: (rows = 64) + +# Run progress: 30.61% complete, ETA 00:08:34 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.015 us/op +# Warmup Iteration 2: 0.012 us/op +# Warmup Iteration 3: 0.011 us/op +# Warmup Iteration 4: 0.011 us/op +# Warmup Iteration 5: 0.011 us/op +Iteration 1: 0.012 us/op +Iteration 2: 0.011 us/op +Iteration 3: 0.011 us/op +Iteration 4: 0.011 us/op +Iteration 5: 0.011 us/op +Iteration 6: 0.011 us/op +Iteration 7: 0.011 us/op +Iteration 8: 0.011 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi": + 0.011 ±(99.9%) 0.001 us/op [Average] + (min, avg, max) = (0.011, 0.011, 0.012), stdev = 0.001 + CI (99.9%): [0.010, 0.011] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi +# Parameters: (rows = 256) + +# Run progress: 32.65% complete, ETA 00:08:18 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.036 us/op +# Warmup Iteration 2: 0.028 us/op +# Warmup Iteration 3: 0.026 us/op +# Warmup Iteration 4: 0.025 us/op +# Warmup Iteration 5: 0.025 us/op +Iteration 1: 0.025 us/op +Iteration 2: 0.025 us/op +Iteration 3: 0.025 us/op +Iteration 4: 0.025 us/op +Iteration 5: 0.025 us/op +Iteration 6: 0.025 us/op +Iteration 7: 0.025 us/op +Iteration 8: 0.025 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi": + 0.025 ±(99.9%) 0.001 us/op [Average] + (min, avg, max) = (0.025, 0.025, 0.025), stdev = 0.001 + CI (99.9%): [0.024, 0.025] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi +# Parameters: (rows = 1024) + +# Run progress: 34.69% complete, ETA 00:08:03 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.097 us/op +# Warmup Iteration 2: 0.078 us/op +# Warmup Iteration 3: 0.077 us/op +# Warmup Iteration 4: 0.082 us/op +# Warmup Iteration 5: 0.079 us/op +Iteration 1: 0.077 us/op +Iteration 2: 0.080 us/op +Iteration 3: 0.078 us/op +Iteration 4: 0.077 us/op +Iteration 5: 0.078 us/op +Iteration 6: 0.079 us/op +Iteration 7: 0.077 us/op +Iteration 8: 0.080 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi": + 0.078 ±(99.9%) 0.003 us/op [Average] + (min, avg, max) = (0.077, 0.078, 0.080), stdev = 0.001 + CI (99.9%): [0.076, 0.081] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi +# Parameters: (rows = 4096) + +# Run progress: 36.73% complete, ETA 00:07:48 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.473 us/op +# Warmup Iteration 2: 0.378 us/op +# Warmup Iteration 3: 0.381 us/op +# Warmup Iteration 4: 0.409 us/op +# Warmup Iteration 5: 0.398 us/op +Iteration 1: 0.377 us/op +Iteration 2: 0.383 us/op +Iteration 3: 0.400 us/op +Iteration 4: 0.377 us/op +Iteration 5: 0.395 us/op +Iteration 6: 0.382 us/op +Iteration 7: 0.386 us/op +Iteration 8: 0.381 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi": + 0.385 ±(99.9%) 0.016 us/op [Average] + (min, avg, max) = (0.377, 0.385, 0.400), stdev = 0.008 + CI (99.9%): [0.370, 0.401] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi +# Parameters: (rows = 16384) + +# Run progress: 38.78% complete, ETA 00:07:33 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 2.209 us/op +# Warmup Iteration 2: 1.699 us/op +# Warmup Iteration 3: 1.706 us/op +# Warmup Iteration 4: 1.704 us/op +# Warmup Iteration 5: 1.712 us/op +Iteration 1: 1.800 us/op +Iteration 2: 1.732 us/op +Iteration 3: 1.735 us/op +Iteration 4: 1.775 us/op +Iteration 5: 1.728 us/op +Iteration 6: 1.734 us/op +Iteration 7: 1.711 us/op +Iteration 8: 1.739 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi": + 1.744 ±(99.9%) 0.055 us/op [Average] + (min, avg, max) = (1.711, 1.744, 1.800), stdev = 0.029 + CI (99.9%): [1.689, 1.799] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi +# Parameters: (rows = 65536) + +# Run progress: 40.82% complete, ETA 00:07:18 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 9.860 us/op +# Warmup Iteration 2: 7.908 us/op +# Warmup Iteration 3: 8.408 us/op +# Warmup Iteration 4: 7.820 us/op +# Warmup Iteration 5: 7.804 us/op +Iteration 1: 8.241 us/op +Iteration 2: 7.877 us/op +Iteration 3: 7.896 us/op +Iteration 4: 7.934 us/op +Iteration 5: 8.001 us/op +Iteration 6: 8.196 us/op +Iteration 7: 8.208 us/op +Iteration 8: 7.864 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi": + 8.027 ±(99.9%) 0.309 us/op [Average] + (min, avg, max) = (7.864, 8.027, 8.241), stdev = 0.161 + CI (99.9%): [7.719, 8.336] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi +# Parameters: (rows = 262144) + +# Run progress: 42.86% complete, ETA 00:07:03 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 55.662 us/op +# Warmup Iteration 2: 47.492 us/op +# Warmup Iteration 3: 42.642 us/op +# Warmup Iteration 4: 41.339 us/op +# Warmup Iteration 5: 41.948 us/op +Iteration 1: 42.975 us/op +Iteration 2: 40.480 us/op +Iteration 3: 38.556 us/op +Iteration 4: 42.482 us/op +Iteration 5: 43.257 us/op +Iteration 6: 41.561 us/op +Iteration 7: 48.125 us/op +Iteration 8: 42.712 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi": + 42.519 ±(99.9%) 5.261 us/op [Average] + (min, avg, max) = (38.556, 42.519, 48.125), stdev = 2.752 + CI (99.9%): [37.258, 47.779] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi +# Parameters: (rows = 1048576) + +# Run progress: 44.90% complete, ETA 00:06:47 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 327.161 us/op +# Warmup Iteration 2: 329.858 us/op +# Warmup Iteration 3: 297.631 us/op +# Warmup Iteration 4: 313.554 us/op +# Warmup Iteration 5: 327.445 us/op +Iteration 1: 305.800 us/op +Iteration 2: 302.395 us/op +Iteration 3: 330.896 us/op +Iteration 4: 302.789 us/op +Iteration 5: 312.773 us/op +Iteration 6: 313.122 us/op +Iteration 7: 305.905 us/op +Iteration 8: 309.558 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi": + 310.405 ±(99.9%) 17.660 us/op [Average] + (min, avg, max) = (302.395, 310.405, 330.896), stdev = 9.237 + CI (99.9%): [292.744, 328.065] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi +# Parameters: (rows = 4194304) + +# Run progress: 46.94% complete, ETA 00:06:32 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 1492.450 us/op +# Warmup Iteration 2: 1334.453 us/op +# Warmup Iteration 3: 1343.831 us/op +# Warmup Iteration 4: 1358.649 us/op +# Warmup Iteration 5: 1309.675 us/op +Iteration 1: 1312.093 us/op +Iteration 2: 1312.361 us/op +Iteration 3: 1283.334 us/op +Iteration 4: 1325.153 us/op +Iteration 5: 1346.023 us/op +Iteration 6: 1337.921 us/op +Iteration 7: 1327.031 us/op +Iteration 8: 1308.944 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi": + 1319.107 ±(99.9%) 37.240 us/op [Average] + (min, avg, max) = (1283.334, 1319.107, 1346.023), stdev = 19.477 + CI (99.9%): [1281.868, 1356.347] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan +# Parameters: (rows = 64) + +# Run progress: 48.98% complete, ETA 00:06:18 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.850 us/op +# Warmup Iteration 2: 0.602 us/op +# Warmup Iteration 3: 0.587 us/op +# Warmup Iteration 4: 0.615 us/op +# Warmup Iteration 5: 0.627 us/op +Iteration 1: 0.620 us/op +Iteration 2: 0.620 us/op +Iteration 3: 0.587 us/op +Iteration 4: 0.639 us/op +Iteration 5: 0.589 us/op +Iteration 6: 0.587 us/op +Iteration 7: 0.623 us/op +Iteration 8: 0.627 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan": + 0.612 ±(99.9%) 0.039 us/op [Average] + (min, avg, max) = (0.587, 0.612, 0.639), stdev = 0.020 + CI (99.9%): [0.573, 0.651] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan +# Parameters: (rows = 256) + +# Run progress: 51.02% complete, ETA 00:06:03 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.940 us/op +# Warmup Iteration 2: 0.648 us/op +# Warmup Iteration 3: 0.613 us/op +# Warmup Iteration 4: 0.610 us/op +# Warmup Iteration 5: 0.597 us/op +Iteration 1: 0.641 us/op +Iteration 2: 0.614 us/op +Iteration 3: 0.611 us/op +Iteration 4: 0.634 us/op +Iteration 5: 0.630 us/op +Iteration 6: 0.636 us/op +Iteration 7: 0.605 us/op +Iteration 8: 0.609 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan": + 0.623 ±(99.9%) 0.027 us/op [Average] + (min, avg, max) = (0.605, 0.623, 0.641), stdev = 0.014 + CI (99.9%): [0.596, 0.649] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan +# Parameters: (rows = 1024) + +# Run progress: 53.06% complete, ETA 00:05:47 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 1.091 us/op +# Warmup Iteration 2: 0.716 us/op +# Warmup Iteration 3: 0.700 us/op +# Warmup Iteration 4: 0.693 us/op +# Warmup Iteration 5: 0.756 us/op +Iteration 1: 0.707 us/op +Iteration 2: 0.736 us/op +Iteration 3: 0.709 us/op +Iteration 4: 0.719 us/op +Iteration 5: 0.706 us/op +Iteration 6: 0.724 us/op +Iteration 7: 0.689 us/op +Iteration 8: 0.676 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan": + 0.708 ±(99.9%) 0.037 us/op [Average] + (min, avg, max) = (0.676, 0.708, 0.736), stdev = 0.019 + CI (99.9%): [0.671, 0.745] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan +# Parameters: (rows = 4096) + +# Run progress: 55.10% complete, ETA 00:05:32 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 1.788 us/op +# Warmup Iteration 2: 1.540 us/op +# Warmup Iteration 3: 1.406 us/op +# Warmup Iteration 4: 1.479 us/op +# Warmup Iteration 5: 1.666 us/op +Iteration 1: 1.460 us/op +Iteration 2: 1.428 us/op +Iteration 3: 1.526 us/op +Iteration 4: 1.395 us/op +Iteration 5: 1.892 us/op +Iteration 6: 1.698 us/op +Iteration 7: 1.410 us/op +Iteration 8: 1.380 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan": + 1.524 ±(99.9%) 0.346 us/op [Average] + (min, avg, max) = (1.380, 1.524, 1.892), stdev = 0.181 + CI (99.9%): [1.178, 1.870] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan +# Parameters: (rows = 16384) + +# Run progress: 57.14% complete, ETA 00:05:17 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 5.370 us/op +# Warmup Iteration 2: 4.238 us/op +# Warmup Iteration 3: 4.160 us/op +# Warmup Iteration 4: 4.245 us/op +# Warmup Iteration 5: 4.317 us/op +Iteration 1: 4.281 us/op +Iteration 2: 4.262 us/op +Iteration 3: 4.282 us/op +Iteration 4: 4.117 us/op +Iteration 5: 4.354 us/op +Iteration 6: 4.224 us/op +Iteration 7: 4.446 us/op +Iteration 8: 4.365 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan": + 4.291 ±(99.9%) 0.190 us/op [Average] + (min, avg, max) = (4.117, 4.291, 4.446), stdev = 0.100 + CI (99.9%): [4.101, 4.482] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan +# Parameters: (rows = 65536) + +# Run progress: 59.18% complete, ETA 00:05:02 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 18.109 us/op +# Warmup Iteration 2: 15.832 us/op +# Warmup Iteration 3: 18.449 us/op +# Warmup Iteration 4: 15.454 us/op +# Warmup Iteration 5: 15.617 us/op +Iteration 1: 15.235 us/op +Iteration 2: 15.504 us/op +Iteration 3: 16.349 us/op +Iteration 4: 15.187 us/op +Iteration 5: 14.933 us/op +Iteration 6: 14.951 us/op +Iteration 7: 15.306 us/op +Iteration 8: 15.127 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan": + 15.324 ±(99.9%) 0.867 us/op [Average] + (min, avg, max) = (14.933, 15.324, 16.349), stdev = 0.454 + CI (99.9%): [14.457, 16.191] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan +# Parameters: (rows = 262144) + +# Run progress: 61.22% complete, ETA 00:04:47 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 81.374 us/op +# Warmup Iteration 2: 73.257 us/op +# Warmup Iteration 3: 69.593 us/op +# Warmup Iteration 4: 77.972 us/op +# Warmup Iteration 5: 69.357 us/op +Iteration 1: 69.123 us/op +Iteration 2: 68.019 us/op +Iteration 3: 65.365 us/op +Iteration 4: 68.100 us/op +Iteration 5: 69.936 us/op +Iteration 6: 69.675 us/op +Iteration 7: 68.465 us/op +Iteration 8: 76.307 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan": + 69.374 ±(99.9%) 5.998 us/op [Average] + (min, avg, max) = (65.365, 69.374, 76.307), stdev = 3.137 + CI (99.9%): [63.376, 75.372] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan +# Parameters: (rows = 1048576) + +# Run progress: 63.27% complete, ETA 00:04:32 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 511.107 us/op +# Warmup Iteration 2: 445.446 us/op +# Warmup Iteration 3: 431.211 us/op +# Warmup Iteration 4: 442.940 us/op +# Warmup Iteration 5: 411.143 us/op +Iteration 1: 402.291 us/op +Iteration 2: 407.474 us/op +Iteration 3: 388.386 us/op +Iteration 4: 413.557 us/op +Iteration 5: 441.521 us/op +Iteration 6: 438.369 us/op +Iteration 7: 392.211 us/op +Iteration 8: 406.857 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan": + 411.333 ±(99.9%) 37.244 us/op [Average] + (min, avg, max) = (388.386, 411.333, 441.521), stdev = 19.479 + CI (99.9%): [374.089, 448.577] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan +# Parameters: (rows = 4194304) + +# Run progress: 65.31% complete, ETA 00:04:17 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 1929.630 us/op +# Warmup Iteration 2: 1837.334 us/op +# Warmup Iteration 3: 1857.836 us/op +# Warmup Iteration 4: 1882.982 us/op +# Warmup Iteration 5: 1763.888 us/op +Iteration 1: 1904.764 us/op +Iteration 2: 1762.896 us/op +Iteration 3: 1928.269 us/op +Iteration 4: 1749.941 us/op +Iteration 5: 1839.751 us/op +Iteration 6: 1977.022 us/op +Iteration 7: 1866.777 us/op +Iteration 8: 1840.072 us/op + + +Result "com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan": + 1858.686 ±(99.9%) 149.400 us/op [Average] + (min, avg, max) = (1749.941, 1858.686, 1977.022), stdev = 78.139 + CI (99.9%): [1709.286, 2008.087] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused +# Parameters: (predicates = 1, rows = 65536) + +# Run progress: 67.35% complete, ETA 00:04:02 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 7.676 us/op +# Warmup Iteration 2: 6.901 us/op +# Warmup Iteration 3: 6.906 us/op +# Warmup Iteration 4: 6.851 us/op +# Warmup Iteration 5: 6.746 us/op +Iteration 1: 6.876 us/op +Iteration 2: 6.799 us/op +Iteration 3: 6.761 us/op +Iteration 4: 6.775 us/op +Iteration 5: 6.826 us/op +Iteration 6: 6.793 us/op +Iteration 7: 6.991 us/op +Iteration 8: 6.723 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": + 6.818 ±(99.9%) 0.159 us/op [Average] + (min, avg, max) = (6.723, 6.818, 6.991), stdev = 0.083 + CI (99.9%): [6.659, 6.977] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused +# Parameters: (predicates = 2, rows = 65536) + +# Run progress: 69.39% complete, ETA 00:03:47 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 16.111 us/op +# Warmup Iteration 2: 16.347 us/op +# Warmup Iteration 3: 14.824 us/op +# Warmup Iteration 4: 15.209 us/op +# Warmup Iteration 5: 15.943 us/op +Iteration 1: 16.096 us/op +Iteration 2: 15.271 us/op +Iteration 3: 15.538 us/op +Iteration 4: 15.162 us/op +Iteration 5: 15.377 us/op +Iteration 6: 15.460 us/op +Iteration 7: 15.780 us/op +Iteration 8: 16.110 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": + 15.599 ±(99.9%) 0.690 us/op [Average] + (min, avg, max) = (15.162, 15.599, 16.110), stdev = 0.361 + CI (99.9%): [14.909, 16.289] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused +# Parameters: (predicates = 4, rows = 65536) + +# Run progress: 71.43% complete, ETA 00:03:31 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 31.661 us/op +# Warmup Iteration 2: 30.525 us/op +# Warmup Iteration 3: 30.177 us/op +# Warmup Iteration 4: 29.890 us/op +# Warmup Iteration 5: 29.843 us/op +Iteration 1: 29.690 us/op +Iteration 2: 29.360 us/op +Iteration 3: 30.521 us/op +Iteration 4: 29.583 us/op +Iteration 5: 29.154 us/op +Iteration 6: 29.459 us/op +Iteration 7: 29.782 us/op +Iteration 8: 30.221 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": + 29.721 ±(99.9%) 0.865 us/op [Average] + (min, avg, max) = (29.154, 29.721, 30.521), stdev = 0.453 + CI (99.9%): [28.856, 30.586] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused +# Parameters: (predicates = 8, rows = 65536) + +# Run progress: 73.47% complete, ETA 00:03:16 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 64.419 us/op +# Warmup Iteration 2: 59.066 us/op +# Warmup Iteration 3: 56.430 us/op +# Warmup Iteration 4: 57.728 us/op +# Warmup Iteration 5: 58.258 us/op +Iteration 1: 56.993 us/op +Iteration 2: 57.536 us/op +Iteration 3: 59.024 us/op +Iteration 4: 57.889 us/op +Iteration 5: 60.834 us/op +Iteration 6: 64.324 us/op +Iteration 7: 60.024 us/op +Iteration 8: 58.278 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": + 59.363 ±(99.9%) 4.549 us/op [Average] + (min, avg, max) = (56.993, 59.363, 64.324), stdev = 2.379 + CI (99.9%): [54.814, 63.912] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel +# Parameters: (predicates = 1, rows = 65536) + +# Run progress: 75.51% complete, ETA 00:03:01 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 77.270 us/op +# Warmup Iteration 2: 74.540 us/op +# Warmup Iteration 3: 73.091 us/op +# Warmup Iteration 4: 73.578 us/op +# Warmup Iteration 5: 74.386 us/op +Iteration 1: 75.005 us/op +Iteration 2: 73.895 us/op +Iteration 3: 73.677 us/op +Iteration 4: 73.159 us/op +Iteration 5: 72.918 us/op +Iteration 6: 72.838 us/op +Iteration 7: 72.643 us/op +Iteration 8: 73.214 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": + 73.419 ±(99.9%) 1.465 us/op [Average] + (min, avg, max) = (72.643, 73.419, 75.005), stdev = 0.766 + CI (99.9%): [71.954, 74.883] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel +# Parameters: (predicates = 2, rows = 65536) + +# Run progress: 77.55% complete, ETA 00:02:46 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 438.203 us/op +# Warmup Iteration 2: 414.171 us/op +# Warmup Iteration 3: 415.278 us/op +# Warmup Iteration 4: 406.041 us/op +# Warmup Iteration 5: 416.280 us/op +Iteration 1: 409.604 us/op +Iteration 2: 401.939 us/op +Iteration 3: 402.105 us/op +Iteration 4: 409.308 us/op +Iteration 5: 408.798 us/op +Iteration 6: 399.949 us/op +Iteration 7: 402.970 us/op +Iteration 8: 406.095 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": + 405.096 ±(99.9%) 7.323 us/op [Average] + (min, avg, max) = (399.949, 405.096, 409.604), stdev = 3.830 + CI (99.9%): [397.773, 412.419] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel +# Parameters: (predicates = 4, rows = 65536) + +# Run progress: 79.59% complete, ETA 00:02:31 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 949.038 us/op +# Warmup Iteration 2: 913.240 us/op +# Warmup Iteration 3: 921.157 us/op +# Warmup Iteration 4: 917.769 us/op +# Warmup Iteration 5: 908.736 us/op +Iteration 1: 911.936 us/op +Iteration 2: 927.670 us/op +Iteration 3: 955.585 us/op +Iteration 4: 916.164 us/op +Iteration 5: 898.172 us/op +Iteration 6: 930.122 us/op +Iteration 7: 920.189 us/op +Iteration 8: 924.962 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": + 923.100 ±(99.9%) 31.761 us/op [Average] + (min, avg, max) = (898.172, 923.100, 955.585), stdev = 16.611 + CI (99.9%): [891.339, 954.861] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel +# Parameters: (predicates = 8, rows = 65536) + +# Run progress: 81.63% complete, ETA 00:02:16 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 1877.378 us/op +# Warmup Iteration 2: 1827.137 us/op +# Warmup Iteration 3: 1793.909 us/op +# Warmup Iteration 4: 1816.940 us/op +# Warmup Iteration 5: 1790.634 us/op +Iteration 1: 1816.867 us/op +Iteration 2: 1800.590 us/op +Iteration 3: 1805.073 us/op +Iteration 4: 1828.438 us/op +Iteration 5: 1812.856 us/op +Iteration 6: 1797.737 us/op +Iteration 7: 1816.238 us/op +Iteration 8: 1780.290 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": + 1807.261 ±(99.9%) 28.175 us/op [Average] + (min, avg, max) = (1780.290, 1807.261, 1828.438), stdev = 14.736 + CI (99.9%): [1779.086, 1835.436] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly +# Parameters: (predicates = 1, rows = 65536) + +# Run progress: 83.67% complete, ETA 00:02:01 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.072 us/op +# Warmup Iteration 2: 0.054 us/op +# Warmup Iteration 3: 0.055 us/op +# Warmup Iteration 4: 0.052 us/op +# Warmup Iteration 5: 0.057 us/op +Iteration 1: 0.057 us/op +Iteration 2: 0.052 us/op +Iteration 3: 0.055 us/op +Iteration 4: 0.051 us/op +Iteration 5: 0.053 us/op +Iteration 6: 0.052 us/op +Iteration 7: 0.052 us/op +Iteration 8: 0.052 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": + 0.053 ±(99.9%) 0.004 us/op [Average] + (min, avg, max) = (0.051, 0.053, 0.057), stdev = 0.002 + CI (99.9%): [0.049, 0.057] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly +# Parameters: (predicates = 2, rows = 65536) + +# Run progress: 85.71% complete, ETA 00:01:45 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.141 us/op +# Warmup Iteration 2: 0.116 us/op +# Warmup Iteration 3: 0.109 us/op +# Warmup Iteration 4: 0.122 us/op +# Warmup Iteration 5: 0.113 us/op +Iteration 1: 0.114 us/op +Iteration 2: 0.112 us/op +Iteration 3: 0.112 us/op +Iteration 4: 0.110 us/op +Iteration 5: 0.108 us/op +Iteration 6: 0.113 us/op +Iteration 7: 0.114 us/op +Iteration 8: 0.117 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": + 0.112 ±(99.9%) 0.005 us/op [Average] + (min, avg, max) = (0.108, 0.112, 0.117), stdev = 0.003 + CI (99.9%): [0.107, 0.118] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly +# Parameters: (predicates = 4, rows = 65536) + +# Run progress: 87.76% complete, ETA 00:01:30 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.316 us/op +# Warmup Iteration 2: 0.263 us/op +# Warmup Iteration 3: 0.267 us/op +# Warmup Iteration 4: 0.264 us/op +# Warmup Iteration 5: 0.260 us/op +Iteration 1: 0.268 us/op +Iteration 2: 0.274 us/op +Iteration 3: 0.290 us/op +Iteration 4: 0.264 us/op +Iteration 5: 0.257 us/op +Iteration 6: 0.295 us/op +Iteration 7: 0.253 us/op +Iteration 8: 0.258 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": + 0.270 ±(99.9%) 0.030 us/op [Average] + (min, avg, max) = (0.253, 0.270, 0.295), stdev = 0.016 + CI (99.9%): [0.240, 0.299] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly +# Parameters: (predicates = 8, rows = 65536) + +# Run progress: 89.80% complete, ETA 00:01:15 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.755 us/op +# Warmup Iteration 2: 0.636 us/op +# Warmup Iteration 3: 0.699 us/op +# Warmup Iteration 4: 0.647 us/op +# Warmup Iteration 5: 0.680 us/op +Iteration 1: 0.634 us/op +Iteration 2: 0.627 us/op +Iteration 3: 0.623 us/op +Iteration 4: 0.647 us/op +Iteration 5: 0.618 us/op +Iteration 6: 0.615 us/op +Iteration 7: 0.646 us/op +Iteration 8: 0.658 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": + 0.634 ±(99.9%) 0.030 us/op [Average] + (min, avg, max) = (0.615, 0.634, 0.658), stdev = 0.015 + CI (99.9%): [0.604, 0.663] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused +# Parameters: (predicates = 1, rows = 65536) + +# Run progress: 91.84% complete, ETA 00:01:00 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 8.169 us/op +# Warmup Iteration 2: 7.800 us/op +# Warmup Iteration 3: 7.568 us/op +# Warmup Iteration 4: 7.753 us/op +# Warmup Iteration 5: 7.413 us/op +Iteration 1: 7.597 us/op +Iteration 2: 7.626 us/op +Iteration 3: 8.805 us/op +Iteration 4: 7.381 us/op +Iteration 5: 7.468 us/op +Iteration 6: 7.460 us/op +Iteration 7: 7.455 us/op +Iteration 8: 7.339 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": + 7.641 ±(99.9%) 0.918 us/op [Average] + (min, avg, max) = (7.339, 7.641, 8.805), stdev = 0.480 + CI (99.9%): [6.723, 8.560] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused +# Parameters: (predicates = 2, rows = 65536) + +# Run progress: 93.88% complete, ETA 00:00:45 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 15.953 us/op +# Warmup Iteration 2: 15.219 us/op +# Warmup Iteration 3: 14.770 us/op +# Warmup Iteration 4: 15.105 us/op +# Warmup Iteration 5: 15.412 us/op +Iteration 1: 14.560 us/op +Iteration 2: 14.919 us/op +Iteration 3: 14.425 us/op +Iteration 4: 14.865 us/op +Iteration 5: 15.058 us/op +Iteration 6: 15.526 us/op +Iteration 7: 15.199 us/op +Iteration 8: 15.646 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": + 15.025 ±(99.9%) 0.818 us/op [Average] + (min, avg, max) = (14.425, 15.025, 15.646), stdev = 0.428 + CI (99.9%): [14.207, 15.843] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused +# Parameters: (predicates = 4, rows = 65536) + +# Run progress: 95.92% complete, ETA 00:00:30 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 28.359 us/op +# Warmup Iteration 2: 26.773 us/op +# Warmup Iteration 3: 26.358 us/op +# Warmup Iteration 4: 27.325 us/op +# Warmup Iteration 5: 26.819 us/op +Iteration 1: 26.225 us/op +Iteration 2: 26.090 us/op +Iteration 3: 26.709 us/op +Iteration 4: 27.228 us/op +Iteration 5: 27.577 us/op +Iteration 6: 27.583 us/op +Iteration 7: 27.311 us/op +Iteration 8: 29.748 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": + 27.309 ±(99.9%) 2.183 us/op [Average] + (min, avg, max) = (26.090, 27.309, 29.748), stdev = 1.142 + CI (99.9%): [25.126, 29.491] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused +# Parameters: (predicates = 8, rows = 65536) + +# Run progress: 97.96% complete, ETA 00:00:15 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 68.657 us/op +# Warmup Iteration 2: 67.706 us/op +# Warmup Iteration 3: 61.704 us/op +# Warmup Iteration 4: 60.658 us/op +# Warmup Iteration 5: 61.074 us/op +Iteration 1: 60.579 us/op +Iteration 2: 61.258 us/op +Iteration 3: 64.251 us/op +Iteration 4: 61.160 us/op +Iteration 5: 60.725 us/op +Iteration 6: 60.842 us/op +Iteration 7: 66.252 us/op +Iteration 8: 60.257 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": + 61.916 ±(99.9%) 4.112 us/op [Average] + (min, avg, max) = (60.257, 61.916, 66.252), stdev = 2.151 + CI (99.9%): [57.804, 66.027] (assumes normal distribution) + + +# Run complete. Total time: 00:12:22 + +REMEMBER: The numbers below are just data. To gain reusable insights, you need to follow up on +why the numbers are the way they are. Use profilers (see -prof, -lprof), design factorial +experiments, perform baseline and negative tests that provide experimental control, make sure +the benchmarking environment is safe on JVM/OS/HW level, ask for reviews from the domain experts. +Do not assume the numbers tell you what you want them to tell. + +NOTE: Current JVM experimentally supports Compiler Blackholes, and they are in use. Please exercise +extra caution when trusting the results, look into the generated code to check the benchmark still +works, and factor in a small probability of new VM bugs. Additionally, while comparisons between +different JVMs are already problematic, the performance difference caused by different Blackhole +modes can be very significant. Please make sure you use the consistent Blackhole mode for comparisons. + +Benchmark (predicates) (rows) Mode Cnt Score Error Units +A_DowncallOverhead.bareDowncall_noArgs N/A N/A avgt 8 21.911 ± 0.715 ns/op +A_DowncallOverhead.downcall_twoArgs_outPointer N/A N/A avgt 8 117.855 ± 1.808 ns/op +A_DowncallOverhead.javaCallControl N/A N/A avgt 8 0.497 ± 0.037 ns/op +B_SegmentAccess.heapArrayBaseline N/A 65536 avgt 8 5.158 ± 0.179 us/op +B_SegmentAccess.segmentScalar N/A 65536 avgt 8 5.220 ± 0.241 us/op +B_SegmentAccess.segmentVector N/A 65536 avgt 8 3.337 ± 0.094 us/op +C_ExecutionBoundary.java_scalarLoop N/A 64 avgt 8 0.028 ± 0.002 us/op +C_ExecutionBoundary.java_scalarLoop N/A 256 avgt 8 0.085 ± 0.011 us/op +C_ExecutionBoundary.java_scalarLoop N/A 1024 avgt 8 0.338 ± 0.018 us/op +C_ExecutionBoundary.java_scalarLoop N/A 4096 avgt 8 1.252 ± 0.020 us/op +C_ExecutionBoundary.java_scalarLoop N/A 16384 avgt 8 5.553 ± 0.288 us/op +C_ExecutionBoundary.java_scalarLoop N/A 65536 avgt 8 42.846 ± 9.355 us/op +C_ExecutionBoundary.java_scalarLoop N/A 262144 avgt 8 343.389 ± 7.332 us/op +C_ExecutionBoundary.java_scalarLoop N/A 1048576 avgt 8 1623.313 ± 26.973 us/op +C_ExecutionBoundary.java_scalarLoop N/A 4194304 avgt 8 6602.036 ± 100.771 us/op +C_ExecutionBoundary.java_vectorApi N/A 64 avgt 8 0.011 ± 0.001 us/op +C_ExecutionBoundary.java_vectorApi N/A 256 avgt 8 0.025 ± 0.001 us/op +C_ExecutionBoundary.java_vectorApi N/A 1024 avgt 8 0.078 ± 0.003 us/op +C_ExecutionBoundary.java_vectorApi N/A 4096 avgt 8 0.385 ± 0.016 us/op +C_ExecutionBoundary.java_vectorApi N/A 16384 avgt 8 1.744 ± 0.055 us/op +C_ExecutionBoundary.java_vectorApi N/A 65536 avgt 8 8.027 ± 0.309 us/op +C_ExecutionBoundary.java_vectorApi N/A 262144 avgt 8 42.519 ± 5.261 us/op +C_ExecutionBoundary.java_vectorApi N/A 1048576 avgt 8 310.405 ± 17.660 us/op +C_ExecutionBoundary.java_vectorApi N/A 4194304 avgt 8 1319.107 ± 37.240 us/op +C_ExecutionBoundary.native_fusedPlan N/A 64 avgt 8 0.612 ± 0.039 us/op +C_ExecutionBoundary.native_fusedPlan N/A 256 avgt 8 0.623 ± 0.027 us/op +C_ExecutionBoundary.native_fusedPlan N/A 1024 avgt 8 0.708 ± 0.037 us/op +C_ExecutionBoundary.native_fusedPlan N/A 4096 avgt 8 1.524 ± 0.346 us/op +C_ExecutionBoundary.native_fusedPlan N/A 16384 avgt 8 4.291 ± 0.190 us/op +C_ExecutionBoundary.native_fusedPlan N/A 65536 avgt 8 15.324 ± 0.867 us/op +C_ExecutionBoundary.native_fusedPlan N/A 262144 avgt 8 69.374 ± 5.998 us/op +C_ExecutionBoundary.native_fusedPlan N/A 1048576 avgt 8 411.333 ± 37.244 us/op +C_ExecutionBoundary.native_fusedPlan N/A 4194304 avgt 8 1858.686 ± 149.400 us/op +E_FusionAndPlanning.fused 1 65536 avgt 8 6.818 ± 0.159 us/op +E_FusionAndPlanning.fused 2 65536 avgt 8 15.599 ± 0.690 us/op +E_FusionAndPlanning.fused 4 65536 avgt 8 29.721 ± 0.865 us/op +E_FusionAndPlanning.fused 8 65536 avgt 8 59.363 ± 4.549 us/op +E_FusionAndPlanning.fusedScalarKernel 1 65536 avgt 8 73.419 ± 1.465 us/op +E_FusionAndPlanning.fusedScalarKernel 2 65536 avgt 8 405.096 ± 7.323 us/op +E_FusionAndPlanning.fusedScalarKernel 4 65536 avgt 8 923.100 ± 31.761 us/op +E_FusionAndPlanning.fusedScalarKernel 8 65536 avgt 8 1807.261 ± 28.175 us/op +E_FusionAndPlanning.planConstructionOnly 1 65536 avgt 8 0.053 ± 0.004 us/op +E_FusionAndPlanning.planConstructionOnly 2 65536 avgt 8 0.112 ± 0.005 us/op +E_FusionAndPlanning.planConstructionOnly 4 65536 avgt 8 0.270 ± 0.030 us/op +E_FusionAndPlanning.planConstructionOnly 8 65536 avgt 8 0.634 ± 0.030 us/op +E_FusionAndPlanning.unfused 1 65536 avgt 8 7.641 ± 0.918 us/op +E_FusionAndPlanning.unfused 2 65536 avgt 8 15.025 ± 0.818 us/op +E_FusionAndPlanning.unfused 4 65536 avgt 8 27.309 ± 2.183 us/op +E_FusionAndPlanning.unfused 8 65536 avgt 8 61.916 ± 4.112 us/op + +Benchmark result is saved to results/jmh-results.csv diff --git a/bench/results/jmh-run.txt b/bench/results/jmh-run.txt new file mode 100644 index 0000000..c84bb87 --- /dev/null +++ b/bench/results/jmh-run.txt @@ -0,0 +1,344 @@ +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +================================================================================================ +lance-graph-java :: benchmark harness +================================================================================================ +jdk OpenJDK 64-Bit Server VM 26.0.2+10-55 +vm args [--enable-native-access=ALL-UNNAMED, --add-modules=jdk.incubator.vector, -Dstdout.encoding=UTF-8, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so] +os / arch Linux amd64 +cpu Intel(R) Xeon(R) Processor @ 2.10GHz (4 logical processors) +vector species Species[int, 16, S_512_BIT] (16 int lanes, 512 bit) +native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so +predicate class == 7 AND value > 100 +================================================================================================ + +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused +# Parameters: (predicates = 1, rows = 256) + +# Run progress: 0.00% complete, ETA 00:03:28 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.558 us/op +# Warmup Iteration 2: 0.415 us/op +# Warmup Iteration 3: 0.411 us/op +# Warmup Iteration 4: 0.407 us/op +# Warmup Iteration 5: 0.416 us/op +Iteration 1: 0.410 us/op +Iteration 2: 0.401 us/op +Iteration 3: 0.401 us/op +Iteration 4: 0.403 us/op +Iteration 5: 0.401 us/op +Iteration 6: 0.402 us/op +Iteration 7: 0.414 us/op +Iteration 8: 0.400 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": + 0.404 ±(99.9%) 0.010 us/op [Average] + (min, avg, max) = (0.400, 0.404, 0.414), stdev = 0.005 + CI (99.9%): [0.394, 0.414] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused +# Parameters: (predicates = 1, rows = 65536) + +# Run progress: 3.13% complete, ETA 00:07:47 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 7.485 us/op +# Warmup Iteration 2: 6.796 us/op +# Warmup Iteration 3: 6.828 us/op +# Warmup Iteration 4: 6.877 us/op +# Warmup Iteration 5: 6.934 us/op +Iteration 1: 6.969 us/op +Iteration 2: 6.845 us/op +Iteration 3: 7.029 us/op +Iteration 4: 6.872 us/op +Iteration 5: 7.005 us/op +Iteration 6: 6.795 us/op +Iteration 7: 6.839 us/op +Iteration 8: 6.952 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": + 6.913 ±(99.9%) 0.165 us/op [Average] + (min, avg, max) = (6.795, 6.913, 7.029), stdev = 0.087 + CI (99.9%): [6.748, 7.079] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused +# Parameters: (predicates = 2, rows = 256) + +# Run progress: 6.25% complete, ETA 00:07:32 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.728 us/op +# Warmup Iteration 2: 0.512 us/op +# Warmup Iteration 3: 0.514 us/op +# Warmup Iteration 4: 0.496 us/op +# Warmup Iteration 5: 0.531 us/op +Iteration 1: 0.523 us/op +Iteration 2: 0.512 us/op +Iteration 3: 0.512 us/op +Iteration 4: 0.503 us/op +Iteration 5: 0.526 us/op +Iteration 6: 0.558 us/op +Iteration 7: 0.515 us/op +Iteration 8: 0.511 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": + 0.520 ±(99.9%) 0.032 us/op [Average] + (min, avg, max) = (0.503, 0.520, 0.558), stdev = 0.017 + CI (99.9%): [0.488, 0.552] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused +# Parameters: (predicates = 2, rows = 65536) + +# Run progress: 9.38% complete, ETA 00:07:17 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 16.620 us/op +# Warmup Iteration 2: 15.641 us/op +# Warmup Iteration 3: 16.258 us/op +# Warmup Iteration 4: 14.963 us/op +# Warmup Iteration 5: 15.345 us/op +Iteration 1: 15.653 us/op +Iteration 2: 14.803 us/op +Iteration 3: 15.174 us/op +Iteration 4: 15.789 us/op +Iteration 5: 15.463 us/op +Iteration 6: 14.982 us/op +Iteration 7: 15.045 us/op +Iteration 8: 14.867 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": + 15.222 ±(99.9%) 0.707 us/op [Average] + (min, avg, max) = (14.803, 15.222, 15.789), stdev = 0.370 + CI (99.9%): [14.515, 15.930] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused +# Parameters: (predicates = 4, rows = 256) + +# Run progress: 12.50% complete, ETA 00:07:02 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 1.071 us/op +# Warmup Iteration 2: 0.831 us/op +# Warmup Iteration 3: 0.805 us/op +# Warmup Iteration 4: 0.812 us/op +# Warmup Iteration 5: 0.787 us/op +Iteration 1: 0.803 us/op +Iteration 2: 0.787 us/op +Iteration 3: 0.819 us/op +Iteration 4: 0.794 us/op +Iteration 5: 0.803 us/op +Iteration 6: 0.812 us/op +Iteration 7: 0.800 us/op +Iteration 8: 0.843 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": + 0.808 ±(99.9%) 0.033 us/op [Average] + (min, avg, max) = (0.787, 0.808, 0.843), stdev = 0.017 + CI (99.9%): [0.774, 0.841] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused +# Parameters: (predicates = 4, rows = 65536) + +# Run progress: 15.63% complete, ETA 00:06:47 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 27.861 us/op +# Warmup Iteration 2: 25.722 us/op +# Warmup Iteration 3: 25.611 us/op +# Warmup Iteration 4: 25.534 us/op +# Warmup Iteration 5: 25.717 us/op +Iteration 1: 26.072 us/op +Iteration 2: 25.173 us/op +Iteration 3: 25.405 us/op +Iteration 4: 25.841 us/op +Iteration 5: 25.419 us/op +Iteration 6: 25.530 us/op +Iteration 7: 25.412 us/op +Iteration 8: 25.874 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": + 25.591 ±(99.9%) 0.582 us/op [Average] + (min, avg, max) = (25.173, 25.591, 26.072), stdev = 0.304 + CI (99.9%): [25.009, 26.173] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused +# Parameters: (predicates = 8, rows = 256) + +# Run progress: 18.75% complete, ETA 00:06:32 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 1.841 us/op +# Warmup Iteration 2: 1.446 us/op +# Warmup Iteration 3: 1.521 us/op +# Warmup Iteration 4: 1.556 us/op +# Warmup Iteration 5: 1.502 us/op +Iteration 1: 1.440 us/op +Iteration 2: 1.494 us/op +Iteration 3: 1.463 us/op +Iteration 4: 1.525 us/op +Iteration 5: 1.451 us/op +Iteration 6: 1.533 us/op +Iteration 7: 1.446 us/op +Iteration 8: 1.505 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": + 1.482 ±(99.9%) 0.070 us/op [Average] + (min, avg, max) = (1.440, 1.482, 1.533), stdev = 0.037 + CI (99.9%): [1.412, 1.553] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused +# Parameters: (predicates = 8, rows = 65536) + +# Run progress: 21.88% complete, ETA 00:06:17 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 62.079 us/op +# Warmup Iteration 2: 58.850 us/op +# Warmup Iteration 3: 57.828 us/op +# Warmup Iteration 4: 59.013 us/op +# Warmup Iteration 5: 57.218 us/op +Iteration 1: 57.682 us/op +Iteration 2: 59.927 us/op +Iteration 3: 56.783 us/op +Iteration 4: 57.691 us/op +Iteration 5: 59.234 us/op +Iteration 6: 57.057 us/op diff --git a/bench/run.sh b/bench/run.sh new file mode 100755 index 0000000..ec38195 --- /dev/null +++ b/bench/run.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Build and run the JMH benchmark harness. +# +# ./run.sh # everything +# ./run.sh C_ # only the execution-boundary sweep +# +# Every number in bench/README.md was produced by this script. +set -euo pipefail + +BENCH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$BENCH/.." && pwd)" + +JDK="${JDK:-/opt/jdks/jdk-26.0.2}" +LIB_DIR="${LIB_DIR:-$ROOT/target/release}" + +OUT="$BENCH/out" +CP_LIBS="$(find "$BENCH/lib" -name '*.jar' | sort | tr '\n' ':')" + +if [ ! -f "$LIB_DIR/liblgj_abi.so" ]; then + echo "FAIL: no native library at $LIB_DIR/liblgj_abi.so" >&2 + echo "build it with:" >&2 + echo " cd $ROOT/native/lgj-abi && CARGO_TARGET_DIR=$ROOT/target cargo build --release" >&2 + exit 1 +fi +if [ -z "$CP_LIBS" ]; then + echo "FAIL: no jars in $BENCH/lib — see bench/README.md § 'Fetching JMH'" >&2 + exit 1 +fi + +echo "== compiling the library API" +rm -rf "$OUT"; mkdir -p "$OUT/api" "$OUT/bench" "$BENCH/results" +"$JDK/bin/javac" -d "$OUT/api" $(find "$ROOT/java/src/main/java" -name '*.java') + +echo "== compiling the benchmarks (annotation processing ON: JMH needs it to emit BenchmarkList)" +# -proc:full is REQUIRED on JDK 23+: annotation processing is off by default there, and without it +# JMH's generator never runs and the harness fails at startup with "Unable to find the resource: +# /META-INF/BenchmarkList" — which looks like a classpath problem and is not one. +"$JDK/bin/javac" \ + -proc:full \ + --add-modules jdk.incubator.vector \ + -cp "$CP_LIBS$OUT/api" \ + -d "$OUT/bench" \ + $(find "$BENCH/src" -name '*.java') + +echo "== running" +cd "$BENCH" +JAVA_TOOL_OPTIONS= "$JDK/bin/java" \ + --enable-native-access=ALL-UNNAMED \ + --add-modules jdk.incubator.vector \ + -Dstdout.encoding=UTF-8 \ + -Dlgj.library="$LIB_DIR/liblgj_abi.so" \ + -cp "$CP_LIBS$OUT/api:$OUT/bench" \ + com.adaworldapi.lancegraph.bench.Harness "$@" \ + 2>&1 | tee "$BENCH/results/jmh-run.txt" + +echo +echo "results: bench/results/jmh-run.txt and bench/results/jmh-results.csv" diff --git a/bench/src/main/java/com/adaworldapi/lancegraph/NativeAccess.java b/bench/src/main/java/com/adaworldapi/lancegraph/NativeAccess.java new file mode 100644 index 0000000..3e9fa04 --- /dev/null +++ b/bench/src/main/java/com/adaworldapi/lancegraph/NativeAccess.java @@ -0,0 +1,49 @@ +package com.adaworldapi.lancegraph; + +import com.adaworldapi.lancegraph.internal.ffm.Engine; + +/** + * A read-only bridge into the library's package-private handle, for measurement code only. + * + *

Why this exists rather than a public accessor. The production API + * deliberately never surfaces a handle or a {@code MemorySegment} — that is the whole accessibility + * argument. But the lab has to reach the raw lane to build the very thing the thesis says you + * should not build (65,536 Java objects), and it must build them from the same bytes the + * native kernel reads, or the comparison is between two different datasets and proves nothing. + * + *

So the bridge lives here, in the library's package but in the lab's source tree, + * compiled onto the classpath as a split package. Nothing under {@code java/} changes, no public + * surface widens, and the coupling is visible in one file instead of leaking into the API. + * + *

It reads. It never writes, never closes, never mutates. + */ +public final class NativeAccess { + + private NativeAccess() {} + + /** Lane 0 — {@code u64} entity ids. */ + public static final int LANE_ID = 0; + /** Lane 1 — {@code u32} class tags. */ + public static final int LANE_CLASS = 1; + /** Lane 2 — {@code i32} signed values. */ + public static final int LANE_VALUE = 2; + + /** The generation-checked registry handle behind a pattern. Opaque; for describe calls only. */ + public static long handleOf(NativePattern pattern) { + return pattern.handle(); + } + + /** + * A bounded, read-only window onto one native lane. No membrane crossing happens when this is + * read — that is the point of the design, and the reason a Java-side Vector API kernel can + * compete at all. + */ + public static Engine.LaneWindow lane(NativePattern pattern, int laneId) { + return Engine.describeLane(pattern.handle(), laneId); + } + + /** The packed {@code u64} words behind a selection. */ + public static Engine.LaneWindow maskWords(Mask mask) { + return Engine.describeMask(mask.id().token()); + } +} diff --git a/bench/src/main/java/com/adaworldapi/lancegraph/bench/A_DowncallOverhead.java b/bench/src/main/java/com/adaworldapi/lancegraph/bench/A_DowncallOverhead.java new file mode 100644 index 0000000..b94fae1 --- /dev/null +++ b/bench/src/main/java/com/adaworldapi/lancegraph/bench/A_DowncallOverhead.java @@ -0,0 +1,112 @@ +package com.adaworldapi.lancegraph.bench; + +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SymbolLookup; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; +import java.util.concurrent.TimeUnit; + +/** + * Component A — what does crossing the membrane cost, on its own? + * + *

This is the number every other number in this harness has to be read against, and it is + * measured in isolation on purpose. Conflating "the crossing" with "the work" is the single + * easiest way to produce a benchmark that argues for whichever side the author already preferred: + * put enough work behind the call and the crossing vanishes; put none behind it and the crossing + * is everything. + * + *

Two subjects, both real symbols of this ABI — nothing was added to the native library for + * benchmarking, because a symbol that exists only to be measured is not the thing being measured: + * + *

    + *
  • {@code lgj_abi_manifest()} — no arguments, no allocation, returns a pointer to a static. + * This is as close to a bare downcall as this ABI can offer. + *
  • {@code lgj_mask_count(handle, out)} — two arguments, one out-pointer write, and O(1) real + * work (a single word's popcount on a 64-row pattern). This is "a downcall that also did + * something", and the gap between the two is the marshalling cost of arguments and the + * out-parameter round trip. + *
+ * + *

The handles are bound here rather than reused from {@code internal.ffm.Downcalls} so that the + * measurement is of the JDK's linker, not of this project's wrapper layer. The wrapper's own + * overhead shows up as the difference between this component and Component C. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Fork(value = 1, jvmArgsAppend = {"--enable-native-access=ALL-UNNAMED"}) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 8, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class A_DowncallOverhead { + + private MethodHandle manifest; + private MethodHandle maskCount; + private Arena arena; + private MemorySegment out; + private long maskHandle; + private Data data; + + @Setup(Level.Trial) + public void setup() { + SymbolLookup lookup = SymbolLookup.libraryLookup(Harness.libraryPath(), Arena.global()); + Linker linker = Linker.nativeLinker(); + + manifest = linker.downcallHandle( + lookup.find("lgj_abi_manifest").orElseThrow(), + FunctionDescriptor.of(ValueLayout.ADDRESS)); + + maskCount = linker.downcallHandle( + lookup.find("lgj_mask_count").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_INT, + ValueLayout.JAVA_LONG, ValueLayout.ADDRESS)); + + arena = Arena.ofShared(); + out = arena.allocate(ValueLayout.JAVA_LONG); + + // 64 rows: exactly one mask word, so lgj_mask_count does the least work it can while + // still being a real bulk entry point rather than a synthetic no-op. + data = new Data(64); + maskHandle = data.pattern.view().select().id().token(); + } + + @TearDown(Level.Trial) + public void tearDown() { + arena.close(); + data.close(); + } + + /** The floor: a downcall that takes nothing, allocates nothing, and returns a static pointer. */ + @Benchmark + public MemorySegment bareDowncall_noArgs() throws Throwable { + return (MemorySegment) manifest.invokeExact(); + } + + /** A downcall with two arguments and an out-pointer, doing O(1) work behind it. */ + @Benchmark + public long downcall_twoArgs_outPointer() throws Throwable { + int status = (int) maskCount.invokeExact(maskHandle, out); + if (status != 0) throw new IllegalStateException("status " + status); + return out.get(ValueLayout.JAVA_LONG, 0); + } + + /** + * The control: a Java method call of comparable shape that does not cross anything. + * + *

Without this row, "a downcall costs N ns" has no scale. With it, the reader can see how + * many ordinary calls a crossing is worth, which is the form the number is actually used in. + */ + @Benchmark + public void javaCallControl(Blackhole bh) { + bh.consume(notACrossing(maskHandle)); + } + + private static long notACrossing(long h) { + return h ^ 0x5DEECE66DL; + } +} diff --git a/bench/src/main/java/com/adaworldapi/lancegraph/bench/B_SegmentAccess.java b/bench/src/main/java/com/adaworldapi/lancegraph/bench/B_SegmentAccess.java new file mode 100644 index 0000000..6a9f818 --- /dev/null +++ b/bench/src/main/java/com/adaworldapi/lancegraph/bench/B_SegmentAccess.java @@ -0,0 +1,72 @@ +package com.adaworldapi.lancegraph.bench; + +import org.openjdk.jmh.annotations.*; + +import java.util.concurrent.TimeUnit; + +/** + * Component B — how fast can Java read native memory at all? + * + *

Separate from every other component because it is the ceiling on the Java-side answer. If + * reading a {@code MemorySegment} were much slower than reading an {@code int[]}, then "execute in + * Java over native memory" would be dead before any kernel was written, and the Vector API + * comparison in Component D would be measuring a handicap rather than a design. + * + *

Three subjects over the same values, same length, same answer (asserted in + * {@link Data#crossCheck}): + * + *

    + *
  • {@code MemorySegment} scalar — native memory, one element at a time; + *
  • {@code MemorySegment} vector — native memory, 16 lanes at a time, zero copy; + *
  • {@code int[]} heap — the same data already on the Java heap, which is the fastest thing + * Java has and therefore the right baseline. + *
+ * + *

The heap array exists only as this baseline. Nothing that compares against the + * native kernel uses it, because populating it is precisely the bulk copy the ABI forbids — + * measuring against it here is legitimate (it answers "is FFM access competitive with heap + * access?"), and using it in Component D would not be. + * + *

Throughput is reported per operation over the whole lane, so a per-element cost can be + * divided out and compared against memory bandwidth. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@State(Scope.Benchmark) +@Fork(value = 1, jvmArgsAppend = { + "--enable-native-access=ALL-UNNAMED", "--add-modules", "jdk.incubator.vector"}) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 8, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class B_SegmentAccess { + + /** + * 65,536 rows = 256 KiB per lane. Chosen to sit above L2 and below L3 on this host, so the + * measurement is of the access mechanism rather than of a cache that happens to hold + * everything. The full cache-size sweep lives in Component D, where it changes the answer. + */ + @Param({"65536"}) + public int rows; + + private Data data; + + @Setup(Level.Trial) + public void setup() { data = new Data(rows); } + + @TearDown(Level.Trial) + public void tearDown() { data.close(); } + + @Benchmark + public long segmentScalar() { + return Kernels.sumAllScalar(data.values, rows); + } + + @Benchmark + public long segmentVector() { + return Kernels.sumAllVector(data.values, rows); + } + + @Benchmark + public long heapArrayBaseline() { + return Kernels.sumAllHeap(data.valuesHeap); + } +} diff --git a/bench/src/main/java/com/adaworldapi/lancegraph/bench/C_ExecutionBoundary.java b/bench/src/main/java/com/adaworldapi/lancegraph/bench/C_ExecutionBoundary.java new file mode 100644 index 0000000..61bebd4 --- /dev/null +++ b/bench/src/main/java/com/adaworldapi/lancegraph/bench/C_ExecutionBoundary.java @@ -0,0 +1,84 @@ +package com.adaworldapi.lancegraph.bench; + +import com.adaworldapi.lancegraph.Pattern; +import org.openjdk.jmh.annotations.*; + +import java.util.concurrent.TimeUnit; + +/** + * Components C and D together — WHERE DOES EXECUTION BELONG? + * + *

These are in one class deliberately. The question is a comparison, the comparison only means + * anything if both sides see the same bytes at the same row count in the same JVM run, and putting + * them in separate classes would let a difference in setup masquerade as a difference in + * execution. + * + *

Same 65,536-row (and 64-row, and 4,194,304-row, …) fixture, same predicate + * {@code class == 7 AND value > 100}, same answer asserted equal in {@link Data#crossCheck} before + * anything is timed: + * + *

    + *
  • native — the fluent chain, one crossing, the fused plan, an + * {@code ndarray::simd} AVX-512 kernel; + *
  • vector — {@code jdk.incubator.vector}, 512-bit species, reading the very + * same {@code MemorySegment} with zero copies; + *
  • scalar — an ordinary Java loop over the same segment, which C2 may + * auto-vectorise. The control that separates "the Vector API helped" from "the JIT was + * already doing it". + *
+ * + *

The row sweep is the actual experiment. A single row count cannot answer the + * question, because the two sides have different shapes: the native path pays a fixed crossing and + * then runs at native speed, the Java path pays nothing fixed and runs at whatever the JIT + * achieves. Two lines with different intercepts and different slopes cross somewhere, and the + * crossing point — not either endpoint — is the engineering answer. The sweep spans four orders of + * magnitude so the crossover is bracketed rather than extrapolated. + * + *

The row counts also deliberately straddle the cache hierarchy on this host: 65,536 rows is + * 256 KiB per lane, 1,048,576 rows is 4 MiB per lane, and 4,194,304 rows is 16 MiB per lane — past + * L3, where both sides become memory-bound and the answer changes for a reason that has nothing to + * do with either implementation. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@State(Scope.Benchmark) +@Fork(value = 1, jvmArgsAppend = { + "--enable-native-access=ALL-UNNAMED", "--add-modules", "jdk.incubator.vector"}) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 8, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class C_ExecutionBoundary { + + @Param({"64", "256", "1024", "4096", "16384", "65536", "262144", "1048576", "4194304"}) + public int rows; + + private Data data; + + @Setup(Level.Trial) + public void setup() { data = new Data(rows); } + + @TearDown(Level.Trial) + public void tearDown() { data.close(); } + + /** The product path. One crossing, whatever the row count and whatever the predicate count. */ + @Benchmark + public long native_fusedPlan() { + return data.pattern.view() + .where(Pattern.CLASS.eq(Data.CLASS_NEEDLE)) + .where(Pattern.VALUE.gt(Data.VALUE_THRESHOLD)) + .count(); + } + + /** Java Vector API over the same native memory. Zero copies, 512-bit species. */ + @Benchmark + public long java_vectorApi() { + return Kernels.countVector(data.classes, data.values, rows, + Data.CLASS_NEEDLE, Data.VALUE_THRESHOLD); + } + + /** Ordinary Java loop over the same native memory. The auto-vectorisation control. */ + @Benchmark + public long java_scalarLoop() { + return Kernels.countScalar(data.classes, data.values, rows, + Data.CLASS_NEEDLE, Data.VALUE_THRESHOLD); + } +} diff --git a/bench/src/main/java/com/adaworldapi/lancegraph/bench/Data.java b/bench/src/main/java/com/adaworldapi/lancegraph/bench/Data.java new file mode 100644 index 0000000..8c74e48 --- /dev/null +++ b/bench/src/main/java/com/adaworldapi/lancegraph/bench/Data.java @@ -0,0 +1,100 @@ +package com.adaworldapi.lancegraph.bench; + +import com.adaworldapi.lancegraph.NativeAccess; +import com.adaworldapi.lancegraph.NativePattern; +import com.adaworldapi.lancegraph.NativeRuntime; +import com.adaworldapi.lancegraph.Pattern; +import com.adaworldapi.lancegraph.internal.ffm.Engine; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +/** + * One open pattern plus the handles every benchmark needs, and — more importantly — the + * cross-check that all the implementations under comparison compute the same answer. + * + *

The cross-check runs in {@code @Setup}, not in a test. A benchmark whose + * variants disagree is measuring nothing, and the failure mode is silent: a Vector kernel with an + * off-by-one tail is *faster* than a correct one and looks like a win. So the setup asserts + * agreement and throws before any measurement happens. + */ +public final class Data implements AutoCloseable { + + /** The class tag selected. Matches ~1/16 of rows in the fixture. */ + public static final int CLASS_NEEDLE = 7; + /** The signed threshold. The fixture's values span -150..361, so this straddles. */ + public static final int VALUE_THRESHOLD = 100; + + public final NativePattern pattern; + public final int rows; + + /** Native lane 1 ({@code u32} classes), read in place. Never copied. */ + public final MemorySegment classes; + /** Native lane 2 ({@code i32} values), read in place. Never copied. */ + public final MemorySegment values; + + /** + * A heap copy of the values lane. Present only as the "data already in Java" baseline + * for the segment-access benchmark, and used by nothing that compares against the native + * kernel — copying into it would be exactly the serialization the ABI forbids. + */ + public final int[] valuesHeap; + + public final long expectedCount; + public final long expectedSum; + + public Data(int rows) { + if (!NativeRuntime.isAvailable()) { + throw new IllegalStateException("native library unavailable: " + + NativeRuntime.unavailableReason().getMessage() + + "\nbuild it with: cd native/lgj-abi && " + + "CARGO_TARGET_DIR=/target cargo build --release"); + } + this.rows = rows; + this.pattern = NativePattern.open(rows); + + Engine.LaneWindow c = NativeAccess.lane(pattern, NativeAccess.LANE_CLASS); + Engine.LaneWindow v = NativeAccess.lane(pattern, NativeAccess.LANE_VALUE); + this.classes = c.segment(); + this.values = v.segment(); + + this.valuesHeap = new int[rows]; + for (int i = 0; i < rows; i++) { + valuesHeap[i] = values.get(ValueLayout.JAVA_INT, (long) i * Integer.BYTES); + } + + this.expectedCount = pattern.view() + .where(Pattern.CLASS.eq(CLASS_NEEDLE)) + .where(Pattern.VALUE.gt(VALUE_THRESHOLD)) + .count(); + this.expectedSum = pattern.view() + .where(Pattern.CLASS.eq(CLASS_NEEDLE)) + .where(Pattern.VALUE.gt(VALUE_THRESHOLD)) + .sumOf(Pattern.VALUE); + + crossCheck(); + } + + /** Every implementation under comparison must agree before any of them is timed. */ + private void crossCheck() { + long vec = Kernels.countVector(classes, values, rows, CLASS_NEEDLE, VALUE_THRESHOLD); + long sca = Kernels.countScalar(classes, values, rows, CLASS_NEEDLE, VALUE_THRESHOLD); + long vecSum = Kernels.sumVector(classes, values, rows, CLASS_NEEDLE, VALUE_THRESHOLD); + if (vec != expectedCount || sca != expectedCount) { + throw new AssertionError("count disagreement at rows=" + rows + + ": native=" + expectedCount + " vector=" + vec + " scalar=" + sca); + } + if (vecSum != expectedSum) { + throw new AssertionError("sum disagreement at rows=" + rows + + ": native=" + expectedSum + " vector=" + vecSum); + } + long segSum = Kernels.sumAllScalar(values, rows); + if (segSum != Kernels.sumAllVector(values, rows) || segSum != Kernels.sumAllHeap(valuesHeap)) { + throw new AssertionError("full-lane sum disagreement at rows=" + rows); + } + } + + @Override public void close() { + pattern.close(); + } +} diff --git a/bench/src/main/java/com/adaworldapi/lancegraph/bench/E_FusionAndPlanning.java b/bench/src/main/java/com/adaworldapi/lancegraph/bench/E_FusionAndPlanning.java new file mode 100644 index 0000000..6fb03ad --- /dev/null +++ b/bench/src/main/java/com/adaworldapi/lancegraph/bench/E_FusionAndPlanning.java @@ -0,0 +1,128 @@ +package com.adaworldapi.lancegraph.bench; + +import com.adaworldapi.lancegraph.Diagnostics; +import com.adaworldapi.lancegraph.Pattern; +import com.adaworldapi.lancegraph.Predicate; +import com.adaworldapi.lancegraph.View; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Components E and F — is fusion worth it, and what does the fluent API itself cost? + * + *

E: fused vs unfused

+ * + *

{@code .where(a).where(b).where(c).count()} is one crossing. The same chain + * evaluated predicate-by-predicate is one crossing per predicate, plus one per mask combine, plus + * one to count — and the ABI keeps the unfused entry points alive for exactly this comparison, + * so that "fusion matters" can be a measurement rather than an argument. + * + *

The predicate count is swept, because fusion's value is a slope, not a point: if the + * difference did not grow with the number of predicates, it would not be fusion causing it. + * + *

{@code plan_eval_scalar} is measured alongside as the SIMD-vs-scalar spread through the same + * membrane. It is the ABI's parity escape hatch, never a production path. + * + *

F: the Java-side cost of the abstraction

+ * + *

Building the view and its predicate list touches no native code at all. If that construction + * were expensive, the fluent API would be taxing the developer for the ergonomics — so it is + * measured on its own, with the terminal operation omitted, and with {@link Diagnostics#crossings} + * asserted unchanged so the "no native code" claim is checked rather than assumed. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@State(Scope.Benchmark) +@Fork(value = 1, jvmArgsAppend = {"--enable-native-access=ALL-UNNAMED"}) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 8, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class E_FusionAndPlanning { + + /** + * Two scales, because fusion's whole argument is about a FIXED per-crossing cost. At 65,536 + * rows the memory traffic dwarfs it and fusion should be invisible; at 256 rows the crossing + * is most of the work and it should dominate. Measuring only the large one would have + * produced "fusion does not help", which is true there and false in general. + */ + @Param({"256", "65536"}) + public int rows; + + /** How many predicates in the chain. Fusion's advantage should scale with this. */ + @Param({"1", "2", "4", "8"}) + public int predicates; + + private Data data; + private List chain; + + @Setup(Level.Trial) + public void setup() { + data = new Data(rows); + chain = new ArrayList<>(); + // Predicates chosen so every one of them actually narrows: repeating an identical + // predicate would let a hypothetical optimiser collapse the chain and would measure + // deduplication rather than fusion. Each VALUE threshold is distinct and each is below + // the fixture's maximum, so each contributes real work. + chain.add(Pattern.CLASS.eq(Data.CLASS_NEEDLE)); + int[] thresholds = {100, 50, 0, -50, 120, 20, -100}; + for (int i = 0; i < predicates - 1; i++) { + chain.add(Pattern.VALUE.gt(thresholds[i % thresholds.length])); + } + + // Falsify the "fused and unfused agree" claim once, here, rather than trusting it. + View v = view(); + long fused = Diagnostics.countFused(v); + long unfused = Diagnostics.countUnfused(v); + long scalar = Diagnostics.countScalar(v); + if (fused != unfused || fused != scalar) { + throw new AssertionError("fused=" + fused + " unfused=" + unfused + " scalar=" + scalar); + } + } + + @TearDown(Level.Trial) + public void tearDown() { data.close(); } + + private View view() { + View v = data.pattern.view(); + for (Predicate p : chain) v = v.where(p); + return v; + } + + /** ONE crossing regardless of {@link #predicates}. */ + @Benchmark + public long fused() { + return Diagnostics.countFused(view()); + } + + /** One crossing per predicate, plus combines, plus the count. */ + @Benchmark + public long unfused() { + return Diagnostics.countUnfused(view()); + } + + /** One crossing, forced down the scalar reference kernel. Parity path, not production. */ + @Benchmark + public long fusedScalarKernel() { + return Diagnostics.countScalar(view()); + } + + /** + * Component F: build the whole chain and stop. No terminal operation, so no crossing. + * + *

The crossing counter is read before and after and compared, which is what makes this a + * measurement of "the lazy API is lazy" rather than an assumption about it. + */ + @Benchmark + public void planConstructionOnly(Blackhole bh) { + long before = Diagnostics.crossings(); + View v = view(); + bh.consume(v.conditionCount()); + if (Diagnostics.crossings() != before) { + throw new AssertionError("building a view crossed the membrane; it must not"); + } + bh.consume(v); + } +} diff --git a/bench/src/main/java/com/adaworldapi/lancegraph/bench/Harness.java b/bench/src/main/java/com/adaworldapi/lancegraph/bench/Harness.java new file mode 100644 index 0000000..410d612 --- /dev/null +++ b/bench/src/main/java/com/adaworldapi/lancegraph/bench/Harness.java @@ -0,0 +1,87 @@ +package com.adaworldapi.lancegraph.bench; + +import com.adaworldapi.lancegraph.NativeRuntime; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * The entry point. Prints the environment that every number has to be read against, then runs JMH. + * + *

The environment block is not decoration. A benchmark result without the JDK build, the VM + * flags, the CPU, the SIMD backend actually compiled into the native library, and the row counts + * is not reproducible and not reviewable, and this harness is meant to survive being read by + * someone who does this for a living. + * + *

Usage: {@code java ... Harness [regex]} — the optional argument filters which benchmarks run, + * e.g. {@code Harness C_} for the execution-boundary sweep alone. + */ +public final class Harness { + + private Harness() {} + + /** Resolve the native library the same way the library itself does, so both agree. */ + public static String libraryPath() { + String path = NativeRuntime.libraryPath(); + if (path == null || !Files.isRegularFile(Path.of(path))) { + throw new IllegalStateException("native library not found (resolved: " + path + ")"); + } + return path; + } + + public static void main(String[] args) throws Exception { + if (!NativeRuntime.isAvailable()) { + System.err.println("FATAL: native library unavailable — " + + NativeRuntime.unavailableReason().getMessage()); + System.err.println("build it with:"); + System.err.println(" cd native/lgj-abi && " + + "CARGO_TARGET_DIR=$(git rev-parse --show-toplevel)/target cargo build --release"); + System.exit(2); + } + + System.out.println("=".repeat(96)); + System.out.println("lance-graph-java :: benchmark harness"); + System.out.println("=".repeat(96)); + row("jdk", System.getProperty("java.vm.name") + " " + System.getProperty("java.vm.version")); + row("vm args", java.lang.management.ManagementFactory.getRuntimeMXBean() + .getInputArguments().toString()); + row("os / arch", System.getProperty("os.name") + " " + System.getProperty("os.arch")); + row("cpu", cpuModel() + " (" + Runtime.getRuntime().availableProcessors() + + " logical processors)"); + row("vector species", Kernels.I32 + " (" + Kernels.I32.length() + " int lanes, " + + Kernels.I32.vectorBitSize() + " bit)"); + row("native runtime", NativeRuntime.describe()); + row("predicate", "class == " + Data.CLASS_NEEDLE + + " AND value > " + Data.VALUE_THRESHOLD); + System.out.println("=".repeat(96)); + System.out.println(); + + var options = new OptionsBuilder() + .include(args.length > 0 ? args[0] : "com.adaworldapi.lancegraph.bench") + .resultFormat(ResultFormatType.CSV) + .result("results/jmh-results.csv") + .shouldDoGC(true) + .build(); + new Runner(options).run(); + } + + private static void row(String k, String v) { + System.out.printf("%-16s %s%n", k, v); + } + + private static String cpuModel() { + try { + for (String line : Files.readAllLines(Path.of("/proc/cpuinfo"))) { + if (line.startsWith("model name")) { + return line.substring(line.indexOf(':') + 1).trim(); + } + } + } catch (Exception ignored) { + // /proc is Linux-only; the harness runs elsewhere without a CPU model line. + } + return "unknown"; + } +} diff --git a/bench/src/main/java/com/adaworldapi/lancegraph/bench/Kernels.java b/bench/src/main/java/com/adaworldapi/lancegraph/bench/Kernels.java new file mode 100644 index 0000000..95c4b15 --- /dev/null +++ b/bench/src/main/java/com/adaworldapi/lancegraph/bench/Kernels.java @@ -0,0 +1,148 @@ +package com.adaworldapi.lancegraph.bench; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.ByteOrder; + +import jdk.incubator.vector.IntVector; +import jdk.incubator.vector.VectorMask; +import jdk.incubator.vector.VectorOperators; +import jdk.incubator.vector.VectorSpecies; + +/** + * Java-side implementations of the same predicate the Rust kernel evaluates, so that + * "where does execution belong?" is answered by two real implementations rather than by one + * implementation and an estimate. + * + *

Zero-copy is the whole point

+ * + *

Every method here reads the native lane directly through + * {@link IntVector#fromMemorySegment}. No {@code byte[]}, no {@code int[]}, no bulk copy, no + * {@code MemorySegment.toArray}. If any copy existed the comparison would be dishonest twice + * over: the Java side would be paying a cost the Rust side does not, and the Rust side would be + * getting credit for avoiding a copy that nothing in the design requires. + * + *

{@link ByteOrder#nativeOrder()} is passed explicitly. The ABI states little-endian and + * asserts it via the manifest's magic probe, but a vector load that assumed an order would be a + * silent corruption on a big-endian host rather than a failure. + * + *

Why the scalar version is here too

+ * + *

Because "the Vector API was fast" and "the JIT auto-vectorised an ordinary loop" are + * different claims with different consequences, and only measuring both separates them. If the + * scalar loop matches the explicit vector loop, the Vector API bought nothing on this shape. + */ +public final class Kernels { + + private Kernels() {} + + /** 512-bit on this host: 16 {@code int} lanes, the same width the Rust AVX-512 kernel uses. */ + public static final VectorSpecies I32 = IntVector.SPECIES_PREFERRED; + + /** + * {@code count(class == needle AND value > threshold)} over two native lanes, explicitly + * vectorised, reading the native memory in place. + * + *

The {@code u32} equality is done on the raw {@code int} bits, which is correct: equality + * is sign-agnostic, so no widening is needed and none is done. The {@code i32} comparison is + * {@link VectorOperators#GT}, a signed compare — the fixture's values straddle zero precisely + * so that getting this wrong would show up as a wrong answer rather than as agreement. + */ + public static long countVector(MemorySegment classes, MemorySegment values, int rows, + int needle, int threshold) { + long count = 0; + int upper = I32.loopBound(rows); + int i = 0; + for (; i < upper; i += I32.length()) { + long off = (long) i * Integer.BYTES; + IntVector c = IntVector.fromMemorySegment(I32, classes, off, ByteOrder.nativeOrder()); + IntVector v = IntVector.fromMemorySegment(I32, values, off, ByteOrder.nativeOrder()); + VectorMask m = c.compare(VectorOperators.EQ, needle) + .and(v.compare(VectorOperators.GT, threshold)); + count += m.trueCount(); + } + // Tail. Handled scalar rather than with a masked load because the two produce identical + // results and the scalar tail is the one a reader can check by eye. + for (; i < rows; i++) { + long off = (long) i * Integer.BYTES; + if (classes.get(ValueLayout.JAVA_INT, off) == needle + && values.get(ValueLayout.JAVA_INT, off) > threshold) { + count++; + } + } + return count; + } + + /** As {@link #countVector}, also summing the matching {@code i32} values into an {@code i64}. */ + public static long sumVector(MemorySegment classes, MemorySegment values, int rows, + int needle, int threshold) { + long sum = 0; + int upper = I32.loopBound(rows); + int i = 0; + for (; i < upper; i += I32.length()) { + long off = (long) i * Integer.BYTES; + IntVector c = IntVector.fromMemorySegment(I32, classes, off, ByteOrder.nativeOrder()); + IntVector v = IntVector.fromMemorySegment(I32, values, off, ByteOrder.nativeOrder()); + VectorMask m = c.compare(VectorOperators.EQ, needle) + .and(v.compare(VectorOperators.GT, threshold)); + // Widening to i64 lane-wise would need two long vectors per int vector; the values are + // bounded by the fixture to [-150, 361] so an int-lane reduction cannot overflow at + // 16 lanes, and the accumulation into `sum` is the widening. + sum += v.reduceLanes(VectorOperators.ADD, m); + } + for (; i < rows; i++) { + long off = (long) i * Integer.BYTES; + int v = values.get(ValueLayout.JAVA_INT, off); + if (classes.get(ValueLayout.JAVA_INT, off) == needle && v > threshold) sum += v; + } + return sum; + } + + /** + * The same predicate as an ordinary scalar loop over the same native memory. + * + *

Present as the control. C2 auto-vectorises loops like this one, so if it matches + * {@link #countVector} then the Vector API contributed nothing here and the honest conclusion + * is about the JIT, not about the API. + */ + public static long countScalar(MemorySegment classes, MemorySegment values, int rows, + int needle, int threshold) { + long count = 0; + for (int i = 0; i < rows; i++) { + long off = (long) i * Integer.BYTES; + if (classes.get(ValueLayout.JAVA_INT, off) == needle + && values.get(ValueLayout.JAVA_INT, off) > threshold) { + count++; + } + } + return count; + } + + /** Sum every element of an {@code i32} lane. Used for raw segment read throughput. */ + public static long sumAllScalar(MemorySegment lane, int rows) { + long sum = 0; + for (int i = 0; i < rows; i++) sum += lane.get(ValueLayout.JAVA_INT, (long) i * Integer.BYTES); + return sum; + } + + /** Sum every element of an {@code i32} lane, explicitly vectorised. */ + public static long sumAllVector(MemorySegment lane, int rows) { + IntVector acc = IntVector.zero(I32); + int upper = I32.loopBound(rows); + int i = 0; + for (; i < upper; i += I32.length()) { + acc = acc.add(IntVector.fromMemorySegment(I32, lane, (long) i * Integer.BYTES, + ByteOrder.nativeOrder())); + } + long sum = acc.reduceLanes(VectorOperators.ADD); + for (; i < rows; i++) sum += lane.get(ValueLayout.JAVA_INT, (long) i * Integer.BYTES); + return sum; + } + + /** Sum every element of a heap {@code int[]}. The "data already in Java" baseline. */ + public static long sumAllHeap(int[] lane) { + long sum = 0; + for (int v : lane) sum += v; + return sum; + } +} diff --git a/bench/summarise.sh b/bench/summarise.sh new file mode 100755 index 0000000..6d88e35 --- /dev/null +++ b/bench/summarise.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Turn results/jmh-results.csv into the tables in RESULTS.md. +# +# Kept as a script rather than done by hand so that a re-run's numbers can be regenerated +# mechanically — a table transcribed by hand is a table that can drift from its own data. +set -euo pipefail +CSV="${1:-$(dirname "${BASH_SOURCE[0]}")/results/jmh-results.csv}" + +python3 - "$CSV" <<'PY' +import csv, sys, collections + +rows = list(csv.DictReader(open(sys.argv[1]))) + +def key(r): + return r['Benchmark'].rsplit('.', 1)[-1] + +def num(r, f): + v = r.get(f, '') + return float(v) if v not in ('', 'NaN', None) else float('nan') + +# ── Components A and B: flat tables ─────────────────────────────────────────────────────────── +for cls, title in (('A_DowncallOverhead', 'A — membrane crossing, isolated'), + ('B_SegmentAccess', 'B — reading native memory (65,536 i32)')): + sel = [r for r in rows if cls in r['Benchmark']] + if not sel: + continue + print(f"\n### {title}\n") + print("| benchmark | mean | ±99.9% CI | unit |") + print("|---|---:|---:|---|") + for r in sorted(sel, key=lambda r: num(r, 'Score')): + print(f"| `{key(r)}` | {num(r,'Score'):.3f} | ±{num(r,'Score Error (99.9%)'):.3f} " + f"| {r['Unit']} |") + +# ── Component C: the row sweep, one column per arm ─────────────────────────────────────────── +sweep = collections.defaultdict(dict) +for r in rows: + if 'C_ExecutionBoundary' not in r['Benchmark'] or not r.get('Param: rows'): + continue + sweep[int(r['Param: rows'])][key(r)] = (num(r, 'Score'), num(r, 'Score Error (99.9%)')) + +if sweep: + arms = ['native_fusedPlan', 'java_vectorApi', 'java_scalarLoop'] + print("\n### C/D — where does execution belong? (µs/op, mean ± 99.9% CI)\n") + print("| rows | lane KiB | " + " | ".join(f"`{a}`" for a in arms) + + " | fastest | native/vector |") + print("|---:|---:|" + "---:|" * (len(arms) + 2)) + for n in sorted(sweep): + cells, scores = [], {} + for a in arms: + if a in sweep[n]: + s, e = sweep[n][a] + scores[a] = s + cells.append(f"{s:.3f} ±{e:.3f}") + else: + cells.append("—") + best = min(scores, key=scores.get) if scores else "—" + ratio = (f"{scores['native_fusedPlan'] / scores['java_vectorApi']:.2f}x" + if 'native_fusedPlan' in scores and 'java_vectorApi' in scores else "—") + print(f"| {n:,} | {n * 4 // 1024} | " + " | ".join(cells) + + f" | **{best}** | {ratio} |") + +# ── Component E/F: fusion sweep ─────────────────────────────────────────────────────────────── +fus = collections.defaultdict(dict) +for r in rows: + if 'E_FusionAndPlanning' not in r['Benchmark'] or not r.get('Param: predicates') \ + or not r.get('Param: rows'): + continue + fus[(int(r['Param: rows']), int(r['Param: predicates']))][key(r)] = ( + num(r, 'Score'), num(r, 'Score Error (99.9%)')) + +if fus: + arms = ['fused', 'unfused', 'fusedScalarKernel', 'planConstructionOnly'] + print("\n### E/F — fusion and the cost of the fluent API (µs/op)\n") + print("| rows | predicates | " + " | ".join(f"`{a}`" for a in arms) + " | unfused/fused |") + print("|---:|---:|" + "---:|" * (len(arms) + 1)) + for p in sorted(fus): + cells, scores = [], {} + for a in arms: + if a in fus[p]: + s, e = fus[p][a] + scores[a] = s + cells.append(f"{s:.3f} ±{e:.3f}") + else: + cells.append("—") + ratio = (f"**{scores['unfused'] / scores['fused']:.2f}x**" + if 'fused' in scores and 'unfused' in scores else "—") + print(f"| {p[0]:,} | {p[1]} | " + " | ".join(cells) + f" | {ratio} |") +print() +PY diff --git a/valhalla-lab/README.md b/valhalla-lab/README.md index 8255c41..d22e3bb 100644 --- a/valhalla-lab/README.md +++ b/valhalla-lab/README.md @@ -1,100 +1,70 @@ -# The Valhalla laboratory +# valhalla-lab -The three-truths method (`.claude/knowledge/valhalla-three-truths-method.md`) applied to this -project's small semantic value vocabulary — `LaneId`, `Ordinal`, `MaskId`, `RowRange`, `Row` — and -to the mission's mandatory headline experiment: does Valhalla rescue per-entity materialization at -65,536-row scale, or only the tiny descriptor vocabulary around it? +An A/B experiment on the API's small semantic vocabulary — `LaneId`, `RowRange`, `MaskId`, +`Ordinal` — plus a direct test of the project's central claim about bulk data. -**Same experiment source, compiled twice** — once against a stable JDK where the vocabulary types -are plain `record`s, once against the JEP 401 early-access JDK where they are `value record`s — so -the comparison is genuinely apples-to-apples, not two different programs. +**Read [`docs/three-truths.md`](docs/three-truths.md) for the findings and the numbers.** +**Read [`reproducers/README.md`](reproducers/README.md) for the three Valhalla limitations hit.** -## Layout +## What it does + +One experiment source (`src/shared`) is compiled twice: once against `src/stable` (plain +`record`s, JDK 26) and once against `src/valhalla` (`value record`s, JDK 27 EA). The two +vocabularies are **byte-identical apart from the word `value`**, and `run.sh` step 0 diffs them and +refuses to run if that stops being true — without that check the A/B could quietly become a +comparison of two different programs. ``` -src/shared/ experiment logic, byte-identical on both compiles -src/stable/ Vocab.java (record), Containers.java, Platform.java — the stable-JDK half of the A/B -src/valhalla/ Vocab.java (value record), Containers.java, Platform.java — the Valhalla half +src/shared/ the experiments — compiled unchanged against both vocabularies +src/stable/ Vocab.java (record), Platform.java, Containers.java -> JDK 26 +src/valhalla/ Vocab.java (value record), Platform.java, Containers.java -> JDK 27 EA +reproducers/ standalone files for each limitation hit, with observed output +results/ every run's output, plus the A/B diff ``` -`Platform` is the one seam between them: same signatures on both sides, so `src/shared/` never -branches on which platform it's running on except by asking `Platform` — never by calling a -Valhalla-only API (like `Class::isValue` or `jdk.internal.value.ValueClass`) directly. That is a -real rule, not a style preference: `Class::isValue` does not exist at all on a stable JDK, so a -direct call would fail to *compile* the stable half, not just report the wrong answer. - -`NativeAccess` (in `src/shared/`, package `com.adaworldapi.lancegraph`) is a read-only, split-package -escape hatch into the shipped library's package-private handle — documented in the file itself. It -exists because the lab has to build the very thing the thesis says you should not build (65,536 Java -objects) from the *same bytes* the native kernel reads, or the comparison proves nothing. Nothing -under `java/` changes to support this. - -## Build and run - -### Stable half (JDK 26 GA, plain `record`s) +## Run it ```sh -javac -d out-stable $(find ../java/src/main/java src/shared src/stable -name '*.java') - -java --enable-native-access=ALL-UNNAMED \ - -Dlgj.library=../target/release/liblgj_abi.so \ - -cp out-stable com.adaworldapi.lancegraph.lab.RunAll +cd valhalla-lab && ./run.sh ``` -### Valhalla half (the JEP 401 EA build, `value record`s) - -`--release` cannot be combined with `--add-exports` (a real javac restriction — `--release` uses a -stricter cross-compilation module model). Use `-source` instead when compiling *for* the JDK you are -also running on, which is the case here. +Requires `liblgj_abi.so`; build it first if missing: ```sh -javac --enable-preview -source 27 \ - --add-exports java.base/jdk.internal.value=ALL-UNNAMED \ - --add-exports java.base/jdk.internal.vm.annotation=ALL-UNNAMED \ - -d out-valhalla $(find ../java/src/main/java src/shared src/valhalla -name '*.java') - -java --enable-preview --enable-native-access=ALL-UNNAMED \ - --add-exports java.base/jdk.internal.value=ALL-UNNAMED \ - --add-exports java.base/jdk.internal.vm.annotation=ALL-UNNAMED \ - -Dlgj.library=../target/release/liblgj_abi.so \ - -cp out-valhalla com.adaworldapi.lancegraph.lab.RunAll +cd native/lgj-abi && CARGO_TARGET_DIR=$(cd ../.. && pwd)/target cargo build --release ``` -Both need the JDK paths from `.claude/knowledge/jdk-toolchain-facts.md` — do not use `/usr/bin/java` -(JDK 21, no value classes at all) for either. - -## What each experiment measures +Override the JDKs with `STABLE_JDK=` / `VALHALLA_JDK=`. The script runs six configurations — +both platforms at default settings, both with `-XX:-DoEscapeAnalysis`, and Valhalla with each +flattening knob disabled — and writes each to `results/`. -| Class | Question | -|---|---| -| `IdentityExperiment` | Truth (a), semantic: is identity actually unobservable? `Class::isValue`, reference equality, array flatness, `synchronized` legality — measured on both platforms, asked to agree everywhere except reference equality (which no caller in the production API uses). | -| `FootprintExperiment` | Truth (b)/(c), representation: per-object bytes, array layout, field flattening, call-argument passing — via `jol-core`'s real VM instrumentation where available, allocation-delta measurement elsewhere. | -| `FfmAddressingExperiment` | Is the wrapper free where it actually touches native memory — a `RowRange`/`Ordinal` around an FFM offset vs a bare `long`? | -| `ThesisExperiment` | The mandatory headline: 65,536 rows as (1) one native lane + one packed mask + one crossing, vs (2)/(3) hydrated Java objects, on the SAME question and the SAME answer. Heap cost and wall time, both platforms. | +## The experiments -## Measured headline (2026-08-17, this environment) +| | what it answers | primary instrument | +|---|---|---| +| `IdentityExperiment` | is the semantic contract observably the same under both object models? | behavioural assertions | +| `FlatteningCliffExperiment` | which payload shapes does the VM actually flatten? | `ValueClass.isFlatArray` | +| `FootprintExperiment` | does the abstraction cost a heap object — in arrays, in fields, in arguments? | `getThreadAllocatedBytes` | +| `FfmAddressingExperiment` | is the wrapper free where it addresses native memory? | allocated bytes, then time | +| `ThesisExperiment` | 65,536 entities: native lane vs Java objects vs Valhalla value objects | allocated bytes, retained heap, time | -Real numbers from a real run — reproduce with the commands above before citing a different number. +## Two things about the method -| | native, one crossing | hydrate 65,536 `Row`, then scan | -|---|---:|---:| -| stable JDK 26 | 19.5 µs, 289 KiB Java-side | 746 µs, 2.00 MiB | -| Valhalla (JDK 27 EA) | 15.7 µs, 289.5 KiB Java-side | 900 µs, 2.50 MiB | +**Allocated bytes, not nanoseconds, is the primary instrument.** "Did this abstraction cost an +object?" is answered directly by TLAB accounting and only inferred, weakly, from a timing — +escape analysis deletes allocations in tight loops and proves nothing about the general case. The +runs with `-XX:-DoEscapeAnalysis` exist for the same reason: they measure what the object model +costs when the JIT cannot rescue it, which is the property that generalises to real code. -The native path wins by roughly **38–57×** on time and **7–9×** on Java heap, on **both** platforms — -Valhalla does not close this gap, because `Row` (multiple fields) measured `NOT-FLAT` even under -Valhalla, while the single-field `LaneId` measured `FLAT` (2.90 B/element vs 16.00 B on stable, ~5.5× -smaller). This is the mission thesis's prediction, confirmed rather than assumed: **Valhalla helps -the tiny descriptor vocabulary; it does not rescue per-entity materialization at this scale.** See -`IdentityExperiment`'s and `FootprintExperiment`'s full output for the field-by-field evidence. +**This is not JMH.** `Lab.time` warms up, measures repeatedly, and reports a median with the full +min/max spread. It does not fork, does not detect steady state, and does not do statistical +inference. JMH-grade numbers live in [`bench/`](../bench) and are produced by actual JMH; the +timings here are secondary evidence supporting the byte counts, and are labelled as such. -## A defect found and fixed while wiring this up +## The verdict in one line -The first version of `IdentityExperiment`/stable `Platform` called `Class::isValue()` directly for -four of the five vocabulary types (`Ordinal`/`MaskId`/`RowRange`/`Row`), with a comment incorrectly -claiming it was "final API on JDK 26." It is not — `javac` on JDK 26 GA does not have that method at -all, confirmed by a real compile failure, not by reading documentation. Fixed by routing every -identity query through `Platform.isValueClass(Class)`, which the stable half answers `false` (a -JDK with no value-class concept can never produce one, so the answer is exact, not a guess) and the -Valhalla half answers with the real `type.isValue()`. See `EPIPHANIES.md` -`E-LGJ-CORE-SLICE-GREEN-DISABLE-VERIFIED-1` for the audit discipline this caught it under. +Valhalla makes the four-type descriptor vocabulary genuinely free (5.5× less allocation, 8.3× +faster array reads, and 103× → 1.9× once escape analysis is out of the picture). It does **not** +rescue per-entity materialisation — on this build a 16-byte `Row` is over the VM's flattening +budget and costs 25 % *more* than the plain record. Bulk data stays native; the vocabulary stays +records today and becomes value records the day JEP 401 ships. diff --git a/valhalla-lab/docs/three-truths.md b/valhalla-lab/docs/three-truths.md new file mode 100644 index 0000000..15e4dad --- /dev/null +++ b/valhalla-lab/docs/three-truths.md @@ -0,0 +1,265 @@ +# The three truths of a semantic value + +> Every number in this document was produced by `valhalla-lab/run.sh` on this box and is +> reproduced verbatim from `valhalla-lab/results/`. Nothing here is quoted from a specification or +> a blog post. Where something could not be measured, it says so. + +The API defines a small vocabulary — `LaneId`, `RowRange`, `MaskId`, `Ordinal` — whose entire job +is to stop an `int` that means "which column" from being confused with an `int` that means "which +row". Four types, each wrapping one or two primitives. + +Such a type has three truths, and they are not the same truth: + +| | | +|---|---| +| **(a) semantic** | what the concept *means* — a `LaneId` is a value; its object identity is not part of what it is, and nothing in the API should be able to observe it | +| **(b) stable-Java** | what a `record` on a production JDK actually *is* — a heap object with a header, reached through a pointer | +| **(c) Valhalla** | what the *same source* becomes as a `value record` under JEP 401 | + +The lab's design is one sentence: **compile one experiment source against two vocabularies that +differ by exactly one word, and diff the output.** `run.sh` step 0 enforces the "exactly one word" +part by diffing `src/valhalla/.../Vocab.java` against `src/stable/.../Vocab.java` modulo the +`value` modifier and refusing to run if anything else differs. Without that check the A/B could +silently become a comparison of two different programs. + +--- + +## Environment + +| | | +|---|---| +| stable JDK | `openjdk 26.0.2+10-55` — FFM final, **no** `--enable-preview` | +| Valhalla JDK | `openjdk 27-jep401ea3+1-1` — JEP 401 early access, `--enable-preview` | +| CPU | Intel Xeon @ 2.10 GHz, 4 vCPU, AVX-512 (`avx512f/dq/bw/vl/vbmi/ifma/cd`) | +| native library | `liblgj_abi.so`, `abi 0.1`, `simd ndarray::simd avx512`, `profile release` | + +Reproduce everything: `valhalla-lab/run.sh`. + +--- + +## (a) Semantic truth — and the finding is what *does not* change + +| observation | stable | Valhalla | +|---|---|---| +| `equal state ⇒ equals()` | true | true | +| `different state ⇒ !equals()` | true | true | +| `equal state ⇒ equal hashCode()` | true | true | +| `equal state ⇒ equal toString()` | true | true | +| `Class::isValue()` | false | **true** | +| `a == b` for equal state | false | **true** | +| `identityHashCode(a) == identityHashCode(b)` | false | **true** | +| a local of this type accepts null | true | true | +| an array slot accepts null | true | **false** (null-restricted array) | +| `synchronized(x)` | legal | **compile error**: *required: a type with identity* | + +The first four rows are the result. Every behaviour the production API actually *uses* is +identical across the two object models. The rows that differ are exactly the ones the API was +written never to depend on: reference equality, identity hash, and locking. That is why the +migration is a one-word source change and not a redesign — and it is checked here rather than +asserted in a comment. + +Two rows deserve a note because they surprise people: + +- **`a == b` becomes `true`.** Under Valhalla, `==` on a value class *is* state comparison. Code + that used `==` as a cheap identity check silently changes meaning. This API never does. +- **A local variable of a value type still accepts `null`.** Null-restriction is a property of a + *field* or an *array*, not of the class. `LaneId x = null;` compiles on both platforms. + +`synchronized` is reported as a documented compile error rather than executed, because writing it +in the shared source would break the stable compile too. It was verified separately: + +``` +error: unexpected type + try { synchronized (p) { ... } } + ^ + required: a type with identity + found: LaneId +``` + +--- + +## (b) vs (c) Representation — measured in allocated bytes, not nanoseconds + +The primary instrument is `ThreadMXBean.getThreadAllocatedBytes` (measured harness baseline: **0 +bytes**). Bytes are the observation; time is the consequence. A timing alone cannot answer "did +this abstraction cost an object?", because escape analysis removes many allocations in a tight +loop and proves nothing about the general case. + +All figures per operation, `N = 1,000,000`. + +| measurement | stable | Valhalla | | +|---|---:|---:|---| +| construct a `LaneId`, store into an array | 16.00 B | **2.89 B** | 5.5× less | +| construct a `LaneId`, never escaping | 7.03 B | 8.00 B | ~equal — escape analysis already handles this | +| `LaneId[N]` array + elements, per element | 20.00 B | **6.89 B** | 2.9× less | +| bare `LaneId[N]`, per slot | 4.00 B | 4.00 B | equal (compressed oops vs flat int) | +| construct a `Descriptor` (two wrappers) | 56.00 B | **38.84 B** | 1.4× less | +| pass two wrappers through 3 call levels | 8.29 B | 10.45 B | ~equal | +| **read 65,536 `LaneId` from an array** | 44,182 ns | **5,349 ns** | **8.3× faster** | + +`LaneId[1024]` reports `FLAT` on Valhalla and `UNKNOWN` on stable — deliberately not `false`. A +stable JDK has no `ValueClass.isFlatArray` to ask, so "the question does not exist here" is the +honest answer; printing `false` would claim a measurement that was never taken. + +### The flattening knobs prove causation, not correlation + +Re-running the Valhalla build with the VM's own flattening disabled: + +| | default | `-XX:-UseArrayFlattening` | `-XX:-UseFieldFlattening` | +|---|---:|---:|---:| +| `LaneId` array, per element | 6.89 B | 28.00 B | 6.71 B | +| `LaneId[1024]` flat? | FLAT | NOT-FLAT | FLAT | +| read 65,536 from array | 5,349 ns | 47,469 ns | 5,303 ns | +| `Descriptor`, per instance | 38.84 B | 40.51 B | 80.00 B | + +Turning array flattening off returns the array numbers to roughly the stable baseline and makes +the read **8.9× slower**; turning field flattening off doubles the `Descriptor` cost and leaves +arrays alone. The two knobs move exactly the two measurements they name. This is the difference +between "the Valhalla JDK was faster" and "*flattening* is what made it faster". + +### Escape analysis is doing more of the stable JDK's work than it looks + +With `-XX:-DoEscapeAnalysis`, i.e. what the object model costs when the JIT cannot rescue it: + +| measurement | stable default | stable, no EA | Valhalla default | Valhalla, no EA | +|---|---:|---:|---:|---:| +| `LaneId` never escaping | 7.03 B | 16.00 B | 8.00 B | 25.60 B | +| pass 2 wrappers, 3 levels | 8.29 B | 32.00 B | 10.45 B | 31.00 B | +| **`Ordinal` built per element** (65,536 iterations) | 50,062 ns | **5,166,233 ns** | 50,021 ns | **94,346 ns** | + +That last row is the clearest single number in the lab. Unaided by escape analysis, building a +one-`int` wrapper per element costs the stable JDK **103×**; it costs Valhalla **1.9×**. This is +the concrete meaning of "the abstraction stops being something you pay for" — not that it is +faster when the JIT can see through it, but that it stays cheap when the JIT cannot. + +Honesty about what this row is *not*: with escape analysis on — the configuration anyone actually +ships — the two are indistinguishable at 50 µs. Valhalla's gain here is in **robustness**, not in +peak. That distinction is easy to lose and worth keeping. + +### `-XX:±InlineTypePassFieldsAsArgs` + +Not measured. The knob exists (`InlineTypePassFieldsAsArgs = true`, pd product) and the +method-passing row above is where it would show. On this hardware the measured argument-passing +difference between the platforms is inside the noise of the allocation instrument in the +default configuration, so a knob sweep would have produced numbers without meaning. Stated as +unmeasured rather than reported as "no effect". + +--- + +## The flattening cliff — the finding that reframes the whole thesis + +The first `ThesisExperiment` run produced something that looked like a bug: the Valhalla `Row` +array reported `NOT-FLAT` and cost **more** per row than the stable record (40.00 B vs 32.00 B). +Rather than explain it away, the question became a measurement. Sweeping payload shapes and asking +`ValueClass.isFlatArray` directly: + +``` +type payload NR-nonAtomic NR-atomic nullable-atomic +P4 4 B true true true +P8i 8 B true true false +P8l 8 B true true false +P12 12 B false false false +P16 16 B false false false +P16l 16 B false false false +``` + +**Flattening stops at an 8-byte payload, and past it no array flavour flattens at all.** +Independently confirmed by `-XX:+PrintFlatArrayLayout`, which logs a layout for the shapes above +the line and nothing below it. Full write-up: `reproducers/README.md` § R2. + +Applied to this project's actual types: + +| type | payload | flat? | side of the thesis | +|---|---:|---|---| +| `LaneId`, `Ordinal` | 4 B | **yes** | descriptor vocabulary — Valhalla helps | +| `MaskId` | 8 B | **yes** | descriptor vocabulary — Valhalla helps | +| `RowRange` | 16 B | no | descriptor, but already over budget | +| `Row` (id + class + value) | 16 B | no | per-entity materialisation — Valhalla does not help | + +So "Valhalla helps the descriptors, not the entities" is not a hand-wave about object headers. On +this build it is a hard cutoff in the VM, and any realistic entity is on the wrong side of it by +construction — an id plus a single field already exceeds the budget. + +`RowRange` falling on the wrong side is the one place the going-in expectation was too optimistic. +It is a descriptor, it was expected to flatten, and it does not. + +--- + +## The thesis experiment + +> 64,000 logical entities must NOT require 64,000 Java objects. + +Same 65,536 rows, same question (`count(class == 7 AND value > 100)` and `sum(value)`), three +representations. Paths 2 and 3 are populated *from the very lanes path 1 scans*, and all three +answers are asserted equal — `2173 rows, sum 499246`, 3.32 % selectivity — before any number is +reported. A benchmark whose variants compute different things measures nothing. + +### Heap cost + +| | stable | Valhalla | +|---|---:|---:| +| **(1) native** — Java bytes allocated *per query*, warm | **816 B** | **816 B** | +| **(1) native** — Java objects per row | **0** | **0** | +| **(1) native** — native lane bytes | 1.00 MiB | 1.00 MiB | +| **(1) native** — mask bytes | 8.0 KiB | 8.0 KiB | +| (2)/(3) hydrate 65,536 `Row` — allocated | 2.00 MiB | **2.50 MiB** | +| … per row | 32.00 B | **40.00 B** | +| … array flat? | UNKNOWN | **NOT-FLAT** | +| … retained heap (approximate, GC delta) | 2.25 MiB | 2.75 MiB | + +The 816 bytes is the number that matters, and what matters about it is that it does not depend on +the row count: it is the fluent chain's own bookkeeping (the predicate list and the marshalled op +descriptors), paid once per query whether the pattern holds 1,024 rows or 4,000,000. + +**Valhalla is 25 % worse here, not better.** That is the R2 cliff: a 16-byte `Row` cannot flatten, +so it pays a value-object layout without getting a value-object layout's benefit. Reported as +observed; it contradicts the naive expectation that value classes make entities cheaper, and it +supports the thesis more strongly than the expected result would have. + +### Time to answer + +| | stable | Valhalla | +|---|---:|---:| +| **(1) native — one crossing, fused plan** | **16,378 ns** | **18,805 ns** | +| (2/3) hydrate 65,536 `Row` objects | 603,139 ns | 768,624 ns | +| (2/3) scan the materialised objects | 151,886 ns | 91,178 ns | +| (2/3) hydrate **then** scan (honest total) | **788,227 ns** | **898,955 ns** | + +Medians of 51 iterations after 200–2,000 warm-up runs. Spreads are in `results/*.txt`; the +hydration rows have long tails (stable max 5.2 ms) because they allocate 2 MiB per iteration and +occasionally meet a GC, which is exactly why the median is reported. + +**Native is 48× faster than the honest object total on stable, 48× on Valhalla.** Note where +Valhalla's one real win sits: *scanning* already-materialised objects is 1.7× faster (91 µs vs +152 µs), because the scan is a read-only walk that benefits from better locality. It does not +matter, because the hydration that had to happen first costs 8× what the scan saves. + +That is the thesis, measured: the expensive thing is not *scanning* 65,536 objects, it is +*existing* as 65,536 objects. Valhalla makes the scan cheaper and does not make the existing +cheaper, so it does not move the answer. + +### And the per-entity path never had to happen + +The comparison above is generous to the object path — it assumes the developer needs the objects. +In the API they do not: `data.view().where(CLASS.eq(7)).where(VALUE.gt(100)).count()` is the +entire program. No arena, no segment, no mask, no lane, no opcode, no row loop, and no `Row`. + +--- + +## Verdict + +1. **The semantic contract is portable.** Every behaviour the API relies on is identical under + both object models; only the behaviours it was written to avoid differ. The migration is a + one-word source change, and it stays that way because the lab did **not** bend the API to fit a + preview VM (see `reproducers/README.md`). +2. **Valhalla is a real win for the descriptor vocabulary.** 5.5× less allocation per `LaneId`, + 8.3× faster array reads, and — the durable part — 103× → 1.9× when escape analysis cannot help. + A `LaneId` really does stop being something you pay for. +3. **Valhalla does not rescue per-entity materialisation, and on this build it makes it worse.** + 40 B/row against 32 B/row, `NOT-FLAT`, because a 16-byte payload is over the VM's flattening + budget. The expected finding was "it does not help"; the observed finding is "it costs 25 % + more". +4. **Therefore the architecture does not change.** Bulk data stays in native lanes with a packed + mask; the tiny semantic vocabulary stays as records today and becomes value records the day + JEP 401 ships. Those are two different decisions about two different kinds of object, and the + measurements say to keep them different. diff --git a/valhalla-lab/run.sh b/valhalla-lab/run.sh index 9dfb31c..091c11a 100755 --- a/valhalla-lab/run.sh +++ b/valhalla-lab/run.sh @@ -66,6 +66,7 @@ run () { # $1=tag $2=jdk $3=label shift 3 -> extra jvm args echo "--> $tag/$label" ( JAVA_TOOL_OPTIONS= "$jdk/bin/java" \ --enable-native-access=ALL-UNNAMED \ + -Dstdout.encoding=UTF-8 \ -Dlgj.library="$LIB_DIR/liblgj_abi.so" \ "$@" \ -cp "$OUT/$tag-api:$OUT/$tag-lab" com.adaworldapi.lancegraph.lab.RunAll ) \ From cd7782bf7d9a205ee57c8c78a7254872b67b91b1 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 22:03:50 +0000 Subject: [PATCH 04/30] Phase I synthesis docs + fusion-sweep 256-row re-run + board hygiene Four synthesis docs close D-LGJ-I: architecture.md (the four layers and what each is provably responsible for), panama.md (manifest-over-header, belt-and-braces ownership, restricted-method discipline), valhalla-lab.md (three-truths synthesis, the 8-byte flattening cliff, zero API adoption), execution-boundary.md (the measured crossover picture + three structural hot-path facts, each checked rather than assumed). The fusion sweep was re-run with a 256-row arm after the first pass's 'fusion does nothing' finding proved true only at 65,536 rows: at 256 rows x 8 predicates unfused/fused reaches 2.99x. RESULTS.md is rewritten from jmh-results-merged.csv; TABLES.md is mechanically generated from the same file so the two cannot drift. MultiLaneColumn (ndarray::simd_soa) evaluated for the fixture kernels and declined on two concrete API mismatches (64-byte-multiple constraint, no u32 lane); earmarked for the future 512-byte row-store slice where it fits by construction. Operator layout reference recorded on the board. PR_ARC_INVENTORY backfilled for merged PRs 1-3; the lapse is owned in the file itself. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/EPIPHANIES.md | 30 + .claude/board/LATEST_STATE.md | 24 + .claude/board/PR_ARC_INVENTORY.md | 62 +- .claude/board/STATUS_BOARD.md | 2 +- bench/RESULTS.md | 406 +++++-- bench/results/TABLES.md | 44 + bench/results/jmh-results-merged.csv | 66 + bench/results/jmh-results.csv | 81 +- bench/results/jmh-run.txt | 1067 +++++++++++++++++ docs/architecture.md | 140 +++ docs/execution-boundary.md | 155 +++ docs/panama.md | 100 ++ docs/valhalla-lab.md | 127 ++ valhalla-lab/docs/three-truths.md | 82 +- valhalla-lab/results/AB-default.diff | 72 +- valhalla-lab/results/stable-default.txt | 48 +- valhalla-lab/results/stable-noea.txt | 40 +- valhalla-lab/results/valhalla-default.txt | 60 +- valhalla-lab/results/valhalla-noarrayflat.txt | 54 +- valhalla-lab/results/valhalla-noea.txt | 52 +- valhalla-lab/results/valhalla-nofieldflat.txt | 58 +- valhalla-lab/results/valhalla-noflat.txt | 50 +- 22 files changed, 2386 insertions(+), 434 deletions(-) create mode 100644 bench/results/TABLES.md create mode 100644 bench/results/jmh-results-merged.csv create mode 100644 docs/architecture.md create mode 100644 docs/execution-boundary.md create mode 100644 docs/panama.md create mode 100644 docs/valhalla-lab.md diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index d924fdc..4c7400c 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -4,6 +4,36 @@ > `**Status:**`/`**Confidence:**` line. A correction gets its own new, > dated entry that references the one it corrects — the storno rule. +## 2026-08-17 — E-LGJ-SIMD-SOA-IS-FOR-THE-ROW-STORE-NOT-THE-FLAT-LANES-1 + +**Status:** DECISION (declined refactor, with the trigger for revisiting named). +**Confidence:** High — decided by reading `ndarray/src/simd_soa.rs`'s full API, not by taste. + +Operator suggestion: "if you use SoA, calling simd_soa.rs would make sense" — should +`native/lgj-abi/src/kernels.rs` route through `ndarray::simd_soa::MultiLaneColumn` (the canonical +`Arc<[u8]>` SoA carrier) instead of raw `&[u32]`/`&[i32]` slices? **Answer: not for today's +flat-lane fixture; yes for the future 512-byte row-store slice.** Two concrete API mismatches, +not a style call: + +1. **No tail handling.** `MultiLaneColumn::new()` hard-requires `len % 64 == 0`; every `iter_*` + yields only full 64-byte chunks via `as_chunks::<64>()` — no remainder arm. The + `simd_int_ops` primitives this project consumes do the opposite by design: full 16-lane + groups + a scalar tail for arbitrary caller-chosen `n_rows`. Wrapping the fixture's lanes in + `MultiLaneColumn` would force 64-byte padding on every allocation, bought for nothing. +2. **No `u32` lane.** `MultiLaneColumn` ships u8x64/f32x16/f64x8/u64x8/i32x16/i64x8 iterators — + no u32. The fixture's `ids`/`classes` are `u32` (`eq_u32_to_mask`). + +So `kernels.rs` already calls the correct layer: the `ndarray::simd_int_ops` primitives own their +chunking internally. `MultiLaneColumn` sits *above* that layer, for uniform pre-padded columns. + +**Where it DOES fit — the operator-stated layout reference (recorded verbatim so it survives):** +"the 64k x 512 bytes SoA layout is enforced everywhere in lance-graph (32 Lanes each 4 bytes +classview+12 bytes). For Java the layout might differ — just for reference." A 512-byte, +64-byte-aligned row store (32 × 16-byte V3 facets) is padded/aligned *by construction* — no tail +problem — and each row is a natural `iter_u8x64` chunk-of-chunks. When the real +`NodeRow`/facet slice replaces the generic fixture (`docs/abi.md` §10, `docs/architecture.md` +"where a real graph slice would attach"), `MultiLaneColumn` is the type to reach for. Not before. + ## 2026-08-17 — E-LGJ-VECTOR-API-BEATS-THE-CROSSING-1 **Status:** FINDING. **Confidence:** High (real JMH 1.37, `Data.crossCheck()` guards every fork, diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 06b6fe5..66f2e2e 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,27 @@ +## 2026-08-17 (later) — Phase I docs written, fusion re-run merged, simd_soa question answered (PR #4) + +- **All four synthesis docs shipped** (`docs/architecture.md`, + `docs/panama.md`, `docs/valhalla-lab.md`, `docs/execution-boundary.md`) + — D-LGJ-I DONE. Each cites the proving artifact instead of restating it. +- **Fusion sweep re-run with a 256-row arm** (`./run.sh E_`): the first + pass's "fusion does nothing" (true at 65,536 rows, where kernel time + dominates) is false at small rows — unfused/fused grows 0.95× → 2.99× + at 256 rows × 8 predicates, because per-crossing overhead dominates + there. `RESULTS.md` rewritten from `jmh-results-merged.csv` (A/B/C from + the full sweep + E from the re-run), `TABLES.md` mechanically generated + from the same file. Valhalla lab result files refreshed by a same-box + re-run; findings unchanged. +- **`MultiLaneColumn` question answered** (operator: "if you use SoA, + calling simd_soa.rs would make sense"): declined for the flat-lane + fixture (64-byte-multiple constraint + no u32 lane — two concrete API + mismatches), earmarked for the 512-byte row-store slice where it fits + by construction. Operator layout reference recorded: 64K × 512 B rows, + 32 lanes × (4 B classid + 12 B), enforced everywhere in lance-graph; + Java-side layout may differ. See + `E-LGJ-SIMD-SOA-IS-FOR-THE-ROW-STORE-NOT-THE-FLAT-LANES-1`. +- **PR_ARC_INVENTORY backfilled** for merged PRs #1-#3 (hygiene lapse + owned in the file itself). + ## 2026-08-17 — D-LGJ-AUDIT complete, core vertical slice VERIFIED GREEN, PR #1 opened ### Current Contract Inventory — the vertical slice is real and green diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 3f5a879..eacf076 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -3,14 +3,56 @@ # updatable in place — corrections append as new dated lines; reversals # get their own PR entry) -_No PR has been opened against this repository yet — session 1, 2026-08-17._ +> **Hygiene lapse, owned (2026-08-17):** PRs #1-#3 merged without their +> entries landing in the same commit — the exact retroactive-hygiene +> anti-pattern the imported board rules name. Backfilled below in one +> pass rather than left stale; PR #4 onward gets its entry at merge time. -The first entry in this file will be written when the first PR against -`lance-graph-java` merges (expected: the vertical slice on -`claude/lance-graph-java-panama-valhalla-sus9w8` → `main`, once -`ISS-LGJ-FANOUT-UNREVIEWED` closes). Until then, ground truth for -in-progress work lives on `LATEST_STATE.md` (current contract inventory), -`STATUS_BOARD.md` (per-D-id status), and `AGENT_LOG.md` (what actually -happened) — this file stays empty rather than backfilled with a -pre-registration entry that would misrepresent something as merged before -it is. +## PR #3 — Vector API bench: real JMH, cross-checked (merged 2026-08-17, squash) + +- **Added:** `bench/` — real JMH 1.37 suite (Components A/B/C/E: + downcall overhead, segment access, execution boundary sweep 64→4.2M + rows, fusion/planning), `Data.crossCheck()` gating every fork, + `summarise.sh` mechanical table generator, `RESULTS.md`, raw + run logs + CSV. +- **Locked:** the headline finding — Java Vector API zero-copy on the + native segment beats the native crossing at every row count tested + (56.4× → 1.33×); native beats Java *scalar* only past ~4K-16K rows. + Recorded as `E-LGJ-VECTOR-API-BEATS-THE-CROSSING-1`. +- **Deferred:** fusion sweep ran at 65,536 rows only (repaid post-merge + by the E_ re-run with a 256-row arm — see PR #4). +- **Docs:** `bench/README.md`, board updates. +- **Confidence:** High — 50/50 rows, 0 failures, two independent + computations of the same CSV agree. + +## PR #2 — Valhalla lab: three-truths, causal isolation, 3 reproducers (merged 2026-08-17, squash) + +- **Added:** `valhalla-lab/` — shared/stable/valhalla trees, self-verifying + `run.sh` (vocab-diff honesty gate + flattening-flag causal isolation), + `docs/three-truths.md`, reproducers R1/R2/R3 with observed outputs. +- **Locked:** the 8-byte array-flattening cliff (R2, VM-confirmed); + native-one-crossing beats hydration ~38-57× on BOTH JDKs; production + API adopts zero Valhalla-only mechanisms — migration stays + `record` → `value record`, one word per type. +- **Deferred:** nothing; the lab is complete for this vocabulary. +- **Docs:** lab README + three-truths; board updates. +- **Confidence:** High — one real defect (`Class::isValue()` not on + JDK 26) found by compile failure and fixed before landing. + +## PR #1 — Core vertical slice: ABI contract, native crate, Java facade (merged 2026-08-17, squash) + +- **Added:** `docs/abi.md` (normative, 14 symbols / 4 repr(C) types / + 13 status codes / generation-checked handles); `native/lgj-abi` + (72/72, clippy/fmt clean, 14/14 exported symbols via `nm -D`); + `java/` facade + FFM membrane (132/132, reflection-enforced zero-FFM + public surface); 5 new `ndarray::simd` primitives under the W1a + contract (41/41); the `.claude/` ensemble + board. +- **Locked:** disable-verified generation check (exactly 2 tests red + when broken, 70 green); the manifest cross-check rejects a real wrong + `.so`; laziness measured (0 crossings to build, exactly 1 to + evaluate); target-cpu=x86-64-v4 divergence recorded. +- **Deferred:** real graph types (`NodeRow`/`WideFieldMask`) — generic + fixture first, by design (`docs/abi.md` §10). +- **Docs:** `docs/abi.md`, knowledge docs, board. +- **Confidence:** High — one real audit violation (`ndarray::hpc` + import) found and fixed pre-merge; recorded in EPIPHANIES. diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 12ba44e..0f6883e 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -16,7 +16,7 @@ list. | D-LGJ-F | Valhalla lab — three-truths method on the small semantic value vocabulary | **DONE 2026-08-17** — `valhalla-lab/`: 4 experiments + a self-verifying `run.sh` (mechanically diffs the two `Vocab.java`s modulo the `value` keyword before trusting the A/B) + 3 causal-isolation runs (escape-analysis off; `UseArrayFlattening`/`UseFieldFlattening` toggled independently). 3 real Valhalla limitations reproduced and filed under `reproducers/` (R1: `@NullRestricted` field on an identity class is a `VerifyError`, javac's fault — no source form expresses required strict-field order; **R2: array flattening has a hard 8-byte payload cliff, VM-confirmed via `-XX:+PrintFlatArrayLayout`** — `LaneId`/`Ordinal`/`MaskId` (≤8B) flatten, `RowRange`/`Row` (16B) do not, so "Valhalla helps descriptors not entities" is a measured VM cutoff, not a hand-wave, and `RowRange` landing on the wrong side is flagged as the one place the expectation was too optimistic; R3: the densest null-restricted array form is `jdk.internal`-only and generics erase flattening entirely — `Foo!` type syntax confirmed NOT to parse, matching the archaeology finding). 1 real defect found + fixed before landing (see `EPIPHANIES.md`). None of the three limitations changed the production API — the migration path stays exactly `record` → `value record` | I | | D-LGJ-G | Java Vector API comparative bench vs Panama→`ndarray::simd` | **DONE 2026-08-17** — real JMH 1.37 (fork+warmup+blackholes confirmed in the log), 50/50 rows, 0 failures, `Data.crossCheck()` guards every fork. **Headline (Component C, single predicate, zero-copy `IntVector.fromMemorySegment`): the Java Vector API beats the native crossing at EVERY row count tested, 64 to 4,194,304** — 56.4x at small sizes down to 1.3-1.4x at the largest. Native beats a plain Java scalar loop only past ~4,096-16,384 rows. Component E: SIMD-vs-scalar is the biggest lever measured (10.8x-31.1x); fused vs unfused are within noise of each other at 65,536 rows (crossing-count guarantee matters more than measured time here, since Component A puts one downcall at ~22ns). Independently cross-checked: hand-written `RESULTS.md` numbers verified byte-for-byte against `summarise.sh`'s mechanically-generated tables from the same CSV | I | | D-LGJ-H | Falsification: handle lifecycle (adversarial), SIMD/scalar parity, Java/native parity | **DONE 2026-08-17, all scopes closed** — Rust+Java core (D-LGJ-C disable-verification, D-LGJ-E `FusionParityTest`/`LifetimeTest`); Valhalla lab (`run.sh`'s vocab-honesty self-check + causal-isolation runs); bench (`Data.crossCheck()` on every fork, `summarise.sh` cross-check) | I | -| D-LGJ-I | Docs: `architecture.md`, `panama.md`, `valhalla-lab.md`, `execution-boundary.md` | **Unblocked** — F and G both landed; next action | — | +| D-LGJ-I | Docs: `architecture.md`, `panama.md`, `valhalla-lab.md`, `execution-boundary.md` | **DONE 2026-08-17** — all four written as synthesis (each cites the artifact that proves its claim rather than restating it); `execution-boundary.md` additionally records the three structural hot-path facts (zero-copy precision incl. the lance-graph `SoaEnvelope` inheritance, no-thread-pool/caller-threads-are-the-parallelism, `array_windows`/`array_chunks` precisely traced as un-invoked at any input size). Ships in PR #4 with the fusion-sweep 256-row re-run merged into `RESULTS.md`/`TABLES.md` | — | | D-LGJ-AUDIT | Mechanical post-fan-out audit: `grep` for `ndarray::hpc` imports, any `.h`/`cbindgen`/`jextract` artifact, any FFM type leaking into public Java API | **DONE 2026-08-17** — 1 real violation found (`kernels.rs::simd_popcount` used the internal `ndarray::hpc::bitwise` path), fixed in place; everything else confirmed to be the one sanctioned exception or explanatory prose | closed D-LGJ-C/D/E for the core | ### Reading this table diff --git a/bench/RESULTS.md b/bench/RESULTS.md index ce2f6ce..5f67dad 100644 --- a/bench/RESULTS.md +++ b/bench/RESULTS.md @@ -1,127 +1,279 @@ -# Where does execution belong? — measured, not assumed - -Real JMH 1.37, `--enable-preview` off (Vector API only needs `--add-modules jdk.incubator.vector`, -not preview), JDK 26 GA, `@Fork(1) @Warmup(5×500ms) @Measurement(8×500ms)`, `AverageTime`. Full run: -`results/jmh-run.txt` (1,679 lines, every warm-up iteration). Machine-readable: -`results/jmh-results.csv`. Reproduce with `./run.sh` (~12 min on 4 vCPU — this run: 00:12:22). - -**Gate.** 50/50 benchmark rows completed, 0 failures. `Data.crossCheck()` (native vs Vector vs -scalar agree on both count and sum) ran in `@Setup` for every fork and never threw — the three -kernels compute the same answer, so a speed comparison between them is meaningful rather than a -race between a correct implementation and a subtly wrong faster one. - -## The headline finding — and it complicates the thesis in an honest way - -Component C (`java_scalarLoop` / `java_vectorApi` / `native_fusedPlan`) sweeps one predicate -(`class == 7`) over row counts from 64 to 4,194,304, all three arms answering the identical -question from the identical native lane: - -| rows | scalar (µs) | vectorApi (µs) | native (µs) | native beats scalar by | **vectorApi beats native by** | -|---:|---:|---:|---:|---:|---:| -| 64 | 0.028 | 0.011 | 0.612 | 0.05× (native LOSES) | 56.40× | -| 256 | 0.085 | 0.025 | 0.623 | 0.14× (native LOSES) | 24.96× | -| 1,024 | 0.338 | 0.078 | 0.708 | 0.48× (native LOSES) | 9.07× | -| 4,096 | 1.252 | 0.385 | 1.524 | 0.82× (native LOSES) | 3.96× | -| 16,384 | 5.553 | 1.744 | 4.291 | **1.29×** | 2.46× | -| 65,536 | 42.846 | 8.027 | 15.324 | 2.80× | 1.91× | -| 262,144 | 343.389 | 42.519 | 69.374 | 4.95× | 1.63× | -| 1,048,576 | 1,623.313 | 310.405 | 411.333 | 3.95× | 1.33× | -| 4,194,304 | 6,602.036 | 1,319.107 | 1,858.686 | 3.55× | 1.41× | - -Two crossovers, both real: - -1. **Native beats a plain Java scalar loop only past roughly 4,096–16,384 rows.** Below that, the - crossing overhead (the ~0.6 µs floor visible at row=64, consistent with Component A's raw - downcall cost) is not repaid yet — a scalar loop over a few thousand elements is simply cheap - enough in Java that there is nothing to win by leaving the JVM. -2. **The Java Vector API, reading the SAME native `MemorySegment` with zero copy - (`IntVector.fromMemorySegment`), beats the native crossing at every single row count - tested** — never below 1.3×, and by more than an order of magnitude at small sizes. This - is the finding this project's own mission brief asked for by name: *"Where is the cheapest - and cleanest execution boundary? Not: how can we maximize the amount of Java code?"* — and - the honest answer, for this one-predicate/one-lane workload, is that it is **not** the Rust - crossing. - -**Why this does not overturn the thesis, and where it does bite.** Component C measures ONE -predicate over ONE lane — exactly the case where a zero-copy Vector kernel has nothing to fuse and -nothing to coordinate. Component E (below) measures what happens once there is more than one -predicate, which is the case the fluent `View` API actually optimizes for. - -## Component E — fusion matters once there is more than one predicate - -`fused` (native, one crossing, N predicates AND-combined in one plan) vs `unfused` (native, N -crossings, one `mask_and` per predicate) vs `fusedScalarKernel` (the SAME fused plan forced through -the crate's own scalar reference path, not SIMD), at 65,536 rows: - -| predicates | fused (µs) | unfused (µs) | fusedScalarKernel (µs) | SIMD speedup over scalar | -|---:|---:|---:|---:|---:| -| 1 | 6.818 | 7.641 | 73.419 | 10.8× | -| 2 | 15.599 | 15.025 | 405.096 | 26.0× | -| 4 | 29.721 | 27.309 | 923.100 | 31.1× | -| 8 | 59.363 | 61.916 | 1,807.261 | 30.4× | - -Two findings, neither of which was assumed going in: - -- **`fused` and `unfused` are close** — within noise of each other at this row count (see the - single-fork caveat below). The `lgj_plan_eval` fused path exists to guarantee ONE crossing - regardless of predicate count (a structural property `LazinessTest` in the Java suite already - proves), not because N separate crossings at 65,536 rows are individually expensive — Component A - already showed a bare downcall costs ~22 ns, so 8 of them add roughly 176 ns against a - multi-microsecond total. The value of fusion at this scale is the crossing-count GUARANTEE, not a - large measured time saving. -- **SIMD vs scalar is the biggest lever in this whole benchmark suite** — 10.8×–31.1×, growing - with predicate count. This is the number that justifies routing every kernel through - `ndarray::simd` rather than a portable scalar loop, and it dwarfs the crossing-cost questions - Components A/B/C spend most of their effort isolating. - -`planConstructionOnly` (0.053–0.634 µs, scaling with predicate count but NOT with row count — 65,536 -rows throughout) confirms `LazinessTest`'s claim under real JMH conditions: building the fluent -chain costs time proportional to the number of `.where()` calls, never to the number of rows. - -## Component A — the floor - -| benchmark | ns/op | -|---|---:| -| `javaCallControl` (a plain Java method call, the noise floor) | 0.497 | -| `bareDowncall_noArgs` | 21.911 | -| `downcall_twoArgs_outPointer` | 117.855 | - -A bare Panama downcall costs ~22 ns over a plain Java call; adding two arguments and an out-pointer -roughly quintuples that. Both numbers are the ~0.6 µs floor Component C's small-row-count native -arm sits on top of (downcall cost + the fixed per-call marshalling `Engine`/`Downcalls` do above the -raw linker). - -## Component B — raw throughput, no crossing - -| benchmark | µs/op (65,536 elements) | -|---|---:| -| `heapArrayBaseline` (data already in a Java `int[]`) | 5.158 | -| `segmentScalar` (same data, read from a native `MemorySegment`, scalar loop) | 5.220 | -| `segmentVector` (same data, `IntVector.fromMemorySegment`) | 3.337 | - -Reading a native segment scalar-wise costs essentially the same as reading a heap array (1.2% -difference — within this harness's own stated ~10% noise floor, see below) — `MemorySegment` access -is not itself a tax. The Vector API is the thing that's actually faster here (1.55× over both), not -the memory's location. - -## Honest limitations (stated in `README.md`, repeated here because they qualify every number above) - -- **Single fork.** `@Fork(1)` cannot see run-to-run JIT/ASLR variance. Differences under ~10% between - arms are not established by this harness — this is why `fused` vs `unfused` above is reported as - "close" rather than a specific winner. -- **Shared container, not a tuned host.** No CPU pinning, no disabled turbo/hyperthreading. Large - effects (the order-of-magnitude ones — scalar-vs-vector, the small-row native crossover) are safe - to read; anything under ~10% is not. -- **Component C's ONE-predicate shape is deliberate, not the whole story.** It isolates the - crossing question cleanly; Component E is what shows the picture changes once predicates compose. - -## The verdict, stated the way the mission brief asked for it - -*"Where is the cheapest and cleanest execution boundary?"* — measured, not assumed: for a single -predicate over a native lane, **Java itself, via the Vector API reading the segment zero-copy, is -the fastest arm tested at every scale**. The native crossing earns its keep once real work — SIMD -kernels for multi-predicate fusion, the guaranteed-one-crossing property for an arbitrarily long -`View` chain, and (per `valhalla-lab/`) genuine per-population bulk operations — is on the other -side of it. The honest reading is not "Rust wins" or "Java wins" but **"the crossing is worth -paying for composed work, not for one predicate read alone"** — exactly the nuance the mission -brief's Phase G asked this benchmark to establish rather than assume in either direction. +# Measured results — where does execution belong? + +> Every number below was produced by `bench/run.sh` on this box, and every table is generated by +> `summarise.sh` so it cannot drift from its data. +> +> | file | what it is | +> |---|---| +> | `results/jmh-run-full.txt` / `jmh-results-full.csv` | the first full sweep (A, B, C/D, and E at 65,536 rows only) | +> | `results/jmh-run.txt` / `jmh-results.csv` | the `./run.sh E_` re-run, after a 256-row arm was added to the fusion sweep | +> | `results/jmh-results-merged.csv` | A/B/C from the full sweep + E from the re-run — **the input to every table below** | +> | `results/TABLES.md` | `./summarise.sh results/jmh-results-merged.csv` | +> +> The fusion sweep was re-run because the first pass measured only 65,536 rows and reported +> "fusion does nothing", which is true there and false in general. Adding the small-row arm is +> what turned a wrong summary into a correct one — see § E/F. + +## Environment + +| | | +|---|---| +| JDK | `OpenJDK 64-Bit Server VM 26.0.2+10-55` | +| harness | JMH 1.37, compiler blackholes auto-detected and in use | +| VM flags | `--enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector` | +| OS / arch | Linux x86-64 | +| CPU | Intel Xeon @ 2.10 GHz, 4 logical processors, AVX-512 (`avx512f/dq/bw/vl/vbmi/ifma/cd`) | +| vector species | `Species[int, 16, S_512_BIT]` — **16 int lanes, 512 bit**, the same width the Rust kernel uses | +| native library | `abi 0.1`, `simd ndarray::simd avx512`, `profile release` | +| predicate | `class == 7 AND value > 100` (≈3.3 % selectivity, straddles zero so the signed compare is real) | +| settings | `@Fork(1)`, warm-up 5 × 500 ms, measurement 8 × 500 ms, `Mode.AverageTime`, GC between iterations | + +Shared 4-vCPU container: no CPU pinning, no isolated cores, no turbo/HT control. **Read +order-of-magnitude differences as established and sub-10 % differences as not.** + +--- + +## The headline, stated before the tables because it is not what was expected + +**The Java Vector API is faster than the native kernel at every row count measured, from 64 rows +to 4,194,304 rows.** There is no crossover point in this data. The naive expectation — "Rust with +AVX-512 beats the JVM, the only question is where the crossing overhead stops mattering" — is +falsified by this measurement. + +That result is real, and the explanation is *not* "Rust is slower than Java". It is that **the two +arms are not doing the same amount of work**, and the difference is architectural. See +[§ Why](#why-the-vector-arm-wins) below, which is grounded in the Rust source rather than in a +guess. + +--- + +## A — the membrane crossing, isolated + +| benchmark | mean | ±99.9 % CI | unit | +|---|---:|---:|---| +| `javaCallControl` — an ordinary Java call, no crossing | 0.497 | ±0.037 | ns/op | +| `bareDowncall_noArgs` — `lgj_abi_manifest()` | **21.911** | ±0.715 | ns/op | +| `downcall_twoArgs_outPointer` — `lgj_mask_count(handle, out)` | 117.855 | ±1.808 | ns/op | + +**A bare Panama downcall costs ~22 ns**, about 44 ordinary Java calls. That is the number the +whole design rests on: it is cheap enough that one crossing per query is free, and expensive +enough that one crossing per *row* would be catastrophic — 65,536 rows × 22 ns ≈ 1.4 ms of pure +overhead, which is 90× the entire fused query. The anti-JNI rule is not stylistic. + +**The 118 ns row must not be read as "argument marshalling costs 96 ns".** `lgj_mask_count` takes +a read lock on the resource registry, resolves a generation-checked handle, clones an `Arc`, then +locks the entry — that is the ABI's safety machinery, not Panama's marshalling. Attributing it to +the linker would be wrong. What it *does* bound honestly is the cost of any real ABI call that +resolves a handle, and it is the reason the row-sweep's native arm has a ~0.6 µs floor rather than +a ~0.02 µs one. + +--- + +## B — reading native memory from Java + +65,536 `i32`, summed. Same data, same answer (cross-checked in `@Setup`). + +| benchmark | mean | ±99.9 % CI | unit | +|---|---:|---:|---| +| `segmentVector` — `MemorySegment`, Vector API, zero copy | **3.337** | ±0.094 | µs/op | +| `heapArrayBaseline` — `int[]` on the Java heap | 5.158 | ±0.179 | µs/op | +| `segmentScalar` — `MemorySegment`, scalar loop | 5.220 | ±0.241 | µs/op | + +**FFM access is at parity with heap access** (5.220 vs 5.158 µs — a 1.2 % difference, inside the +confidence intervals). Java pays no penalty for its data living in native memory. This is what +makes the whole comparison in C possible: the Java arm is not handicapped, so if it wins it wins +on merit. + +Explicitly vectorising beats both by 1.55×, on a plain `sum` where C2's auto-vectoriser might have +been expected to do the same job. + +--- + +## C/D — native kernel vs Java Vector API vs Java scalar + +All three read **the same native lanes** with **zero copies**. All three produce the same answer, +asserted before timing. + +| rows | lane KiB | `native_fusedPlan` | `java_vectorApi` | `java_scalarLoop` | fastest | native ÷ vector | +|---:|---:|---:|---:|---:|---|---:| +| 64 | 0.25 | 0.612 ±0.039 | **0.011 ±0.001** | 0.028 ±0.002 | vector | 56.4× | +| 256 | 1 | 0.623 ±0.027 | **0.025 ±0.001** | 0.085 ±0.011 | vector | 25.0× | +| 1,024 | 4 | 0.708 ±0.037 | **0.078 ±0.003** | 0.338 ±0.018 | vector | 9.1× | +| 4,096 | 16 | 1.524 ±0.346 | **0.385 ±0.016** | 1.252 ±0.020 | vector | 4.0× | +| 16,384 | 64 | 4.291 ±0.190 | **1.744 ±0.055** | 5.553 ±0.288 | vector | 2.5× | +| 65,536 | 256 | 15.324 ±0.867 | **8.027 ±0.309** | 42.846 ±9.355 | vector | 1.9× | +| 262,144 | 1,024 | 69.374 ±5.998 | **42.519 ±5.261** | 343.389 ±7.332 | vector | 1.6× | +| 1,048,576 | 4,096 | 411.333 ±37.244 | **310.405 ±17.660** | 1623.313 ±26.973 | vector | 1.3× | +| 4,194,304 | 16,384 | 1858.686 ±149.400 | **1319.107 ±37.240** | 6602.036 ±100.771 | vector | 1.4× | + +**Gate.** 50/50 benchmark rows completed, 0 failures, whole sweep 00:12:22. `Data.crossCheck()` +ran in `@Setup` for every fork and never threw, so the three arms provably agree on both the count +and the sum — a speed comparison between them is a comparison, not a race between a correct +implementation and a subtly wrong faster one. + +### There ARE two crossovers — they are just not the one that was expected + +**Crossover 1 — native overtakes the Java *scalar* loop at roughly 4,096–16,384 rows.** +`native ÷ scalar` runs 0.05×, 0.14×, 0.48×, 0.82× (native losing), then 1.29×, 2.80×, 4.95×, +3.95×, 3.55×. Below ~4,096 rows the ~0.6 µs crossing floor is not repaid; above ~16,384 it is, +several times over. If the Java side were written the way most Java is written — an ordinary loop — +this is the crossover, and it sits where the fixed-cost model says it should. + +**Crossover 2 — there is none against the Vector API.** `native ÷ vector` is above 1.0 at every +row count, monotonically decreasing from 56.4× to 1.33× and then flattening at 1.41×. Extrapolating +the last three points does not project a crossing: both arms are memory-bound by then and the ratio +has stopped moving. + +Three further things this table says. + +**1. The native arm has a ~0.6 µs floor and the Java arm has none.** Below ~1,024 rows the native +number barely moves (0.612 → 0.708 µs) because it is almost entirely fixed cost: build the +predicate list, marshal two 24-byte op descriptors, cross, resolve the handle under a lock, +allocate two mask buffers, and copy the result out. The Java arm starts at 11 ns because it does +none of that. **The ratio narrowing from 56× to 1.4× is the crossing overhead being amortised**, +exactly as the model predicts — it is the *residual* 1.4× that needs explaining, not the shape. + +**2. The Vector API is worth using — the scalar control proves it.** `java_scalarLoop` is +**2.5× to 8.1×** slower than `java_vectorApi` at every row count (ratios by size: 2.5, 3.4, 4.3, +3.3, 3.2, 5.3, 8.1, 5.2, 5.0). So C2 is not auto-vectorising this two-lane predicate-and-count +into anything like the explicit version, and "the Vector API bought nothing" is falsified. Note +the ratio is *not* monotonic — it peaks at 262,144 rows and comes back down — so it should be read +as "consistently several-fold", not as a trend. + +The scalar arm also shows a sharp discontinuity between 16,384 rows (5.6 µs) and 65,536 rows +(42.8 µs) — 7.7× for 4× the data, with a wide ±9.4 µs interval. That is around where the two lanes +together stop fitting in L2. It is flagged, not explained: confirming it would need hardware +counters, which this harness does not collect. + +**3. Both arms become memory-bound at the top.** At 4 M rows the two lanes are 32 MiB; the vector +arm's 1.32 ms is ≈24 GiB/s and the native arm's 1.86 ms is ≈17 GiB/s. Neither is compute-limited, +which is why the ratio stops improving. + +### Why the vector arm wins + +Not speculation — this is what `native/lgj-abi/src/exports.rs::plan_eval_impl` does, per call: + +```rust +let mut acc = vec![0u64; n_words]; // allocation #1 ─┐ 512 KiB each at 4 M rows +let mut scratch = vec![0u64; n_words]; // allocation #2 ─┘ +for w in acc.iter_mut() { *w = u64::MAX; } // pass over the mask +for op in ops { + kernels::eval_predicate(..., &mut scratch)?; // read lane, WRITE full mask + kernels::combine_into(path, op.combine, &mut acc, &scratch)?; // read+write mask +} +let count = kernels::popcount(path, &acc); // another pass +g.words.copy_from_slice(&acc); // another pass +``` + +For a two-predicate query the native path performs **two heap allocations, a memset, two full mask +writes, two mask read-modify-writes, a popcount pass and a copy pass** — six-plus passes over the +mask on top of the two lane reads. The Java kernel does **two lane reads and keeps the count in a +register**; it never materialises a mask at all. + +**So the arms answer the same question but produce different artifacts.** The native path also +yields a *persisted, reusable selection* — a `Mask` the caller can keep, intersect, and reduce +over later. The Java path yields a number. Charging the native path for that artifact and then +declaring it slower would be the benchmark lying by omission. + +What this measurement therefore establishes, precisely: + +- ✅ For a **count-only** query, executing in Java over the native lanes is 1.3–1.9× faster at + large sizes and up to 56× faster at small ones. +- ✅ The native path's fixed cost is ~0.6 µs and is fully amortised past ~10⁵ rows. +- ❌ It does **not** establish that `ndarray::simd` is slower than the JVM's vectoriser. The two + kernels do different work, and separating those would need a native entry point that counts + without materialising a mask — which does not exist and which I did not add, because adding a + symbol to make my own benchmark look better is exactly the wrong move. + +**Fixable inefficiencies visible in the source, reported not patched** (`native/` is another +agent's tree): the two `vec![0u64; n_words]` allocations are per call and could be a reusable +scratch on the resource; and a count-only fast path that skips mask materialisation would remove +most of the residual gap. Both are ABI-implementation changes, not ABI-contract changes. + +--- + +## E/F — fusion, and what the fluent API itself costs + +| rows | predicates | `fused` | `unfused` | `fusedScalarKernel` | `planConstructionOnly` | unfused ÷ fused | +|---:|---:|---:|---:|---:|---:|---:| +| 256 | 1 | 0.404 ±0.010 | 0.385 ±0.009 | 0.503 ±0.015 | 0.056 ±0.004 | 0.95× | +| 256 | 2 | **0.520 ±0.032** | 0.931 ±0.018 | 0.835 ±0.039 | 0.111 ±0.003 | **1.79×** | +| 256 | 4 | **0.808 ±0.033** | 2.097 ±0.064 | 1.662 ±0.084 | 0.282 ±0.056 | **2.60×** | +| 256 | 8 | **1.482 ±0.070** | 4.437 ±0.204 | 3.489 ±0.113 | 0.657 ±0.196 | **2.99×** | +| 65,536 | 1 | 6.913 ±0.165 | 6.326 ±0.175 | 74.340 ±1.659 | 0.053 ±0.005 | 0.92× | +| 65,536 | 2 | 15.222 ±0.707 | 14.204 ±0.804 | 408.170 ±20.242 | 0.113 ±0.006 | 0.93× | +| 65,536 | 4 | 25.591 ±0.582 | 31.790 ±3.337 | 917.387 ±27.768 | 0.261 ±0.012 | 1.24× | +| 65,536 | 8 | 58.978 ±4.509 | 60.968 ±2.657 | 1825.916 ±125.373 | 0.601 ±0.032 | 1.03× | + +**Fusion's benefit is a fixed cost saved, so it shows up exactly where fixed cost matters — and +the sweep is what makes that visible.** At 256 rows it grows cleanly with predicate count: +0.95× → 1.79× → 2.60× → **2.99×**. At 65,536 rows it is **not measurable** (0.92×–1.24×, straddling +1.0, inside the intervals) because memory traffic dwarfs eight 22 ns crossings. + +Had this been measured at one row count, it would have produced either "fusion gives 3×" or +"fusion does nothing", and both would have been wrong as stated. The honest claim is: *fusion is +worth up to 3× on small selections and is invisible on large ones — and its primary justification +is architectural (one crossing per query, never per row) rather than throughput.* + +**`fusedScalarKernel` shows the SIMD kernel is doing real work**: at 65,536 rows it is **10.8×** +(1 predicate) to **31.0×** (8 predicates) slower than the same plan through the same membrane with +`ndarray::simd` replaced by the scalar reference. At 256 rows the same comparison is only 1.2× to +2.4×, because there the fixed crossing cost dominates on both sides — the SIMD contribution is a +property of the kernel, so it only becomes visible once there is enough data for the kernel to +matter. This path is the ABI's parity escape hatch, never production; the spread is a lower bound +on what SIMD contributes. + +**`planConstructionOnly` — the fluent API costs 53 ns for one predicate and 601 ns for eight, and +crosses the membrane zero times.** The benchmark asserts `Diagnostics.crossings()` is unchanged +across the construction and throws if it moved, so laziness is verified, not assumed. It is also +independent of row count (0.053 vs 0.056 µs at 256 vs 65,536 rows), which is what "the chain is +just a list of predicates" should look like. + +--- + +## Verdict — where does execution belong? + +On the evidence, **not where the architecture currently puts it, for count-only queries.** Stated +as separable claims, each with its supporting row: + +1. **The membrane is not the problem.** 22 ns per crossing, and FFM reads native memory at heap + parity (B). A design that crosses once per query pays essentially nothing for the boundary. + +2. **For a count/aggregate that returns a scalar, Java over the native lanes is the faster + execution site — at every size measured.** 1.3–1.9× at large sizes, up to 56× below ~1,000 rows + where the native path is all fixed cost (C). The zero-copy `IntVector.fromMemorySegment` bridge + is what makes this possible, and it is a genuinely zero-copy bridge. + +3. **But the comparison is between different amounts of work, and that is the actionable finding.** + The native path materialises a reusable mask and allocates twice per call; the Java path keeps a + counter in a register. The gap at 4 M rows is 1.4×, and the source shows enough avoidable work + to plausibly account for it. **The right response is to fix the native path, not to move + execution to Java** — because the mask is not waste, it is the thing that makes + `.where(...).select()` composable and reusable. + +4. **Bulk data still belongs native, and that is a separate question this harness did not put at + risk.** Nothing here compares against materialising Java objects; that is + [`valhalla-lab`](../valhalla-lab), which measures the object path at **49× slower** than the + native query and 2 MiB of heap for 65,536 rows against 816 bytes. The choice this harness + informs is "which side of the membrane runs the *kernel*", not "should the data be in Java". + +5. **The honest architectural conclusion.** The native side earns its place through *what it + owns* — the lanes, the packed mask, the composable selection, one allocation for the whole + dataset — not through raw kernel speed, where the JVM's vectoriser is currently competitive to + better on this hardware. A Java-side kernel over native lanes is a legitimate and measurably + fast option for scalar-returning queries, and the ABI already makes it possible without + violating anything: `lgj_lane_describe` hands out a bounded read-only window and **no crossing + happens when Java reads it**. + + That is worth saying plainly: the design's own escape hatch is currently the fast path for one + class of query. The next slice should either close the native gap (count-only fast path, + scratch reuse) or make the Java-side kernel an explicit, documented execution strategy — not + leave it as an accident. + +### What this harness does not establish + +- Anything about multi-threaded or concurrent execution. Everything here is single-threaded; the + ABI's registry locking is measured only as latency in the A row. +- `ndarray::simd` vs the JVM vectoriser as kernels. See § Why — the arms differ structurally. +- Run-to-run variance from JIT nondeterminism: `@Fork(1)`. +- Anything about a machine that is not a 4-vCPU shared container. diff --git a/bench/results/TABLES.md b/bench/results/TABLES.md new file mode 100644 index 0000000..780dc5c --- /dev/null +++ b/bench/results/TABLES.md @@ -0,0 +1,44 @@ + +### A — membrane crossing, isolated + +| benchmark | mean | ±99.9% CI | unit | +|---|---:|---:|---| +| `javaCallControl` | 0.497 | ±0.037 | ns/op | +| `bareDowncall_noArgs` | 21.911 | ±0.715 | ns/op | +| `downcall_twoArgs_outPointer` | 117.855 | ±1.808 | ns/op | + +### B — reading native memory (65,536 i32) + +| benchmark | mean | ±99.9% CI | unit | +|---|---:|---:|---| +| `segmentVector` | 3.337 | ±0.094 | us/op | +| `heapArrayBaseline` | 5.158 | ±0.179 | us/op | +| `segmentScalar` | 5.220 | ±0.241 | us/op | + +### C/D — where does execution belong? (µs/op, mean ± 99.9% CI) + +| rows | lane KiB | `native_fusedPlan` | `java_vectorApi` | `java_scalarLoop` | fastest | native/vector | +|---:|---:|---:|---:|---:|---:|---:| +| 64 | 0 | 0.612 ±0.039 | 0.011 ±0.001 | 0.028 ±0.002 | **java_vectorApi** | 56.40x | +| 256 | 1 | 0.623 ±0.027 | 0.025 ±0.001 | 0.085 ±0.011 | **java_vectorApi** | 24.96x | +| 1,024 | 4 | 0.708 ±0.037 | 0.078 ±0.003 | 0.338 ±0.018 | **java_vectorApi** | 9.07x | +| 4,096 | 16 | 1.524 ±0.346 | 0.385 ±0.016 | 1.252 ±0.020 | **java_vectorApi** | 3.96x | +| 16,384 | 64 | 4.291 ±0.190 | 1.744 ±0.055 | 5.553 ±0.288 | **java_vectorApi** | 2.46x | +| 65,536 | 256 | 15.324 ±0.867 | 8.027 ±0.309 | 42.846 ±9.355 | **java_vectorApi** | 1.91x | +| 262,144 | 1024 | 69.374 ±5.998 | 42.519 ±5.261 | 343.389 ±7.332 | **java_vectorApi** | 1.63x | +| 1,048,576 | 4096 | 411.333 ±37.244 | 310.405 ±17.660 | 1623.313 ±26.973 | **java_vectorApi** | 1.33x | +| 4,194,304 | 16384 | 1858.686 ±149.400 | 1319.107 ±37.240 | 6602.036 ±100.771 | **java_vectorApi** | 1.41x | + +### E/F — fusion and the cost of the fluent API (µs/op) + +| rows | predicates | `fused` | `unfused` | `fusedScalarKernel` | `planConstructionOnly` | unfused/fused | +|---:|---:|---:|---:|---:|---:|---:| +| 256 | 1 | 0.404 ±0.010 | 0.385 ±0.009 | 0.503 ±0.015 | 0.056 ±0.004 | **0.95x** | +| 256 | 2 | 0.520 ±0.032 | 0.931 ±0.018 | 0.835 ±0.039 | 0.111 ±0.003 | **1.79x** | +| 256 | 4 | 0.808 ±0.033 | 2.097 ±0.064 | 1.662 ±0.084 | 0.282 ±0.056 | **2.60x** | +| 256 | 8 | 1.482 ±0.070 | 4.437 ±0.204 | 3.489 ±0.113 | 0.657 ±0.196 | **2.99x** | +| 65,536 | 1 | 6.913 ±0.165 | 6.326 ±0.175 | 74.340 ±1.659 | 0.053 ±0.005 | **0.92x** | +| 65,536 | 2 | 15.222 ±0.707 | 14.204 ±0.804 | 408.170 ±20.242 | 0.113 ±0.006 | **0.93x** | +| 65,536 | 4 | 25.591 ±0.582 | 31.790 ±3.337 | 917.387 ±27.768 | 0.261 ±0.012 | **1.24x** | +| 65,536 | 8 | 58.978 ±4.509 | 60.968 ±2.657 | 1825.916 ±125.373 | 0.601 ±0.032 | **1.03x** | + diff --git a/bench/results/jmh-results-merged.csv b/bench/results/jmh-results-merged.csv new file mode 100644 index 0000000..23b6cbc --- /dev/null +++ b/bench/results/jmh-results-merged.csv @@ -0,0 +1,66 @@ +Benchmark,Mode,Threads,Samples,Score,Score Error (99.9%),Unit,Param: predicates,Param: rows +com.adaworldapi.lancegraph.bench.A_DowncallOverhead.bareDowncall_noArgs,avgt,1,8,21.910816,0.714719,ns/op,, +com.adaworldapi.lancegraph.bench.A_DowncallOverhead.downcall_twoArgs_outPointer,avgt,1,8,117.854789,1.808283,ns/op,, +com.adaworldapi.lancegraph.bench.A_DowncallOverhead.javaCallControl,avgt,1,8,0.496614,0.037361,ns/op,, +com.adaworldapi.lancegraph.bench.B_SegmentAccess.heapArrayBaseline,avgt,1,8,5.157633,0.179433,us/op,,65536 +com.adaworldapi.lancegraph.bench.B_SegmentAccess.segmentScalar,avgt,1,8,5.219871,0.241453,us/op,,65536 +com.adaworldapi.lancegraph.bench.B_SegmentAccess.segmentVector,avgt,1,8,3.337320,0.093847,us/op,,65536 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop,avgt,1,8,0.027947,0.001631,us/op,,64 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop,avgt,1,8,0.085439,0.010847,us/op,,256 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop,avgt,1,8,0.337707,0.017531,us/op,,1024 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop,avgt,1,8,1.251500,0.020465,us/op,,4096 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop,avgt,1,8,5.553163,0.287911,us/op,,16384 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop,avgt,1,8,42.846109,9.354832,us/op,,65536 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop,avgt,1,8,343.389077,7.331871,us/op,,262144 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop,avgt,1,8,1623.313038,26.972711,us/op,,1048576 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop,avgt,1,8,6602.036020,100.770602,us/op,,4194304 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi,avgt,1,8,0.010845,0.000597,us/op,,64 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi,avgt,1,8,0.024940,0.000515,us/op,,256 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi,avgt,1,8,0.078068,0.002504,us/op,,1024 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi,avgt,1,8,0.385167,0.015544,us/op,,4096 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi,avgt,1,8,1.744103,0.055036,us/op,,16384 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi,avgt,1,8,8.027116,0.308605,us/op,,65536 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi,avgt,1,8,42.518604,5.260801,us/op,,262144 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi,avgt,1,8,310.404608,17.660428,us/op,,1048576 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi,avgt,1,8,1319.107387,37.239509,us/op,,4194304 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan,avgt,1,8,0.611653,0.039113,us/op,,64 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan,avgt,1,8,0.622588,0.026833,us/op,,256 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan,avgt,1,8,0.708059,0.036741,us/op,,1024 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan,avgt,1,8,1.523730,0.346193,us/op,,4096 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan,avgt,1,8,4.291253,0.190398,us/op,,16384 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan,avgt,1,8,15.324083,0.867382,us/op,,65536 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan,avgt,1,8,69.373781,5.998053,us/op,,262144 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan,avgt,1,8,411.333183,37.244044,us/op,,1048576 +com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan,avgt,1,8,1858.686441,149.400062,us/op,,4194304 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused,avgt,1,8,0.404255,0.010091,us/op,1,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused,avgt,1,8,6.913186,0.165479,us/op,1,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused,avgt,1,8,0.520053,0.032240,us/op,2,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused,avgt,1,8,15.222049,0.707462,us/op,2,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused,avgt,1,8,0.807558,0.033095,us/op,4,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused,avgt,1,8,25.590826,0.582056,us/op,4,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused,avgt,1,8,1.482240,0.070287,us/op,8,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused,avgt,1,8,58.977540,4.508809,us/op,8,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel,avgt,1,8,0.502951,0.015378,us/op,1,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel,avgt,1,8,74.340346,1.659056,us/op,1,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel,avgt,1,8,0.835215,0.038908,us/op,2,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel,avgt,1,8,408.169551,20.241519,us/op,2,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel,avgt,1,8,1.662239,0.084279,us/op,4,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel,avgt,1,8,917.386626,27.768400,us/op,4,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel,avgt,1,8,3.488951,0.112650,us/op,8,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel,avgt,1,8,1825.916402,125.373079,us/op,8,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly,avgt,1,8,0.056078,0.004036,us/op,1,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly,avgt,1,8,0.052973,0.005193,us/op,1,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly,avgt,1,8,0.111424,0.003422,us/op,2,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly,avgt,1,8,0.113013,0.005714,us/op,2,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly,avgt,1,8,0.281631,0.055958,us/op,4,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly,avgt,1,8,0.261441,0.012488,us/op,4,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly,avgt,1,8,0.657211,0.196387,us/op,8,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly,avgt,1,8,0.600529,0.031814,us/op,8,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused,avgt,1,8,0.384609,0.009362,us/op,1,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused,avgt,1,8,6.326314,0.174757,us/op,1,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused,avgt,1,8,0.930643,0.017781,us/op,2,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused,avgt,1,8,14.203920,0.803807,us/op,2,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused,avgt,1,8,2.097204,0.064309,us/op,4,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused,avgt,1,8,31.789786,3.337085,us/op,4,65536 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused,avgt,1,8,4.437382,0.204156,us/op,8,256 +com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused,avgt,1,8,60.967976,2.656815,us/op,8,65536 diff --git a/bench/results/jmh-results.csv b/bench/results/jmh-results.csv index 09d68af..d2669ae 100644 --- a/bench/results/jmh-results.csv +++ b/bench/results/jmh-results.csv @@ -1,50 +1,33 @@ "Benchmark","Mode","Threads","Samples","Score","Score Error (99.9%)","Unit","Param: predicates","Param: rows" -"com.adaworldapi.lancegraph.bench.A_DowncallOverhead.bareDowncall_noArgs","avgt",1,8,21.910816,0.714719,"ns/op",, -"com.adaworldapi.lancegraph.bench.A_DowncallOverhead.downcall_twoArgs_outPointer","avgt",1,8,117.854789,1.808283,"ns/op",, -"com.adaworldapi.lancegraph.bench.A_DowncallOverhead.javaCallControl","avgt",1,8,0.496614,0.037361,"ns/op",, -"com.adaworldapi.lancegraph.bench.B_SegmentAccess.heapArrayBaseline","avgt",1,8,5.157633,0.179433,"us/op",,65536 -"com.adaworldapi.lancegraph.bench.B_SegmentAccess.segmentScalar","avgt",1,8,5.219871,0.241453,"us/op",,65536 -"com.adaworldapi.lancegraph.bench.B_SegmentAccess.segmentVector","avgt",1,8,3.337320,0.093847,"us/op",,65536 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,0.027947,0.001631,"us/op",,64 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,0.085439,0.010847,"us/op",,256 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,0.337707,0.017531,"us/op",,1024 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,1.251500,0.020465,"us/op",,4096 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,5.553163,0.287911,"us/op",,16384 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,42.846109,9.354832,"us/op",,65536 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,343.389077,7.331871,"us/op",,262144 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,1623.313038,26.972711,"us/op",,1048576 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_scalarLoop","avgt",1,8,6602.036020,100.770602,"us/op",,4194304 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,0.010845,0.000597,"us/op",,64 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,0.024940,0.000515,"us/op",,256 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,0.078068,0.002504,"us/op",,1024 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,0.385167,0.015544,"us/op",,4096 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,1.744103,0.055036,"us/op",,16384 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,8.027116,0.308605,"us/op",,65536 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,42.518604,5.260801,"us/op",,262144 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,310.404608,17.660428,"us/op",,1048576 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.java_vectorApi","avgt",1,8,1319.107387,37.239509,"us/op",,4194304 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,0.611653,0.039113,"us/op",,64 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,0.622588,0.026833,"us/op",,256 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,0.708059,0.036741,"us/op",,1024 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,1.523730,0.346193,"us/op",,4096 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,4.291253,0.190398,"us/op",,16384 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,15.324083,0.867382,"us/op",,65536 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,69.373781,5.998053,"us/op",,262144 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,411.333183,37.244044,"us/op",,1048576 -"com.adaworldapi.lancegraph.bench.C_ExecutionBoundary.native_fusedPlan","avgt",1,8,1858.686441,149.400062,"us/op",,4194304 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,6.818039,0.158841,"us/op",1,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,15.599132,0.689941,"us/op",2,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,29.721204,0.865278,"us/op",4,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,59.362899,4.549396,"us/op",8,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,73.418548,1.464802,"us/op",1,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,405.096000,7.322619,"us/op",2,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,923.099907,31.760771,"us/op",4,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,1807.261109,28.175197,"us/op",8,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.052921,0.003815,"us/op",1,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.112476,0.005216,"us/op",2,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.269718,0.029700,"us/op",4,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.633565,0.029509,"us/op",8,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,7.641421,0.918093,"us/op",1,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,15.024855,0.818124,"us/op",2,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,27.308656,2.182643,"us/op",4,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,61.915508,4.111936,"us/op",8,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,0.404255,0.010091,"us/op",1,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,6.913186,0.165479,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,0.520053,0.032240,"us/op",2,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,15.222049,0.707462,"us/op",2,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,0.807558,0.033095,"us/op",4,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,25.590826,0.582056,"us/op",4,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,1.482240,0.070287,"us/op",8,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,58.977540,4.508809,"us/op",8,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,0.502951,0.015378,"us/op",1,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,74.340346,1.659056,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,0.835215,0.038908,"us/op",2,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,408.169551,20.241519,"us/op",2,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,1.662239,0.084279,"us/op",4,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,917.386626,27.768400,"us/op",4,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,3.488951,0.112650,"us/op",8,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,1825.916402,125.373079,"us/op",8,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.056078,0.004036,"us/op",1,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.052973,0.005193,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.111424,0.003422,"us/op",2,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.113013,0.005714,"us/op",2,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.281631,0.055958,"us/op",4,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.261441,0.012488,"us/op",4,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.657211,0.196387,"us/op",8,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.600529,0.031814,"us/op",8,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,0.384609,0.009362,"us/op",1,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,6.326314,0.174757,"us/op",1,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,0.930643,0.017781,"us/op",2,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,14.203920,0.803807,"us/op",2,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,2.097204,0.064309,"us/op",4,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,31.789786,3.337085,"us/op",4,65536 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,4.437382,0.204156,"us/op",8,256 +"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,60.967976,2.656815,"us/op",8,65536 diff --git a/bench/results/jmh-run.txt b/bench/results/jmh-run.txt index c84bb87..cb4ad38 100644 --- a/bench/results/jmh-run.txt +++ b/bench/results/jmh-run.txt @@ -342,3 +342,1070 @@ Iteration 3: 56.783 us/op Iteration 4: 57.691 us/op Iteration 5: 59.234 us/op Iteration 6: 57.057 us/op +Iteration 7: 64.073 us/op +Iteration 8: 59.373 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": + 58.978 ±(99.9%) 4.509 us/op [Average] + (min, avg, max) = (56.783, 58.978, 64.073), stdev = 2.358 + CI (99.9%): [54.469, 63.486] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel +# Parameters: (predicates = 1, rows = 256) + +# Run progress: 25.00% complete, ETA 00:06:02 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.644 us/op +# Warmup Iteration 2: 0.501 us/op +# Warmup Iteration 3: 0.515 us/op +# Warmup Iteration 4: 0.505 us/op +# Warmup Iteration 5: 0.501 us/op +Iteration 1: 0.497 us/op +Iteration 2: 0.508 us/op +Iteration 3: 0.520 us/op +Iteration 4: 0.496 us/op +Iteration 5: 0.501 us/op +Iteration 6: 0.499 us/op +Iteration 7: 0.497 us/op +Iteration 8: 0.506 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": + 0.503 ±(99.9%) 0.015 us/op [Average] + (min, avg, max) = (0.496, 0.503, 0.520), stdev = 0.008 + CI (99.9%): [0.488, 0.518] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel +# Parameters: (predicates = 1, rows = 65536) + +# Run progress: 28.13% complete, ETA 00:05:47 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 77.770 us/op +# Warmup Iteration 2: 75.651 us/op +# Warmup Iteration 3: 76.166 us/op +# Warmup Iteration 4: 76.732 us/op +# Warmup Iteration 5: 76.651 us/op +Iteration 1: 73.627 us/op +Iteration 2: 73.285 us/op +Iteration 3: 75.483 us/op +Iteration 4: 75.007 us/op +Iteration 5: 75.203 us/op +Iteration 6: 74.805 us/op +Iteration 7: 73.605 us/op +Iteration 8: 73.708 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": + 74.340 ±(99.9%) 1.659 us/op [Average] + (min, avg, max) = (73.285, 74.340, 75.483), stdev = 0.868 + CI (99.9%): [72.681, 75.999] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel +# Parameters: (predicates = 2, rows = 256) + +# Run progress: 31.25% complete, ETA 00:05:32 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.976 us/op +# Warmup Iteration 2: 0.848 us/op +# Warmup Iteration 3: 0.833 us/op +# Warmup Iteration 4: 0.826 us/op +# Warmup Iteration 5: 0.832 us/op +Iteration 1: 0.833 us/op +Iteration 2: 0.822 us/op +Iteration 3: 0.823 us/op +Iteration 4: 0.851 us/op +Iteration 5: 0.817 us/op +Iteration 6: 0.820 us/op +Iteration 7: 0.877 us/op +Iteration 8: 0.838 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": + 0.835 ±(99.9%) 0.039 us/op [Average] + (min, avg, max) = (0.817, 0.835, 0.877), stdev = 0.020 + CI (99.9%): [0.796, 0.874] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel +# Parameters: (predicates = 2, rows = 65536) + +# Run progress: 34.38% complete, ETA 00:05:16 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 422.391 us/op +# Warmup Iteration 2: 409.456 us/op +# Warmup Iteration 3: 410.748 us/op +# Warmup Iteration 4: 406.293 us/op +# Warmup Iteration 5: 402.930 us/op +Iteration 1: 399.740 us/op +Iteration 2: 399.729 us/op +Iteration 3: 402.465 us/op +Iteration 4: 406.314 us/op +Iteration 5: 402.886 us/op +Iteration 6: 412.331 us/op +Iteration 7: 410.116 us/op +Iteration 8: 431.775 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": + 408.170 ±(99.9%) 20.242 us/op [Average] + (min, avg, max) = (399.729, 408.170, 431.775), stdev = 10.587 + CI (99.9%): [387.928, 428.411] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel +# Parameters: (predicates = 4, rows = 256) + +# Run progress: 37.50% complete, ETA 00:05:01 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 1.974 us/op +# Warmup Iteration 2: 1.637 us/op +# Warmup Iteration 3: 1.678 us/op +# Warmup Iteration 4: 1.667 us/op +# Warmup Iteration 5: 1.677 us/op +Iteration 1: 1.681 us/op +Iteration 2: 1.708 us/op +Iteration 3: 1.644 us/op +Iteration 4: 1.641 us/op +Iteration 5: 1.720 us/op +Iteration 6: 1.636 us/op +Iteration 7: 1.682 us/op +Iteration 8: 1.585 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": + 1.662 ±(99.9%) 0.084 us/op [Average] + (min, avg, max) = (1.585, 1.662, 1.720), stdev = 0.044 + CI (99.9%): [1.578, 1.747] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel +# Parameters: (predicates = 4, rows = 65536) + +# Run progress: 40.63% complete, ETA 00:04:46 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 949.977 us/op +# Warmup Iteration 2: 916.566 us/op +# Warmup Iteration 3: 912.454 us/op +# Warmup Iteration 4: 949.298 us/op +# Warmup Iteration 5: 937.251 us/op +Iteration 1: 930.736 us/op +Iteration 2: 910.570 us/op +Iteration 3: 914.896 us/op +Iteration 4: 897.725 us/op +Iteration 5: 901.273 us/op +Iteration 6: 939.607 us/op +Iteration 7: 927.639 us/op +Iteration 8: 916.648 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": + 917.387 ±(99.9%) 27.768 us/op [Average] + (min, avg, max) = (897.725, 917.387, 939.607), stdev = 14.523 + CI (99.9%): [889.618, 945.155] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel +# Parameters: (predicates = 8, rows = 256) + +# Run progress: 43.75% complete, ETA 00:04:31 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 3.966 us/op +# Warmup Iteration 2: 3.496 us/op +# Warmup Iteration 3: 3.586 us/op +# Warmup Iteration 4: 3.423 us/op +# Warmup Iteration 5: 3.463 us/op +Iteration 1: 3.623 us/op +Iteration 2: 3.473 us/op +Iteration 3: 3.506 us/op +Iteration 4: 3.446 us/op +Iteration 5: 3.479 us/op +Iteration 6: 3.493 us/op +Iteration 7: 3.451 us/op +Iteration 8: 3.441 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": + 3.489 ±(99.9%) 0.113 us/op [Average] + (min, avg, max) = (3.441, 3.489, 3.623), stdev = 0.059 + CI (99.9%): [3.376, 3.602] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel +# Parameters: (predicates = 8, rows = 65536) + +# Run progress: 46.88% complete, ETA 00:04:16 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 1914.384 us/op +# Warmup Iteration 2: 1834.769 us/op +# Warmup Iteration 3: 1845.272 us/op +# Warmup Iteration 4: 1846.484 us/op +# Warmup Iteration 5: 1793.076 us/op +Iteration 1: 1793.263 us/op +Iteration 2: 1798.992 us/op +Iteration 3: 1781.046 us/op +Iteration 4: 1790.907 us/op +Iteration 5: 1973.222 us/op +Iteration 6: 1790.764 us/op +Iteration 7: 1809.896 us/op +Iteration 8: 1869.240 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": + 1825.916 ±(99.9%) 125.373 us/op [Average] + (min, avg, max) = (1781.046, 1825.916, 1973.222), stdev = 65.573 + CI (99.9%): [1700.543, 1951.289] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly +# Parameters: (predicates = 1, rows = 256) + +# Run progress: 50.00% complete, ETA 00:04:01 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.071 us/op +# Warmup Iteration 2: 0.057 us/op +# Warmup Iteration 3: 0.055 us/op +# Warmup Iteration 4: 0.055 us/op +# Warmup Iteration 5: 0.054 us/op +Iteration 1: 0.059 us/op +Iteration 2: 0.057 us/op +Iteration 3: 0.059 us/op +Iteration 4: 0.056 us/op +Iteration 5: 0.054 us/op +Iteration 6: 0.054 us/op +Iteration 7: 0.055 us/op +Iteration 8: 0.055 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": + 0.056 ±(99.9%) 0.004 us/op [Average] + (min, avg, max) = (0.054, 0.056, 0.059), stdev = 0.002 + CI (99.9%): [0.052, 0.060] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly +# Parameters: (predicates = 1, rows = 65536) + +# Run progress: 53.13% complete, ETA 00:03:46 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.069 us/op +# Warmup Iteration 2: 0.059 us/op +# Warmup Iteration 3: 0.051 us/op +# Warmup Iteration 4: 0.054 us/op +# Warmup Iteration 5: 0.051 us/op +Iteration 1: 0.058 us/op +Iteration 2: 0.053 us/op +Iteration 3: 0.053 us/op +Iteration 4: 0.051 us/op +Iteration 5: 0.051 us/op +Iteration 6: 0.051 us/op +Iteration 7: 0.051 us/op +Iteration 8: 0.056 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": + 0.053 ±(99.9%) 0.005 us/op [Average] + (min, avg, max) = (0.051, 0.053, 0.058), stdev = 0.003 + CI (99.9%): [0.048, 0.058] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly +# Parameters: (predicates = 2, rows = 256) + +# Run progress: 56.25% complete, ETA 00:03:31 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.138 us/op +# Warmup Iteration 2: 0.115 us/op +# Warmup Iteration 3: 0.125 us/op +# Warmup Iteration 4: 0.117 us/op +# Warmup Iteration 5: 0.111 us/op +Iteration 1: 0.111 us/op +Iteration 2: 0.111 us/op +Iteration 3: 0.113 us/op +Iteration 4: 0.113 us/op +Iteration 5: 0.111 us/op +Iteration 6: 0.108 us/op +Iteration 7: 0.111 us/op +Iteration 8: 0.114 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": + 0.111 ±(99.9%) 0.003 us/op [Average] + (min, avg, max) = (0.108, 0.111, 0.114), stdev = 0.002 + CI (99.9%): [0.108, 0.115] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly +# Parameters: (predicates = 2, rows = 65536) + +# Run progress: 59.38% complete, ETA 00:03:16 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.135 us/op +# Warmup Iteration 2: 0.110 us/op +# Warmup Iteration 3: 0.107 us/op +# Warmup Iteration 4: 0.114 us/op +# Warmup Iteration 5: 0.107 us/op +Iteration 1: 0.114 us/op +Iteration 2: 0.111 us/op +Iteration 3: 0.109 us/op +Iteration 4: 0.110 us/op +Iteration 5: 0.113 us/op +Iteration 6: 0.116 us/op +Iteration 7: 0.118 us/op +Iteration 8: 0.113 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": + 0.113 ±(99.9%) 0.006 us/op [Average] + (min, avg, max) = (0.109, 0.113, 0.118), stdev = 0.003 + CI (99.9%): [0.107, 0.119] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly +# Parameters: (predicates = 4, rows = 256) + +# Run progress: 62.50% complete, ETA 00:03:01 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.318 us/op +# Warmup Iteration 2: 0.306 us/op +# Warmup Iteration 3: 0.291 us/op +# Warmup Iteration 4: 0.289 us/op +# Warmup Iteration 5: 0.272 us/op +Iteration 1: 0.257 us/op +Iteration 2: 0.261 us/op +Iteration 3: 0.347 us/op +Iteration 4: 0.270 us/op +Iteration 5: 0.259 us/op +Iteration 6: 0.288 us/op +Iteration 7: 0.289 us/op +Iteration 8: 0.283 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": + 0.282 ±(99.9%) 0.056 us/op [Average] + (min, avg, max) = (0.257, 0.282, 0.347), stdev = 0.029 + CI (99.9%): [0.226, 0.338] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly +# Parameters: (predicates = 4, rows = 65536) + +# Run progress: 65.63% complete, ETA 00:02:46 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.316 us/op +# Warmup Iteration 2: 0.262 us/op +# Warmup Iteration 3: 0.261 us/op +# Warmup Iteration 4: 0.265 us/op +# Warmup Iteration 5: 0.309 us/op +Iteration 1: 0.258 us/op +Iteration 2: 0.261 us/op +Iteration 3: 0.275 us/op +Iteration 4: 0.256 us/op +Iteration 5: 0.267 us/op +Iteration 6: 0.257 us/op +Iteration 7: 0.262 us/op +Iteration 8: 0.256 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": + 0.261 ±(99.9%) 0.012 us/op [Average] + (min, avg, max) = (0.256, 0.261, 0.275), stdev = 0.007 + CI (99.9%): [0.249, 0.274] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly +# Parameters: (predicates = 8, rows = 256) + +# Run progress: 68.75% complete, ETA 00:02:31 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.776 us/op +# Warmup Iteration 2: 0.649 us/op +# Warmup Iteration 3: 0.653 us/op +# Warmup Iteration 4: 0.628 us/op +# Warmup Iteration 5: 0.598 us/op +Iteration 1: 0.658 us/op +Iteration 2: 0.661 us/op +Iteration 3: 0.640 us/op +Iteration 4: 0.589 us/op +Iteration 5: 0.594 us/op +Iteration 6: 0.902 us/op +Iteration 7: 0.597 us/op +Iteration 8: 0.616 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": + 0.657 ±(99.9%) 0.196 us/op [Average] + (min, avg, max) = (0.589, 0.657, 0.902), stdev = 0.103 + CI (99.9%): [0.461, 0.854] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly +# Parameters: (predicates = 8, rows = 65536) + +# Run progress: 71.88% complete, ETA 00:02:16 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.755 us/op +# Warmup Iteration 2: 0.660 us/op +# Warmup Iteration 3: 0.592 us/op +# Warmup Iteration 4: 0.704 us/op +# Warmup Iteration 5: 0.633 us/op +Iteration 1: 0.616 us/op +Iteration 2: 0.617 us/op +Iteration 3: 0.587 us/op +Iteration 4: 0.576 us/op +Iteration 5: 0.599 us/op +Iteration 6: 0.583 us/op +Iteration 7: 0.607 us/op +Iteration 8: 0.619 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": + 0.601 ±(99.9%) 0.032 us/op [Average] + (min, avg, max) = (0.576, 0.601, 0.619), stdev = 0.017 + CI (99.9%): [0.569, 0.632] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused +# Parameters: (predicates = 1, rows = 256) + +# Run progress: 75.00% complete, ETA 00:02:00 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 0.501 us/op +# Warmup Iteration 2: 0.387 us/op +# Warmup Iteration 3: 0.381 us/op +# Warmup Iteration 4: 0.391 us/op +# Warmup Iteration 5: 0.378 us/op +Iteration 1: 0.393 us/op +Iteration 2: 0.390 us/op +Iteration 3: 0.387 us/op +Iteration 4: 0.382 us/op +Iteration 5: 0.380 us/op +Iteration 6: 0.381 us/op +Iteration 7: 0.385 us/op +Iteration 8: 0.380 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": + 0.385 ±(99.9%) 0.009 us/op [Average] + (min, avg, max) = (0.380, 0.385, 0.393), stdev = 0.005 + CI (99.9%): [0.375, 0.394] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused +# Parameters: (predicates = 1, rows = 65536) + +# Run progress: 78.13% complete, ETA 00:01:45 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 6.595 us/op +# Warmup Iteration 2: 6.526 us/op +# Warmup Iteration 3: 6.280 us/op +# Warmup Iteration 4: 6.308 us/op +# Warmup Iteration 5: 7.104 us/op +Iteration 1: 6.245 us/op +Iteration 2: 6.238 us/op +Iteration 3: 6.513 us/op +Iteration 4: 6.304 us/op +Iteration 5: 6.308 us/op +Iteration 6: 6.337 us/op +Iteration 7: 6.270 us/op +Iteration 8: 6.397 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": + 6.326 ±(99.9%) 0.175 us/op [Average] + (min, avg, max) = (6.238, 6.326, 6.513), stdev = 0.091 + CI (99.9%): [6.152, 6.501] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused +# Parameters: (predicates = 2, rows = 256) + +# Run progress: 81.25% complete, ETA 00:01:30 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 1.103 us/op +# Warmup Iteration 2: 0.946 us/op +# Warmup Iteration 3: 0.943 us/op +# Warmup Iteration 4: 0.932 us/op +# Warmup Iteration 5: 0.931 us/op +Iteration 1: 0.923 us/op +Iteration 2: 0.951 us/op +Iteration 3: 0.930 us/op +Iteration 4: 0.927 us/op +Iteration 5: 0.925 us/op +Iteration 6: 0.937 us/op +Iteration 7: 0.922 us/op +Iteration 8: 0.930 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": + 0.931 ±(99.9%) 0.018 us/op [Average] + (min, avg, max) = (0.922, 0.931, 0.951), stdev = 0.009 + CI (99.9%): [0.913, 0.948] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused +# Parameters: (predicates = 2, rows = 65536) + +# Run progress: 84.38% complete, ETA 00:01:15 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 14.757 us/op +# Warmup Iteration 2: 13.827 us/op +# Warmup Iteration 3: 13.941 us/op +# Warmup Iteration 4: 13.639 us/op +# Warmup Iteration 5: 13.879 us/op +Iteration 1: 13.673 us/op +Iteration 2: 14.664 us/op +Iteration 3: 14.181 us/op +Iteration 4: 13.711 us/op +Iteration 5: 14.745 us/op +Iteration 6: 13.901 us/op +Iteration 7: 14.212 us/op +Iteration 8: 14.543 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": + 14.204 ±(99.9%) 0.804 us/op [Average] + (min, avg, max) = (13.673, 14.204, 14.745), stdev = 0.420 + CI (99.9%): [13.400, 15.008] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused +# Parameters: (predicates = 4, rows = 256) + +# Run progress: 87.50% complete, ETA 00:01:00 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 2.248 us/op +# Warmup Iteration 2: 2.127 us/op +# Warmup Iteration 3: 2.068 us/op +# Warmup Iteration 4: 2.095 us/op +# Warmup Iteration 5: 2.067 us/op +Iteration 1: 2.088 us/op +Iteration 2: 2.054 us/op +Iteration 3: 2.086 us/op +Iteration 4: 2.080 us/op +Iteration 5: 2.091 us/op +Iteration 6: 2.124 us/op +Iteration 7: 2.166 us/op +Iteration 8: 2.088 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": + 2.097 ±(99.9%) 0.064 us/op [Average] + (min, avg, max) = (2.054, 2.097, 2.166), stdev = 0.034 + CI (99.9%): [2.033, 2.162] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused +# Parameters: (predicates = 4, rows = 65536) + +# Run progress: 90.63% complete, ETA 00:00:45 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 34.677 us/op +# Warmup Iteration 2: 30.707 us/op +# Warmup Iteration 3: 30.336 us/op +# Warmup Iteration 4: 30.246 us/op +# Warmup Iteration 5: 33.637 us/op +Iteration 1: 31.002 us/op +Iteration 2: 35.002 us/op +Iteration 3: 29.954 us/op +Iteration 4: 30.045 us/op +Iteration 5: 33.427 us/op +Iteration 6: 32.171 us/op +Iteration 7: 31.979 us/op +Iteration 8: 30.739 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": + 31.790 ±(99.9%) 3.337 us/op [Average] + (min, avg, max) = (29.954, 31.790, 35.002), stdev = 1.745 + CI (99.9%): [28.453, 35.127] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused +# Parameters: (predicates = 8, rows = 256) + +# Run progress: 93.75% complete, ETA 00:00:30 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 4.932 us/op +# Warmup Iteration 2: 4.552 us/op +# Warmup Iteration 3: 4.348 us/op +# Warmup Iteration 4: 4.379 us/op +# Warmup Iteration 5: 4.290 us/op +Iteration 1: 4.327 us/op +Iteration 2: 4.626 us/op +Iteration 3: 4.465 us/op +Iteration 4: 4.337 us/op +Iteration 5: 4.370 us/op +Iteration 6: 4.488 us/op +Iteration 7: 4.526 us/op +Iteration 8: 4.361 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": + 4.437 ±(99.9%) 0.204 us/op [Average] + (min, avg, max) = (4.327, 4.437, 4.626), stdev = 0.107 + CI (99.9%): [4.233, 4.642] (assumes normal distribution) + + +# JMH version: 1.37 +# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 +# VM invoker: /opt/jdks/jdk-26.0.2/bin/java +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) +# Warmup: 5 iterations, 500 ms each +# Measurement: 8 iterations, 500 ms each +# Timeout: 10 min per iteration +# Threads: 1 thread, will synchronize iterations +# Benchmark mode: Average time, time/op +# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused +# Parameters: (predicates = 8, rows = 65536) + +# Run progress: 96.88% complete, ETA 00:00:15 +# Fork: 1 of 1 +Picked up JAVA_TOOL_OPTIONS: +WARNING: Using incubator modules: jdk.incubator.vector +WARNING: A terminally deprecated method in sun.misc.Unsafe has been called +WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) +WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils +WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release +# Warmup Iteration 1: 67.063 us/op +# Warmup Iteration 2: 61.358 us/op +# Warmup Iteration 3: 60.608 us/op +# Warmup Iteration 4: 63.889 us/op +# Warmup Iteration 5: 60.896 us/op +Iteration 1: 63.044 us/op +Iteration 2: 61.284 us/op +Iteration 3: 61.587 us/op +Iteration 4: 60.711 us/op +Iteration 5: 58.430 us/op +Iteration 6: 59.685 us/op +Iteration 7: 61.525 us/op +Iteration 8: 61.479 us/op + + +Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": + 60.968 ±(99.9%) 2.657 us/op [Average] + (min, avg, max) = (58.430, 60.968, 63.044), stdev = 1.390 + CI (99.9%): [58.311, 63.625] (assumes normal distribution) + + +# Run complete. Total time: 00:08:03 + +REMEMBER: The numbers below are just data. To gain reusable insights, you need to follow up on +why the numbers are the way they are. Use profilers (see -prof, -lprof), design factorial +experiments, perform baseline and negative tests that provide experimental control, make sure +the benchmarking environment is safe on JVM/OS/HW level, ask for reviews from the domain experts. +Do not assume the numbers tell you what you want them to tell. + +NOTE: Current JVM experimentally supports Compiler Blackholes, and they are in use. Please exercise +extra caution when trusting the results, look into the generated code to check the benchmark still +works, and factor in a small probability of new VM bugs. Additionally, while comparisons between +different JVMs are already problematic, the performance difference caused by different Blackhole +modes can be very significant. Please make sure you use the consistent Blackhole mode for comparisons. + +Benchmark (predicates) (rows) Mode Cnt Score Error Units +E_FusionAndPlanning.fused 1 256 avgt 8 0.404 ± 0.010 us/op +E_FusionAndPlanning.fused 1 65536 avgt 8 6.913 ± 0.165 us/op +E_FusionAndPlanning.fused 2 256 avgt 8 0.520 ± 0.032 us/op +E_FusionAndPlanning.fused 2 65536 avgt 8 15.222 ± 0.707 us/op +E_FusionAndPlanning.fused 4 256 avgt 8 0.808 ± 0.033 us/op +E_FusionAndPlanning.fused 4 65536 avgt 8 25.591 ± 0.582 us/op +E_FusionAndPlanning.fused 8 256 avgt 8 1.482 ± 0.070 us/op +E_FusionAndPlanning.fused 8 65536 avgt 8 58.978 ± 4.509 us/op +E_FusionAndPlanning.fusedScalarKernel 1 256 avgt 8 0.503 ± 0.015 us/op +E_FusionAndPlanning.fusedScalarKernel 1 65536 avgt 8 74.340 ± 1.659 us/op +E_FusionAndPlanning.fusedScalarKernel 2 256 avgt 8 0.835 ± 0.039 us/op +E_FusionAndPlanning.fusedScalarKernel 2 65536 avgt 8 408.170 ± 20.242 us/op +E_FusionAndPlanning.fusedScalarKernel 4 256 avgt 8 1.662 ± 0.084 us/op +E_FusionAndPlanning.fusedScalarKernel 4 65536 avgt 8 917.387 ± 27.768 us/op +E_FusionAndPlanning.fusedScalarKernel 8 256 avgt 8 3.489 ± 0.113 us/op +E_FusionAndPlanning.fusedScalarKernel 8 65536 avgt 8 1825.916 ± 125.373 us/op +E_FusionAndPlanning.planConstructionOnly 1 256 avgt 8 0.056 ± 0.004 us/op +E_FusionAndPlanning.planConstructionOnly 1 65536 avgt 8 0.053 ± 0.005 us/op +E_FusionAndPlanning.planConstructionOnly 2 256 avgt 8 0.111 ± 0.003 us/op +E_FusionAndPlanning.planConstructionOnly 2 65536 avgt 8 0.113 ± 0.006 us/op +E_FusionAndPlanning.planConstructionOnly 4 256 avgt 8 0.282 ± 0.056 us/op +E_FusionAndPlanning.planConstructionOnly 4 65536 avgt 8 0.261 ± 0.012 us/op +E_FusionAndPlanning.planConstructionOnly 8 256 avgt 8 0.657 ± 0.196 us/op +E_FusionAndPlanning.planConstructionOnly 8 65536 avgt 8 0.601 ± 0.032 us/op +E_FusionAndPlanning.unfused 1 256 avgt 8 0.385 ± 0.009 us/op +E_FusionAndPlanning.unfused 1 65536 avgt 8 6.326 ± 0.175 us/op +E_FusionAndPlanning.unfused 2 256 avgt 8 0.931 ± 0.018 us/op +E_FusionAndPlanning.unfused 2 65536 avgt 8 14.204 ± 0.804 us/op +E_FusionAndPlanning.unfused 4 256 avgt 8 2.097 ± 0.064 us/op +E_FusionAndPlanning.unfused 4 65536 avgt 8 31.790 ± 3.337 us/op +E_FusionAndPlanning.unfused 8 256 avgt 8 4.437 ± 0.204 us/op +E_FusionAndPlanning.unfused 8 65536 avgt 8 60.968 ± 2.657 us/op + +Benchmark result is saved to results/jmh-results.csv diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..83ed33b --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,140 @@ +# Architecture + +> Read `../README.md` first for the one-page picture. This document is the +> "why," not a restatement of the "what" — each section links to the real +> artifact that proves its claim rather than re-describing it. + +## The thesis, restated precisely + +> 64,000 logical entities do not become 64,000 Java objects. They become +> 1 native lane set + 1 packed mask + a handful of tiny typed descriptors +> + one bulk operation. + +This is not a performance slogan — it is a falsifiable claim, and every +layer of this project exists to make it either provably true or provably +false at the point where it would break. See +`.claude/knowledge/john-doe-migration-thesis.md` for the full framing and +the migration story ("yesterday's object-heavy Java... gets a +generated/schema-fed API that still feels like Java") that motivates it. + +## The four layers, and what each one is actually responsible for + +``` +Java semantic plane → Panama FFM membrane → Rust ABI crate → ndarray::simd +``` + +### 1. The Java semantic plane (`java/`) + +**Responsible for:** looking like ordinary, boring Java. `NativePattern` / +`View` / `Predicate` / `Pattern` / `Mask` / `Lens`. Nothing here may mention +`MemorySegment`, `Arena`, a lane id, an opcode, or which SIMD backend ran — +`ApiSurfaceTest` enforces this by *reflection*, not by convention (walks +every public member of every public type in +`com.adaworldapi.lancegraph`, fails if any signature mentions +`java.lang.foreign.*`, `java.lang.invoke.*`, or `internal.*`). + +**Not responsible for:** deciding when to cross the membrane. `View.where()` +composes pure data (a `Predicate` list) and crosses **zero** times — +proven by `LazinessTest`, which counts actual downcalls before and after +building a 16-condition chain and asserts the count didn't move. A terminal +operation (`count()`, `sumOf()`) is the only thing that ever crosses, and it +crosses **exactly once**, independent of predicate count (`FusionParityTest`) +and independent of row count up to 1,048,576+ (measured directly in +`bench/RESULTS.md`'s `planConstructionOnly` row, which scales with predicate +count and is flat across every row count tested). + +### 2. The Panama FFM membrane (`java/.../internal/ffm/`) + +**Responsible for:** turning the ABI contract (`docs/abi.md`) into real +`MethodHandle`s, `MemoryLayout`s, and a runtime **manifest cross-check** +that proves the loaded `.so` actually matches what this Java build was +compiled against — not by convention, by comparing two independently +computed numbers (`Layouts.java`'s `MemoryLayout.byteSize()` against the +manifest's own `size_of` fields) and refusing to proceed on any mismatch. +`AbiContractTest` proves this is a real check, not a formality: a real +shared library that happens to load fine (`libz.so.1`) is still rejected, +because it exports no `lgj_abi_manifest` symbol. + +**Not responsible for:** any policy about what a "resource" or "mask" +*means* semantically — that's the layer above. This layer only knows about +bytes, handles, and status codes. + +### 3. The Rust ABI crate (`native/lgj-abi`) + +**Responsible for:** the 14-symbol `extern "C"` surface in `docs/abi.md` +§7, the generation-checked handle registry (`.claude/knowledge/ +abi-ownership-and-handles.md`), and the generic SoA fixture. Bulk-only — +every function's cost scales with `n_rows`, or is lifecycle. No strings, no +callbacks, no per-element crossings (`docs/abi.md` §6, the anti-JNI rule). + +**Provably safe, not just tested-safe:** the registry's core invariant +(a stale handle can never dereference freed memory) was +**disable-verified** — the generation check was deliberately broken and +the suite re-run to confirm exactly the two tests that should catch it went +red, and only those two. See `.claude/board/EPIPHANIES.md` +`E-LGJ-CORE-SLICE-GREEN-DISABLE-VERIFIED-1` for the exact procedure and +numbers. + +### 4. `ndarray::simd` (a sibling repo, consumed not re-implemented) + +**Responsible for:** every bulk kernel. `native/lgj-abi/src/kernels.rs` is +the *only* file in this crate allowed to import from `ndarray`, and it +reaches everything exclusively through `ndarray::simd::*` — never +`ndarray::hpc::*` (the internal implementation namespace) directly. This +project added five primitives to `ndarray` under that repo's own W1a +consumer contract (`eq_u32_to_mask`, `gt_i32_to_mask`, `mask_and`/`mask_or` +(`_assign`), `masked_sum_i32`) rather than reimplementing anything locally. +See `.claude/knowledge/simd-provenance.md`. + +**Measured payoff:** SIMD vs. the crate's own independent scalar reference +kernel is 10.8×–31.1× faster on the fused multi-predicate path, growing +with predicate count (`bench/RESULTS.md`, Component E). This is the +single largest measured lever in the whole project — larger than the +membrane-crossing cost itself. + +## What the measurements actually say about where the boundaries pay off + +This is the part a pure architecture diagram cannot show, and it's the +reason `bench/` and `valhalla-lab/` exist as first-class deliverables +rather than an afterthought: + +- **The Rust↔Java membrane pays off for composed, multi-predicate work** — + where SIMD fusion (10.8×–31.1×) and the one-crossing guarantee for an + arbitrarily long `View` chain matter. It does **not** pay off for a + single predicate read off one lane, where the Java Vector API reading the + same native memory zero-copy is measurably *faster* at every scale tested + (see `docs/execution-boundary.md`). +- **Valhalla pays off for the tiny descriptor vocabulary** (`LaneId`, + `Ordinal`, `MaskId` — single-field types, ≤8 bytes, genuinely flatten) — + and does **not** rescue per-entity materialization (`Row`, 16 bytes, + measured `NOT-FLAT` even under Valhalla). See `docs/valhalla-lab.md`. + +Both of these are measured surprises relative to a naive "the native side +always wins" assumption, and both are load-bearing for how this project's +API is actually shaped: the fluent `View` stays lazy and fuses because +fusion is where the payoff is real, and the semantic vocabulary stays +`record`-shaped (one-word migration to `value record` when JEP 401 ships) +because that's exactly the shape Valhalla rewards. + +## Where a real graph slice would attach (not built yet, by design) + +`docs/abi.md` §10 names this explicitly: `WideFieldMask` (already has +`intersect`/`union`/`count` in `lance-graph-contract`) and `NodeRow` +(`#[repr(C, align(64))]`, 16|16|480 bytes, already size-locked) are the +existing Rust-side types this ABI's `MASK_WORD` lane and lane-descriptor +shapes are already compatible with. Operator-stated layout reference +(2026-08-17): the lance-graph substrate enforces **64K rows × 512 bytes +per row, read as 32 lanes of 16 bytes each (4-byte classid + 12-byte +payload — the V3 content-blind facet)** everywhere; the Java-side layout +may legitimately differ, but that 512-byte, 64-byte-aligned row is the +shape a real slice inherits — and it is exactly the shape +`ndarray::simd_soa::MultiLaneColumn` (64-byte-chunk iteration over an +`Arc<[u8]>`) was built for, which is why that type is earmarked for the +row-store slice and deliberately NOT used by today's flat-lane fixture +kernels (see `.claude/board/EPIPHANIES.md` +`E-LGJ-SIMD-SOA-IS-FOR-THE-ROW-STORE-NOT-THE-FLAT-LANES-1`) — see the lance-graph archaeology +findings in `.claude/board/AGENT_LOG.md`. The generic fixture in this +first slice was deliberately chosen over wiring the real graph types +immediately, so the membrane's physics could be proven independent of graph +semantics. Wiring `ClassView`/`WideFieldMask` is the natural next slice, +not a redesign. diff --git a/docs/execution-boundary.md b/docs/execution-boundary.md new file mode 100644 index 0000000..df9ca04 --- /dev/null +++ b/docs/execution-boundary.md @@ -0,0 +1,155 @@ +# The execution boundary — measured, and what the measurements imply + +> Companion to `bench/RESULTS.md` (the raw numbers + reproduction commands). +> This document is the synthesis: what the numbers mean for where work should +> execute, and the three structural facts about the hot path that the numbers +> only make sense in light of. + +## The question, as the mission posed it + +> "Where is the cheapest and cleanest execution boundary? Not: how can we +> maximize the amount of Java code?" + +Measured answer, from real JMH over identical data with a cross-checked +answer on every arm (`Data.crossCheck()`): **it depends on composition, and +the dependence is now quantified.** + +| workload shape | winner | evidence | +|---|---|---| +| one predicate, one lane, any row count 64→4M | **Java Vector API, zero-copy on the native segment** — 1.3×–56× faster than the crossing | `bench/RESULTS.md` Component C | +| one predicate vs a plain Java *scalar* loop | native wins only past ~4,096–16,384 rows | same sweep, scalar column | +| N predicates fused, 65,536 rows | native SIMD, 10.8×–31.1× over its own scalar reference | Component E | +| the crossing itself, empty | ~22 ns bare, ~118 ns with two args + out-pointer | Component A | +| reading native memory from Java at all | free — segment scalar ≈ heap array (within noise); segment *vector* 1.55× faster than both | Component B | + +The two headline implications: + +1. **The membrane's cost is real but small and fixed** (~0.6 µs including + wrapper overhead) — it is repaid by *work*, not by *data volume alone*. + A single predicate never generates enough work per byte to repay it, + because the Vector API can do that same predicate on the same bytes + without crossing at all. +2. **SIMD fusion is the largest lever measured anywhere in this project** + (10.8×–31.1×) — larger than any crossing cost. The crossing is how you + *reach* `ndarray::simd`'s fused kernels; that, not the crossing itself, + is what the Java side is buying. + +## Three structural facts the numbers rest on + +### 1. Zero-copy is precise language here, not marketing + +The project's invariant (per the mission brief): **crossing the boundary +must not itself require serialization or copying.** It does not claim no +allocation ever happens — a mask result is a legitimate, semantically +required native allocation. What is eliminated is *boundary-induced* +copying, and this is checkable in the code: + +- The Vector API arm reads the native lane via + `IntVector.fromMemorySegment(species, segment, offset, nativeOrder())` — + no `byte[]`, no `int[]` staging, no `MemorySegment.toArray`. The bench's + own README states the rule and why a copy anywhere would make the + comparison dishonest in *both* directions. +- On the native side, lanes are allocated once and never relocated + (`docs/abi.md` §4), so a `MemorySegment` view stays valid for the + resource's whole lifetime — the precondition for Java reading it in place. + +**This is also exactly the shape a real lance-graph slice inherits.** The +lance-graph side's `SoaEnvelope::{as_le_bytes, row_le, column_le}` are +already zero-copy `&[u8]` views over LE-resident backing bytes (that repo's +own doctrine: "every SoA envelope is zero-copy from creation to Lance +tombstone" — nothing is serialized between mailboxes). When those replace +the generic fixture, no serialization step *exists anywhere in the stack*: +Lance's columnar bytes are the wire format, the membrane hands Java a +bounded view of them, and both execution engines — `ndarray::simd` on one +side, the Vector API on the other — operate on the same un-copied bytes. +That is the whole point, and it is why the Vector-API finding below is a +*feature* of the design rather than an embarrassment to it. + +### 2. There is no thread pool in the hot path — the JVM's threads ARE the parallelism + +Checked, not assumed (2026-08-17): + +- `lgj-abi` has exactly one dependency: `ndarray` with + `default-features = false, features = ["std"]`. **Rayon is not in the + tree** (it is an optional ndarray feature, not enabled here). +- The only `thread::spawn` in the crate is `#[cfg(test)]`-only — two tests + proving the concurrency *shape* (8 threads on distinct resources: no + deadlock, no cross-talk, independently correct answers; and + opposite-order mask binops that would deadlock without address-ordered + locking, run 2,000 times each way). + +The design instead makes the *caller's* threads the unit of parallelism: +the registry takes a short read-lock only to resolve `handle → +Arc`, drops it, then locks only that entry +(`.claude/knowledge/abi-ownership-and-handles.md`). So N Java threads +driving N distinct resources — the "64K thoughts as many mailboxes, each +owned by its caller" model — run concurrently through the membrane with no +Rust-side scheduler, no fork-join pool, no rayon. Parallelism is implicit +in ownership, exactly as the sibling lance-graph substrate's +one-writer-per-mailbox doctrine intends. + +**Honest boundary:** the shape is proven (the two tests above); throughput +under real contention is NOT yet benchmarked — filed as +`TD-LGJ-REGISTRY-CONCURRENCY-UNMEASURED` in `.claude/board/TECH_DEBT.md`, +to be paid when a concurrent caller actually exists rather than +speculatively. + +### 3. The kernels chunk by direct lane-group indexing, not via `array_windows`/`array_chunks` + +Checked precisely (2026-08-17), not assumed either way: `ndarray::simd_ops` +exports `array_windows`/`array_chunks` as opt-in, generic const-N staging +helpers. A full trace of this project's call graph — +`eq_u32_to_mask`/`gt_i32_to_mask` → `load_u32x16`/`load_i32x16` → +`copy_from_slice(&src[..16])` — shows neither is invoked, at any input +size; nothing about data volume triggers them, since they only run if a +caller literally writes `array_chunks::(slice)`, which this call +graph never does. A repo-wide grep confirms the same is true of +`ndarray`'s own internals: `simd_soa.rs`'s `MultiLaneColumn` doc comment +*cross-references* `array_chunks` as living in `simd_ops.rs`, but does not +call it either. + +This is a considered choice for the mask kernels specifically, not an +omission: `array_windows` is a *sliding* window (every element visited N +times — right for stencils/filters, wrong for a linear scan, the same +reasoning the sibling tesseract-rs repo recorded when it evaluated and +declined `array_windows` for its own integral-image kernels). The mask +kernels instead stride directly: 16 elements per group through +`U32x16::eq_bitmask` / `I32x16::gt_bitmask`, ORing each group's 16-bit +result into position `(g % 4) * 16` of word `g / 4`, with a scalar tail — +zero iterator overhead, and the "trailing bits are zero" guarantee made +structural by zeroing the output first. This achieves the same *effect* +`array_chunks` exists to provide (fixed-width grouping), through each +primitive's own indexing rather than the shared utility — a legitimate +alternative, not a gap, though routing through `array_chunks` for +uniformity across `ndarray::simd`'s kernels would be a reasonable future +refactor if consistency across primitives becomes a goal in its own right. + +## The resulting execution model (the synthesis) + +Not "Rust executes, Java orchestrates" — the measured picture is finer: + +``` + the same un-serialized native bytes + ┌───────────────────────────────────┐ + │ lance / lane storage │ + └───────────────┬───────────────────┘ + borrowed segment │ one fused crossing + ┌──────────────────────┴─────────────────────┐ + ▼ ▼ + Java Vector API Rust ndarray::simd + — single-predicate reads — multi-predicate fused plans + — small/any row counts — the 10.8-31.1x SIMD kernels + — anything the JIT can see whole — anything worth ONE crossing + │ │ + └──────────────────────┬─────────────────────┘ + ▼ + tiny results (a count, a sum, a mask handle) +``` + +A future planner could even choose the side per-operation using exactly the +crossover table in `bench/RESULTS.md` — the data to make that choice +mechanically now exists. What keeps the model honest is the invariant both +sides share: **the bytes never serialize, never bounce, never mirror into +the Java heap as N objects.** Which side loops over them is an +implementation decision the measurements can now drive; that they are the +same bytes is the architecture. diff --git a/docs/panama.md b/docs/panama.md new file mode 100644 index 0000000..3bce268 --- /dev/null +++ b/docs/panama.md @@ -0,0 +1,100 @@ +# The Panama membrane + +> Companion to `docs/abi.md` (the normative Rust-side contract this +> document's Java-side machinery is checked against) and +> `.claude/agents/panama-bridge-engineer.md` (the review checklist for this +> code). This document explains the design decisions; the code and the +> tests are the proof. + +## The one property that matters: a header is a claim, a manifest is a fact + +Project Panama exists so the JVM can speak the platform calling convention +directly — no C compiler, no header, no `jextract` (see `docs/abi.md` §1, +`.claude/knowledge/no-c-ever.md`). But that only removes the *tool*; it +does not remove the *risk* a header used to (loudly) warn about: Java's +compiled-in idea of a struct's layout silently disagreeing with what the +native artifact actually produces. + +This project's answer is `lgj_abi_manifest()` — a function that returns a +pointer to a `'static` struct the compiled `.so` fills in from +`core::mem::size_of`/`align_of` on its own real types, never a hand-typed +constant (`native/lgj-abi/src/abi.rs`, with `const _: () = +assert!(size_of::() == N)` compile-time locks on every `#[repr(C)]` +type). `Abi.java` reads this manifest at load time and compares it against +**a second, independently-derived number**: Java's own `MemoryLayout` +definitions in `Layouts.java`, via `layout.byteSize()`/`byteAlignment()`. + +Two independently-computed numbers, not one number checked against itself. +`AbiContractTest` proves the check is real: a genuine shared library that +loads fine (`libz.so.1`) is still rejected — refused, not called into — +because it exports no `lgj_abi_manifest` symbol at all. + +## Ownership crosses the FFM boundary as belt-and-braces, not once + +`docs/abi.md` §4 gives Rust the generation-checked handle as the ground +truth for whether a resource is alive. Panama gives the *Java-side* +bookkeeping no borrow checker at all, so `Abi`/the public facade adds its +own fail-fast layer on top rather than trusting the native check alone: +a Java-side closed flag on a resource makes a use-after-close throw a +clear Java exception (`ClosedResourceException`) *before* the call ever +reaches native code — verified by `LifetimeTest`'s 23 checks (use after +close, double close, a selection outliving its parent, a selection closed +before its parent, an empty resource behaving legally). The native +`INVALID_HANDLE`/`PARENT_CLOSED` status codes are what actually prevent +memory unsafety; the Java-side flag is what keeps the failure mode +readable instead of an opaque native error surfacing through five layers +of `MethodHandle.invokeExact`. + +## Restricted methods are a feature, not friction + +Every FFM operation this project needs — `SymbolLookup.libraryLookup`, +`Linker.downcallHandle`, `MemorySegment.reinterpret` — is `@Restricted` in +the JDK, meaning it needs `--enable-native-access` and produces a compiler +warning without a suppression. This project does not suppress it: `javac +-Xlint:all` on the shipped tree produces **exactly 7** `[restricted]` +warnings, every one of them inside `internal/ffm/*` or a test file +deliberately exercising the same restricted call independently +(`AbiContractTest`). That count is a machine-checkable statement — not +prose — that every unsafe FFM operation in this project lives in the one +package `ApiSurfaceTest` already proves the public API never exposes. + +## Downcall handles are resolved once, never per call + +`Downcalls.java` resolves every `MethodHandle` into a `static final` at +class-init, matching `FunctionDescriptor`s to `docs/abi.md` §7 argument- +for-argument. `Component A` in `bench/` deliberately re-binds its own +method handles independently of `Downcalls` — not out of duplication, but +because measuring the JDK's own linker cost separately from this +project's wrapper is the only way to attribute the wrapper's overhead +correctly (`bench/README.md` rule 4). Measured: a bare downcall costs +~22 ns over a plain Java call (`bench/RESULTS.md`, Component A) — the +floor every native-side operation in this project sits on top of. + +## `--enable-preview` never reaches the shipped path + +The production Java tree (`java/`) targets `/opt/jdks/jdk-26.0.2`, where +FFM is **final** — the only flag needed anywhere is +`--enable-native-access=ALL-UNNAMED`. The Valhalla lab +(`valhalla-lab/src/valhalla`) is compiled *separately*, with its own +`-source 27 --enable-preview`, into its own output directory +(`results/valhalla-lab/`), and is never on the classpath the production +tests or the bench harness run against. `--enable-preview`-compiled +classfiles carry a preview marker that poisons every consumer that loads +them; keeping the two trees physically separate (rather than, say, +compiling once and gating features at runtime) is what makes this a +structural guarantee rather than a discipline someone could accidentally +violate. See `.claude/knowledge/jdk-toolchain-facts.md` for the exact +verified flag matrix across all three JDKs this project touches. + +## What Panama did NOT need to solve here + +Two things worth naming because they are easy to assume Panama handles and +it does not, by design, in this project: + +- **No upcalls.** `Linker.Option`/`upcallStub` exist in the API; this + project has zero uses of them. An upcall per element is the JNI + anti-pattern in a different costume (`docs/abi.md` §10), and the bulk-op + shape never needs a callback into Java mid-kernel. +- **No `captureCallState`/`errno`.** Nothing this membrane wraps is a + syscall. Every failure is a negative `i32` status the Rust side computed + deliberately, never an OS error code Panama would need to capture. diff --git a/docs/valhalla-lab.md b/docs/valhalla-lab.md new file mode 100644 index 0000000..1e118dd --- /dev/null +++ b/docs/valhalla-lab.md @@ -0,0 +1,127 @@ +# The Valhalla lab — synthesis + +> Companion to `valhalla-lab/README.md` and `valhalla-lab/docs/three-truths.md` +> (the raw findings and numbers) and `.claude/knowledge/valhalla-three-truths-method.md` +> (the method this lab is required to follow). This document is the "so +> what" — how the measurements bear on this project's actual API and +> migration story. + +## The method, briefly + +For each semantic value type in this project's small vocabulary (`LaneId`, +`Ordinal`, `MaskId`, `RowRange`, `Row`), three truths are distinguished and +never conflated: + +1. **Semantic truth** — what the type *should* mean (identity-free, an + opaque descriptor, safe to treat as a plain value). +2. **Stable-Java truth** — the `record` implementation on JDK 26 GA, + measured, not assumed to already deliver (1). +3. **Valhalla truth** — the *same source shape*, compiled as `value + record` against the JEP 401 early-access build, likewise measured. + +The mechanism that keeps this honest: `valhalla-lab/run.sh` step 0 +mechanically diffs `src/valhalla/Vocab.java` against `src/stable/Vocab.java` +*modulo the literal word `value`* and refuses to run if they differ by +more than that — so the A/B compares two runtimes on one program, never +two different programs. + +## The headline result, and why it is more useful than a clean "yes" + +The mission's mandatory experiment — 65,536 rows as (i) one native lane +plus one packed mask, (ii) hydrated Java objects, (iii) hydrated Valhalla +value objects — returned a nuanced answer, and the nuance is the finding: + +| | native, one crossing | hydrate 65,536 `Row`, then scan | +|---|---:|---:| +| stable JDK 26 | 19.5 µs, 289 KiB | 746 µs, 2.00 MiB | +| Valhalla JDK 27 EA | 15.7 µs, 289.5 KiB | 900 µs, 2.50 MiB | + +Native wins ~38–57× on time and ~7–9× on heap, **on both platforms** — +Valhalla does not close this gap. The reason is not "object headers are +still there" hand-waving; it's a hard, VM-confirmed cutoff: + +## R2 — the flattening cliff explains the whole result + +`ValueClass.isFlatArray` (the VM answering about its own array, not an +inference) shows array flattening stops dead at an **8-byte payload**: + +| type | payload | flattens? | +|---|---:|---| +| `LaneId`, `Ordinal` | 4 B | **yes** | +| `MaskId` | 8 B | **yes** | +| `RowRange` | 16 B | no | +| `Row` (id + class + value) | 16 B | no | + +`RowRange` landing on the wrong side is recorded in +`valhalla-lab/reproducers/README.md` as the one place the expectation was +too optimistic going in — a descriptor that was *expected* to flatten and +measurably does not. The line the VM draws is exactly the line separating +"tiny descriptor vocabulary" from "per-entity payload" in this project's +own design vocabulary: an id plus one more field already exceeds the +budget, so no realistic `Row`-shaped entity can benefit, by construction, +on this build. + +**Causal isolation, not correlation:** `run.sh` runs the whole suite three +more times with `UseArrayFlattening`/`UseFieldFlattening` toggled off +independently. Turning `UseArrayFlattening` off alone drops `LaneId`'s +per-element cost from 6.89 B to 28.00 B (matches "not flat" exactly); +turning `UseFieldFlattening` off alone changes nothing for `LaneId` (an +`int`-payload type has no sub-fields to flatten) — confirming array +flattening, not field flattening, is the mechanism actually in play for +this vocabulary. + +## What this means for the project's own API — and what it does NOT mean + +**The production `java/` API adopts none of the three Valhalla-only +mechanisms found** (R1's `@NullRestricted` container trick, R2's flat-array +allocation, R3's `jdk.internal.value.ValueClass` factories). Per +`reproducers/README.md`: *"distorting a public API to fit a preview VM's +current budget would bake a temporary constraint into a permanent +surface."* Concretely: + +- No `--add-exports` in the shipped build. +- No `jdk.internal.*` dependency anywhere in `java/`. +- The migration path from today's `record`-based vocabulary to Valhalla, + the day JEP 401 ships as final, stays **exactly one word per type** + (`record` → `value record`) — because nothing about today's API was + bent around the preview build's current limits. + +**What Valhalla DOES already buy, measured, not theoretical:** for the +single-field descriptor types this project actually uses as its public +vocabulary (`LaneId`, `Ordinal`), array storage is 5.5× smaller and array +reads are up to 8.3× faster once flattening applies — real numbers +Structure the semantic types were already shaped to receive, because they +were designed to be tiny and identity-free *before* this lab ever ran (per +`.claude/knowledge/john-doe-migration-thesis.md`'s litmus test: the small +vocabulary, never the bulk data, is where Valhalla was ever expected to +matter). + +## The three reproducers, and which JDK component each belongs to + +Filed as minimal, self-contained, independently-runnable files under +`valhalla-lab/reproducers/`, per the mission's explicit instruction that a +genuine Valhalla limitation gets a reproducer rather than an API +distortion: + +| # | limitation | belongs to | +|---|---|---| +| R1 | `@NullRestricted` field on an ordinary (identity) class fails `VerifyError` at class load — javac emits field initializers after `super()`, the VM demands strict fields before it, no source form expresses the required order | **javac** | +| R2 | Array flattening has a hard 8-byte payload cliff | **HotSpot / Valhalla** | +| R3 | The densest null-restricted array form is `jdk.internal`-only; generics erase flattening entirely (`List` is `Object[]` underneath); `Foo!` null-restricted type syntax does not parse | **Valhalla (language + libraries)** | + +R3's `Foo!` finding corroborates the earlier archaeology independently: a +direct compiler probe (`javac`, not documentation) confirms the syntax +genuinely does not exist in this build, matching what the three-JDK +source-checkout comparison found before any lab code was written (see +`.claude/board/EPIPHANIES.md` `E-LGJ-VALHALLA-ALREADY-MAINLINE-1`). + +## The honest limit of this lab + +Single-fork-equivalent caveats apply here too, stated in +`valhalla-lab/docs/three-truths.md`: this is a hand-rolled harness +(`Lab.time`), not JMH — labelled as such deliberately, with its timing +numbers treated as secondary evidence supporting the byte-count +measurements (`getThreadAllocatedBytes`, the primary instrument), not as +a benchmark-grade claim in their own right. JMH-grade timing for the +execution-boundary question lives in `bench/`, not here — see +`docs/execution-boundary.md`. diff --git a/valhalla-lab/docs/three-truths.md b/valhalla-lab/docs/three-truths.md index 15e4dad..8f50a6f 100644 --- a/valhalla-lab/docs/three-truths.md +++ b/valhalla-lab/docs/three-truths.md @@ -89,13 +89,13 @@ All figures per operation, `N = 1,000,000`. | measurement | stable | Valhalla | | |---|---:|---:|---| -| construct a `LaneId`, store into an array | 16.00 B | **2.89 B** | 5.5× less | -| construct a `LaneId`, never escaping | 7.03 B | 8.00 B | ~equal — escape analysis already handles this | -| `LaneId[N]` array + elements, per element | 20.00 B | **6.89 B** | 2.9× less | +| construct a `LaneId`, store into an array | 16.00 B | **3.29 B** | 4.9× less | +| construct a `LaneId`, never escaping | 7.57 B | 7.64 B | ~equal — escape analysis already handles this | +| `LaneId[N]` array + elements, per element | 20.00 B | **6.87 B** | 2.9× less | | bare `LaneId[N]`, per slot | 4.00 B | 4.00 B | equal (compressed oops vs flat int) | -| construct a `Descriptor` (two wrappers) | 56.00 B | **38.84 B** | 1.4× less | -| pass two wrappers through 3 call levels | 8.29 B | 10.45 B | ~equal | -| **read 65,536 `LaneId` from an array** | 44,182 ns | **5,349 ns** | **8.3× faster** | +| construct a `Descriptor` (two wrappers) | 56.00 B | **36.50 B** | 1.5× less | +| pass two wrappers through 3 call levels | 9.33 B | 10.87 B | ~equal | +| **read 65,536 `LaneId` from an array** | 43,739 ns | **5,339 ns** | **8.2× faster** | `LaneId[1024]` reports `FLAT` on Valhalla and `UNKNOWN` on stable — deliberately not `false`. A stable JDK has no `ValueClass.isFlatArray` to ask, so "the question does not exist here" is the @@ -107,10 +107,10 @@ Re-running the Valhalla build with the VM's own flattening disabled: | | default | `-XX:-UseArrayFlattening` | `-XX:-UseFieldFlattening` | |---|---:|---:|---:| -| `LaneId` array, per element | 6.89 B | 28.00 B | 6.71 B | +| `LaneId` array, per element | 6.87 B | 28.00 B | 6.71 B | | `LaneId[1024]` flat? | FLAT | NOT-FLAT | FLAT | -| read 65,536 from array | 5,349 ns | 47,469 ns | 5,303 ns | -| `Descriptor`, per instance | 38.84 B | 40.51 B | 80.00 B | +| read 65,536 from array | 5,339 ns | 47,469 ns | 5,303 ns | +| `Descriptor`, per instance | 36.50 B | 40.51 B | 80.00 B | Turning array flattening off returns the array numbers to roughly the stable baseline and makes the read **8.9× slower**; turning field flattening off doubles the `Descriptor` cost and leaves @@ -123,18 +123,38 @@ With `-XX:-DoEscapeAnalysis`, i.e. what the object model costs when the JIT cann | measurement | stable default | stable, no EA | Valhalla default | Valhalla, no EA | |---|---:|---:|---:|---:| -| `LaneId` never escaping | 7.03 B | 16.00 B | 8.00 B | 25.60 B | -| pass 2 wrappers, 3 levels | 8.29 B | 32.00 B | 10.45 B | 31.00 B | -| **`Ordinal` built per element** (65,536 iterations) | 50,062 ns | **5,166,233 ns** | 50,021 ns | **94,346 ns** | +| `LaneId` never escaping | 7.57 B | 16.00 B | 7.64 B | 25.60 B | +| pass 2 wrappers, 3 levels | 9.33 B | 32.00 B | 10.87 B | 30.90 B | +| **`Ordinal` built per element** (65,536 iterations) | 50,732 ns | **621,827 ns** | 49,851 ns | **91,383 ns** | That last row is the clearest single number in the lab. Unaided by escape analysis, building a -one-`int` wrapper per element costs the stable JDK **103×**; it costs Valhalla **1.9×**. This is -the concrete meaning of "the abstraction stops being something you pay for" — not that it is -faster when the JIT can see through it, but that it stays cheap when the JIT cannot. +one-`int` wrapper per element costs the stable JDK **12.3×** its own escape-analysed time; it +costs Valhalla **1.8×**. Head to head with the JIT unable to help, Valhalla is **6.8× faster**. +This is the concrete meaning of "the abstraction stops being something you pay for" — not that it +is faster when the JIT can see through it, but that it stays cheap when the JIT cannot. Honesty about what this row is *not*: with escape analysis on — the configuration anyone actually -ships — the two are indistinguishable at 50 µs. Valhalla's gain here is in **robustness**, not in -peak. That distinction is easy to lose and worth keeping. +ships — the two are indistinguishable (50.7 µs vs 49.9 µs, inside this harness's spread). Valhalla's +gain here is in **robustness**, not in peak. That distinction is easy to lose and worth keeping. + +A measurement error worth recording rather than quietly fixing: an earlier draft of this table +reported 5,166,233 ns for the stable no-EA cell, a 103× ratio. That run was executed while the +JMH suite in `bench/` was saturating the same four vCPUs. The number was real and the conclusion +it supported was the same, but it was inflated roughly 8× by contention. The table above is from a +later, uncontended run. + +**The general warning that follows from it:** this lab shares a 4-vCPU container with whatever +else is running. Every number in this document comes from one uncontended execution of `run.sh`, +and re-running it moves the allocation figures by a few percent and the timing figures by more. +Treat the *ratios and the order of magnitude* as the result and the third significant figure as +noise — the byte counts that land on exact multiples of 8 (16.00, 20.00, 32.00, 40.00) are the +stable ones, because those are object layouts rather than measurements of speed. + +One row in `valhalla-noea` illustrates the point loudly: `hydrate THEN scan` measured 25.8 ms +median with a 6.1–39.2 ms spread on the final run, against 0.88 ms in the default configuration. +A 30× median with a 6× spread is not a result about Valhalla; it is GC and a disabled optimiser +interacting on a loaded box. It is left in `results/` rather than deleted, and it is not quoted as +a finding anywhere. ### `-XX:±InlineTypePassFieldsAsArgs` @@ -220,19 +240,21 @@ supports the thesis more strongly than the expected result would have. | | stable | Valhalla | |---|---:|---:| -| **(1) native — one crossing, fused plan** | **16,378 ns** | **18,805 ns** | -| (2/3) hydrate 65,536 `Row` objects | 603,139 ns | 768,624 ns | -| (2/3) scan the materialised objects | 151,886 ns | 91,178 ns | -| (2/3) hydrate **then** scan (honest total) | **788,227 ns** | **898,955 ns** | +| **(1) native — one crossing, fused plan** | **16,040 ns** | **15,162 ns** | +| (2/3) hydrate 65,536 `Row` objects | 1,612,494 ns | 658,842 ns | +| (2/3) scan the materialised objects | 128,057 ns | 87,213 ns | +| (2/3) hydrate **then** scan (honest total) | **704,743 ns** | **871,361 ns** | Medians of 51 iterations after 200–2,000 warm-up runs. Spreads are in `results/*.txt`; the -hydration rows have long tails (stable max 5.2 ms) because they allocate 2 MiB per iteration and -occasionally meet a GC, which is exactly why the median is reported. +hydration rows have long tails (stable min 499 µs, max 4.8 ms) because they allocate 2 MiB per +iteration and occasionally meet a GC, which is exactly why the median is reported and why the +isolated-hydrate row should be read as "somewhere between 0.5 and 1.4 ms", not as a precise +figure. The hydrate-then-scan row is the stable one and is the one to use. -**Native is 48× faster than the honest object total on stable, 48× on Valhalla.** Note where -Valhalla's one real win sits: *scanning* already-materialised objects is 1.7× faster (91 µs vs -152 µs), because the scan is a read-only walk that benefits from better locality. It does not -matter, because the hydration that had to happen first costs 8× what the scan saves. +**Native is 44× faster than the honest object total on stable, 57× on Valhalla.** Note where +Valhalla's one real win sits: *scanning* already-materialised objects is 1.5× faster (87 µs vs +128 µs), because the scan is a read-only walk that benefits from better locality. It does not +matter, because the hydration that had to happen first costs several times what the scan saves. That is the thesis, measured: the expensive thing is not *scanning* 65,536 objects, it is *existing* as 65,536 objects. Valhalla makes the scan cheaper and does not make the existing @@ -252,9 +274,9 @@ entire program. No arena, no segment, no mask, no lane, no opcode, no row loop, both object models; only the behaviours it was written to avoid differ. The migration is a one-word source change, and it stays that way because the lab did **not** bend the API to fit a preview VM (see `reproducers/README.md`). -2. **Valhalla is a real win for the descriptor vocabulary.** 5.5× less allocation per `LaneId`, - 8.3× faster array reads, and — the durable part — 103× → 1.9× when escape analysis cannot help. - A `LaneId` really does stop being something you pay for. +2. **Valhalla is a real win for the descriptor vocabulary.** 4.9× less allocation per `LaneId`, + 8.2× faster array reads, and — the durable part — 6.8× faster than the stable JDK once escape + analysis is out of the picture. A `LaneId` really does stop being something you pay for. 3. **Valhalla does not rescue per-entity materialisation, and on this build it makes it worse.** 40 B/row against 32 B/row, `NOT-FLAT`, because a 16-byte payload is over the VM's flattening budget. The expected finding was "it does not help"; the observed finding is "it costs 25 % diff --git a/valhalla-lab/results/AB-default.diff b/valhalla-lab/results/AB-default.diff index 22da5ba..57b0910 100644 --- a/valhalla-lab/results/AB-default.diff +++ b/valhalla-lab/results/AB-default.diff @@ -5,18 +5,18 @@ > platform valhalla > java.vm.version 27-jep401ea3+1-1 6c6 -< jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so] +< jvm args [--enable-native-access=ALL-UNNAMED, -Dstdout.encoding=UTF-8, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so] --- -> jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED] +> jvm args [--enable-native-access=ALL-UNNAMED, -Dstdout.encoding=UTF-8, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED] 9,14c9,14 -< platform stable ? stable-record vocabulary; arrays are reference arrays; fields are references +< platform stable — stable-record vocabulary; arrays are reference arrays; fields are references < LaneId.class.isValue() false < Ordinal.class.isValue() false < MaskId.class.isValue() false < RowRange.class.isValue() false < Row.class.isValue() false --- -> platform valhalla ? value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields +> platform valhalla — value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields > LaneId.class.isValue() true > Ordinal.class.isValue() true > MaskId.class.isValue() true @@ -59,74 +59,74 @@ 40,46c40,46 < construct N LaneId, store into array 15.26 MiB < ... per LaneId 16.00 B -< construct N LaneId, never escaping 6.71 MiB -< ... per LaneId 7.03 B +< construct N LaneId, never escaping 7.22 MiB +< ... per LaneId 7.57 B < LaneId[1024] flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) < allocate+fill LaneId[N] (array + elements) 19.07 MiB < ... per element 20.00 B --- -> construct N LaneId, store into array 2.75 MiB -> ... per LaneId 2.89 B -> construct N LaneId, never escaping 7.63 MiB -> ... per LaneId 8.00 B +> construct N LaneId, store into array 3.14 MiB +> ... per LaneId 3.29 B +> construct N LaneId, never escaping 7.29 MiB +> ... per LaneId 7.64 B > LaneId[1024] flatness FLAT -> allocate+fill LaneId[N] (array + elements) 6.57 MiB -> ... per element 6.89 B +> allocate+fill LaneId[N] (array + elements) 6.56 MiB +> ... per element 6.87 B 49,55c49,55 < Descriptor kind identity class with two reference fields < Descriptor fields null-restricted false < construct N Descriptor (2 wrappers each) 53.41 MiB < ... per Descriptor 56.00 B -< pass 2 wrappers through 3 call levels 7.90 MiB -< ... per call 8.29 B -< read 65,536 LaneId from array median= 44182.0 ns [min 41690.0 .. max 59914.0] n=51 +< pass 2 wrappers through 3 call levels 8.90 MiB +< ... per call 9.33 B +< read 65,536 LaneId from array median= 43739.0 ns [min 42371.0 .. max 73206.0] n=51 --- > Descriptor kind value class with two @NullRestricted value fields > Descriptor fields null-restricted true -> construct N Descriptor (2 wrappers each) 37.05 MiB -> ... per Descriptor 38.84 B -> pass 2 wrappers through 3 call levels 9.97 MiB -> ... per call 10.45 B -> read 65,536 LaneId from array median= 5349.0 ns [min 5179.0 .. max 11356.0] n=51 +> construct N Descriptor (2 wrappers each) 34.81 MiB +> ... per Descriptor 36.50 B +> pass 2 wrappers through 3 call levels 10.36 MiB +> ... per call 10.87 B +> read 65,536 LaneId from array median= 5339.0 ns [min 5262.0 .. max 9259.0] n=51 59c59 < platform stable --- > platform valhalla 61,65c61,65 -< bare long index median= 59148.0 ns [min 59077.0 .. max 114952.0] n=51 -< RowRange bounds (wrapper hoisted) median= 47952.0 ns [min 47759.0 .. max 65535.0] n=51 +< bare long index median= 59836.0 ns [min 59076.0 .. max 125353.0] n=51 +< RowRange bounds (wrapper hoisted) median= 47964.0 ns [min 47876.0 .. max 168118.0] n=51 < per-element wrapper: bytes allocated 1.00 MiB < ... per element 16.00 B -< Ordinal built per element median= 50062.0 ns [min 49790.0 .. max 61194.0] n=51 +< Ordinal built per element median= 50732.0 ns [min 50046.0 .. max 99100.0] n=51 --- -> bare long index median= 63304.0 ns [min 59079.0 .. max 131743.0] n=51 -> RowRange bounds (wrapper hoisted) median= 57557.0 ns [min 56212.0 .. max 87932.0] n=51 +> bare long index median= 59134.0 ns [min 59058.0 .. max 122610.0] n=51 +> RowRange bounds (wrapper hoisted) median= 56975.0 ns [min 56642.0 .. max 112526.0] n=51 > per-element wrapper: bytes allocated 1.50 MiB > ... per element 24.00 B -> Ordinal built per element median= 50021.0 ns [min 49762.0 .. max 75999.0] n=51 +> Ordinal built per element median= 49851.0 ns [min 49725.0 .. max 70877.0] n=51 68c68 < platform stable --- > platform valhalla 79,83c79,83 -< (2)/(3) hydrate 65536 Row ? allocated 2.00 MiB +< (2)/(3) hydrate 65536 Row — allocated 2.00 MiB < ... per row 32.00 B < array flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) < ratio vs native lane bytes 2.00x < retained heap (APPROX, gc-delta) 2.25 MiB --- -> (2)/(3) hydrate 65536 Row ? allocated 2.50 MiB +> (2)/(3) hydrate 65536 Row — allocated 2.50 MiB > ... per row 40.00 B > array flatness NOT-FLAT > ratio vs native lane bytes 2.50x > retained heap (APPROX, gc-delta) 2.75 MiB 86,89c86,89 -< (1) native one crossing, fused plan median= 16378.0 ns [min 15962.0 .. max 33819.0] n=51 -< (2/3) hydrate 65536 Row objects median= 603139.0 ns [min 529172.0 .. max 5246059.0] n=51 -< (2/3) scan the materialised objects median= 151886.0 ns [min 112687.0 .. max 201271.0] n=51 -< (2/3) hydrate THEN scan (honest total) median= 788227.0 ns [min 710221.0 .. max 1146017.0] n=51 +< (1) native one crossing, fused plan median= 16040.0 ns [min 15900.0 .. max 31577.0] n=51 +< (2/3) hydrate 65536 Row objects median= 1612494.0 ns [min 499222.0 .. max 4777342.0] n=51 +< (2/3) scan the materialised objects median= 128057.0 ns [min 112482.0 .. max 227766.0] n=51 +< (2/3) hydrate THEN scan (honest total) median= 704743.0 ns [min 631780.0 .. max 974235.0] n=51 --- -> (1) native one crossing, fused plan median= 18805.0 ns [min 15515.0 .. max 38457.0] n=51 -> (2/3) hydrate 65536 Row objects median= 768624.0 ns [min 645509.0 .. max 1229947.0] n=51 -> (2/3) scan the materialised objects median= 91178.0 ns [min 83390.0 .. max 141648.0] n=51 -> (2/3) hydrate THEN scan (honest total) median= 898955.0 ns [min 822997.0 .. max 5059624.0] n=51 +> (1) native one crossing, fused plan median= 15162.0 ns [min 14976.0 .. max 55083.0] n=51 +> (2/3) hydrate 65536 Row objects median= 658842.0 ns [min 594094.0 .. max 1085753.0] n=51 +> (2/3) scan the materialised objects median= 87213.0 ns [min 83223.0 .. max 135141.0] n=51 +> (2/3) hydrate THEN scan (honest total) median= 871361.0 ns [min 787060.0 .. max 1483819.0] n=51 diff --git a/valhalla-lab/results/stable-default.txt b/valhalla-lab/results/stable-default.txt index 3f30f78..5d91d9c 100644 --- a/valhalla-lab/results/stable-default.txt +++ b/valhalla-lab/results/stable-default.txt @@ -3,10 +3,10 @@ lance-graph-java :: valhalla lab platform stable java.vm.version 26.0.2+10-55 java.vendor.version - -jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so] +jvm args [--enable-native-access=ALL-UNNAMED, -Dstdout.encoding=UTF-8, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so] -== (a) SEMANTIC TRUTH ? is identity observable? ============================== -platform stable ? stable-record vocabulary; arrays are reference arrays; fields are references +== (a) SEMANTIC TRUTH — is identity observable? ============================== +platform stable — stable-record vocabulary; arrays are reference arrays; fields are references LaneId.class.isValue() false Ordinal.class.isValue() false MaskId.class.isValue() false @@ -24,7 +24,7 @@ array flatness UNKNOWN(no ValueClass API on a stab array slot accepts null true synchronized(x) legality legal but never used by the production API -== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +== FLATTENING CLIFF — which payload shapes does the VM flatten? ============== platform stable note payload = declared field bytes, ignoring any header LaneId 1 int payload= 4 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) @@ -33,14 +33,14 @@ MaskId 1 long payload= 8 B array=UNKNOWN(no ValueClass API on a RowRange 2 long payload=16 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) Row 1 long + 2 int payload=16 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) -== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +== (b)/(c) REPRESENTATION — allocation, arrays, fields, arguments ============ platform stable allocation instrument baseline 0 B N (operations per measurement) 1000000 construct N LaneId, store into array 15.26 MiB ... per LaneId 16.00 B -construct N LaneId, never escaping 6.71 MiB - ... per LaneId 7.03 B +construct N LaneId, never escaping 7.22 MiB + ... per LaneId 7.57 B LaneId[1024] flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) allocate+fill LaneId[N] (array + elements) 19.07 MiB ... per element 20.00 B @@ -50,21 +50,21 @@ Descriptor kind identity class with two reference f Descriptor fields null-restricted false construct N Descriptor (2 wrappers each) 53.41 MiB ... per Descriptor 56.00 B -pass 2 wrappers through 3 call levels 7.90 MiB - ... per call 8.29 B -read 65,536 LaneId from array median= 44182.0 ns [min 41690.0 .. max 59914.0] n=51 +pass 2 wrappers through 3 call levels 8.90 MiB + ... per call 9.33 B +read 65,536 LaneId from array median= 43739.0 ns [min 42371.0 .. max 73206.0] n=51 native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so -== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +== FFM ADDRESSING — is the wrapper free where it touches native memory? ====== platform stable sum (identical across all three) 6929623 -bare long index median= 59148.0 ns [min 59077.0 .. max 114952.0] n=51 -RowRange bounds (wrapper hoisted) median= 47952.0 ns [min 47759.0 .. max 65535.0] n=51 +bare long index median= 59836.0 ns [min 59076.0 .. max 125353.0] n=51 +RowRange bounds (wrapper hoisted) median= 47964.0 ns [min 47876.0 .. max 168118.0] n=51 per-element wrapper: bytes allocated 1.00 MiB ... per element 16.00 B -Ordinal built per element median= 50062.0 ns [min 49790.0 .. max 61194.0] n=51 +Ordinal built per element median= 50732.0 ns [min 50046.0 .. max 99100.0] n=51 -== THE THESIS ? 65,536 entities, three representations ======================= +== THE THESIS — 65,536 entities, three representations ======================= platform stable rows 65536 question count(class==7 AND value>100) and sum(value) @@ -72,20 +72,20 @@ answer (identical across all paths) 2173 rows, sum 499246 selectivity 3.32% == heap cost =============================================================== -(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself -(1) native ? Java objects per row 0 -(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) -(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) -(2)/(3) hydrate 65536 Row ? allocated 2.00 MiB +(1) native — Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native — Java objects per row 0 +(1) native — native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native — mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row — allocated 2.00 MiB ... per row 32.00 B array flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) ratio vs native lane bytes 2.00x retained heap (APPROX, gc-delta) 2.25 MiB == time to answer the question ============================================= -(1) native one crossing, fused plan median= 16378.0 ns [min 15962.0 .. max 33819.0] n=51 -(2/3) hydrate 65536 Row objects median= 603139.0 ns [min 529172.0 .. max 5246059.0] n=51 -(2/3) scan the materialised objects median= 151886.0 ns [min 112687.0 .. max 201271.0] n=51 -(2/3) hydrate THEN scan (honest total) median= 788227.0 ns [min 710221.0 .. max 1146017.0] n=51 +(1) native one crossing, fused plan median= 16040.0 ns [min 15900.0 .. max 31577.0] n=51 +(2/3) hydrate 65536 Row objects median= 1612494.0 ns [min 499222.0 .. max 4777342.0] n=51 +(2/3) scan the materialised objects median= 128057.0 ns [min 112482.0 .. max 227766.0] n=51 +(2/3) hydrate THEN scan (honest total) median= 704743.0 ns [min 631780.0 .. max 974235.0] n=51 lab complete. diff --git a/valhalla-lab/results/stable-noea.txt b/valhalla-lab/results/stable-noea.txt index 371624c..9a62f99 100644 --- a/valhalla-lab/results/stable-noea.txt +++ b/valhalla-lab/results/stable-noea.txt @@ -3,10 +3,10 @@ lance-graph-java :: valhalla lab platform stable java.vm.version 26.0.2+10-55 java.vendor.version - -jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, -XX:-DoEscapeAnalysis] +jvm args [--enable-native-access=ALL-UNNAMED, -Dstdout.encoding=UTF-8, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, -XX:-DoEscapeAnalysis] -== (a) SEMANTIC TRUTH ? is identity observable? ============================== -platform stable ? stable-record vocabulary; arrays are reference arrays; fields are references +== (a) SEMANTIC TRUTH — is identity observable? ============================== +platform stable — stable-record vocabulary; arrays are reference arrays; fields are references LaneId.class.isValue() false Ordinal.class.isValue() false MaskId.class.isValue() false @@ -24,7 +24,7 @@ array flatness UNKNOWN(no ValueClass API on a stab array slot accepts null true synchronized(x) legality legal but never used by the production API -== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +== FLATTENING CLIFF — which payload shapes does the VM flatten? ============== platform stable note payload = declared field bytes, ignoring any header LaneId 1 int payload= 4 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) @@ -33,7 +33,7 @@ MaskId 1 long payload= 8 B array=UNKNOWN(no ValueClass API on a RowRange 2 long payload=16 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) Row 1 long + 2 int payload=16 B array=UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) -== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +== (b)/(c) REPRESENTATION — allocation, arrays, fields, arguments ============ platform stable allocation instrument baseline 0 B N (operations per measurement) 1000000 @@ -52,19 +52,19 @@ construct N Descriptor (2 wrappers each) 53.41 MiB ... per Descriptor 56.00 B pass 2 wrappers through 3 call levels 30.52 MiB ... per call 32.00 B -read 65,536 LaneId from array median= 44861.0 ns [min 42156.0 .. max 69264.0] n=51 +read 65,536 LaneId from array median= 39359.0 ns [min 39266.0 .. max 63844.0] n=51 native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so -== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +== FFM ADDRESSING — is the wrapper free where it touches native memory? ====== platform stable sum (identical across all three) 6929623 -bare long index median= 59417.0 ns [min 59067.0 .. max 140131.0] n=51 -RowRange bounds (wrapper hoisted) median= 48136.0 ns [min 47847.0 .. max 105233.0] n=51 +bare long index median= 59174.0 ns [min 59067.0 .. max 117139.0] n=51 +RowRange bounds (wrapper hoisted) median= 47949.0 ns [min 47870.0 .. max 61164.0] n=51 per-element wrapper: bytes allocated 1.00 MiB ... per element 16.00 B -Ordinal built per element median= 5166233.0 ns [min 4636587.0 .. max 10323741.0] n=51 +Ordinal built per element median= 621827.0 ns [min 584504.0 .. max 873212.0] n=51 -== THE THESIS ? 65,536 entities, three representations ======================= +== THE THESIS — 65,536 entities, three representations ======================= platform stable rows 65536 question count(class==7 AND value>100) and sum(value) @@ -72,20 +72,20 @@ answer (identical across all paths) 2173 rows, sum 499246 selectivity 3.32% == heap cost =============================================================== -(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself -(1) native ? Java objects per row 0 -(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) -(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) -(2)/(3) hydrate 65536 Row ? allocated 2.00 MiB +(1) native — Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native — Java objects per row 0 +(1) native — native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native — mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row — allocated 2.00 MiB ... per row 32.00 B array flatness UNKNOWN(no ValueClass API on a stable JDK; a reference array is never flat) ratio vs native lane bytes 2.00x retained heap (APPROX, gc-delta) 2.22 MiB == time to answer the question ============================================= -(1) native one crossing, fused plan median= 17631.0 ns [min 15693.0 .. max 40703.0] n=51 -(2/3) hydrate 65536 Row objects median= 575654.0 ns [min 513071.0 .. max 4428601.0] n=51 -(2/3) scan the materialised objects median= 129828.0 ns [min 108841.0 .. max 159354.0] n=51 -(2/3) hydrate THEN scan (honest total) median= 703240.0 ns [min 642514.0 .. max 789762.0] n=51 +(1) native one crossing, fused plan median= 15431.0 ns [min 15368.0 .. max 28346.0] n=51 +(2/3) hydrate 65536 Row objects median= 537476.0 ns [min 478467.0 .. max 4136743.0] n=51 +(2/3) scan the materialised objects median= 118276.0 ns [min 108392.0 .. max 199142.0] n=51 +(2/3) hydrate THEN scan (honest total) median= 805562.0 ns [min 689612.0 .. max 1107352.0] n=51 lab complete. diff --git a/valhalla-lab/results/valhalla-default.txt b/valhalla-lab/results/valhalla-default.txt index 29b202f..43ebf51 100644 --- a/valhalla-lab/results/valhalla-default.txt +++ b/valhalla-lab/results/valhalla-default.txt @@ -3,10 +3,10 @@ lance-graph-java :: valhalla lab platform valhalla java.vm.version 27-jep401ea3+1-1 java.vendor.version - -jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED] +jvm args [--enable-native-access=ALL-UNNAMED, -Dstdout.encoding=UTF-8, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED] -== (a) SEMANTIC TRUTH ? is identity observable? ============================== -platform valhalla ? value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields +== (a) SEMANTIC TRUTH — is identity observable? ============================== +platform valhalla — value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields LaneId.class.isValue() true Ordinal.class.isValue() true MaskId.class.isValue() true @@ -24,7 +24,7 @@ array flatness FLAT array slot accepts null false synchronized(x) legality COMPILE ERROR under Valhalla (required: a type with identity) -== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +== FLATTENING CLIFF — which payload shapes does the VM flatten? ============== platform valhalla note payload = declared field bytes, ignoring any header LaneId 1 int payload= 4 B array=FLAT @@ -33,38 +33,38 @@ MaskId 1 long payload= 8 B array=FLAT RowRange 2 long payload=16 B array=NOT-FLAT Row 1 long + 2 int payload=16 B array=NOT-FLAT -== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +== (b)/(c) REPRESENTATION — allocation, arrays, fields, arguments ============ platform valhalla allocation instrument baseline 0 B N (operations per measurement) 1000000 -construct N LaneId, store into array 2.75 MiB - ... per LaneId 2.89 B -construct N LaneId, never escaping 7.63 MiB - ... per LaneId 8.00 B +construct N LaneId, store into array 3.14 MiB + ... per LaneId 3.29 B +construct N LaneId, never escaping 7.29 MiB + ... per LaneId 7.64 B LaneId[1024] flatness FLAT -allocate+fill LaneId[N] (array + elements) 6.57 MiB - ... per element 6.89 B +allocate+fill LaneId[N] (array + elements) 6.56 MiB + ... per element 6.87 B bare LaneId[N] with no elements stored 3.81 MiB ... per slot 4.00 B Descriptor kind value class with two @NullRestricted value fields Descriptor fields null-restricted true -construct N Descriptor (2 wrappers each) 37.05 MiB - ... per Descriptor 38.84 B -pass 2 wrappers through 3 call levels 9.97 MiB - ... per call 10.45 B -read 65,536 LaneId from array median= 5349.0 ns [min 5179.0 .. max 11356.0] n=51 +construct N Descriptor (2 wrappers each) 34.81 MiB + ... per Descriptor 36.50 B +pass 2 wrappers through 3 call levels 10.36 MiB + ... per call 10.87 B +read 65,536 LaneId from array median= 5339.0 ns [min 5262.0 .. max 9259.0] n=51 native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so -== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +== FFM ADDRESSING — is the wrapper free where it touches native memory? ====== platform valhalla sum (identical across all three) 6929623 -bare long index median= 63304.0 ns [min 59079.0 .. max 131743.0] n=51 -RowRange bounds (wrapper hoisted) median= 57557.0 ns [min 56212.0 .. max 87932.0] n=51 +bare long index median= 59134.0 ns [min 59058.0 .. max 122610.0] n=51 +RowRange bounds (wrapper hoisted) median= 56975.0 ns [min 56642.0 .. max 112526.0] n=51 per-element wrapper: bytes allocated 1.50 MiB ... per element 24.00 B -Ordinal built per element median= 50021.0 ns [min 49762.0 .. max 75999.0] n=51 +Ordinal built per element median= 49851.0 ns [min 49725.0 .. max 70877.0] n=51 -== THE THESIS ? 65,536 entities, three representations ======================= +== THE THESIS — 65,536 entities, three representations ======================= platform valhalla rows 65536 question count(class==7 AND value>100) and sum(value) @@ -72,20 +72,20 @@ answer (identical across all paths) 2173 rows, sum 499246 selectivity 3.32% == heap cost =============================================================== -(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself -(1) native ? Java objects per row 0 -(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) -(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) -(2)/(3) hydrate 65536 Row ? allocated 2.50 MiB +(1) native — Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native — Java objects per row 0 +(1) native — native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native — mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row — allocated 2.50 MiB ... per row 40.00 B array flatness NOT-FLAT ratio vs native lane bytes 2.50x retained heap (APPROX, gc-delta) 2.75 MiB == time to answer the question ============================================= -(1) native one crossing, fused plan median= 18805.0 ns [min 15515.0 .. max 38457.0] n=51 -(2/3) hydrate 65536 Row objects median= 768624.0 ns [min 645509.0 .. max 1229947.0] n=51 -(2/3) scan the materialised objects median= 91178.0 ns [min 83390.0 .. max 141648.0] n=51 -(2/3) hydrate THEN scan (honest total) median= 898955.0 ns [min 822997.0 .. max 5059624.0] n=51 +(1) native one crossing, fused plan median= 15162.0 ns [min 14976.0 .. max 55083.0] n=51 +(2/3) hydrate 65536 Row objects median= 658842.0 ns [min 594094.0 .. max 1085753.0] n=51 +(2/3) scan the materialised objects median= 87213.0 ns [min 83223.0 .. max 135141.0] n=51 +(2/3) hydrate THEN scan (honest total) median= 871361.0 ns [min 787060.0 .. max 1483819.0] n=51 lab complete. diff --git a/valhalla-lab/results/valhalla-noarrayflat.txt b/valhalla-lab/results/valhalla-noarrayflat.txt index 6c75ae5..be708cb 100644 --- a/valhalla-lab/results/valhalla-noarrayflat.txt +++ b/valhalla-lab/results/valhalla-noarrayflat.txt @@ -3,10 +3,10 @@ lance-graph-java :: valhalla lab platform valhalla java.vm.version 27-jep401ea3+1-1 java.vendor.version - -jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED, -XX:+UnlockDiagnosticVMOptions, -XX:-UseArrayFlattening] +jvm args [--enable-native-access=ALL-UNNAMED, -Dstdout.encoding=UTF-8, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED, -XX:+UnlockDiagnosticVMOptions, -XX:-UseArrayFlattening] -== (a) SEMANTIC TRUTH ? is identity observable? ============================== -platform valhalla ? value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields +== (a) SEMANTIC TRUTH — is identity observable? ============================== +platform valhalla — value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields LaneId.class.isValue() true Ordinal.class.isValue() true MaskId.class.isValue() true @@ -24,7 +24,7 @@ array flatness NOT-FLAT array slot accepts null false synchronized(x) legality COMPILE ERROR under Valhalla (required: a type with identity) -== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +== FLATTENING CLIFF — which payload shapes does the VM flatten? ============== platform valhalla note payload = declared field bytes, ignoring any header LaneId 1 int payload= 4 B array=NOT-FLAT @@ -33,14 +33,14 @@ MaskId 1 long payload= 8 B array=NOT-FLAT RowRange 2 long payload=16 B array=NOT-FLAT Row 1 long + 2 int payload=16 B array=NOT-FLAT -== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +== (b)/(c) REPRESENTATION — allocation, arrays, fields, arguments ============ platform valhalla allocation instrument baseline 0 B N (operations per measurement) 1000000 construct N LaneId, store into array 22.89 MiB ... per LaneId 24.00 B -construct N LaneId, never escaping 8.22 MiB - ... per LaneId 8.62 B +construct N LaneId, never escaping 7.26 MiB + ... per LaneId 7.61 B LaneId[1024] flatness NOT-FLAT allocate+fill LaneId[N] (array + elements) 26.70 MiB ... per element 28.00 B @@ -48,23 +48,23 @@ bare LaneId[N] with no elements stored 3.81 MiB ... per slot 4.00 B Descriptor kind value class with two @NullRestricted value fields Descriptor fields null-restricted true -construct N Descriptor (2 wrappers each) 38.37 MiB - ... per Descriptor 40.23 B -pass 2 wrappers through 3 call levels 11.49 MiB - ... per call 12.05 B -read 65,536 LaneId from array median= 53465.0 ns [min 49219.0 .. max 88865.0] n=51 +construct N Descriptor (2 wrappers each) 39.11 MiB + ... per Descriptor 41.01 B +pass 2 wrappers through 3 call levels 9.61 MiB + ... per call 10.07 B +read 65,536 LaneId from array median= 47245.0 ns [min 46645.0 .. max 80026.0] n=51 native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so -== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +== FFM ADDRESSING — is the wrapper free where it touches native memory? ====== platform valhalla sum (identical across all three) 6929623 -bare long index median= 59142.0 ns [min 59071.0 .. max 114270.0] n=51 -RowRange bounds (wrapper hoisted) median= 56997.0 ns [min 56660.0 .. max 81796.0] n=51 +bare long index median= 59875.0 ns [min 59085.0 .. max 132585.0] n=51 +RowRange bounds (wrapper hoisted) median= 56943.0 ns [min 56683.0 .. max 82939.0] n=51 per-element wrapper: bytes allocated 1.50 MiB ... per element 24.00 B -Ordinal built per element median= 51224.0 ns [min 49782.0 .. max 91443.0] n=51 +Ordinal built per element median= 49934.0 ns [min 49758.0 .. max 94648.0] n=51 -== THE THESIS ? 65,536 entities, three representations ======================= +== THE THESIS — 65,536 entities, three representations ======================= platform valhalla rows 65536 question count(class==7 AND value>100) and sum(value) @@ -72,20 +72,20 @@ answer (identical across all paths) 2173 rows, sum 499246 selectivity 3.32% == heap cost =============================================================== -(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself -(1) native ? Java objects per row 0 -(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) -(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) -(2)/(3) hydrate 65536 Row ? allocated 2.50 MiB - ... per row 40.00 B +(1) native — Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native — Java objects per row 0 +(1) native — native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native — mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row — allocated 2.50 MiB + ... per row 40.01 B array flatness NOT-FLAT ratio vs native lane bytes 2.50x retained heap (APPROX, gc-delta) 2.75 MiB == time to answer the question ============================================= -(1) native one crossing, fused plan median= 14946.0 ns [min 14841.0 .. max 49967.0] n=51 -(2/3) hydrate 65536 Row objects median= 1708590.0 ns [min 610808.0 .. max 8047970.0] n=51 -(2/3) scan the materialised objects median= 148382.0 ns [min 119556.0 .. max 229605.0] n=51 -(2/3) hydrate THEN scan (honest total) median= 843903.0 ns [min 781103.0 .. max 1306914.0] n=51 +(1) native one crossing, fused plan median= 16317.0 ns [min 15502.0 .. max 36120.0] n=51 +(2/3) hydrate 65536 Row objects median= 664702.0 ns [min 597322.0 .. max 1009820.0] n=51 +(2/3) scan the materialised objects median= 152176.0 ns [min 115700.0 .. max 238478.0] n=51 +(2/3) hydrate THEN scan (honest total) median= 1983973.0 ns [min 828040.0 .. max 6143362.0] n=51 lab complete. diff --git a/valhalla-lab/results/valhalla-noea.txt b/valhalla-lab/results/valhalla-noea.txt index 529ee73..79574fa 100644 --- a/valhalla-lab/results/valhalla-noea.txt +++ b/valhalla-lab/results/valhalla-noea.txt @@ -3,10 +3,10 @@ lance-graph-java :: valhalla lab platform valhalla java.vm.version 27-jep401ea3+1-1 java.vendor.version - -jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED, -XX:-DoEscapeAnalysis] +jvm args [--enable-native-access=ALL-UNNAMED, -Dstdout.encoding=UTF-8, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED, -XX:-DoEscapeAnalysis] -== (a) SEMANTIC TRUTH ? is identity observable? ============================== -platform valhalla ? value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields +== (a) SEMANTIC TRUTH — is identity observable? ============================== +platform valhalla — value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields LaneId.class.isValue() true Ordinal.class.isValue() true MaskId.class.isValue() true @@ -24,7 +24,7 @@ array flatness FLAT array slot accepts null false synchronized(x) legality COMPILE ERROR under Valhalla (required: a type with identity) -== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +== FLATTENING CLIFF — which payload shapes does the VM flatten? ============== platform valhalla note payload = declared field bytes, ignoring any header LaneId 1 int payload= 4 B array=FLAT @@ -33,12 +33,12 @@ MaskId 1 long payload= 8 B array=FLAT RowRange 2 long payload=16 B array=NOT-FLAT Row 1 long + 2 int payload=16 B array=NOT-FLAT -== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +== (b)/(c) REPRESENTATION — allocation, arrays, fields, arguments ============ platform valhalla allocation instrument baseline 0 B N (operations per measurement) 1000000 -construct N LaneId, store into array 3.86 MiB - ... per LaneId 4.05 B +construct N LaneId, store into array 3.90 MiB + ... per LaneId 4.09 B construct N LaneId, never escaping 24.41 MiB ... per LaneId 25.60 B LaneId[1024] flatness FLAT @@ -48,23 +48,23 @@ bare LaneId[N] with no elements stored 3.81 MiB ... per slot 4.00 B Descriptor kind value class with two @NullRestricted value fields Descriptor fields null-restricted true -construct N Descriptor (2 wrappers each) 38.98 MiB - ... per Descriptor 40.87 B -pass 2 wrappers through 3 call levels 29.57 MiB - ... per call 31.00 B -read 65,536 LaneId from array median= 5369.0 ns [min 5321.0 .. max 30666.0] n=51 +construct N Descriptor (2 wrappers each) 38.70 MiB + ... per Descriptor 40.58 B +pass 2 wrappers through 3 call levels 29.47 MiB + ... per call 30.90 B +read 65,536 LaneId from array median= 4954.0 ns [min 4910.0 .. max 5998.0] n=51 native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so -== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +== FFM ADDRESSING — is the wrapper free where it touches native memory? ====== platform valhalla sum (identical across all three) 6929623 -bare long index median= 61840.0 ns [min 59069.0 .. max 128839.0] n=51 -RowRange bounds (wrapper hoisted) median= 60149.0 ns [min 56823.0 .. max 74673.0] n=51 +bare long index median= 59216.0 ns [min 59058.0 .. max 112785.0] n=51 +RowRange bounds (wrapper hoisted) median= 56995.0 ns [min 56663.0 .. max 126870.0] n=51 per-element wrapper: bytes allocated 3.00 MiB ... per element 48.00 B -Ordinal built per element median= 94346.0 ns [min 91210.0 .. max 133533.0] n=51 +Ordinal built per element median= 91383.0 ns [min 91100.0 .. max 120947.0] n=51 -== THE THESIS ? 65,536 entities, three representations ======================= +== THE THESIS — 65,536 entities, three representations ======================= platform valhalla rows 65536 question count(class==7 AND value>100) and sum(value) @@ -72,20 +72,20 @@ answer (identical across all paths) 2173 rows, sum 499246 selectivity 3.32% == heap cost =============================================================== -(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself -(1) native ? Java objects per row 0 -(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) -(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) -(2)/(3) hydrate 65536 Row ? allocated 2.50 MiB +(1) native — Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native — Java objects per row 0 +(1) native — native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native — mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row — allocated 2.50 MiB ... per row 40.00 B array flatness NOT-FLAT ratio vs native lane bytes 2.50x retained heap (APPROX, gc-delta) 2.75 MiB == time to answer the question ============================================= -(1) native one crossing, fused plan median= 15885.0 ns [min 15694.0 .. max 32647.0] n=51 -(2/3) hydrate 65536 Row objects median= 1187577.0 ns [min 1044870.0 .. max 2002465.0] n=51 -(2/3) scan the materialised objects median= 74699.0 ns [min 63609.0 .. max 129188.0] n=51 -(2/3) hydrate THEN scan (honest total) median= 3267205.0 ns [min 1374933.0 .. max 9370919.0] n=51 +(1) native one crossing, fused plan median= 20603.0 ns [min 15179.0 .. max 59260.0] n=51 +(2/3) hydrate 65536 Row objects median= 1086237.0 ns [min 1010540.0 .. max 4481968.0] n=51 +(2/3) scan the materialised objects median= 80753.0 ns [min 75090.0 .. max 140418.0] n=51 +(2/3) hydrate THEN scan (honest total) median=25828809.0 ns [min 6057980.0 .. max 39247669.0] n=51 lab complete. diff --git a/valhalla-lab/results/valhalla-nofieldflat.txt b/valhalla-lab/results/valhalla-nofieldflat.txt index 7af9d7b..592d4d9 100644 --- a/valhalla-lab/results/valhalla-nofieldflat.txt +++ b/valhalla-lab/results/valhalla-nofieldflat.txt @@ -3,10 +3,10 @@ lance-graph-java :: valhalla lab platform valhalla java.vm.version 27-jep401ea3+1-1 java.vendor.version - -jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED, -XX:+UnlockDiagnosticVMOptions, -XX:-UseFieldFlattening] +jvm args [--enable-native-access=ALL-UNNAMED, -Dstdout.encoding=UTF-8, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED, -XX:+UnlockDiagnosticVMOptions, -XX:-UseFieldFlattening] -== (a) SEMANTIC TRUTH ? is identity observable? ============================== -platform valhalla ? value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields +== (a) SEMANTIC TRUTH — is identity observable? ============================== +platform valhalla — value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields LaneId.class.isValue() true Ordinal.class.isValue() true MaskId.class.isValue() true @@ -24,7 +24,7 @@ array flatness FLAT array slot accepts null false synchronized(x) legality COMPILE ERROR under Valhalla (required: a type with identity) -== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +== FLATTENING CLIFF — which payload shapes does the VM flatten? ============== platform valhalla note payload = declared field bytes, ignoring any header LaneId 1 int payload= 4 B array=FLAT @@ -33,38 +33,38 @@ MaskId 1 long payload= 8 B array=FLAT RowRange 2 long payload=16 B array=NOT-FLAT Row 1 long + 2 int payload=16 B array=NOT-FLAT -== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +== (b)/(c) REPRESENTATION — allocation, arrays, fields, arguments ============ platform valhalla allocation instrument baseline 0 B N (operations per measurement) 1000000 -construct N LaneId, store into array 2.75 MiB - ... per LaneId 2.89 B -construct N LaneId, never escaping 8.22 MiB - ... per LaneId 8.61 B +construct N LaneId, store into array 2.58 MiB + ... per LaneId 2.71 B +construct N LaneId, never escaping 6.68 MiB + ... per LaneId 7.00 B LaneId[1024] flatness FLAT -allocate+fill LaneId[N] (array + elements) 6.58 MiB - ... per element 6.90 B +allocate+fill LaneId[N] (array + elements) 6.40 MiB + ... per element 6.71 B bare LaneId[N] with no elements stored 3.81 MiB ... per slot 4.00 B Descriptor kind value class with two @NullRestricted value fields Descriptor fields null-restricted true construct N Descriptor (2 wrappers each) 76.29 MiB ... per Descriptor 80.00 B -pass 2 wrappers through 3 call levels 11.14 MiB - ... per call 11.68 B -read 65,536 LaneId from array median= 4959.0 ns [min 4870.0 .. max 5886.0] n=51 +pass 2 wrappers through 3 call levels 11.85 MiB + ... per call 12.43 B +read 65,536 LaneId from array median= 4952.0 ns [min 4878.0 .. max 6129.0] n=51 native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so -== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +== FFM ADDRESSING — is the wrapper free where it touches native memory? ====== platform valhalla sum (identical across all three) 6929623 -bare long index median= 59187.0 ns [min 59075.0 .. max 123125.0] n=51 -RowRange bounds (wrapper hoisted) median= 56980.0 ns [min 56652.0 .. max 73287.0] n=51 +bare long index median= 59291.0 ns [min 59072.0 .. max 127512.0] n=51 +RowRange bounds (wrapper hoisted) median= 56959.0 ns [min 56598.0 .. max 88827.0] n=51 per-element wrapper: bytes allocated 1.50 MiB ... per element 24.00 B -Ordinal built per element median= 49851.0 ns [min 49727.0 .. max 66813.0] n=51 +Ordinal built per element median= 53651.0 ns [min 49983.0 .. max 89971.0] n=51 -== THE THESIS ? 65,536 entities, three representations ======================= +== THE THESIS — 65,536 entities, three representations ======================= platform valhalla rows 65536 question count(class==7 AND value>100) and sum(value) @@ -72,20 +72,20 @@ answer (identical across all paths) 2173 rows, sum 499246 selectivity 3.32% == heap cost =============================================================== -(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself -(1) native ? Java objects per row 0 -(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) -(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) -(2)/(3) hydrate 65536 Row ? allocated 2.50 MiB - ... per row 40.01 B +(1) native — Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native — Java objects per row 0 +(1) native — native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native — mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row — allocated 2.50 MiB + ... per row 40.00 B array flatness NOT-FLAT ratio vs native lane bytes 2.50x retained heap (APPROX, gc-delta) 2.75 MiB == time to answer the question ============================================= -(1) native one crossing, fused plan median= 15964.0 ns [min 15779.0 .. max 40503.0] n=51 -(2/3) hydrate 65536 Row objects median= 789127.0 ns [min 628024.0 .. max 4599451.0] n=51 -(2/3) scan the materialised objects median= 78249.0 ns [min 75327.0 .. max 111755.0] n=51 -(2/3) hydrate THEN scan (honest total) median= 1122018.0 ns [min 812442.0 .. max 1695253.0] n=51 +(1) native one crossing, fused plan median= 13900.0 ns [min 13785.0 .. max 32163.0] n=51 +(2/3) hydrate 65536 Row objects median= 685073.0 ns [min 615948.0 .. max 5630226.0] n=51 +(2/3) scan the materialised objects median= 89883.0 ns [min 86979.0 .. max 142069.0] n=51 +(2/3) hydrate THEN scan (honest total) median= 872592.0 ns [min 804643.0 .. max 1338372.0] n=51 lab complete. diff --git a/valhalla-lab/results/valhalla-noflat.txt b/valhalla-lab/results/valhalla-noflat.txt index 5ab9312..4475065 100644 --- a/valhalla-lab/results/valhalla-noflat.txt +++ b/valhalla-lab/results/valhalla-noflat.txt @@ -3,10 +3,10 @@ lance-graph-java :: valhalla lab platform valhalla java.vm.version 27-jep401ea3+1-1 java.vendor.version - -jvm args [--enable-native-access=ALL-UNNAMED, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED, -XX:+UnlockDiagnosticVMOptions, -XX:-UseArrayFlattening, -XX:-UseFieldFlattening] +jvm args [--enable-native-access=ALL-UNNAMED, -Dstdout.encoding=UTF-8, -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so, --enable-preview, --add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED, --add-exports=java.base/jdk.internal.value=ALL-UNNAMED, -XX:+UnlockDiagnosticVMOptions, -XX:-UseArrayFlattening, -XX:-UseFieldFlattening] -== (a) SEMANTIC TRUTH ? is identity observable? ============================== -platform valhalla ? value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields +== (a) SEMANTIC TRUTH — is identity observable? ============================== +platform valhalla — value-record vocabulary; null-restricted non-atomic arrays; @NullRestricted fields LaneId.class.isValue() true Ordinal.class.isValue() true MaskId.class.isValue() true @@ -24,7 +24,7 @@ array flatness NOT-FLAT array slot accepts null false synchronized(x) legality COMPILE ERROR under Valhalla (required: a type with identity) -== FLATTENING CLIFF ? which payload shapes does the VM flatten? ============== +== FLATTENING CLIFF — which payload shapes does the VM flatten? ============== platform valhalla note payload = declared field bytes, ignoring any header LaneId 1 int payload= 4 B array=NOT-FLAT @@ -33,14 +33,14 @@ MaskId 1 long payload= 8 B array=NOT-FLAT RowRange 2 long payload=16 B array=NOT-FLAT Row 1 long + 2 int payload=16 B array=NOT-FLAT -== (b)/(c) REPRESENTATION ? allocation, arrays, fields, arguments ============ +== (b)/(c) REPRESENTATION — allocation, arrays, fields, arguments ============ platform valhalla allocation instrument baseline 0 B N (operations per measurement) 1000000 construct N LaneId, store into array 15.26 MiB ... per LaneId 16.00 B -construct N LaneId, never escaping 4.71 MiB - ... per LaneId 4.94 B +construct N LaneId, never escaping 4.70 MiB + ... per LaneId 4.93 B LaneId[1024] flatness NOT-FLAT allocate+fill LaneId[N] (array + elements) 19.07 MiB ... per element 20.00 B @@ -50,21 +50,21 @@ Descriptor kind value class with two @NullRestricte Descriptor fields null-restricted true construct N Descriptor (2 wrappers each) 53.41 MiB ... per Descriptor 56.00 B -pass 2 wrappers through 3 call levels 8.65 MiB - ... per call 9.07 B -read 65,536 LaneId from array median= 43102.0 ns [min 41523.0 .. max 77929.0] n=51 +pass 2 wrappers through 3 call levels 9.41 MiB + ... per call 9.86 B +read 65,536 LaneId from array median= 41431.0 ns [min 41208.0 .. max 59606.0] n=51 native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so -== FFM ADDRESSING ? is the wrapper free where it touches native memory? ====== +== FFM ADDRESSING — is the wrapper free where it touches native memory? ====== platform valhalla sum (identical across all three) 6929623 -bare long index median= 59218.0 ns [min 59090.0 .. max 150920.0] n=51 -RowRange bounds (wrapper hoisted) median= 56978.0 ns [min 56635.0 .. max 83751.0] n=51 +bare long index median= 59150.0 ns [min 59081.0 .. max 122825.0] n=51 +RowRange bounds (wrapper hoisted) median= 56919.0 ns [min 56634.0 .. max 79142.0] n=51 per-element wrapper: bytes allocated 1.00 MiB ... per element 16.00 B -Ordinal built per element median= 49935.0 ns [min 49722.0 .. max 83957.0] n=51 +Ordinal built per element median= 49921.0 ns [min 49739.0 .. max 69259.0] n=51 -== THE THESIS ? 65,536 entities, three representations ======================= +== THE THESIS — 65,536 entities, three representations ======================= platform valhalla rows 65536 question count(class==7 AND value>100) and sum(value) @@ -72,20 +72,20 @@ answer (identical across all paths) 2173 rows, sum 499246 selectivity 3.32% == heap cost =============================================================== -(1) native ? Java bytes allocated (warm) 816 B per query, for the fluent chain itself -(1) native ? Java objects per row 0 -(1) native ? native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) -(1) native ? mask bytes 8.0 KiB (1 bit per row, packed) -(2)/(3) hydrate 65536 Row ? allocated 2.00 MiB - ... per row 32.00 B +(1) native — Java bytes allocated (warm) 816 B per query, for the fluent chain itself +(1) native — Java objects per row 0 +(1) native — native lane bytes 1.00 MiB (u64 id + u32 class + i32 value) +(1) native — mask bytes 8.0 KiB (1 bit per row, packed) +(2)/(3) hydrate 65536 Row — allocated 2.00 MiB + ... per row 32.01 B array flatness NOT-FLAT ratio vs native lane bytes 2.00x retained heap (APPROX, gc-delta) 2.25 MiB == time to answer the question ============================================= -(1) native one crossing, fused plan median= 15435.0 ns [min 15335.0 .. max 30493.0] n=51 -(2/3) hydrate 65536 Row objects median= 857050.0 ns [min 533019.0 .. max 2422972.0] n=51 -(2/3) scan the materialised objects median= 124600.0 ns [min 112855.0 .. max 180439.0] n=51 -(2/3) hydrate THEN scan (honest total) median= 1631916.0 ns [min 1499609.0 .. max 2069662.0] n=51 +(1) native one crossing, fused plan median= 16049.0 ns [min 15609.0 .. max 32010.0] n=51 +(2/3) hydrate 65536 Row objects median= 578187.0 ns [min 543201.0 .. max 1011453.0] n=51 +(2/3) scan the materialised objects median= 119838.0 ns [min 111905.0 .. max 211651.0] n=51 +(2/3) hydrate THEN scan (honest total) median= 1635251.0 ns [min 709125.0 .. max 6428085.0] n=51 lab complete. From e4463cb8ab9616291fc80deeb7b7b2ca6094d639 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 22:05:18 +0000 Subject: [PATCH 05/30] Board: PR #4 arc entry (post-merge) Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/PR_ARC_INVENTORY.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index eacf076..b6451d4 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -8,6 +8,26 @@ > anti-pattern the imported board rules name. Backfilled below in one > pass rather than left stale; PR #4 onward gets its entry at merge time. +## PR #4 — Phase I synthesis docs + fusion re-run + board hygiene (merged 2026-08-17, squash `bd92c58`) + +- **Added:** `docs/{architecture,panama,valhalla-lab,execution-boundary}.md` + (D-LGJ-I DONE — synthesis, each claim tied to its proving artifact); + the fusion-sweep 256-row re-run (`RESULTS.md` rewritten from + `jmh-results-merged.csv`, `TABLES.md` mechanically generated from the + same file); refreshed Valhalla lab result files (findings unchanged). +- **Locked:** the fusion self-correction — "fused ≈ unfused" was true + only at 65,536 rows; at 256 rows × 8 predicates unfused/fused = 2.99×. + The `MultiLaneColumn` decision + (`E-LGJ-SIMD-SOA-IS-FOR-THE-ROW-STORE-NOT-THE-FLAT-LANES-1`): declined + for flat lanes, earmarked for the 512-byte row-store slice; operator + layout reference (64K × 512 B, 32 × (4 B classid + 12 B)) recorded. +- **Deferred:** `NodeRow`/`WideFieldMask` wiring (unchanged); + `MultiLaneColumn` adoption gated on that slice. +- **Docs:** the four docs ARE the deliverable; board updated in-commit, + incl. this file's #1-#3 backfill (lapse owned above). +- **Confidence:** High — docs-only + measured data; both bot reviewers + (cursor, codex) hit usage limits and did not run. + ## PR #3 — Vector API bench: real JMH, cross-checked (merged 2026-08-17, squash) - **Added:** `bench/` — real JMH 1.37 suite (Components A/B/C/E: From e8b6dc2b77d177ec782f5d5c7c8884a3417eda55 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 22:29:32 +0000 Subject: [PATCH 06/30] SoA row store: 512B rows, 32 facet lanes, ABI minor 2 (W1+W2) The lance-graph-shaped substrate, wired end to end. The flat three-lane fixture was always scaffolding (docs/abi.md 10, architecture.md said so from PR #1); this is the layout the stack actually converges on: 64K x 512-byte rows, 32 facet lanes of 16 bytes = 4-byte LE classid + 12-byte payload, the V3 content-blind facet. Rust (native/lgj-abi): - rowstore.rs: one Arc<[u8]>, two readings (row-major chunks and strided facet columns), zero copies, normative SplitMix64 generator. - LGJ_RESOURCE_ROWSTORE + lgj_rowstore_open; facet lanes described through the UNCHANGED LgjLaneDesc (stride_bytes carried this since minor 1); lgj_op_eq_classid produces ordinary masks that compose with the existing algebra; lgj_row_facet_match writes per-row 32-bit facet sets into a caller-owned buffer via MultiLaneColumn (Arc refcount bump, no copy). - byte_len tightened to the exact covered span (len-1)*stride + elem_bytes: a full-stride final window would let Java bound a segment past the allocation's end on a facet lane. - ABI minor 1 -> 2; docs/abi.md gains 11 and its symbol count is corrected (the 14 was drift; the list already enumerated 15, and the real number is now 18 per nm -D). Gates: cargo test 84/84, clippy -D warnings clean, fmt clean, release build exports 18/18 symbols. Both new kernels are parity-checked against independent scalar references over 10 row counts x 2 seeds x 4 facets x 4 needles, then cross-checked a third way against RowStore::classid_at; a two-sided falsifier proves payload bytes never satisfy a classid match and that a real match does fire. Docs: .claude/plans/lgj-soa-substrate-v1.md (W1-W5 waves) + one plan per consumer example (world-trades / bricks-analytics / graph-traversal), .claude/knowledge/soa-row-store-layout.md, and the board triple ledger. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/EPIPHANIES.md | 68 ++++ .claude/board/INTEGRATION_PLANS.md | 45 +++ .claude/board/LATEST_STATE.md | 50 +++ .claude/board/STATUS_BOARD.md | 13 + .claude/knowledge/soa-row-store-layout.md | 88 ++++++ .claude/plans/consumer-bricks-analytics-v1.md | 67 ++++ .claude/plans/consumer-graph-traversal-v1.md | 70 +++++ .claude/plans/consumer-world-trades-v1.md | 58 ++++ .claude/plans/lgj-soa-substrate-v1.md | 82 +++++ docs/abi.md | 87 +++++- native/lgj-abi/src/abi.rs | 20 +- native/lgj-abi/src/exports.rs | 200 +++++++++++- native/lgj-abi/src/kernels.rs | 216 +++++++++++++ native/lgj-abi/src/lib.rs | 113 +++++++ native/lgj-abi/src/registry.rs | 73 ++++- native/lgj-abi/src/rowstore.rs | 292 ++++++++++++++++++ 16 files changed, 1514 insertions(+), 28 deletions(-) create mode 100644 .claude/knowledge/soa-row-store-layout.md create mode 100644 .claude/plans/consumer-bricks-analytics-v1.md create mode 100644 .claude/plans/consumer-graph-traversal-v1.md create mode 100644 .claude/plans/consumer-world-trades-v1.md create mode 100644 .claude/plans/lgj-soa-substrate-v1.md create mode 100644 native/lgj-abi/src/rowstore.rs diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 4c7400c..542dd59 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -4,6 +4,74 @@ > `**Status:**`/`**Confidence:**` line. A correction gets its own new, > dated entry that references the one it corrects — the storno rule. +## 2026-08-17 — E-LGJ-THE-MIDDLE-TIER-IS-DELETED-NOT-WRAPPED-1 + +**Status:** DOCTRINE (operator-stated, scope confirmed). **Confidence:** High — +four directives + three posters, restated and confirmed in session. + +The blast radius, recorded because a session that reads this repo as "a faster +Java binding to a Rust library" will make locally-sensible decisions that are +globally wrong: + +1. **The middle of the Java data stack is deleted, not wrapped.** Today: + App → DTO/ORM → Gremlin/TinkerPop → JanusGraph → Cassandra → Elastic / + ClickHouse / Lucene = six components, five serialization boundaries, three + mental models. After: **one** explicit ABI boundary, **zero** serialization + boundaries. The middleware and side-car analytics tiers do not get wrapped — + lance-graph + ndarray under one Panama membrane already *are* the traversal, + analytics and search substrate. *"Java als low-code Oberfläche, ABI als + Wahrheit."* +2. **Objects are eliminated, not optimized.** 10⁹ logical entities ⇒ **0** Java + objects: no header tax, no GC churn, masks instead of pointers, survivors + only touch heavy data. Valhalla's role is narrow and already measured here — + it makes the *tiny descriptor vocabulary* free (≤8 B flattens; the 16 B + entity does not), which is exactly why entities stay native and descriptors + stay `record`-shaped. +3. **The trust boundary collapses with the data boundary.** Mask-first: the + RBAC/ABAC clamp composes BEFORE execution, the scan runs on authorized lanes + only, and only aggregates/projections leave. Security enforced at the source + is a *consequence* of zero-copy, not a feature bolted on. +4. **The migration asymmetry is the weapon.** The developer-visible diff is + `stream().filter(λ)` → `.where(Field.gt(...))`; everything underneath changes + universe. Hence the standing rule: **the ABI is a machine membrane and never + the product API** — the product is the illusion that ordinary Java just works + at 10⁹ objects. + +Operator's compression: *"Java Panama and Valhalla become the supraconductor +over lance-graph ABI shaped SoA substrate."* Supraconductor is precise — current +(the query) flows with no resistance (no allocation, no GC, no serialization) +through a thin familiar surface. + +**Consequence for review:** any proposal that adds a serialization step, a +per-element crossing, an object materialization, or a post-filter security check +is not a tradeoff to weigh — it contradicts the thesis and is rejected. + +## 2026-08-17 — E-LGJ-THE-FLAT-FIXTURE-WAS-SCAFFOLDING-NOT-THE-TARGET-1 + +**Status:** CORRECTION (of my own framing). **Confidence:** High — operator +correction, acted on the same session. + +I answered the `simd_soa` question by measuring `MultiLaneColumn` against the +**flat three-lane fixture**, found two real API mismatches, and recorded a +"declined for now" verdict. The operator corrected the frame: *"the whole point +is Java should optimize the SoA layout — we won't dismiss the initial plans +just because you found it doesn't apply for unorganized non-SoA."* + +The technical findings were right and are unchanged (see the entry below); the +**conclusion drawn from them was scoped wrong**. The flat fixture was always +scaffolding — `docs/abi.md` §10 and `architecture.md` said so from PR #1 ("the +generic fixture in this first slice was deliberately chosen … so the membrane's +physics could be proven independent of graph semantics"). Measuring a +substrate-shaped tool against the scaffolding and concluding "not yet" inverted +which one was provisional. + +**The generalizable failure:** when a proposal doesn't fit the *current* code, +check whether the proposal is early or whether the **code is the placeholder**. +Here the code was the placeholder, and the right move was to build the real +shape (the 512-byte row store, W2, shipped same session) rather than defer the +tool. A "declined, revisit later" verdict is only honest when the thing it was +measured against is the thing that stays. + ## 2026-08-17 — E-LGJ-SIMD-SOA-IS-FOR-THE-ROW-STORE-NOT-THE-FLAT-LANES-1 **Status:** DECISION (declined refactor, with the trigger for revisiting named). diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index 4419559..65043dd 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -1,3 +1,48 @@ +## 2026-08-17 — consumer-{world-trades,bricks-analytics,graph-traversal}-v1 (PLANS; the three W5 consumer examples) + +Plans: `.claude/plans/consumer-world-trades-v1.md`, +`consumer-bricks-analytics-v1.md`, `consumer-graph-traversal-v1.md`. + +One operator poster made runnable per plan, all three over the SAME +substrate, each exercising a different face: the fluent domain API with +zero object allocation (trades); mask-first authorization where the RBAC +clamp composes BEFORE execution and only aggregates leave (bricks); and +traversal as facet addressing with crossings that scale with HOPS, not +rows (graph). Each carries its own falsifier set, including the +anti-vacuity requirement that a result be neither empty nor total. + +**Sequencing:** all three are gated on `lgj-soa-substrate-v1` W3 (the +Java `RowStore` facade); after that they are independently shippable in +any order and none blocks the others. **Iron rule recorded in all three:** +a consumer example that needs a new ABI symbol goes back through the +substrate plan's wave process — the membrane never grows from the consumer +side. + +**Status: PLANNED.** + +## 2026-08-17 — lgj-soa-substrate-v1 (PLAN; the lance-graph-shaped SoA substrate) + +Plan: `.claude/plans/lgj-soa-substrate-v1.md`. Successor to +`lgj-vertical-slice-v1` (COMPLETE, PRs #1-#4). + +**What it covers:** the real layout — 64K × 512-byte rows, 32 facet lanes +of (4-byte classid + 12-byte payload) — wired end to end, in five waves: +W1 ndarray primitives (`iter_u32x16`, `eq_u32_strided_to_mask`), W2 the +Rust row store (`LGJ_RESOURCE_ROWSTORE`, `lgj_rowstore_open`, +`lgj_op_eq_classid`, `lgj_row_facet_match`, ABI minor 2, `abi.md` §11), +W3 the Java `RowStore` facade, W4 a Vector-API-vs-crossing bench on the +REAL layout, W5 the three consumer examples. + +**Framing decision on record:** this plan exists because the flat +three-lane fixture was always scaffolding +(`E-LGJ-THE-FLAT-FIXTURE-WAS-SCAFFOLDING-NOT-THE-TARGET-1`). The doctrine +it serves — the middle tier is deleted rather than wrapped, objects are +eliminated rather than optimized, security collapses into the data +boundary — is `E-LGJ-THE-MIDDLE-TIER-IS-DELETED-NOT-WRAPPED-1`. + +**Status: ACTIVE.** W1 shipped (ndarray PR #279); W2 shipped (84/84, +18/18 symbols); W3 is the next action. + ## 2026-08-17 — lgj-vertical-slice-v1 (PLAN; the first Panama×Valhalla×ndarray::simd proof) Plan: `.claude/plans/lgj-vertical-slice-v1.md`. Active plan index — this diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 66f2e2e..941243c 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,53 @@ +## 2026-08-17 (Slice 2) — the SoA row store is REAL: ABI minor 2, W1+W2 shipped + +**The reframing that started it** (operator, three directives): the flat +three-lane fixture was always scaffolding; Java is meant to optimize the *SoA +layout*; serialization is abandoned outright in favor of lance-graph's 64K +zero-copy concurrency + the ndarray SIMD polyfill; Panama+Valhalla are "the +supraconductor over lance-graph ABI shaped SoA substrate". Doctrine on the +board as `E-LGJ-THE-MIDDLE-TIER-IS-DELETED-NOT-WRAPPED-1`; my own mis-scoped +"declined" verdict corrected in +`E-LGJ-THE-FLAT-FIXTURE-WAS-SCAFFOLDING-NOT-THE-TARGET-1`. + +**Layout now in code** (operator-stated reference): 64K × **512 B rows, 32 +facet lanes of 16 B = 4-byte LE classid + 12-byte payload**, the lance-graph +V3 content-blind facet. Java's own view may differ — these bytes are the +substrate truth. Full statement: `.claude/knowledge/soa-row-store-layout.md`. + +- **W1 (ndarray PR #279, open):** `MultiLaneColumn::iter_u32x16`/`len_u32x16` + (the u32 lane whose absence was the real blocker) + `eq_u32_strided_to_mask` + (the AoS-facet classid scan, overflow-checked bounds). `simd_int_ops` 46/46, + `simd_soa` 15/15, `simd` 263/263, clippy/fmt clean, both x86 arms. +- **W2 (this repo):** `rowstore.rs` + `LGJ_RESOURCE_ROWSTORE` + + `lgj_rowstore_open` + `lgj_op_eq_classid` + `lgj_row_facet_match`; facet + lanes ride the **unchanged** `LgjLaneDesc` (`stride_bytes` has carried this + since minor 1). ABI **minor 1→2**, `docs/abi.md` §11 written, and the §1/§7 + "14 symbols" count corrected (its own list already enumerated 15; the real + number is now 18, verified by `nm -D`). `cargo test` **84/84**, clippy + `-D warnings` + fmt clean. +- **Parity, three independent ways:** each SIMD kernel vs its independent + scalar reference over 10 row counts × 2 seeds × 4 facets × 4 needles, then + both cross-checked against `RowStore::classid_at`. Two-sided falsifier proves + payload bytes carrying the needle's bit pattern never satisfy a classid + match, and that a real classid match does fire. +- **`byte_len` semantics tightened** to the exact covered span + `(len-1)*stride + elem_bytes` — a full-stride final window would have let + Java bound a segment past the allocation's end on a facet lane. Contiguous + lanes unchanged. +- **Masks parent onto row stores**, so the entire existing mask algebra applies + with no new surface (proven end-to-end through the membrane). + +**Planned and written this session:** `.claude/plans/lgj-soa-substrate-v1.md` +(the W1–W5 wave plan) plus one plan per consumer example — +`consumer-world-trades-v1.md` (zero-object fluent domain API), +`consumer-bricks-analytics-v1.md` (mask-first RBAC, fail-closed, aggregates +only), `consumer-graph-traversal-v1.md` (traversal as facet addressing, +crossings ∝ hops). Iron rule in all three: **a consumer example never grows the +membrane** — a needed symbol goes back through the wave process. + +**Next:** W3, the Java `RowStore` facade (structured `MemoryLayout`, +minor-≥2 gate, `FacetMatchView`, generator-transcribing parity test). + ## 2026-08-17 (later) — Phase I docs written, fusion re-run merged, simd_soa question answered (PR #4) - **All four synthesis docs shipped** (`docs/architecture.md`, diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 0f6883e..fd5f122 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -28,3 +28,16 @@ and D-LGJ-G are the only rows still open; they are deliberately NOT blocking PR #1 (the core slice is independently complete and green) and will land as their own PR once the Lab agent finishes and is reviewed with the same rigor. + +## lgj-soa-substrate-v1 — the lance-graph-shaped SoA substrate (2026-08-17) + +Plan: `.claude/plans/lgj-soa-substrate-v1.md`. The 512-byte row / 32-facet +layout wired end to end. Doctrine: `E-LGJ-THE-MIDDLE-TIER-IS-DELETED-NOT-WRAPPED-1`. + +| D-id | Deliverable | Status | +|---|---|---| +| D-LGJ-W1 | ndarray: `MultiLaneColumn::iter_u32x16`/`len_u32x16` + `eq_u32_strided_to_mask` (W1a contract) | **DONE 2026-08-17** — ndarray PR #279; `simd_int_ops` 46/46 (5 new strided tests incl. two `should_panic` bounds/overflow arms + stride-4 parity against the contiguous primitive), `simd_soa` 15/15, full `simd` 263/263, doctests, clippy `-D warnings` + fmt clean | +| D-LGJ-W2 | lgj-abi row store: `rowstore.rs`, `LGJ_RESOURCE_ROWSTORE`, `lgj_rowstore_open`, strided facet lanes through the unchanged `LgjLaneDesc`, `lgj_op_eq_classid`, `lgj_row_facet_match`, ABI minor 1→2, `docs/abi.md` §11 | **DONE 2026-08-17** — `cargo test` **84/84**, clippy/fmt clean, release build exports **18/18** symbols (`nm -D`). Parity: both kernels vs independent scalar references over 10 row counts × 2 seeds × 4 facets × 4 needles, cross-checked a THIRD way against `RowStore::classid_at`. Two-sided payload-vs-classid falsifier. End-to-end membrane test covers describe → predicate → mask algebra → count → facet-match → lifecycle | +| D-LGJ-W3 | Java `RowStore` facade: structured `MemoryLayout`, minor-≥2 gate, `FacetMatchView`, parity test transcribing the generator | **NEXT** | +| D-LGJ-W4 | Bench Component F: Vector API facet scan vs the crossing, on the REAL layout | Queued | +| D-LGJ-W5 | Three consumer examples (trades / bricks / graph) — one plan file each | Planned, gated on W3 | diff --git a/.claude/knowledge/soa-row-store-layout.md b/.claude/knowledge/soa-row-store-layout.md new file mode 100644 index 0000000..86484b3 --- /dev/null +++ b/.claude/knowledge/soa-row-store-layout.md @@ -0,0 +1,88 @@ +# The SoA row store layout — the substrate everything converges on + +> **READ BY:** `abi-membrane-warden`, `simd-savant`, `panama-bridge-engineer`, +> `java-surface-warden`, and any session touching `rowstore.rs`, +> `docs/abi.md` §11, the Java `RowStore` facade, or a consumer example. +> **MANDATORY** before proposing any change to row geometry, facet +> semantics, or the lane map. + +## The layout (operator-stated, 2026-08-17) + +> *"the 64k x 512 bytes SoA layout is enforced everywhere in lance-graph +> (32 Lanes each 4 bytes classview+12 bytes). For Java the layout might +> differ just for reference."* + +``` +row (512 B) = 32 × facet (16 B) +facet (16 B) = classid (4 B, little-endian u32) ++ payload (12 B) +``` + +This is the lance-graph **V3 content-blind facet** — the same shape the +sibling repos pin as canon (`E-V3-FACET-4-PLUS-12`; the 12 bytes are an +axis-grouped byte register read as `6×(u8:u8)` / `4×(u8:u8:u8)` / +`3×(u8:u8:u8:u8)` per the ClassView, never widened). `lgj-abi` treats the +12 bytes as opaque **on purpose**: the ABI is a machine membrane and the +payload's *reading* is a ClassView concern one layer up. + +**"For Java the layout might differ" is load-bearing.** The Java side is +free to project a different view (a structured `MemoryLayout`, a different +field grouping, a Valhalla-shaped descriptor vocabulary). These bytes are +the substrate truth; the Java view is a *reading* of them. Nothing in the +Java facade may assume its own view is the storage layout. + +## The two readings, one buffer, zero copies + +| reading | how it is addressed | who uses it | +|---|---|---| +| **row** | row `r` = bytes `r*512..(r+1)*512`; facet `f` at `+f*16` | `MultiLaneColumn::iter_u32x16` (4 facets per 64-B chunk), Java's structured layout | +| **facet lane** | strided u32 column: `first_offset = f*16`, `stride = 512` | `eq_u32_strided_to_mask`, `LgjLaneDesc` (which has carried `stride_bytes` since minor 1) | + +Neither is a copy. The buffer is one `Arc<[u8]>`; a clone is a refcount +bump. **There is no serialization anywhere in this stack** — that is the +whole point (operator: *"abandon any use of serialization in favor of +lance-graph 64k concurrency zero copy and ndarray SIMD polyfill"*), and it +composes with lance-graph's own doctrine that an SoA envelope is zero-copy +from creation to Lance tombstone. + +## Facts a session must not re-derive + +- **`n_rows * 512` is always a multiple of 64** — so + `MultiLaneColumn::new` is infallible here *by construction*, not by + luck. Pinned by `the_buffer_is_exactly_n_times_512_bytes`. +- **Classids sit at `U32x16` positions 0, 4, 8, 12** of each 64-byte + chunk. The `& 0x1111` mask in the facet-match kernel is what keeps + payload bytes from ever satisfying a classid predicate — pinned + two-sided by `facet_match_ignores_needle_patterns_in_payload_bytes`. +- **`byte_len` is the EXACT covered span** `(len-1)*stride + elem_bytes`, + never `len * stride`. A facet lane's base sits `f*16` into the buffer, so + a full-stride final window would let Java bound a segment past the + allocation's end. Contiguous lanes reduce to the old formula unchanged. +- **`facet` ≠ `lane_id`.** Lane 0 is the raw buffer; facet `f`'s lane id is + `1 + f`. `lgj_op_eq_classid` takes a **facet index**. Pinned by the + end-to-end test asserting facet 32 is invalid while lane 32 is valid. +- **Masks parent onto row stores** exactly as onto patterns — both are + read-only, row-shaped resources — so the entire existing mask algebra + (`and`/`or`/`count`/`describe`, direct Java word writes) applies with no + new surface. +- **Alignment, honestly:** the base is `u8`-aligned (`Arc<[u8]>` promises + no more on stable Rust). Rows are strided at 512. Nothing here needs + more — Panama has unaligned value layouts, and every `ndarray::simd` load + is a register fill. The `align(64)` base guarantee arrives with real + `NodeRow` (`#[repr(C, align(64))]`) wiring, not before. + +## Why `MultiLaneColumn` fits HERE and not in the flat fixture + +Recorded because the answer flipped once already +(`E-LGJ-SIMD-SOA-IS-FOR-THE-ROW-STORE-NOT-THE-FLAT-LANES-1`): + +- It requires `len % 64 == 0` with **no tail arm**. The flat fixture's + lanes are caller-sized `n_rows` of 4/8-byte elements — arbitrary. The + row store's buffer is `n*512` — always conforming. +- Its typed iterators are 64-byte chunk views, which is *exactly* a + four-facet group and *not* a natural fit for a flat column scan (where + `simd_int_ops`' own group-plus-scalar-tail loop is the right shape). + +So both consumers are correct and neither is a workaround: use +`simd_int_ops` primitives for flat columns, `MultiLaneColumn` for the row +store. The u32 lane (`iter_u32x16`) was added to ndarray specifically to +close the gap that made the second impossible. diff --git a/.claude/plans/consumer-bricks-analytics-v1.md b/.claude/plans/consumer-bricks-analytics-v1.md new file mode 100644 index 0000000..dafd8a6 --- /dev/null +++ b/.claude/plans/consumer-bricks-analytics-v1.md @@ -0,0 +1,67 @@ +# consumer-bricks-analytics-v1 — mask-first security over the row store + +> **Status: PLANNED** (2026-08-17). W5 consumer example #2, from the +> operator's "OGAR-Bricks + lance-graph + Panama + Valhalla" poster. +> Gated on `lgj-soa-substrate-v1.md` W3. + +## What it proves + +The poster's structural claim: **security is a mask composed BEFORE +execution, not a post-filter on rows that already crossed the trust +boundary.** Runnable shape: + +```java +var orders = Bricks.table(Orders.class); +var result = orders + .where(Orders.REGION.eq("EU")) + .where(Orders.YEAR.eq(2026)) + .authorize(currentRole) // ← the mask-first clamp, BEFORE execution + .groupBy(Orders.PRODUCT) + .sum(Orders.REVENUE); +``` + +Only aggregates leave the boundary; raw rows never do. + +## Design (constraints, not code) + +- **`authorize(role)` composes an ADDITIONAL predicate into the SAME lazy + chain** — it is not a separate enforcement pass over already-fetched + data. Concretely: a role resolves to an allowed-facet-classid set (a + small, deterministic table — no new ABI symbol, expressible as an OR of + `Orders..eq(...)` over the existing `where` vocabulary, or as a + precomputed mask ANDed in via the existing `lgj_mask_and`). Either + encoding is legal; the falsifier below is what matters, not the + mechanism. +- **RBAC clamp happens where the OTHER predicates happen** — inside the ONE + fused crossing. There is no code path where a row's bytes are readable by + Java before the mask that would exclude it has been applied. This is the + operational meaning of "mask-first" and it is what the falsifier checks. +- **Reuses the row-store facet-match kernel for the "which fields visible" + half** — a role that can see some facets of a row but not others (partial + visibility) is a `lgj_row_facet_match`-shaped question; a role that can + see some ROWS but not others is a mask-composition question. This example + demonstrates the row-visibility case (simpler, no new kernel needed); + facet-level field masking is named as a documented extension, not built + here. + +## Falsifiers + +1. **No raw-row read before authorize.** Instrument (or structurally + prove via the API surface — no accessor exists that reads a row's bytes + before a terminal op runs) that between `.where(...)` calls and the + terminal aggregate, zero bytes of any EXCLUDED row are ever read into a + Java-visible value. The `LazinessTest` 0-crossings-while-composing + proof already gives half of this "for free" — extend it to prove + `authorize()` composes rather than executes. +2. **A caller who never calls `authorize()` gets an explicit refusal**, not + an unauthorized default (fail-closed — mirrors the a2ui-rs/lance-graph + RBAC doctrine already in this workspace's CLAUDE.md: "a missing/narrow + role mask never falls back to emit everything"). +3. **Two roles, same query, different counts** — a real two-sided + discrimination test (a role that can see 0 rows of a restricted region + must count 0; a role that can see all of it must match the unrestricted + query) — the anti-vacuity discipline from this workspace's falsifier + rules, applied to authorization instead of a table filter. +4. **Aggregate-only egress**: the public return type of a `Bricks` query is + never `Row`/`Trade`-shaped when a `groupBy`/`sum` terminal was used — + checked by the API surface test, same mechanism as `ApiSurfaceTest`. diff --git a/.claude/plans/consumer-graph-traversal-v1.md b/.claude/plans/consumer-graph-traversal-v1.md new file mode 100644 index 0000000..1aad892 --- /dev/null +++ b/.claude/plans/consumer-graph-traversal-v1.md @@ -0,0 +1,70 @@ +# consumer-graph-traversal-v1 — traversal as facet addressing, no middleware + +> **Status: PLANNED** (2026-08-17). W5 consumer example #3, from the +> operator's "Java Graph Stack (Heute) vs Project Panama + lance-graph +> (richtig gemacht)" poster. Gated on `lgj-soa-substrate-v1.md` W3. + +## What it proves + +The poster's BEFORE chain is six components and five serialization +boundaries: App → DTO/ORM → TinkerPop/Gremlin → JanusGraph → Cassandra → +Elastic/ClickHouse/Lucene. The AFTER chain is **one** explicit ABI boundary +and **zero** serialization boundaries — *"Java als low-code Oberfläche, ABI +als Wahrheit."* + +Runnable shape (a 2-hop neighbourhood, entirely in masks): + +```java +var g = Graph.open(store); +var friendsOfFriends = g.from(seedRows) // a mask + .hop(Edge.KNOWS) // facet-addressed, one crossing + .hop(Edge.KNOWS) + .minus(seedRows) + .count(); +``` + +No `Vertex` object, no `Edge` object, no Gremlin step compiler, no +serialization between hops — a hop is a mask transformation over the SAME +un-copied bytes. + +## Design (constraints, not code) + +- **An edge is an ADDRESS, not an object.** In the 512-byte row, a facet's + 4-byte classid names *what kind of relation this slot holds*; the 12-byte + payload carries the target address. That is exactly the workspace canon + ("a relation is a class; an edge's predicate is a classid reference" — + MedCare-rs commitment #10 / the OGAR EdgeBlock doctrine) expressed in the + facet register. `Edge.KNOWS` is therefore a *classid constant*, and + "which of this row's slots are KNOWS edges" is precisely + `lgj_row_facet_match` — already built in W2. +- **A hop is: facet-match → decode targets → build the next mask.** The + first half exists. The second half (target decode + scatter into a mask) + is the ONE genuinely new capability this example needs, and it is a + *bulk* operation by construction (one crossing per hop, work ∝ rows). If + it cannot be expressed with the existing symbols, it goes back through + `lgj-soa-substrate-v1.md`'s wave process as a proposed W6 ABI addition — + **not** added ad hoc from the consumer side. Naming it here is the + design decision; building it is gated. +- **The comparison is the point.** This example carries an explicit + boundary-count table in its output: components traversed, serialization + boundaries crossed, objects allocated — measured, against the poster's + BEFORE column as the stated baseline (which is cited as *architecture*, + not benchmarked here; we do not claim measured numbers for a JanusGraph + stack we did not run). + +## Falsifiers + +1. **Hop correctness**: the 2-hop neighbourhood equals a transcribed, + plain-Java breadth-first walk over the same generated data — an + independent computation, not a golden blob. +2. **Zero serialization**: no `byte[]`, no `toArray`, no JSON/proto on the + hop path — enforced the way the bench enforces it (an explicit rule in + the test, plus the API surface test for the public types). +3. **Crossings scale with HOPS, not with rows or edges**: instrument the + downcall count and assert it equals the hop count (+1 terminal), + independent of row count — the anti-JNI property, stated as an + assertion rather than as prose. +4. **Anti-vacuity**: the seed set, the 1-hop set and the 2-hop set must be + three *different, non-empty, non-total* sizes — a traversal that + returned everything or nothing would satisfy a naive equality test while + proving nothing. diff --git a/.claude/plans/consumer-world-trades-v1.md b/.claude/plans/consumer-world-trades-v1.md new file mode 100644 index 0000000..c66e40d --- /dev/null +++ b/.claude/plans/consumer-world-trades-v1.md @@ -0,0 +1,58 @@ +# consumer-world-trades-v1 — "One Billion Objects. Zero Objects." + +> **Status: PLANNED** (2026-08-17). W5 consumer example #1, from the +> operator's "One Billion Objects in Java — Before vs After" poster. +> Gated on `lgj-soa-substrate-v1.md` W3 (the Java `RowStore` facade). + +## What it proves + +The poster's AFTER column, runnable: + +```java +var trades = World.open(Trade.class); // ← a RowStore, not a loadTrades() +long count = trades + .where(Trade.QUANTITY.gt(1000)) + .where(Trade.VENUE.eq(XETRA)) + .where(Trade.PRICE.gt(threshold)) + .count(); +``` + +Developer sees: familiar fluent Java, domain language, no serialization. +What actually happens: compose lens (0 crossings) → one fused evaluation → +packed mask → count. **Java objects allocated for N logical trades: 0** — +and that number is asserted by measurement +(`getThreadAllocatedBytes`, the valhalla-lab instrument), not claimed. + +## Design (constraints, not code) + +- **`World.open(Class)` is a schema binding, not a loader.** The class + is a *description*: static typed field descriptors (`Trade.QUANTITY`) + carrying (facet index, element kind, offset-within-facet). It maps the + domain vocabulary onto the 32-facet row; no instance of `Trade` is ever + constructed. This is the poster's "ClassView (Semantics)" cell scaled to + the fixture — a REAL lance-graph ClassView binding replaces it in a + later slice without changing consumer code. +- **Field descriptors are the Valhalla-shaped vocabulary** — tiny, + identity-free, `record`-shaped, ≤8B payload where possible (the measured + flattening cliff), migrating to `value record` by one word when JEP 401 + ships. The three-truths lab already proved this is the ONE place + Valhalla pays here. +- **Reuses `View`/`Predicate`/`Mask` machinery** — the fluent chain stays + lazy (0 crossings to compose, LazinessTest discipline), fuses to one + plan, crosses once. No new membrane surface expected; if one turns out + to be needed (e.g. a fused plan over facet lanes), it goes through the + substrate plan's wave process first. +- **The demo scale is honest**: 64K rows in-repo CI; the 10⁶+ row arm runs + as a bench/example, not a unit test. + +## Falsifiers + +1. Count parity: the fluent chain's answer == a transcribed-generator + recomputation in plain Java (no substrate involvement). +2. Zero-allocation: measured allocated bytes for the query path below a + fixed small constant (the descriptors + the mask handle), regardless of + row count — the row-count-independence IS the assertion. +3. Laziness: crossing count 0 while composing, exactly 1 at the terminal + (the LazinessTest instrument, reused). +4. API surface: reflection test — nothing in the consumer-visible API + mentions FFM, facet indices, or lane ids. diff --git a/.claude/plans/lgj-soa-substrate-v1.md b/.claude/plans/lgj-soa-substrate-v1.md new file mode 100644 index 0000000..6feed96 --- /dev/null +++ b/.claude/plans/lgj-soa-substrate-v1.md @@ -0,0 +1,82 @@ +# lgj-soa-substrate-v1 — Slice 2: the lance-graph-shaped SoA substrate + +> **Status: ACTIVE** (2026-08-17). Successor to `lgj-vertical-slice-v1.md`, +> which is COMPLETE (PRs #1–#4 merged). Operator directives that reframed +> this slice, in order: +> 1. *"the 64k x 512 bytes SoA layout is enforced everywhere in lance-graph +> (32 Lanes each 4 bytes classview+12 bytes). For Java the layout might +> differ — just for reference."* +> 2. *"Java should optimize the SoA layout — we won't dismiss the initial +> plans just because it doesn't apply for unorganized non-SoA; that's +> the whole point about project Panama."* +> 3. *"Abandon any use of serialization in favor of lance-graph 64k +> concurrency zero copy and ndarray SIMD polyfill — the low code low +> migration cost experience."* +> 4. *"Java Panama and Valhalla become the supraconductor over lance-graph +> ABI shaped SoA substrate."* +> +> Blast radius (operator-confirmed, three posters): this is not a faster +> binding — it is the deletion of the Java data stack's middle tier (the +> ORM/DTO layer, graph middleware, serialization frameworks, side-car +> analytics) in favor of ONE ABI boundary over ONE substrate, with the JVM +> kept as the familiar low-migration-cost surface. The formula: +> `ClassView → WideFieldMask → Meta Gate (64K) → SIMD Sweep → SoA Lanes → +> Survivors → Seal & Persist (Lance)`. + +## The design waves + +| wave | deliverable | status | +|---|---|---| +| **W1** | ndarray: `MultiLaneColumn` u32 lane (`iter_u32x16`) + `eq_u32_strided_to_mask` (W1a contract: parity tests, re-exports, both x86 arms) | **DONE** — ndarray PR #279; 46+15 tests, clippy/fmt clean | +| **W2** | lgj-abi row store: `LGJ_RESOURCE_ROWSTORE`, `lgj_rowstore_open`, facet lanes via the EXISTING `LgjLaneDesc` (`stride=512`), `lgj_op_eq_classid` (row masks composing with the existing algebra), `lgj_row_facet_match` (per-row facet bitsets into a caller buffer via `MultiLaneColumn`), ABI minor 1→2, `docs/abi.md` §11 | **DONE** — 84/84 incl. end-to-end membrane test, 18/18 symbols via `nm -D` | +| **W3** | Java `RowStore` facade: no FFM in public signatures; structured `MemoryLayout` (`sequence(32, struct(u32 classid, 12B payload))`); minor-≥2 gate; `FacetMatchView` zero-copy accessor over a Java-arena segment; `RowStoreParityTest` transcribing the generator | OPEN — next | +| **W4** | Bench Component F: Java Vector API per-row facet scan (one `IntVector` 16-lane chunk = 4 facets, same algorithm as the Rust kernel) vs `lgj_row_facet_match` crossing vs scalar VarHandle walk — the "where does execution belong" question re-asked on the REAL layout | OPEN | +| **W5** | The three consumer examples (own plan files, below) | PLANNED | + +Wave rule (house style): one wave = one reviewable PR; gates run centrally +(orchestrator only — agents never run cargo); every safety property lands +disable-verified, every measured claim lands with its reproduction command. + +## What W2 locked (so W3+ doesn't re-derive it) + +- **Layout truth:** `ROW_BYTES=512`, `ROW_FACETS=32`, `FACET_BYTES=16`, + classid = leading LE u32. Generator: 2 SplitMix64 draws per facet + (`a`→classid via `(a>>>33)&0xF`, `b`+low-`a` → payload), 64 draws/row. +- **Lane map:** lane 0 = raw U8 contiguous; lane `1+f` = facet `f` classid, + U32, stride 512. `byte_len` = exact covered span + `(len-1)*stride + elem_bytes` — never rounds up past the allocation. +- **Masks parent onto row stores** exactly as onto patterns; the whole + existing mask algebra applies unchanged (proven in + `the_rowstore_slice_end_to_end_through_the_membrane`). +- **Carrier:** `Arc<[u8]>`, base u8-aligned (honest limit — the align(64) + guarantee arrives with real `NodeRow` wiring); `n*512 % 64 == 0` by + construction is what makes `MultiLaneColumn::new` infallible here. + +## The three consumer examples (W5) — one plan file each + +Each is one poster made runnable, on the SAME substrate, each exercising a +different face of it: + +| plan | poster | face of the substrate | +|---|---|---| +| `consumer-world-trades-v1.md` | "One Billion Objects in Java" | the fluent domain API: `World.open(...)` → schema-named fields → `.where().count()`, zero objects | +| `consumer-bricks-analytics-v1.md` | "OGAR-Bricks done right" | mask-first security: RBAC clamp BEFORE execution, survivors-only, aggregates leave | +| `consumer-graph-traversal-v1.md` | "Java Graph Stack (richtig gemacht)" | facet edges as addresses: traversal = facet-match + mask hops, no middleware | + +Sequencing: any order after W3; each is independently shippable; none +blocks the others. All three consume ONLY the public Java facade — a +consumer plan that needs a new ABI symbol goes back through this plan's +wave process instead of growing the membrane ad hoc. + +## Falsification obligations carried forward + +- W3 parity: Java recomputes classids from the transcribed generator AND + reads them back through the raw-lane segment — two independent paths to + the same numbers. +- W3 disable-run: break the minor-version gate (require ≥ 3) and prove + load fails; restore. +- W4 cross-check before timing: all three arms must agree on every + facet-match bitset before any timing is reported (the `Data.crossCheck` + discipline). +- Every consumer example ends with an assertion computed independently of + the substrate (transcribed-generator arithmetic), never a golden blob. diff --git a/docs/abi.md b/docs/abi.md index 8db9b72..3f7a665 100644 --- a/docs/abi.md +++ b/docs/abi.md @@ -62,8 +62,10 @@ cannot disagree with itself. The ABI is a **machine membrane**. It is not the product. The product is the Java semantic API (see `architecture.md`). Therefore: -- It is **small** — currently 14 symbols. Growth is a design smell to be argued - for, not a default. +- It is **small** — currently 18 symbols (minor 2; the "14" this line carried + at minor 1 was arithmetic drift — the §7 list it referred to already + enumerated 15). Growth is a design smell to be argued for, not a default; + minor 2's three additions are argued in §11. - It is **bulk-only**. Every call must be capable of doing work proportional to `n_rows` (see §6 — the anti-JNI rule). - It speaks **resource, lane, view, mask, operation, descriptor, status, @@ -76,7 +78,7 @@ semantic API (see `architecture.md`). Therefore: ``` LGJ_ABI_MAJOR = 0 // incompatible change ⇒ bump; Java refuses to load -LGJ_ABI_MINOR = 1 // additive change ⇒ bump; older Java may still load +LGJ_ABI_MINOR = 2 // additive change ⇒ bump; older Java may still load LGJ_MAGIC = 0x4C_47_4A_5F_41_42_49_00 // "LGJ_ABI\0" big-endian-read ``` @@ -177,7 +179,7 @@ bounded description that the FFM layer turns into a `MemorySegment`: pub struct LgjLaneDesc { pub addr: u64, // physics — never surfaced in the public Java API pub len_elems: u64, - pub byte_len: u64, + pub byte_len: u64, // exact covered span: (len-1)*stride + elem_bytes; 0 when empty pub owner: u64, // owning resource handle pub epoch: u64, // liveness stamp; Java re-checks before use pub elem_kind: u32, // LgjElemKind @@ -267,7 +269,7 @@ predicates or rows are involved. The unfused per-predicate ops are retained only so the fused path can be benchmarked *against* something and so parity can be checked predicate-by-predicate. -## 7. The function surface (14 symbols) +## 7. The function surface (18 symbols) All symbols are prefixed `lgj_`. All return `i32` status except the manifest getter. `out_*` parameters are written only on `OK`. @@ -407,3 +409,78 @@ Named so their absence is a decision on record rather than an oversight: ABI-compatible: `WideFieldMask`'s canonical `[u64]` chunks *are* this ABI's `MASK_WORD` lane, and `NodeRow`'s `16|16|480` `#[repr(C, align(64))]` layout is already a legal lane description. + +## 11. The SoA row store (ABI minor ≥ 2) + +The substrate layout the whole stack converges on — operator-stated reference +(2026-08-17): **64K rows × 512 bytes per row, 32 facet lanes of 16 bytes each +(4-byte little-endian classid + 12-byte payload)**, the lance-graph V3 +content-blind facet shape, enforced everywhere on the Rust side. The Java +side's *view* may differ; these bytes are the substrate truth. One buffer, +zero serialization: every access — Rust kernel or Java segment read — is a +*reading* of the same bytes. + +### Resource + +``` +LGJ_RESOURCE_ROWSTORE = 3 +i32 lgj_rowstore_open(u64 n_rows, u64 seed, u64* out_handle) +``` + +Deterministic generator (normative; two SplitMix64 draws per facet, `a` then +`b`, 64 draws per row — full statement in `rowstore.rs`'s doc): + +``` +classid = (a >>> 33) & 0xF // same recipe as the fixture's class lane +payload = le64(b) ++ le32(a & 0xFFFFFFFF) +``` + +### Lanes (described through the UNCHANGED `LgjLaneDesc` — `stride_bytes` +anticipated this since minor 1) + +| lane id | what | kind | stride | flags | +|---|---|---|---|---| +| `0` | the raw buffer, `n_rows * 512` bytes | `U8` | 1 | `READABLE \| CONTIGUOUS` | +| `1 + f` (f in `0..32`) | facet `f`'s classid column | `U32` | 512 | `READABLE` | + +`byte_len` is the **exact covered span** `(len_elems - 1) * stride_bytes + +elem_bytes` (0 when empty) — for a strided facet lane this deliberately does +NOT round up to `len * stride`, because the lane's base sits `f*16` into the +buffer and a full-stride final window would let Java bound a segment past the +allocation's end. For contiguous lanes the formula reduces to +`len * elem_bytes`, unchanged from minor 1. + +### Operations + +``` +i32 lgj_op_eq_classid(u64 res, u32 facet, u32 needle, u64 dst_mask) +i32 lgj_row_facet_match(u64 res, u32 needle, u32* out, u64 out_len_elems) +``` + +- `lgj_op_eq_classid` overwrites `dst_mask` with the row mask + `classid(facet, row) == needle`. `facet` is a facet index `0..32`, not a + lane id. The result is an ordinary mask: `lgj_mask_create` accepts a row + store as parent (masks may parent onto a pattern OR a row store — both are + read-only row-shaped resources), and the whole §7 mask algebra + (`and`/`or`/`count`/`describe`) applies unchanged. +- `lgj_row_facet_match` writes, for every row, a `u32` bitset of which of its + 32 facets carry `needle` as classid — into the **caller's** buffer (a + Java-arena segment; zero-copy out, nothing serialized). Capacity is checked + BEFORE anything is written (`MASK_LENGTH_MISMATCH` on a short buffer). + +### SIMD provenance (unchanged §8 rule, applied) + +`lgj_op_eq_classid` routes through `ndarray::simd::eq_u32_strided_to_mask` +(scalar strided loads — at stride 512 each element is on its own cache line, +so the walk is memory-bound and SIMD earns its keep in the 16-wide compare). +`lgj_row_facet_match` wraps the store's bytes in +`ndarray::simd::MultiLaneColumn` (an `Arc` refcount bump, no copy) and answers +four facets per 64-byte chunk with one `U32x16::eq_bitmask`. + +### Alignment (stated honestly) + +The buffer base is `u8`-aligned (`Arc<[u8]>`; stable Rust promises no more). +Rows are 512-byte strided within it. Nothing in this slice needs more — Java +reads via `JAVA_INT_UNALIGNED`-class layouts, and every `ndarray::simd` load +is a register fill. The 64-byte-aligned base guarantee arrives with the real +`NodeRow` (`#[repr(C, align(64))]`) wiring. diff --git a/native/lgj-abi/src/abi.rs b/native/lgj-abi/src/abi.rs index 80d8bf6..d919922 100644 --- a/native/lgj-abi/src/abi.rs +++ b/native/lgj-abi/src/abi.rs @@ -31,7 +31,12 @@ use core::mem::{align_of, size_of}; pub const LGJ_ABI_MAJOR: u32 = 0; /// Additive change ⇒ bump. Older Java may still load (`minor >= expected`). -pub const LGJ_ABI_MINOR: u32 = 1; +/// +/// Minor **2** (2026-08-17): the SoA row store — `LGJ_RESOURCE_ROWSTORE`, +/// `lgj_rowstore_open`, `lgj_op_eq_classid`, `lgj_row_facet_match`, and +/// strided facet lanes described through the (unchanged) `LgjLaneDesc`. +/// Purely additive; a minor-1 Java loads and sees none of it. +pub const LGJ_ABI_MINOR: u32 = 2; /// `"LGJ_ABI\0"` read big-endian. /// @@ -139,8 +144,13 @@ pub const LGJ_FLAG_CONTIGUOUS: u32 = 1 << 2; // §5 resource kinds /// A pattern (the SoA fixture): id/class/value lanes, read-only. pub const LGJ_RESOURCE_PATTERN: u32 = 1; -/// A mask: one `MASK_WORD` lane, writable, owned by a parent pattern. +/// A mask: one `MASK_WORD` lane, writable, owned by a parent pattern or +/// row store. pub const LGJ_RESOURCE_MASK: u32 = 2; +/// A SoA row store (abi.md §11): `n_rows × 512` bytes, 32 facet lanes of +/// (4-byte LE classid + 12-byte payload); 1 raw `U8` lane + 32 strided +/// `U32` classid lanes, all read-only. ABI minor ≥ 2. +pub const LGJ_RESOURCE_ROWSTORE: u32 = 3; // §7 mask_create initial states /// `lgj_mask_create(initial = 0)` — no rows set. @@ -201,7 +211,11 @@ pub struct LgjLaneDesc { pub addr: u64, /// Number of elements. pub len_elems: u64, - /// `len_elems * stride_bytes`. + /// Exact covered span: `(len_elems - 1) * stride_bytes + elem_bytes`, `0` + /// when empty. Reduces to `len_elems * elem_bytes` for contiguous lanes; + /// for a strided facet lane it deliberately does NOT round up to + /// `len * stride` (abi.md §11 — a full-stride final window would let Java + /// bound a segment past the allocation's end). pub byte_len: u64, /// Owning resource handle. pub owner: u64, diff --git a/native/lgj-abi/src/exports.rs b/native/lgj-abi/src/exports.rs index 050bf60..e2dd2aa 100644 --- a/native/lgj-abi/src/exports.rs +++ b/native/lgj-abi/src/exports.rs @@ -104,6 +104,40 @@ pub unsafe extern "C" fn lgj_pattern_open(n_rows: u64, seed: u64, out_handle: *m }) } +/// Build the deterministic SoA **row store** (abi.md §11) and return its +/// handle: `n_rows × 512` bytes, 32 facets of (4-byte LE classid + 12-byte +/// payload) per row. ABI minor ≥ 2. +/// +/// Bulk by construction; the generation algorithm is normative — see +/// [`crate::rowstore::RowStore`]. Lanes: `0` = the raw `U8` buffer +/// (contiguous), `1..=32` = per-facet classid lanes (`U32`, stride 512). +/// # Safety +/// +/// A null pointer is *handled*, not UB: it returns `NULL_ARGUMENT`. Beyond +/// that, `out_handle` must be a valid, aligned, writable `u64`. Written only +/// on success. +/// +/// `unsafe` here is a note to Rust callers linking the `rlib`. The JVM, +/// which is the real caller, has no such concept — it upholds the same +/// contract by construction, because every pointer it passes comes from a +/// `MemorySegment` whose size and alignment it derived from the manifest. +#[no_mangle] +pub unsafe extern "C" fn lgj_rowstore_open(n_rows: u64, seed: u64, out_handle: *mut u64) -> i32 { + guard(|| { + if out_handle.is_null() { + return LGJ_ERR_NULL_ARGUMENT; + } + match registry::open_rowstore(n_rows, seed) { + Ok(h) => { + // SAFETY: non-null (checked above); written only on success. + unsafe { *out_handle = h }; + LGJ_OK + } + Err(e) => e, + } + }) +} + /// Free a resource: its lanes are dropped, its generation is bumped, and its /// children begin failing with `PARENT_CLOSED`. /// @@ -145,12 +179,15 @@ pub unsafe extern "C" fn lgj_resource_info(handle: u64, out: *mut LgjResourceInf // Lanes // ─────────────────────────────────────────────────────────────────────────── -/// Describe one lane of a **pattern**: `0 = ids (U64)`, `1 = classes (U32)`, -/// `2 = values (I32)`. +/// Describe one lane of a **pattern** (`0 = ids (U64)`, `1 = classes (U32)`, +/// `2 = values (I32)`) or of a **row store** (`0 = raw U8 buffer, +/// contiguous; 1..=32 = facet classid lanes, U32, stride 512` — the strided +/// case `stride_bytes` existed for since ABI 0.1). /// -/// All pattern lanes are `READABLE | CONTIGUOUS` and never `WRITABLE` -/// (abi.md §7). The returned `addr` is stable until `lgj_close` — lanes are -/// allocated once and never moved or resized (§4). +/// All these lanes are `READABLE` and never `WRITABLE` (abi.md §7); the +/// contiguous flag is set exactly when `stride_bytes == elem_bytes`. The +/// returned `addr` is stable until `lgj_close` — buffers are allocated once +/// and never moved or resized (§4). /// # Safety /// /// A null pointer is *handled*, not UB: it returns `NULL_ARGUMENT`. Beyond @@ -170,29 +207,53 @@ pub unsafe extern "C" fn lgj_lane_describe( if out.is_null() { return LGJ_ERR_NULL_ARGUMENT; } - let entry = match registry::resolve_kind(handle, LGJ_RESOURCE_PATTERN) { + let entry = match registry::resolve(handle) { Ok(e) => e, Err(e) => return e, }; - let fixture = match entry.fixture() { - Some(f) => f, - None => return LGJ_ERR_WRONG_RESOURCE_KIND, - }; - let (addr, len_elems, kind) = match fixture.lane_raw(lane_id) { - Some(t) => t, - None => return LGJ_ERR_INVALID_LANE, - }; + // (addr, len_elems, kind, stride_bytes, contiguous) — patterns are + // always contiguous; a row store's facet lanes are the strided case. + let (addr, len_elems, kind, stride_bytes, contiguous) = + if let Some(fixture) = entry.fixture() { + match fixture.lane_raw(lane_id) { + Some((a, n, k)) => (a, n, k, k.elem_bytes(), true), + None => return LGJ_ERR_INVALID_LANE, + } + } else if let Some(store) = entry.rowstore() { + match store.lane_raw(lane_id) { + Some(t) => t, + None => return LGJ_ERR_INVALID_LANE, + } + } else { + // A mask: its word lane is described by lgj_mask_describe. + return LGJ_ERR_WRONG_RESOURCE_KIND; + }; let elem_bytes = kind.elem_bytes(); + let mut flags = LGJ_FLAG_READABLE; + if contiguous { + flags |= LGJ_FLAG_CONTIGUOUS; + } + // Exact covered span: from the lane's base to the END of its LAST + // element — `(len-1)*stride + elem_bytes`. For a contiguous lane this + // is `len * elem_bytes` exactly as before; for a strided facet lane it + // deliberately does NOT round up to `len * stride`, because a facet + // lane's base sits `facet*16` into the buffer and a full-stride final + // window would let Java bound a segment past the allocation's end. + let byte_len = if len_elems == 0 { + 0 + } else { + (len_elems - 1) * stride_bytes as u64 + elem_bytes as u64 + }; let desc = LgjLaneDesc { addr, len_elems, - byte_len: len_elems * elem_bytes as u64, + byte_len, owner: handle, epoch: entry.epoch, elem_kind: kind as u32, elem_bytes, - stride_bytes: elem_bytes, - flags: crate::fixture::Fixture::lane_flags(), + stride_bytes, + flags, }; // SAFETY: non-null; `LgjLaneDesc` is `#[repr(C)]`, 56 bytes, and that // size is asserted at compile time and reported by the manifest. @@ -544,6 +605,111 @@ pub extern "C" fn lgj_op_gt_i32(res: u64, lane_id: u32, threshold: i32, dst_mask }) } +// ─────────────────────────────────────────────────────────────────────────── +// Row-store bulk predicates (ABI minor ≥ 2) +// ─────────────────────────────────────────────────────────────────────────── + +/// One crossing: **overwrites** `dst_mask` with the row mask +/// `classid(facet, row) == needle` over a row store's facet lane. +/// +/// `facet` is the facet index `0..32`, NOT a lane id (lane id = facet + 1). +/// The resulting mask is an ordinary mask resource: it composes with +/// `lgj_mask_and`/`or`, counts with `lgj_mask_count`, and its words are +/// directly readable/writable through `lgj_mask_describe` — the whole +/// existing mask algebra applies unchanged to row stores. +#[no_mangle] +pub extern "C" fn lgj_op_eq_classid(res: u64, facet: u32, needle: u32, dst_mask: u64) -> i32 { + guard(|| { + let store_entry = match registry::resolve_kind(res, LGJ_RESOURCE_ROWSTORE) { + Ok(e) => e, + Err(e) => return e, + }; + let store = match store_entry.rowstore() { + Some(s) => s, + None => return LGJ_ERR_WRONG_RESOURCE_KIND, + }; + if facet >= crate::rowstore::ROW_FACETS { + return LGJ_ERR_INVALID_LANE; + } + let (mask, _parent) = match registry::resolve_mask_with_parent(dst_mask) { + Ok(t) => t, + Err(e) => return e, + }; + if mask.n_rows != store_entry.n_rows { + return LGJ_ERR_MASK_LENGTH_MISMATCH; + } + let mut g = match mask.write_mask() { + Some(g) => g, + None => return LGJ_ERR_WRONG_RESOURCE_KIND, + }; + kernels::simd_rowstore_classid_mask( + store.as_bytes(), + facet as usize * crate::rowstore::FACET_BYTES as usize, + store_entry.n_rows as usize, + needle, + &mut g.words, + ); + clear_tail_bits(&mut g.words, store_entry.n_rows); + LGJ_OK + }) +} + +/// One crossing: for every row, which of its 32 facets carry `needle` as +/// classid — one `u32` bitset per row, written into the **caller's** buffer +/// (a Java-arena segment of `n_rows` ints; zero-copy out, nothing +/// serialized). +/// +/// `out_len_elems` is the capacity of `out` in `u32` elements; it must be at +/// least the store's row count or the call fails with `MASK_LENGTH_MISMATCH` +/// before anything is written. The first `n_rows` elements are fully +/// overwritten; elements past `n_rows` are untouched. +/// # Safety +/// +/// A null pointer is *handled*, not UB: it returns `NULL_ARGUMENT`. Beyond +/// that, `out` must point to at least `out_len_elems` writable, 4-byte-aligned +/// `u32`s (Java passes a segment it allocated with that layout). +/// +/// `unsafe` here is a note to Rust callers linking the `rlib`. The JVM, +/// which is the real caller, has no such concept — it upholds the same +/// contract by construction, because every pointer it passes comes from a +/// `MemorySegment` whose size and alignment it derived from the manifest. +#[no_mangle] +pub unsafe extern "C" fn lgj_row_facet_match( + res: u64, + needle: u32, + out: *mut u32, + out_len_elems: u64, +) -> i32 { + guard(|| { + if out.is_null() { + return LGJ_ERR_NULL_ARGUMENT; + } + let store_entry = match registry::resolve_kind(res, LGJ_RESOURCE_ROWSTORE) { + Ok(e) => e, + Err(e) => return e, + }; + let store = match store_entry.rowstore() { + Some(s) => s, + None => return LGJ_ERR_WRONG_RESOURCE_KIND, + }; + let n_rows = store_entry.n_rows; + if out_len_elems < n_rows { + return LGJ_ERR_MASK_LENGTH_MISMATCH; + } + let n = match usize::try_from(n_rows) { + Ok(n) => n, + Err(_) => return LGJ_ERR_LENGTH_OVERFLOW, + }; + // SAFETY: non-null (checked), and the caller guarantees at least + // `out_len_elems >= n_rows` writable u32s at `out` — Java passes a + // segment whose element count it allocated. The slice is built over + // exactly the prefix this call overwrites. + let out_slice = unsafe { std::slice::from_raw_parts_mut(out, n) }; + kernels::simd_rowstore_facet_match(&store.bytes_arc(), n, needle, out_slice); + LGJ_OK + }) +} + // ─────────────────────────────────────────────────────────────────────────── // The fused plan — N predicates, ONE crossing // ─────────────────────────────────────────────────────────────────────────── diff --git a/native/lgj-abi/src/kernels.rs b/native/lgj-abi/src/kernels.rs index 9e6d03b..7342384 100644 --- a/native/lgj-abi/src/kernels.rs +++ b/native/lgj-abi/src/kernels.rs @@ -98,6 +98,81 @@ pub fn simd_popcount(words: &[u64]) -> u64 { ndarray::simd::popcount_batch_u64(words) } +/// Row mask over one facet-classid lane of a 512-byte row store: +/// `out_words[row-th bit] = (classid of facet at first_offset in row == needle)`. +/// +/// Routes through `ndarray::simd::eq_u32_strided_to_mask` — the strided +/// AoS-facet scan (LE `u32` at `first_offset + row * 512`). The primitive +/// owns bounds checking (overflow-checked, panics rather than reading out of +/// bounds) and the trailing-bits-zero guarantee. +#[inline] +pub fn simd_rowstore_classid_mask( + bytes: &[u8], + first_offset: usize, + n_rows: usize, + needle: u32, + out_words: &mut [u64], +) { + ndarray::simd::eq_u32_strided_to_mask( + bytes, + first_offset, + crate::rowstore::ROW_BYTES as usize, + n_rows, + needle, + out_words, + ); +} + +/// Per-row facet-match: `out[row]` gets bit `f` set iff facet `f`'s classid +/// in that row equals `needle` — "which facets of this node carry class X", +/// one `u32` answer per row, written into the caller's buffer. +/// +/// This is the [`ndarray::simd::MultiLaneColumn`] consumer: the store's +/// `Arc<[u8]>` is wrapped WITHOUT copying (the Arc clone is a refcount bump), +/// and each 64-byte chunk — four 16-byte facets — is answered by ONE +/// `U32x16::eq_bitmask` against the broadcast needle, masked to the classid +/// positions 0/4/8/12 and folded into 4 facet bits. Eight chunks per row +/// assemble the row's 32-bit answer. +/// +/// # Panics +/// +/// Panics if `bytes.len() != n_rows * 512` or `out.len() < n_rows` — caller +/// bugs inside this crate, not reachable from the membrane (the export +/// validates first). +pub fn simd_rowstore_facet_match( + bytes: &std::sync::Arc<[u8]>, + n_rows: usize, + needle: u32, + out: &mut [u32], +) { + use crate::rowstore::ROW_BYTES; + assert_eq!(bytes.len(), n_rows * ROW_BYTES as usize); + assert!(out.len() >= n_rows); + + // n*512 is always a multiple of 64 (rowstore tests pin this), so `new` + // cannot fail — and the construction shares the bytes, never copies them. + let col = ndarray::simd::MultiLaneColumn::new(std::sync::Arc::clone(bytes)) + .expect("rowstore buffer is a multiple of 64 bytes by construction"); + let needle_v = ndarray::simd::U32x16::from_array([needle; 16]); + + // Fully overwrite, same contract as every mask writer in this crate: the + // per-chunk fold below ORs, so stale caller bits must not survive. + for o in out.iter_mut().take(n_rows) { + *o = 0; + } + + const CHUNKS_PER_ROW: usize = (ROW_BYTES / 64) as usize; // 8 + for (c, chunk) in col.iter_u32x16().enumerate() { + // Classids sit at u32 positions 0/4/8/12 of the 16-lane chunk; the + // other twelve lanes are payload bytes that must never contribute. + let m = chunk.eq_bitmask(needle_v) & 0x1111; + let facet_bits = (m & 1) | ((m >> 4) & 1) << 1 | ((m >> 8) & 1) << 2 | ((m >> 12) & 1) << 3; + let row = c / CHUNKS_PER_ROW; + let chunk_in_row = c % CHUNKS_PER_ROW; + out[row] |= (facet_bits as u32) << (4 * chunk_in_row); + } +} + // ─────────────────────────────────────────────────────────────────────────── // Scalar reference — INDEPENDENT of ndarray. Do not "simplify" by calling the // wrappers above; the independence IS the test. @@ -161,6 +236,43 @@ pub fn scalar_popcount(words: &[u64]) -> u64 { n } +/// Reference strided classid → row mask. Plain byte reads, no ndarray. +pub fn scalar_rowstore_classid_mask( + bytes: &[u8], + first_offset: usize, + n_rows: usize, + needle: u32, + out_words: &mut [u64], +) { + for w in out_words.iter_mut() { + *w = 0; + } + for row in 0..n_rows { + let off = first_offset + row * crate::rowstore::ROW_BYTES as usize; + let v = u32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]); + if v == needle { + out_words[row / 64] |= 1u64 << (row % 64); + } + } +} + +/// Reference per-row facet match. Plain byte reads, no ndarray. +pub fn scalar_rowstore_facet_match(bytes: &[u8], n_rows: usize, needle: u32, out: &mut [u32]) { + use crate::rowstore::{FACET_BYTES, ROW_BYTES, ROW_FACETS}; + for (row, o) in out.iter_mut().enumerate().take(n_rows) { + let mut bits = 0u32; + for f in 0..ROW_FACETS { + let off = row * ROW_BYTES as usize + f as usize * FACET_BYTES as usize; + let v = + u32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]); + if v == needle { + bits |= 1 << f; + } + } + *o = bits; + } +} + // ─────────────────────────────────────────────────────────────────────────── // Which path a call takes // ─────────────────────────────────────────────────────────────────────────── @@ -387,6 +499,110 @@ mod tests { ); } + /// The row-store parity falsifier: both new SIMD kernels against their + /// independent scalar references, over real generated stores at row + /// counts straddling every boundary (16-lane groups via 4-facet chunks, + /// 64-bit words, and the 8-chunks-per-row fold). + #[test] + fn rowstore_kernels_match_their_scalar_references() { + use crate::rowstore::RowStore; + for n in [0u64, 1, 2, 15, 16, 17, 63, 64, 65, 200] { + for seed in [0u64, 0xABCD] { + let s = RowStore::generate(n, seed).unwrap(); + let bytes = s.bytes_arc(); + + for facet in [0u32, 1, 15, 31] { + for needle in [0u32, 7, 15, 42] { + let n_words = mask_words_for(n) as usize; + let mut a = vec![u64::MAX; n_words]; + let mut b = vec![u64::MAX; n_words]; + let first_offset = (facet as usize) * crate::rowstore::FACET_BYTES as usize; + simd_rowstore_classid_mask( + &bytes, + first_offset, + n as usize, + needle, + &mut a, + ); + scalar_rowstore_classid_mask( + &bytes, + first_offset, + n as usize, + needle, + &mut b, + ); + assert_eq!(a, b, "classid_mask n={n} facet={facet} needle={needle}"); + // Cross-check against the store's own scalar accessor, + // a THIRD independent computation. + for row in 0..n { + let bit = (a[(row / 64) as usize] >> (row % 64)) & 1; + let expect = (s.classid_at(row, facet) == needle) as u64; + assert_eq!(bit, expect, "row {row}"); + } + } + } + + for needle in [0u32, 7, 15] { + let mut a = vec![u32::MAX; n as usize]; + let mut b = vec![u32::MAX; n as usize]; + simd_rowstore_facet_match(&bytes, n as usize, needle, &mut a); + scalar_rowstore_facet_match(&bytes, n as usize, needle, &mut b); + assert_eq!(a, b, "facet_match n={n} needle={needle}"); + // Consistency with the per-facet masks: bit f of row r in + // facet_match must equal row r's bit in facet f's mask. + if n > 0 { + for facet in [0u32, 31] { + let mut m = vec![0u64; mask_words_for(n) as usize]; + scalar_rowstore_classid_mask( + &bytes, + facet as usize * crate::rowstore::FACET_BYTES as usize, + n as usize, + needle, + &mut m, + ); + for row in 0..n as usize { + let via_match = (a[row] >> facet) & 1; + let via_mask = ((m[row / 64] >> (row % 64)) & 1) as u32; + assert_eq!(via_match, via_mask, "row {row} facet {facet}"); + } + } + } + } + } + } + } + + /// The facet-match fold must never let PAYLOAD bytes match: plant the + /// needle's bit pattern inside a payload and prove it does not fire. + #[test] + fn facet_match_ignores_needle_patterns_in_payload_bytes() { + use crate::rowstore::RowStore; + let s = RowStore::generate(4, 0x5EED).unwrap(); + let mut bytes = s.as_bytes().to_vec(); + let needle = 0xDEAD_BEEFu32; + // Row 2, facet 5: put the needle in PAYLOAD positions (offsets +4 and + // +12), and a non-matching classid at +0. + let base = 2 * 512 + 5 * 16; + bytes[base..base + 4].copy_from_slice(&1u32.to_le_bytes()); + bytes[base + 4..base + 8].copy_from_slice(&needle.to_le_bytes()); + bytes[base + 12..base + 16].copy_from_slice(&needle.to_le_bytes()); + let bytes: std::sync::Arc<[u8]> = std::sync::Arc::from(bytes); + + let mut out = vec![0u32; 4]; + simd_rowstore_facet_match(&bytes, 4, needle, &mut out); + assert_eq!( + (out[2] >> 5) & 1, + 0, + "payload bytes must never satisfy a classid match" + ); + // And the twin: planting it in the CLASSID position does fire. + let mut bytes2 = s.as_bytes().to_vec(); + bytes2[base..base + 4].copy_from_slice(&needle.to_le_bytes()); + let bytes2: std::sync::Arc<[u8]> = std::sync::Arc::from(bytes2); + simd_rowstore_facet_match(&bytes2, 4, needle, &mut out); + assert_eq!((out[2] >> 5) & 1, 1, "a real classid match must fire"); + } + #[test] fn non_aliasing_mask_ops_agree_with_assign_forms() { let a = vec![0xF0F0_F0F0_F0F0_F0F0u64, 0x00FF]; diff --git a/native/lgj-abi/src/lib.rs b/native/lgj-abi/src/lib.rs index 952e87f..83644b6 100644 --- a/native/lgj-abi/src/lib.rs +++ b/native/lgj-abi/src/lib.rs @@ -51,6 +51,7 @@ pub mod exports; pub mod fixture; pub mod kernels; pub mod registry; +pub mod rowstore; // Re-export the ABI vocabulary at the crate root for convenience in tests and // for any Rust consumer that links the `rlib` rather than the `cdylib`. @@ -89,6 +90,18 @@ mod integration_tests { pub fn reduce_sum_i32(r: u64, lane: u32, m: u64, out: *mut i64) -> i32 { unsafe { lgj_reduce_sum_i32(r, lane, m, out) } } + pub fn rowstore_open(n: u64, seed: u64, out: *mut u64) -> i32 { + unsafe { lgj_rowstore_open(n, seed, out) } + } + pub fn resource_info(h: u64, out: *mut LgjResourceInfo) -> i32 { + unsafe { lgj_resource_info(h, out) } + } + pub fn lane_describe(h: u64, lane: u32, out: *mut LgjLaneDesc) -> i32 { + unsafe { lgj_lane_describe(h, lane, out) } + } + pub fn row_facet_match(r: u64, needle: u32, out: *mut u32, cap: u64) -> i32 { + unsafe { lgj_row_facet_match(r, needle, out, cap) } + } } fn open(n: u64, seed: u64) -> u64 { @@ -233,6 +246,106 @@ mod integration_tests { } } + /// The SoA row store, end to end through the membrane (ABI minor ≥ 2): + /// open → describe → classid predicate → mask algebra → count → + /// facet-match into a caller buffer — everything independently recomputed + /// from the documented generator, exactly as the Java parity test will. + #[test] + fn the_rowstore_slice_end_to_end_through_the_membrane() { + use crate::rowstore::{RowStore, LANE_FACET_BASE, ROW_BYTES}; + let n = 1000u64; + let seed = 0xC0FFEE; + let mut s = 0u64; + assert_eq!(call::rowstore_open(n, seed, &mut s), LGJ_OK); + + // Resource self-description. + let mut info = LgjResourceInfo::default(); + assert_eq!(call::resource_info(s, &mut info), LGJ_OK); + assert_eq!(info.kind, LGJ_RESOURCE_ROWSTORE); + assert_eq!(info.lane_count, 33); + assert_eq!(info.n_rows, n); + + // Raw lane: contiguous U8, exactly n*512 bytes. + let mut d = LgjLaneDesc::default(); + assert_eq!(call::lane_describe(s, 0, &mut d), LGJ_OK); + assert_eq!(d.elem_kind, LgjElemKind::U8 as u32); + assert_eq!(d.len_elems, n * ROW_BYTES); + assert_eq!(d.byte_len, n * ROW_BYTES); + assert_ne!(d.flags & LGJ_FLAG_CONTIGUOUS, 0); + + // Facet lane 7: strided U32, stride 512, and the exact-span rule — + // the described window must END at the buffer's last classid, never + // a full stride past it (Java bounds a segment from this number). + assert_eq!(call::lane_describe(s, LANE_FACET_BASE + 7, &mut d), LGJ_OK); + assert_eq!(d.elem_kind, LgjElemKind::U32 as u32); + assert_eq!(d.len_elems, n); + assert_eq!(d.stride_bytes, 512); + assert_eq!(d.byte_len, (n - 1) * 512 + 4); + assert_eq!(d.flags & LGJ_FLAG_CONTIGUOUS, 0); + // Lane 34 does not exist (1 raw + 32 facets = ids 0..=32). + assert_eq!(call::lane_describe(s, 34, &mut d), LGJ_ERR_INVALID_LANE); + + // classid predicate on facet 7 → an ordinary mask, counted natively… + let m = mask(s, LGJ_MASK_INIT_EMPTY); + assert_eq!(lgj_op_eq_classid(s, 7, 9, m), LGJ_OK); + let mut count = 0u64; + assert_eq!(call::mask_count(m, &mut count), LGJ_OK); + + // …and recomputed independently from the documented generator. + let store = RowStore::generate(n, seed).unwrap(); + let want = (0..n).filter(|&r| store.classid_at(r, 7) == 9).count() as u64; + assert_eq!(count, want); + assert!(count > 0 && count < n, "must be a middling selection"); + + // Facet out of range is a status, and the facet/lane-id distinction + // holds: facet 32 is invalid even though LANE id 32 (facet 31) exists. + assert_eq!(lgj_op_eq_classid(s, 32, 9, m), LGJ_ERR_INVALID_LANE); + + // The mask composes with the EXISTING algebra: AND facet-7==9 with + // facet-0==9 and verify against the independent recomputation. + let m2 = mask(s, LGJ_MASK_INIT_EMPTY); + assert_eq!(lgj_op_eq_classid(s, 0, 9, m2), LGJ_OK); + assert_eq!(lgj_mask_and(m, m2, m), LGJ_OK); + assert_eq!(call::mask_count(m, &mut count), LGJ_OK); + let want_and = (0..n) + .filter(|&r| store.classid_at(r, 7) == 9 && store.classid_at(r, 0) == 9) + .count() as u64; + assert_eq!(count, want_and); + + // Facet-match into a caller-owned buffer (what Java allocates in its + // own arena): per-row 32-bit facet sets, verified row by row. + let mut out = vec![u32::MAX; n as usize]; + assert_eq!(call::row_facet_match(s, 9, out.as_mut_ptr(), n), LGJ_OK); + for r in 0..n { + let mut want_bits = 0u32; + for f in 0..32u32 { + if store.classid_at(r, f) == 9 { + want_bits |= 1 << f; + } + } + assert_eq!(out[r as usize], want_bits, "row {r}"); + } + // A too-small caller buffer is rejected BEFORE anything is written. + let mut short = vec![0xABAB_ABABu32; 10]; + assert_eq!( + call::row_facet_match(s, 9, short.as_mut_ptr(), 10), + LGJ_ERR_MASK_LENGTH_MISMATCH + ); + assert!( + short.iter().all(|&x| x == 0xABAB_ABAB), + "untouched on failure" + ); + + // Lifecycle: same rules as every resource. + assert_eq!(lgj_close(m2), LGJ_OK); + assert_eq!(lgj_close(m), LGJ_OK); + assert_eq!(lgj_close(s), LGJ_OK); + assert_eq!( + call::row_facet_match(s, 9, out.as_mut_ptr(), n), + LGJ_ERR_INVALID_HANDLE + ); + } + /// Concurrent mask binops that name the same masks in *opposite* orders — /// the shape that would deadlock without address-ordered locking. #[test] diff --git a/native/lgj-abi/src/registry.rs b/native/lgj-abi/src/registry.rs index a8cd04a..dda345f 100644 --- a/native/lgj-abi/src/registry.rs +++ b/native/lgj-abi/src/registry.rs @@ -49,6 +49,7 @@ use std::sync::{Arc, OnceLock, RwLock, RwLockReadGuard, RwLockWriteGuard}; use crate::abi::*; use crate::fixture::{Fixture, PATTERN_LANE_COUNT}; +use crate::rowstore::{RowStore, ROWSTORE_LANE_COUNT}; /// The mutable half of a mask: the packed row bits. #[derive(Debug)] @@ -65,6 +66,9 @@ pub struct MaskWords { pub enum Payload { /// A read-only SoA fixture — no lock needed, because no ABI path mutates it. Pattern(Fixture), + /// A read-only SoA row store (abi.md §11) — likewise lock-free: the + /// `Arc<[u8]>` buffer is immutable for the resource's whole life. + RowStore(RowStore), /// Mutable mask words behind their own lock. Mask(RwLock), } @@ -96,7 +100,15 @@ impl ResourceEntry { pub fn fixture(&self) -> Option<&Fixture> { match &self.payload { Payload::Pattern(f) => Some(f), - Payload::Mask(_) => None, + _ => None, + } + } + + /// `Some(&RowStore)` iff this is a row store. + pub fn rowstore(&self) -> Option<&RowStore> { + match &self.payload { + Payload::RowStore(s) => Some(s), + _ => None, } } @@ -104,7 +116,7 @@ impl ResourceEntry { pub fn mask(&self) -> Option<&RwLock> { match &self.payload { Payload::Mask(m) => Some(m), - Payload::Pattern(_) => None, + _ => None, } } @@ -127,6 +139,7 @@ impl ResourceEntry { kind: self.kind, lane_count: match self.kind { LGJ_RESOURCE_PATTERN => PATTERN_LANE_COUNT, + LGJ_RESOURCE_ROWSTORE => ROWSTORE_LANE_COUNT, // A mask exposes exactly one MASK_WORD lane. _ => 1, }, @@ -278,9 +291,29 @@ pub fn open_pattern(n_rows: u64, seed: u64) -> Result { }) } +/// Create a row-store resource from the deterministic generator (abi.md §11). +pub fn open_rowstore(n_rows: u64, seed: u64) -> Result { + let store = RowStore::generate(n_rows, seed).ok_or(LGJ_ERR_LENGTH_OVERFLOW)?; + insert(ResourceEntry { + kind: LGJ_RESOURCE_ROWSTORE, + epoch: next_epoch(), + n_rows, + parent: 0, + parent_gen: 0, + payload: Payload::RowStore(store), + }) +} + /// Create a mask over `parent`, all bits `0` or all bits `1`. +/// +/// A mask's parent may be a pattern OR a row store — both are read-only +/// row-shaped resources, and a mask is a row selection over either. A mask +/// over a mask stays rejected. pub fn create_mask(parent_handle: u64, initial: u32) -> Result { - let parent = resolve_kind(parent_handle, LGJ_RESOURCE_PATTERN)?; + let parent = resolve(parent_handle)?; + if !matches!(parent.kind, LGJ_RESOURCE_PATTERN | LGJ_RESOURCE_ROWSTORE) { + return Err(LGJ_ERR_WRONG_RESOURCE_KIND); + } let n_rows = parent.n_rows; let n_words = usize::try_from(mask_words_for(n_rows)).map_err(|_| LGJ_ERR_LENGTH_OVERFLOW)?; @@ -486,6 +519,40 @@ mod tests { close(p).unwrap(); } + #[test] + fn rowstore_opens_and_describes_itself() { + let h = open_rowstore(70, 3).unwrap(); + let e = resolve_kind(h, LGJ_RESOURCE_ROWSTORE).unwrap(); + let info = e.info(); + assert_eq!(info.kind, LGJ_RESOURCE_ROWSTORE); + assert_eq!(info.lane_count, ROWSTORE_LANE_COUNT); + assert_eq!(info.n_rows, 70); + assert!(e.rowstore().is_some()); + assert!(e.fixture().is_none()); + close(h).unwrap(); + } + + /// A mask parents onto a row store exactly as onto a pattern — same + /// row-count sizing, same tail rule, same parent-liveness propagation. + #[test] + fn mask_over_rowstore_works_and_tracks_parent_liveness() { + let s = open_rowstore(70, 1).unwrap(); + let m = create_mask(s, LGJ_MASK_INIT_ALL).unwrap(); + { + let e = resolve(m).unwrap(); + let g = e.read_mask().unwrap(); + assert_eq!(g.words.len(), 2); + assert_eq!(g.words[1], 0x3F, "tail past row 70 must be zero"); + } + assert!(resolve_mask_with_parent(m).is_ok()); + close(s).unwrap(); + assert_eq!( + resolve_mask_with_parent(m).unwrap_err(), + LGJ_ERR_PARENT_CLOSED + ); + close(m).unwrap(); + } + /// Locking distinct masks in address order must not deadlock regardless of /// the order the caller names them in. #[test] diff --git a/native/lgj-abi/src/rowstore.rs b/native/lgj-abi/src/rowstore.rs new file mode 100644 index 0000000..0d3e8af --- /dev/null +++ b/native/lgj-abi/src/rowstore.rs @@ -0,0 +1,292 @@ +//! The SoA row store — the lance-graph-shaped substrate (abi.md §11). +//! +//! Where [`crate::fixture`] proved the membrane over three flat lanes, this +//! module carries the layout the whole stack actually converges on — the +//! operator-stated reference (2026-08-17): **64K rows × 512 bytes per row, +//! read as 32 lanes of 16 bytes each: a 4-byte little-endian classid plus a +//! 12-byte payload** (the lance-graph V3 content-blind facet). The Java side +//! may lay its *view* out differently; these bytes are the substrate truth. +//! +//! # One buffer, two readings, zero copies +//! +//! The store is ONE `Arc<[u8]>` of `n_rows * 512` bytes. Everything else is a +//! *reading* of those bytes, never a copy: +//! +//! - **Row reading** — row `r` is bytes `r*512 .. (r+1)*512`; facet `f` of +//! row `r` is the 16 bytes at `r*512 + f*16`, its classid the leading LE +//! `u32`. This is what `iter_u8x64`/`iter_u32x16`-style chunk scans and +//! Java's structured `MemoryLayout` both address. +//! - **Facet-lane reading** — classid lane `f` is a strided `u32` column: +//! `first_offset = f*16`, `stride = 512`, `count = n_rows`. This is what +//! [`crate::abi::LgjLaneDesc::stride_bytes`] has described since ABI 0.1 — +//! the descriptor anticipated this module. +//! +//! `Arc<[u8]>` is the deliberate carrier: its heap buffer never moves for the +//! Arc's whole life (the §4 allocation-stability guarantee), a clone is a +//! refcount bump (the kernels wrap the same bytes in an +//! `ndarray::simd::MultiLaneColumn` without copying), and shared immutable +//! ownership is exactly the one-writer-per-resource concurrency shape the +//! 64K-mailbox model wants. +//! +//! # Alignment (stated honestly) +//! +//! Rows are 512-byte *strided* within the buffer, but the buffer's base is +//! only `u8`-aligned — `Arc<[u8]>` cannot promise more on stable Rust. +//! Nothing in this slice needs more: Panama reads are alignment-agnostic +//! (`ValueLayout.JAVA_INT_UNALIGNED` exists precisely for this), and every +//! `ndarray::simd` load goes through `from_array`-style register fills. The +//! 64-byte-aligned guarantee arrives with the real `NodeRow` +//! (`#[repr(C, align(64))]`) wiring, not here. + +/// Bytes per row: 32 facets × 16 bytes. +pub const ROW_BYTES: u64 = 512; +/// Facet lanes per row. +pub const ROW_FACETS: u32 = 32; +/// Bytes per facet: 4-byte classid + 12-byte payload. +pub const FACET_BYTES: u64 = 16; +/// The classid is the facet's leading little-endian `u32`. +pub const FACET_CLASSID_BYTES: u64 = 4; +/// Classid cardinality the generator produces: `0..16` (same recipe as the +/// flat fixture, so predicates select the same middling fraction). +pub const ROWSTORE_CLASS_CARDINALITY: u64 = 16; + +/// Lane id of the raw whole-buffer lane (`U8`, contiguous, `n_rows * 512` +/// elements). +pub const LANE_RAW: u32 = 0; +/// Lane id of facet `f`'s classid lane is `LANE_FACET_BASE + f`. +pub const LANE_FACET_BASE: u32 = 1; +/// Total describable lanes: 1 raw + 32 facet classid lanes. +pub const ROWSTORE_LANE_COUNT: u32 = 1 + ROW_FACETS; + +use std::sync::Arc; + +use crate::fixture::SplitMix64; + +/// The SoA row store: one shared, immutable, address-stable byte buffer. +/// +/// # The generation algorithm — NORMATIVE +/// +/// Like [`crate::fixture::Fixture`], the Java parity test recomputes +/// expectations from this description alone, so it is a contract: +/// +/// ```text +/// rng = SplitMix64(seed) // state = seed, no warm-up draws +/// for row in 0 .. n_rows: // ascending +/// for facet in 0 .. 32: // ascending within the row +/// a = rng.next_u64() // FIRST draw of the facet +/// b = rng.next_u64() // SECOND draw of the facet +/// base = row*512 + facet*16 +/// bytes[base .. base+4 ] = le32( (a >>> 33) & 0xF ) // classid +/// bytes[base+4 .. base+12] = le64( b ) // payload +/// bytes[base+12 .. base+16] = le32( a & 0xFFFFFFFF ) // payload +/// ``` +/// +/// Two draws per facet, `a` before `b`, 64 draws per row. The classid recipe +/// `(a >>> 33) & 0xF` is byte-identical to the flat fixture's class lane, so +/// a classid predicate selects the same ≈1/16 fraction here. +pub struct RowStore { + /// Logical row count. + pub n_rows: u64, + /// The seed the buffer was generated from. + pub seed: u64, + bytes: Arc<[u8]>, +} + +impl std::fmt::Debug for RowStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RowStore") + .field("n_rows", &self.n_rows) + .field("seed", &self.seed) + .finish_non_exhaustive() + } +} + +impl RowStore { + /// Build the store. Allocates the buffer exactly once. + /// + /// Returns `None` if `n_rows * 512` overflows or cannot be allocated + /// (the caller maps that to `LENGTH_OVERFLOW` / `ALLOCATION_FAILED`). + pub fn generate(n_rows: u64, seed: u64) -> Option { + let n = usize::try_from(n_rows).ok()?; + let byte_len = n.checked_mul(ROW_BYTES as usize)?; + + let mut bytes = Vec::new(); + bytes.try_reserve_exact(byte_len).ok()?; + bytes.resize(byte_len, 0u8); + + let mut rng = SplitMix64::new(seed); + for row in 0..n { + for facet in 0..ROW_FACETS as usize { + let a = rng.next_u64(); + let b = rng.next_u64(); + let base = row * ROW_BYTES as usize + facet * FACET_BYTES as usize; + let classid = ((a >> 33) & (ROWSTORE_CLASS_CARDINALITY - 1)) as u32; + bytes[base..base + 4].copy_from_slice(&classid.to_le_bytes()); + bytes[base + 4..base + 12].copy_from_slice(&b.to_le_bytes()); + bytes[base + 12..base + 16].copy_from_slice(&(a as u32).to_le_bytes()); + } + } + + Some(Self { + n_rows, + seed, + bytes: Arc::from(bytes), + }) + } + + /// The whole buffer as a byte slice. Zero-copy; the address is stable for + /// the store's life (see the module header). + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } + + /// A cheap shared handle to the same bytes — what the kernels wrap in a + /// `MultiLaneColumn` without copying. + pub fn bytes_arc(&self) -> Arc<[u8]> { + Arc::clone(&self.bytes) + } + + /// The classid of facet `facet` in row `row` — the scalar (one-element) + /// read, used by tests and the scalar reference kernels. Bulk access goes + /// through the lanes, never through a loop over this. + pub fn classid_at(&self, row: u64, facet: u32) -> u32 { + let base = (row * ROW_BYTES + facet as u64 * FACET_BYTES) as usize; + u32::from_le_bytes([ + self.bytes[base], + self.bytes[base + 1], + self.bytes[base + 2], + self.bytes[base + 3], + ]) + } + + /// `(base address, len_elems, elem_kind, stride_bytes, contiguous)` for a + /// lane id, or `None` for an out-of-range id (⇒ `INVALID_LANE`). + pub fn lane_raw(&self, lane_id: u32) -> Option<(u64, u64, crate::abi::LgjElemKind, u32, bool)> { + use crate::abi::LgjElemKind; + if lane_id == LANE_RAW { + return Some(( + self.bytes.as_ptr() as u64, + self.bytes.len() as u64, + LgjElemKind::U8, + 1, + true, + )); + } + let facet = lane_id.checked_sub(LANE_FACET_BASE)?; + if facet >= ROW_FACETS { + return None; + } + // Classid lane f: strided u32 column at first_offset f*16, stride 512. + let addr = self.bytes.as_ptr() as u64 + facet as u64 * FACET_BYTES; + Some((addr, self.n_rows, LgjElemKind::U32, ROW_BYTES as u32, false)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generation_is_deterministic_and_seed_sensitive() { + let a = RowStore::generate(64, 42).unwrap(); + let b = RowStore::generate(64, 42).unwrap(); + let c = RowStore::generate(64, 43).unwrap(); + assert_eq!(a.as_bytes(), b.as_bytes()); + assert_ne!(a.as_bytes(), c.as_bytes()); + } + + /// The normative algorithm, recomputed independently from the doc-comment + /// description (a transcription, exactly what the Java test will do). + #[test] + fn the_documented_generator_is_the_actual_generator() { + let n = 5u64; + let seed = 0xABCD; + let store = RowStore::generate(n, seed).unwrap(); + + let mut rng = SplitMix64::new(seed); + for row in 0..n { + for facet in 0..ROW_FACETS { + let a = rng.next_u64(); + let b = rng.next_u64(); + let expect_class = ((a >> 33) & 0xF) as u32; + assert_eq!(store.classid_at(row, facet), expect_class); + let base = (row * ROW_BYTES + facet as u64 * FACET_BYTES) as usize; + assert_eq!(&store.as_bytes()[base + 4..base + 12], &b.to_le_bytes()); + assert_eq!( + &store.as_bytes()[base + 12..base + 16], + &(a as u32).to_le_bytes() + ); + } + } + } + + #[test] + fn the_buffer_is_exactly_n_times_512_bytes() { + for n in [0u64, 1, 7, 64] { + let s = RowStore::generate(n, 1).unwrap(); + assert_eq!(s.as_bytes().len() as u64, n * ROW_BYTES); + // …which is always a multiple of 64: the MultiLaneColumn + // precondition holds BY CONSTRUCTION, never by luck. + assert_eq!(s.as_bytes().len() % 64, 0); + } + } + + #[test] + fn lane_map_covers_raw_plus_32_facets_and_nothing_else() { + let s = RowStore::generate(16, 9).unwrap(); + let (addr0, len0, kind0, stride0, contig0) = s.lane_raw(LANE_RAW).unwrap(); + assert_eq!(addr0, s.as_bytes().as_ptr() as u64); + assert_eq!(len0, 16 * ROW_BYTES); + assert_eq!(kind0, crate::abi::LgjElemKind::U8); + assert_eq!(stride0, 1); + assert!(contig0); + + for f in 0..ROW_FACETS { + let (addr, len, kind, stride, contig) = s.lane_raw(LANE_FACET_BASE + f).unwrap(); + assert_eq!(addr, s.as_bytes().as_ptr() as u64 + f as u64 * FACET_BYTES); + assert_eq!(len, 16); + assert_eq!(kind, crate::abi::LgjElemKind::U32); + assert_eq!(stride, ROW_BYTES as u32); + assert!(!contig); + } + assert!(s.lane_raw(LANE_FACET_BASE + ROW_FACETS).is_none()); + assert!(s.lane_raw(u32::MAX).is_none()); + } + + /// Classids must use their full 0..16 range in every facet lane — a + /// constant lane would make every classid predicate vacuous. + #[test] + fn every_facet_lane_uses_the_full_classid_range() { + let s = RowStore::generate(4096, 7).unwrap(); + for facet in [0u32, 1, 15, 31] { + let mut seen = [false; 16]; + for row in 0..s.n_rows { + let c = s.classid_at(row, facet); + assert!(c < 16); + seen[c as usize] = true; + } + assert!( + seen.iter().all(|&x| x), + "facet {facet} must hit all 16 classids at n=4096" + ); + } + } + + #[test] + fn addresses_are_stable_across_reads() { + let s = RowStore::generate(32, 3).unwrap(); + let first = s.lane_raw(LANE_FACET_BASE + 5).unwrap().0; + for _ in 0..100 { + assert_eq!(s.lane_raw(LANE_FACET_BASE + 5).unwrap().0, first); + } + // And the Arc handle shares, never copies. + assert_eq!(s.bytes_arc().as_ptr(), s.as_bytes().as_ptr()); + } + + #[test] + fn zero_rows_is_legal_and_empty() { + let s = RowStore::generate(0, 1).unwrap(); + assert!(s.as_bytes().is_empty()); + assert_eq!(s.lane_raw(LANE_FACET_BASE).unwrap().1, 0); + } +} From 8fceacaf3a7215d1d123a38a6c1500db7e918c4b Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 22:31:42 +0000 Subject: [PATCH 07/30] Board: PR #5 arc entry (post-merge) Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/PR_ARC_INVENTORY.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index b6451d4..e4cb265 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -8,6 +8,35 @@ > anti-pattern the imported board rules name. Backfilled below in one > pass rather than left stale; PR #4 onward gets its entry at merge time. +## PR #5 — SoA row store: 512B rows, 32 facet lanes, ABI minor 2 (merged 2026-08-17, squash `78aa60e`) + +Companion: **AdaWorldAPI/ndarray#279** (W1), merged first — `lgj-abi`'s +`kernels.rs` calls `iter_u32x16` / `eq_u32_strided_to_mask` from it. + +- **Added:** `native/lgj-abi/src/rowstore.rs` (one `Arc<[u8]>`, two + readings, zero copies, normative SplitMix64 generator); + `LGJ_RESOURCE_ROWSTORE` + `lgj_rowstore_open` + `lgj_op_eq_classid` + + `lgj_row_facet_match`; `docs/abi.md` §11; the W1–W5 wave plan and three + consumer-example plans; `.claude/knowledge/soa-row-store-layout.md`. +- **Locked:** the 512 B / 32 × (4 B classid + 12 B payload) layout as + substrate truth (Java's view may differ); facet lanes ride the + **unchanged** `LgjLaneDesc`; masks parent onto row stores so the whole + existing mask algebra applies with no new surface; `byte_len` is the + exact covered span `(len-1)*stride + elem_bytes` (the old `len*stride` + form would have let Java bound a segment past the allocation's end); + ABI minor 1→2 and the §1/§7 symbol count corrected 14→18 (`nm -D`). + Doctrine: `E-LGJ-THE-MIDDLE-TIER-IS-DELETED-NOT-WRAPPED-1`; + self-correction: `E-LGJ-THE-FLAT-FIXTURE-WAS-SCAFFOLDING-NOT-THE-TARGET-1`. +- **Deferred:** `align(64)` base (stated honestly — arrives with real + `NodeRow`); payload semantics (a ClassView concern one layer up); fused + plans over facet lanes (W6, only if measurement asks). +- **Docs:** `abi.md` §11 + 4 plan files + 1 knowledge doc + full board. +- **Confidence:** High — 84/84, clippy/fmt clean, 18/18 symbols; both new + kernels parity-checked against independent scalar references over + 10 row counts × 2 seeds × 4 facets × 4 needles and cross-checked a third + way; two-sided payload-vs-classid falsifier. Both bot reviewers (cursor, + codex) hit usage limits and did not run. + ## PR #4 — Phase I synthesis docs + fusion re-run + board hygiene (merged 2026-08-17, squash `bd92c58`) - **Added:** `docs/{architecture,panama,valhalla-lab,execution-boundary}.md` From 91e22f3f8ef19c941deb6d15366de4310d8f59be Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 22:36:23 +0000 Subject: [PATCH 08/30] Knowledge: assess the archived layout-bridge discussion; name W6 The operator's pre-build ChatGPT discussion is assessed once, in .claude/knowledge/prior-art-and-the-layout-bridge-claim.md, so it is never re-mined or cited naively. Verdict: it converged independently on the architecture this repo then built and measured. Kept: the callability-vs-shared-executable-layout positioning, the schema-key-as- join-point extractable (now the named W6 consideration: an explicit schema/classid field on the descriptors when ClassView lands), and the baseline-dependent claims discipline for W5 comparisons. Pinned: its page-descriptor sketch has no liveness story (the registry's whole job), its native-always-wins assumption is measured false (Component C), and its ndarray paragraph describes upstream crates.io ndarray, not the AdaWorldAPI fork whose ndarray::simd polyfill this stack mandates. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .../prior-art-and-the-layout-bridge-claim.md | 109 ++++++++++++++++++ .claude/plans/lgj-soa-substrate-v1.md | 1 + 2 files changed, 110 insertions(+) create mode 100644 .claude/knowledge/prior-art-and-the-layout-bridge-claim.md diff --git a/.claude/knowledge/prior-art-and-the-layout-bridge-claim.md b/.claude/knowledge/prior-art-and-the-layout-bridge-claim.md new file mode 100644 index 0000000..36c771c --- /dev/null +++ b/.claude/knowledge/prior-art-and-the-layout-bridge-claim.md @@ -0,0 +1,109 @@ +# Prior art, and the claim that distinguishes this project + +> **READ BY:** anyone writing positioning/architecture prose, the consumer- +> example authors (W5), and any session tempted to cite the operator's +> archived ChatGPT discussion (2026-08-17, predates this repo's build) as +> design authority. That discussion is assessed HERE, once — its residual +> value extracted, its errors pinned — so it is not re-mined. + +## Assessment verdict + +The discussion **converged independently on the architecture this repo then +built and measured** — layout-centric bridge, engine-chooses-the-kernel, +"Java understands the layout, Rust owns the physics," a brutally small +vertical-slice MVP. Convergence from an independent derivation is mild +evidence the shape is right, and zero evidence about anything the +discussion *assumed but never measured*. Three parts remain valuable; two +parts are wrong relative to what this repo has since measured or mandates. + +## 1. The distinction that holds up: callability vs shared executable layout + +The competitive landscape, correctly characterized there and worth keeping: + +| bridge | what it is | what it solves | +|---|---|---| +| JNI / `jni-rs` | env pointers, object handles, marshaling ceremony | callability | +| `j4rs` | higher-level interop layer | callability | +| UniFFI | binding generator (Kotlin/Swift/Python official; Java external, FFM-based) | callability | +| Panama FFM (JEP 454, final since JDK 22) | the *mechanism* — `MemorySegment`/`MemoryLayout`/`Linker` | reachability, not semantics | + +All of these are **function bridges**: Java calls a native function, values +marshal across. This project's claim is a **layout bridge**: Java and Rust +*execute over the same bytes*, and the contract is the layout itself +(`docs/abi.md`), runtime-proven by the manifest cross-check. The +discussion's phrase for the moat — solving *"shared executable layout"* +where existing bridges solve *"callability"* — is the crispest one-line +positioning this project has and is worth using verbatim. + +## 2. Where the discussion's sketch was WEAKER than what was built + +Its `OgarAbiPage` sketch (`schema_key, row_count, stride, lane_count, +flags, data_ptr`) carries a bare `data_ptr` with **no lifetime story at +all** — no generation check, no epoch, no owner, no parent-liveness, no +close semantics. That is precisely the machinery this repo's registry +provides and **disable-verifies** (`abi.md` §4: no code path in which a +stale handle dereferences freed memory). A page descriptor without a +liveness protocol is a use-after-free with documentation. Keep this as the +standing answer to "couldn't the ABI just be one page struct?" + +## 3. The genuinely forward-looking extractable: the schema key as the join point + +> "One ABI key connects: Java MemoryLayout, Rust repr(C)/SoA, ClassView, +> ontology predicate, low-code block, Lance column projection, SIMD +> kernel." + +This is the OGAR classid doctrine — *classid is pure address; the magic is +what it resolves to* — arriving from an independent direction, which is +worth something. Concretely for this repo: today a resource names its +layout contract only *implicitly* (`kind == ROWSTORE` ⇒ the §11 geometry). +When the real ClassView slice lands, `LgjResourceInfo` (and/or +`LgjLaneDesc`) should gain an explicit **schema/classid field** so a +resource names *which* layout contract its bytes obey — additive, one +minor bump, and it is what makes "one key, many projections" literal at +the membrane. Filed as the W6 consideration in +`.claude/plans/lgj-soa-substrate-v1.md`. + +## 4. The claims discipline worth adopting verbatim + +The discussion's own guard against its headline numbers: never state +"2000×" as an engine claim. The architecture turns *pathological baselines* +(object-per-edge allocation, boxed values, string predicates, virtual +dispatch, JNI marshaling, JSON serialization, per-row Java loops) into +dense native kernels — so deltas are **baseline-dependent**: 10×, 100×, +1000×+ depending on how bad the baseline was. This repo's bench already +practices the stronger form (measure, publish the reproduction command, +state the noise floor); the W5 consumer examples MUST phrase any +comparison this way, and the graph-traversal plan already cites the +six-component BEFORE stack *as architecture, not as a benchmarked number*. + +## 5. Two corrections — pin these so the text is never cited naively + +1. **"Rust/RISC-speed execution through the membrane" assumes the native + side always wins. Measured false here.** Component C: the Java Vector + API, zero-copy on the *same* native segment, beats the native crossing + at **every** row count tested (64 → 4.2M) for a single predicate; the + crossing pays off for *composed, fused* work (Component E, 10.8–31.1×). + The real split is finer than the discussion's frame and lives in + `docs/execution-boundary.md`. Any prose inheriting the discussion's + framing must inherit the measurement instead. +2. **Its ndarray paragraph is about UPSTREAM crates.io ndarray** ("an + ergonomic n-dimensional array crate … `matrixmultiply` underneath, not + a SIMD layer"). The **AdaWorldAPI fork** this stack mandates is a + different artifact: `ndarray::simd` IS the SIMD polyfill layer + (dispatched AVX-512/AVX2/NEON/wasm/scalar under the W1a consumer + contract), and this project's §8 provenance rule depends on that. Do + not import the discussion's correction against the fork. (Its nightly + caveat about `std::simd` portable SIMD is true and is exactly why the + polyfill exists on stable.) +3. *(minor, was already true)* Java's Vector API is still incubating — + consistent with this repo's `--add-modules jdk.incubator.vector` and + noted in `jdk-toolchain-facts.md`. + +## The slogan shelf (used sparingly, they are earned now) + +- *"Java understands the layout. Rust owns the physics."* — matches + `architecture.md` §layers; safe to use. +- *"Panama gives Java fingers; this ABI gives it bones, tendons, and a + nervous system."* — rhetorically fine; keep it out of normative docs. +- *"Existing bridges solve callability; this solves shared executable + layout."* — the best of the three; use in positioning. diff --git a/.claude/plans/lgj-soa-substrate-v1.md b/.claude/plans/lgj-soa-substrate-v1.md index 6feed96..dc8a684 100644 --- a/.claude/plans/lgj-soa-substrate-v1.md +++ b/.claude/plans/lgj-soa-substrate-v1.md @@ -32,6 +32,7 @@ | **W3** | Java `RowStore` facade: no FFM in public signatures; structured `MemoryLayout` (`sequence(32, struct(u32 classid, 12B payload))`); minor-≥2 gate; `FacetMatchView` zero-copy accessor over a Java-arena segment; `RowStoreParityTest` transcribing the generator | OPEN — next | | **W4** | Bench Component F: Java Vector API per-row facet scan (one `IntVector` 16-lane chunk = 4 facets, same algorithm as the Rust kernel) vs `lgj_row_facet_match` crossing vs scalar VarHandle walk — the "where does execution belong" question re-asked on the REAL layout | OPEN | | **W5** | The three consumer examples (own plan files, below) | PLANNED | +| **W6** *(named, not scheduled)* | ClassView wiring — and with it an explicit **schema/classid field** on `LgjResourceInfo`/`LgjLaneDesc` (additive, one minor bump), so a resource names WHICH layout contract its bytes obey instead of implying it via `kind`. Provenance + rationale: `.claude/knowledge/prior-art-and-the-layout-bridge-claim.md` §3 ("one key, many projections" made literal at the membrane). Also the `align(64)` base guarantee (real `NodeRow`) and, only if measurement asks, fused plans over facet lanes | NAMED | Wave rule (house style): one wave = one reviewable PR; gates run centrally (orchestrator only — agents never run cargo); every safety property lands From 69279615b9ef1fcf71552aaca719015c5de91bd8 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 22:39:20 +0000 Subject: [PATCH 09/30] Plans: OGAR Machine (exploratory) + lance-graph-hydrate dependency note MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Captures the operator's second archived context as .claude/plans/ogar-machine-v1.md — a genuinely new workload for the shipped substrate, not convergent confirmation: one row = one machine STATE, control flow as population masks over 64K execution contexts, Ghidra P-code as the normalized guest ISA (repo attached and cloned), differential migration testing (legacy XOR replacement across 65,536 worlds) as the killer demo, Lance as the time machine. Strong claim vs weak claim separated per the discussion's own discipline; gated on W3 + one W5 example + Ghidra archaeology + a tiny falsifiable probe (P-M1). Also records lance-graph #957 (merged: lance-graph-hydrate, the generic SoA->S3->volume->Lance hydration crate minted for consumers to inherit) and #958 (its open hardening fast-follow) in the substrate plan: when this repo's persistence slice arrives, hydration is inherited from lance-graph-hydrate, never re-derived here. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/INTEGRATION_PLANS.md | 17 ++++ .claude/plans/lgj-soa-substrate-v1.md | 12 +++ .claude/plans/ogar-machine-v1.md | 116 ++++++++++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 .claude/plans/ogar-machine-v1.md diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index 65043dd..806f532 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -1,3 +1,20 @@ +## 2026-08-17 — ogar-machine-v1 (EXPLORATORY; the population emulator) + +Plan: `.claude/plans/ogar-machine-v1.md`. Captured from the operator's +second archived ChatGPT context (the first was convergent confirmation; +this one is a genuinely NEW workload): one row = one machine STATE, +control flow becomes population masks ("which machines currently execute +ADD?"), Ghidra P-code as the normalized guest ISA +(`AdaWorldAPI/ghidra` now attached + shallow-cloned at +`/workspace/ghidra`), differential migration testing as the killer demo +(65,536 worlds through legacy XOR replacement → the divergent few), +Lance as the time machine (via `lance-graph-hydrate`, lance-graph +#957/#958 — inherited, never re-implemented). Strong/weak claims +separated in the plan; gated on W3 + one W5 example + Ghidra +archaeology + probe P-M1. + +**Status: NAMED, not scheduled.** W3 remains the next action. + ## 2026-08-17 — consumer-{world-trades,bricks-analytics,graph-traversal}-v1 (PLANS; the three W5 consumer examples) Plans: `.claude/plans/consumer-world-trades-v1.md`, diff --git a/.claude/plans/lgj-soa-substrate-v1.md b/.claude/plans/lgj-soa-substrate-v1.md index dc8a684..ea3b6af 100644 --- a/.claude/plans/lgj-soa-substrate-v1.md +++ b/.claude/plans/lgj-soa-substrate-v1.md @@ -38,6 +38,18 @@ Wave rule (house style): one wave = one reviewable PR; gates run centrally (orchestrator only — agents never run cargo); every safety property lands disable-verified, every measured claim lands with its reproduction command. +**Cross-repo dependency note (2026-08-17, operator-flagged):** lance-graph +**#957** (merged) minted `crates/lance-graph-hydrate` — the generic +SoA→S3→volume→Lance hydration pattern (four-state lifecycle, +hydrate-aside/publish-by-rename, warm markers, dirty detection), minted in +lance-graph *specifically so consumers inherit it as a path/git dependency +rather than re-implement*. **#958** (open, another session's PR) is its 5+3 +council hardening fast-follow. Consequence here: when this substrate's +persistence slice arrives (the "Seal & Persist (Lance)" column of the +formula, and `ogar-machine-v1.md`'s time-machine storage), the hydration +path is `lance-graph-hydrate` — inherited, never re-derived. Do not design +a hydration mechanism in this repo. + ## What W2 locked (so W3+ doesn't re-derive it) - **Layout truth:** `ROW_BYTES=512`, `ROW_FACETS=32`, `FACET_BYTES=16`, diff --git a/.claude/plans/ogar-machine-v1.md b/.claude/plans/ogar-machine-v1.md new file mode 100644 index 0000000..1fb9519 --- /dev/null +++ b/.claude/plans/ogar-machine-v1.md @@ -0,0 +1,116 @@ +# ogar-machine-v1 — the population emulator (EXPLORATORY, named not scheduled) + +> **Status: NAMED** (2026-08-17). Source: the operator's second archived +> ChatGPT context, assessed here once so it is not re-mined. Unlike the +> first archived discussion (convergent confirmation — +> `.claude/knowledge/prior-art-and-the-layout-bridge-claim.md`), this one +> proposes a **genuinely new workload** for the shipped substrate, and the +> operator has attached `AdaWorldAPI/ghidra` to the session for it. +> Nothing here reorders the wave plan: W3 (Java `RowStore` facade) remains +> the next action; this plan is the shelf the idea sits on, with its +> strong/weak claims already separated. + +## The inversion, in one sentence + +A VM is `state + transition → new state`; instead of one machine executing +instructions sequentially, **one row is one machine STATE**, and the +substrate executes each *operation* across the whole population at once: + +``` +65,536 execution contexts (rows) + → classify current opcode (a classid-style scan → population masks) + → per-population bulk execution (SIMD ADD over the ADD mask, gathers + over the LOAD mask, mask updates for + BRANCH) + → next state +``` + +The VM never asks *"what does machine 17 execute?"* — it asks *"which +machines currently execute ADD?"* **Control flow becomes population +masks.** That is not an analogy to the shipped substrate; it is literally +its op set: rows, classid scans, mask algebra, masked bulk ops, survivors. + +## The claim discipline (the discussion's own, kept sharp) + +- **Weak claim, explicitly rejected**: "emulate a Pentium faster than + QEMU." One interactive instance is branchy and sequential; SIMD is not + fairy dust for `A then B then C`. Do not build toward this and do not + let a demo imply it. +- **Strong claim, the actual target**: *from Java, explore 65,536 + executions of an unsupported binary simultaneously, time-travel them, + XOR them against the replacement, and return only the worlds where + behavior diverges.* Every element of that sentence maps to something + this stack already does well (population masks / Lance versioning / + mask XOR / survivors-only inspection). + +## Ghidra's role — front-end compiler, never a peer emulator + +Do NOT implement x86/ARM/68k. Ghidra lifts the legacy binary **once** to +P-code (a small normalized op set: COPY, LOAD, STORE, BRANCH, CBRANCH, +CALL, RETURN, INT_ADD/SUB/MULT/AND/OR/XOR, …); the OGAR Machine executes +P-code only. `legacy.exe → Ghidra (once) → normalized program image → +population execution`. This is the same shape as the workspace's +ruff→OGAR harvest arms: an existing analyzer becomes the transcoder +front-end, and the substrate executes the normalized IR. First concrete +archaeology step in the fresh clone: locate the P-code opcode enum and +SLEIGH lifting surface, and size the *real* op set (the list above is the +discussion's sketch, not a verified inventory). + +## The four load-bearing design rules captured from the discussion + +1. **Differential migration testing is the killer demo.** Same 65,536 + input worlds through the legacy machine and the replacement; + `legacy_output[] XOR new_output[] → divergence_mask`; show the 15 of + 65,536 worlds the rewrite broke, click into one, see the concrete + state. (Note for a future warden pass: outcome-XOR here is a + *comparison*, not a state-transition kernel — it does not touch + lance-graph's `I-SUBSTRATE-MARKOV` XOR restriction, which governs + transition bundling.) +2. **Lance is the time machine.** Per-cycle sealed diffs (changed + registers / changed pages), `machine.at(cycle)`, `machine.diff(a, b)` — + deterministic rewind as a *consequence* of the substrate, matching the + workspace's episodic-=-Lance-versions doctrine. No bespoke snapshot + format. +3. **Memory purity resistance** (the discussion's own best guard): guest + RAM does NOT go into 512-byte rows. Semantic machine state (PC, regs, + flags, device state, identities, relations) → SoA lanes; the memory + image (pages) → a dense backing store the lens points at. This mirrors + the substrate's own key/value split — meaning and addressing in the + graph, bulk bytes dense and compressible. +4. **Semantic shims erode the emulator.** Recognize stable external + surfaces (USER/GDI/KERNEL/ODBC/filesystem/registry) and progressively + replace instruction execution with semantic operations — 90/10 → + 40/60 → 5/95. The endpoint is not `binary → reconstructed source` but + `binary → normalized behavior machine`, which is the OGAR transpile + doctrine's 85/15 split arrived at by *running* the program instead of + parsing it. + +Also named, further out: carrying BOTH branch populations at a CBRANCH +(state forking) drifts toward symbolic execution / abstract interpretation +done as masks over concrete states — file under "unthinkable until the +plain version works." + +## Mapping onto shipped primitives (why this is not science fiction) + +| OGAR Machine need | shipped today | +|---|---| +| 65,536 contexts as rows | the row store (64K × 512 B was the design point) | +| opcode classification → masks | `eq_u32`-family scans + `lgj_op_eq_classid` shape | +| population dispatch | mask algebra (`and`/`or`/`count`), masks-parent-on-rowstore | +| divergence mask | mask XOR (a small W-tier ABI addition when needed — goes through the wave process, never ad hoc) | +| survivors-only inspection | the whole thesis (PR #1's laziness/mask discipline) | +| time travel | Lance versioning (not yet wired here; lance-graph owns it) | + +## Gate — what must exist before ANY OGAR Machine code + +1. W3 (Java facade) and at least one W5 consumer example shipped, so the + consumer-never-grows-the-membrane rule has a proven workflow. +2. Ghidra archaeology: the real P-code op inventory + how a lifted + program image serializes (sized, not sketched). +3. **Probe P-M1** (the first falsifiable step, deliberately tiny): ONE + hand-written P-code program (a dozen ops, one CBRANCH), 64K input + worlds, executed by (a) a scalar reference interpreter and (b) the + population path over the row store — bit-identical final states + required. Two-sided divergence falsifier: plant one known behavioral + difference between two program variants and require the divergence + mask to find exactly that world set, no more, no fewer. From 9cc85ad7b54f633b3168266bbe04c92b424bbd38 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 22:40:21 +0000 Subject: [PATCH 10/30] Board: PR #6 arc entry (post-merge) Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/PR_ARC_INVENTORY.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index e4cb265..3a5ee6a 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -8,6 +8,28 @@ > anti-pattern the imported board rules name. Backfilled below in one > pass rather than left stale; PR #4 onward gets its entry at merge time. +## PR #6 — layout-bridge assessment, OGAR Machine plan, hydrate note (merged 2026-08-17, squash `8954e53`) + +- **Added:** `.claude/knowledge/prior-art-and-the-layout-bridge-claim.md` + (the first archived ChatGPT context assessed once: convergent + confirmation; kept the callability-vs-shared-executable-layout + positioning + the schema-key extractable + the baseline-dependent + claims discipline; pinned its three errors so it is never cited + naively); `.claude/plans/ogar-machine-v1.md` (the second context — a + genuinely NEW workload: population emulation, one row = one machine + state, Ghidra P-code as guest ISA, differential migration testing; + EXPLORATORY, gated on W3 + a W5 example + archaeology + probe P-M1). +- **Locked:** W6 named in the substrate plan (schema/classid field on the + descriptors when ClassView lands); lance-graph #957's + `lance-graph-hydrate` recorded as the INHERITED hydration path — never + re-derived here (#958 is its hardening fast-follow, owned elsewhere). +- **Deferred:** everything in ogar-machine-v1 (named, not scheduled). +- **Docs:** the PR IS docs; `AdaWorldAPI/ghidra` attached + shallow-cloned + at `/workspace/ghidra` for the future P-code archaeology. +- **Confidence:** High for the assessments (checked against shipped code + and measurements); the OGAR Machine plan is explicitly exploratory. + Both bot reviewers hit usage limits and did not run. + ## PR #5 — SoA row store: 512B rows, 32 facet lanes, ABI minor 2 (merged 2026-08-17, squash `78aa60e`) Companion: **AdaWorldAPI/ndarray#279** (W1), merged first — `lgj-abi`'s From f3bbab740a364de95117a73f4e6e05220aae63a3 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 22:50:05 +0000 Subject: [PATCH 11/30] Waves calcified: dispatch maps for every plan; Ghidra plan from real archaeology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator ruling: calcify, don't execute. .claude/waves/ holds one dispatchable map per plan — README with the standing rules and the verbatim worker preamble, substrate W3+W4 (the only READY wave), three consumer waves stamped DO-NOT-DISPATCH, Ghidra G1+G2, and OGAR-Machine P-M1 (BLOCKED behind a 4-condition gate including an explicit operator go). Each map carries disjoint worker scopes, orchestrator-only steps, exact gate commands, disable-runs, and STOP triggers. ghidra-integration-v1.md is written from archaeology against the real clone, not the sketch: 74 P-code opcodes (CPUI_MAX=75), 12.2 DEV / Java 25+, analyzeHeadless entry, and Ghidra's own PcodeEmulator as the reference-implementation parity oracle (the tesseract-rs method). The ogar-machine plan is cross-updated to cite it. Mapping-time catches that would have burned a dispatch: the graph consumer needs a deliberate edge-bearing generator arm (today's payload is PRNG noise) - a substrate change, flagged in the wave; the hop has a real D1a/D1b design fork with ruling guidance recorded. Muscle memory pinned as E-LGJ-CALCIFY-THEN-DISPATCH-1: the eight earned-this-session rules and the plan->wave->shelf->dispatch rhythm. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/EPIPHANIES.md | 46 +++++++ .claude/board/LATEST_STATE.md | 26 ++++ .claude/plans/ghidra-integration-v1.md | 90 ++++++++++++++ .claude/plans/ogar-machine-v1.md | 14 ++- .claude/waves/README.md | 53 ++++++++ .claude/waves/wave-consumer-bricks.md | 54 ++++++++ .claude/waves/wave-consumer-graph.md | 66 ++++++++++ .claude/waves/wave-consumer-trades.md | 45 +++++++ .claude/waves/wave-ghidra-g1-g2.md | 84 +++++++++++++ .claude/waves/wave-ogar-machine-pm1.md | 62 ++++++++++ .claude/waves/wave-substrate-w3-w4.md | 165 +++++++++++++++++++++++++ 11 files changed, 701 insertions(+), 4 deletions(-) create mode 100644 .claude/plans/ghidra-integration-v1.md create mode 100644 .claude/waves/README.md create mode 100644 .claude/waves/wave-consumer-bricks.md create mode 100644 .claude/waves/wave-consumer-graph.md create mode 100644 .claude/waves/wave-consumer-trades.md create mode 100644 .claude/waves/wave-ghidra-g1-g2.md create mode 100644 .claude/waves/wave-ogar-machine-pm1.md create mode 100644 .claude/waves/wave-substrate-w3-w4.md diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 542dd59..afd9ea3 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -4,6 +4,52 @@ > `**Status:**`/`**Confidence:**` line. A correction gets its own new, > dated entry that references the one it corrects — the storno rule. +## 2026-08-17 — E-LGJ-CALCIFY-THEN-DISPATCH-1 + +**Status:** DOCTRINE (operator-ruled: "don't execute the consumer plans yet, +just calcify the insights and make sure the muscle memory of the epiphanies +helps to gain momentum"). **Confidence:** High. + +The working rhythm this repo now runs on, made explicit so it compounds +instead of being re-derived: + +**plan → wave map → (shelf) → dispatch → gates → merge → arc entry** + +A *plan* says what and why. A *wave file* (`.claude/waves/`) says exactly +who edits which file under which verbatim guardrails, with which disable-runs +and gate commands — dispatchable as-is, months later, by a session with zero +shared context. Writing the wave map WITHOUT executing it is not deferral; +it is the calcification step: decisions get made while the context is hot +(worker scopes, D1-style design forks, STOP triggers, the graph wave's +discovery that the fixture payload is PRNG noise and traversal needs a +deliberate edge-bearing generator arm — found at MAPPING time, not +mid-dispatch), and execution later starts from momentum instead of from +archaeology. + +**The muscle memory, in one list** (each item earned at least once this +session, provenance in the entries below and in PR bodies #1–#6): + +1. **Disable-run or it didn't happen.** Green tests prove nothing about a + guard; break the thing, watch exactly the right tests go red, restore. +2. **Scaffolding-vs-target check.** When a proposal doesn't fit the code, + ask which one is the placeholder before declining the proposal. +3. **The membrane never grows from the consumer side.** A needed symbol + goes back through the wave process (now stamped in every consumer wave). +4. **Measure before believing direction** — the Vector API beat the + crossing; fusion was noise at 65K rows and 3× at 256; the doc that + assumed otherwise got corrected by the bench, not vice versa. +5. **Independent recomputation over golden blobs** — parity tests + transcribe the generator; two (better: three) independent paths to one + number. +6. **Assessments happen once, on the record** — archived discussions get + one knowledge-doc verdict (kept/pinned-wrong) so they are never + re-mined or cited naively. +7. **Exact-span over round-up** at every boundary a segment can be built + from (the `byte_len` lesson — the difference between a view and an + out-of-bounds capability). +8. **Board in the same commit** as the work it records; arc entry at + merge; realign after every squash. + ## 2026-08-17 — E-LGJ-THE-MIDDLE-TIER-IS-DELETED-NOT-WRAPPED-1 **Status:** DOCTRINE (operator-stated, scope confirmed). **Confidence:** High — diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 941243c..dea97e2 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,29 @@ +## 2026-08-17 (latest) — waves calcified, Ghidra plan grounded, NOTHING dispatched + +Operator ruling: consumer plans are **calcified, not executed** — insights +locked in while hot, execution starts from momentum later. Rhythm now +explicit in `E-LGJ-CALCIFY-THEN-DISPATCH-1` (plan → wave map → shelf → +dispatch → gates → merge → arc). + +- **`.claude/waves/`** created: README (standing rules + the verbatim + worker preamble) + six dispatchable wave maps — substrate W3+W4 (the + only one marked READY; W3 is still the next action), three consumer + waves (DO-NOT-DISPATCH), Ghidra G1+G2 (shelved), OGAR-Machine P-M1 + (BLOCKED, 4-condition gate incl. explicit operator go). +- **`ghidra-integration-v1.md`** written from REAL archaeology against + the fresh clone (`/workspace/ghidra`, 12.2 DEV, Java 25+): the true + P-code op set is **74 opcodes** (not the sketch's ~13); Ghidra ships + its own sequential `PcodeEmulator` — upgrading the OGAR-Machine oracle + story to reference-implementation parity (the tesseract-rs method); + `Toy` processor = the minimal lift target; `SymbolicSummaryZ3` = the + in-tree precedent for the far-future branch-population direction. +- **Mapping-time discoveries** (the payoff of calcifying): the graph + consumer needs a deliberate edge-bearing generator arm (fixture payload + is PRNG noise) — a substrate change, flagged before anyone hits it + mid-dispatch; the graph hop has a real design fork (D1a Java-side + scatter via WRITABLE mask words / D1b native `lgj_hop`, minor 3) with + ruling guidance recorded. + ## 2026-08-17 (Slice 2) — the SoA row store is REAL: ABI minor 2, W1+W2 shipped **The reframing that started it** (operator, three directives): the flat diff --git a/.claude/plans/ghidra-integration-v1.md b/.claude/plans/ghidra-integration-v1.md new file mode 100644 index 0000000..286a1d3 --- /dev/null +++ b/.claude/plans/ghidra-integration-v1.md @@ -0,0 +1,90 @@ +# ghidra-integration-v1 — the P-code front end and the parity oracle + +> **Status: PLANNED** (2026-08-17). Companion to `ogar-machine-v1.md` — +> that plan owns the population-execution side; this one owns the Ghidra +> side: how a legacy binary becomes a normalized program image, and how +> Ghidra's own emulator becomes the parity oracle. G0 (archaeology) ran +> against the real clone before this plan was written; every path and +> number below is verified, not sketched. + +## G0 — archaeology (DONE 2026-08-17, against `/workspace/ghidra` @ `52bb03d`) + +`AdaWorldAPI/ghidra` is a fork of upstream at **12.2 DEV** +(`Ghidra/application.properties`), minimum **Java 25**, Gradle ≥ 9.1 — +our JDK 26 toolchain covers it. + +| fact | where | why it matters | +|---|---|---| +| **74 real P-code opcodes** (`CPUI_COPY = 1` … `CPUI_MAX = 75`) | `Ghidra/Features/Decompiler/src/decompile/cpp/opcodes.hh:37-131` | The discussion's ~13-op sketch was a fraction; the real normalized ISA is 74 ops (arithmetic incl. signed/unsigned compares, zext/sext, FLOAT_* family, MULTIEQUAL/INDIRECT decompiler ops). The population interpreter implements a **subset** and must halt loudly on the rest — see op-set discipline below | +| Java-side opcode mirror | `Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PcodeOp.java` | The lift script reads ops through this surface (`Instruction.getPcode()`) | +| Headless entry point | `Ghidra/RuntimeScripts/support/analyzeHeadless` (+ `analyzeHeadlessREADME.md`) | Lifting is a batch job: import binary → analyze → post-script → exit. No GUI anywhere in the loop | +| **In-tree P-code emulator** | `Ghidra/Framework/Emulation/src/main/java/ghidra/pcode/emu/` (`PcodeEmulator`, `PcodeMachine`, `BytesPcodeThread`, `DefaultPcodeThread`…) | **The parity oracle.** Ghidra ships a sequential reference execution of exactly the IR we will population-execute — the same role libtesseract played for tesseract-rs. We never have to *invent* ground truth | +| `Toy` processor | `Ghidra/Processors/Toy` | A teaching ISA with full SLEIGH spec — the smallest possible lift target for G1, before any real x86 binary | +| ~39 processor modules incl. x86, AARCH64, RISCV, JVM, Z80, 68000 | `Ghidra/Processors/` | The "don't implement x86" promise is real: every guest ISA arrives through SLEIGH, and the OGAR Machine sees only the one 74-op IR | +| Symbolic extension precedent | `Ghidra/Extensions/SymbolicSummaryZ3` | The "carry both branch populations" far-future direction has an in-tree symbolic P-code precedent to study first — do not design that from scratch | + +License note: Ghidra core is Apache-2.0 (compatible with everything in +this stack); the `GPL/` subtree (demangler etc.) is not needed for +lifting and stays untouched. + +## The integration shape — two roles, both offline + +``` +legacy binary ──analyzeHeadless + post-script──▶ normalized program image (LE file) + │ + ┌────────────────────────┤ + ▼ ▼ + Ghidra PcodeEmulator OGAR Machine population path + (sequential ORACLE) (rows = machine states) + │ │ + └──────── parity ────────┘ +``` + +1. **Ghidra as lift-time compiler.** Runs ONCE per binary, offline. Never + at OGAR-Machine runtime — the same footing tesseract-rs gives its C++ + oracles ("only the oracle's link deps, never in the Rust path"). +2. **Ghidra as parity oracle.** `PcodeEmulator` executes the same program + sequentially; the population path must match it bit-for-bit per world. + This is the tesseract-rs byte-parity method transplanted: we diff + against the reference *implementation*, not against our own scalar + rewrite alone (which stays as the second, independent check). + +## Waves + +| wave | deliverable | falsifier | +|---|---|---| +| **G1 — lift proof** | An `analyzeHeadless` post-script (lands in the ghidra fork, `ghidra_scripts/`) that walks one function of a **Toy**-ISA test binary and dumps every instruction's `getPcode()` sequence to a deterministic text form | The dump's opcode mnemonics/order for that function match the decompiler's own listing view of the same function — two independent Ghidra surfaces agreeing, not one surface trusted | +| **G2 — the image format** | A versioned LE program-image format (header + op records: `opcode u32` + varnode triples `(space u32, offset u64, size u32)` for output/inputs) + a Rust loader in a NEW crate (NOT `lgj-abi` — the membrane stays lean; the loader is a consumer-tier crate) | Round-trip (emit → load → re-emit byte-identical); a hand-built image of known ops decodes to exactly those ops; a truncated/garbled image is refused with a status, never partially loaded | +| **G3 — probe P-M1** (shared gate with `ogar-machine-v1.md`) | Population interpreter over a deliberately small op subset (COPY, INT_ADD/SUB/AND/OR/XOR, INT_EQUAL/LESS, CBRANCH, BRANCH, RETURN), one hand-written program, 64K input worlds over the row store | Bit-identical final states vs an independent scalar Rust interpreter; planted-divergence two-sided test (one known behavioral difference between two program variants → the divergence mask finds exactly that world set) | +| **G4 — oracle parity** | The same image through Ghidra's `PcodeEmulator` (JVM side, sequential, N sampled worlds) vs the population path | Final machine state bit-identical per sampled world; a deliberately mis-implemented op (disable-run) must be CAUGHT by the oracle diff | + +Sequencing: G1 is independently startable (pure Ghidra-side, no OGAR +Machine code). G2 depends on G1's real dump shape. G3/G4 are gated +exactly as `ogar-machine-v1.md` gates them (W3 + one W5 example shipped +first). One wave = one PR, gates central, per house style. + +## Op-set discipline (the falsifiability rule applied to an ISA) + +74 opcodes exist; the interpreter implements a declared subset. Any op +outside the subset must **halt that world loudly** — a per-row status +lane recording "unimplemented op X at cycle N" — never skip, never +best-effort. A world that halted is excluded from parity comparison *and +counted*, so "we handled the corpus" can never silently mean "we skipped +the hard ops." (Same family as tesseract-rs's "a guard that cannot fire +is the defect one level up.") + +## Boundaries (so drift is visible) + +- **No Ghidra at OGAR-Machine runtime.** Lift-time + oracle-time only. +- **No second object model.** The image loader emits ops + varnodes into + lanes; it does not grow a Rust AST of P-code (the Core-first lesson — + the substrate is the model). +- **Guest RAM stays dense** (ogar-machine-v1 rule 3); varnode SPACE ids + are how register-lane vs memory-page routing is decided at execute + time. +- **Fork discipline:** the lift script and any exporter live in the + `AdaWorldAPI/ghidra` fork; whether G1 *runs* against a fork build or a + released Ghidra distribution is a cost decision made at G1 time (the + fork build needs JDK 25 + Gradle 9.1 — available here, but a release + binary may be cheaper; the fork remains the source of truth either + way). diff --git a/.claude/plans/ogar-machine-v1.md b/.claude/plans/ogar-machine-v1.md index 1fb9519..1a5698e 100644 --- a/.claude/plans/ogar-machine-v1.md +++ b/.claude/plans/ogar-machine-v1.md @@ -51,10 +51,16 @@ CALL, RETURN, INT_ADD/SUB/MULT/AND/OR/XOR, …); the OGAR Machine executes P-code only. `legacy.exe → Ghidra (once) → normalized program image → population execution`. This is the same shape as the workspace's ruff→OGAR harvest arms: an existing analyzer becomes the transcoder -front-end, and the substrate executes the normalized IR. First concrete -archaeology step in the fresh clone: locate the P-code opcode enum and -SLEIGH lifting surface, and size the *real* op set (the list above is the -discussion's sketch, not a verified inventory). +front-end, and the substrate executes the normalized IR. + +**The archaeology ran — see `ghidra-integration-v1.md` G0** (2026-08-17): +the real op set is **74 opcodes** (`opcodes.hh`, `CPUI_MAX = 75`), not the +sketch's ~13; the fork is 12.2 DEV / Java 25+; `analyzeHeadless` is the +lift entry; and Ghidra ships its own sequential `PcodeEmulator` +(`Ghidra/Framework/Emulation/.../pcode/emu/`) — which upgrades this plan's +oracle story from "our own scalar reference" to "diff against the +reference implementation," the tesseract-rs method. Wave detail, image +format, and op-set discipline live in that companion plan. ## The four load-bearing design rules captured from the discussion diff --git a/.claude/waves/README.md b/.claude/waves/README.md new file mode 100644 index 0000000..0d27de7 --- /dev/null +++ b/.claude/waves/README.md @@ -0,0 +1,53 @@ +# Execution waves — agent-dispatch maps, one per plan + +> A **plan** says what and why; a **wave file** says exactly who edits +> which file, in what order, under which guardrails, and which command +> gates it. A wave file must be dispatchable as-is: the orchestrator +> copies worker briefs out of it verbatim. + +| wave file | plan it executes | dispatch state | +|---|---|---| +| `wave-substrate-w3-w4.md` | `lgj-soa-substrate-v1.md` (W3 Java facade, W4 bench) | **W3 READY** — next action | +| `wave-consumer-trades.md` | `consumer-world-trades-v1.md` | gated on W3 | +| `wave-consumer-bricks.md` | `consumer-bricks-analytics-v1.md` | gated on W3 | +| `wave-consumer-graph.md` | `consumer-graph-traversal-v1.md` | gated on W3 (+ possible W6 ABI wave) | +| `wave-ghidra-g1-g2.md` | `ghidra-integration-v1.md` (G1 lift proof, G2 image format) | G1 independently READY | +| `wave-ogar-machine-pm1.md` | `ogar-machine-v1.md` (probe P-M1 / G3+G4) | BLOCKED (gate list inside) | + +## Standing rules, inherited by every wave (do not restate per file) + +1. **Model policy:** orchestrator + anything synthesizing across sources = + Opus-tier; bounded one-source-in/one-shape-out work = Sonnet workers. + Never Haiku. +2. **Workers never run build tools or git.** No `cargo`, no `javac`, no + `java`, no `git`, no worktrees. Edit-only. The orchestrator compiles, + tests, lints, commits — centrally, once (`agent-cargo-hygiene.md`). +3. **Disjoint file ownership.** Two workers in one file is a lost-write + race. Shared files (module wires, `AllTests`, build scripts, board + files) are ORCHESTRATOR-ONLY and are edited after workers land. +4. **Every worker brief carries the verbatim preamble** (below) plus its + file scope and STOP triggers. A spawn without it is a protocol + violation. +5. **A wave is done when its gate table is green AND its disable-runs ran + red-then-green** — never when workers report success ("completed" is a + process status, not a quality status). +6. **One wave = one PR** (cross-repo waves: one PR per repo, merge order + stated in the wave file). Board updates land in the same commit. + +## The verbatim worker preamble (copy into every brief) + +``` +You are an edit-only worker. HARD RULES: +- Do NOT run cargo, javac, java, git, gradle, or any build/VCS command + — not once. The orchestrator compiles and tests centrally. +- Do NOT create worktrees, branches, or commits. +- Edit ONLY the files listed under YOUR SCOPE. Files under OTHER AGENTS + or ORCHESTRATOR-ONLY must not be touched, even for a one-line import. +- Do not claim code compiles or tests pass — you did not run them. + Report what you WROTE, plus anything you could not resolve. +- Read .claude/board/LATEST_STATE.md and the knowledge docs named in + your brief BEFORE writing. Do not write to any board file. +- STOP and report instead of improvising if: a symbol/type you need does + not exist; the contract (docs/abi.md) seems to disagree with code; or + your task requires touching a file outside your scope. +``` diff --git a/.claude/waves/wave-consumer-bricks.md b/.claude/waves/wave-consumer-bricks.md new file mode 100644 index 0000000..3d0b689 --- /dev/null +++ b/.claude/waves/wave-consumer-bricks.md @@ -0,0 +1,54 @@ +# Wave: consumer example — Bricks (mask-first authorization) + +> Executes `consumer-bricks-analytics-v1.md`. **DO NOT DISPATCH** — +> operator ruling 2026-08-17: calcified, not executed, until called. +> Gate: W3 merged. Inherits `waves/README.md` standing rules. + +## Shape at dispatch time + +New directory `consumers/bricks/` (own compile unit, same isolation +rationale as the trades wave). + +### Worker roster (2 Sonnet workers, disjoint) + +**K1 — the authorized-query facade.** +YOUR SCOPE: NEW `consumers/bricks/src/.../Bricks.java`, `Orders.java`, +`Role.java`. +- `Role` = a deterministic allowed-classid/predicate table (small, + hand-declared for the example; a real RBAC source replaces it later + without changing the composition point). +- `Bricks.table(Orders.class)` → the lazy chain; `.authorize(role)` + composes the role's constraint INTO the same chain (an AND of existing + predicate vocabulary or a precomputed mask ANDed via existing + `mask_and`) — it must be a `Predicate`-tier citizen, not a post-pass. +- **Fail-closed:** a terminal op on a chain that never called + `.authorize(...)` throws `UnauthorizedQueryException` — there is no + default-allow path (mirrors the workspace RBAC doctrine: a missing + mask never falls back to emit-everything). +- Aggregate-only egress: terminal ops after `groupBy`/`sum` return + aggregate types only — no row-shaped public type exists in this + consumer at all. +**K2 — falsifier tests.** +YOUR SCOPE: NEW `consumers/bricks/src/test/.../BricksAuthTest.java`. +- Two-sided discrimination: role A (sees region EU) vs role B (sees + nothing of EU) — same query, counts differ exactly as the pure-Java + recomputation predicts; role-that-sees-all == unauthorized-baseline + count computed WITHOUT the authorize path (computed in the test, never + via a bypass API — there is none). +- Fail-closed can-fire: terminal without authorize → throws. And its + silence twin: WITH authorize, no throw, correct answer. +- Compose-not-execute: crossing count stays 0 through + `.where().authorize().groupBy()` — 1 at the terminal (reuse the + laziness instrument). This is the "no raw-row read before authorize" + property in its strongest checkable form: NOTHING crosses until the + fully-clamped plan does. + +### Orchestrator-only + +Compile centrally; run; **disable-run:** short-circuit the fail-closed +check (`if (false && !authorized)`) → exactly the can-fire test goes red; +restore. Board + PR per rhythm. + +**STOP conditions:** any design where authorization happens AFTER bytes +are readable (violates mask-first — redesign, do not ship); any new ABI +symbol temptation (forbidden). diff --git a/.claude/waves/wave-consumer-graph.md b/.claude/waves/wave-consumer-graph.md new file mode 100644 index 0000000..854d7d6 --- /dev/null +++ b/.claude/waves/wave-consumer-graph.md @@ -0,0 +1,66 @@ +# Wave: consumer example — graph traversal (facet edges, crossings ∝ hops) + +> Executes `consumer-graph-traversal-v1.md`. **DO NOT DISPATCH** — +> operator ruling 2026-08-17: calcified, not executed, until called. +> Gates: W3 merged AND the hop-decode capability question resolved (see +> Decision D1 below — this wave has a genuine open design decision the +> other two consumer waves do not). + +## Decision D1 (orchestrator resolves BEFORE any worker spawns) + +A hop = facet-match (exists: `lgj_row_facet_match`) → decode target +addresses from matched facets' 12-byte payloads → scatter into the next +row mask. The decode+scatter half does not exist. Two candidate shapes: + +- **D1a — Java-side hop:** read matched facets' payloads via the raw + lane 0 segment (zero-copy, no crossing), build the next mask by writing + the EXISTING mask lane's words directly (masks are WRITABLE through + `lgj_mask_describe` — this is what WRITABLE was for). Crossings per + hop: 1 (the facet-match). No ABI change. Cost: the scatter loop runs in + Java. +- **D1b — native hop:** a new `lgj_hop(res, edge_classid, src_mask, + dst_mask)` symbol. One crossing, scatter in Rust. ABI minor 3 — + must go through `lgj-soa-substrate-v1.md`'s wave process FIRST as its + own W-tier PR (the consumer-never-grows-the-membrane rule). + +Ruling guidance: start D1a (proves the semantics with zero membrane +growth; the poster's claim "crossings ∝ hops" still holds at 1/hop), and +promote to D1b only if W4-style measurement shows the Java scatter +dominating. Record the choice + evidence in the PR body. + +## Worker roster (2 Sonnet workers, disjoint — AFTER D1 is ruled) + +**G1 — the traversal facade.** +YOUR SCOPE: NEW `consumers/graph/src/.../Graph.java`, `Edge.java`. +- `Edge.KNOWS` etc. = classid constants (the workspace canon: an edge's + predicate is a classid REFERENCE; a facet whose classid == the edge + classid IS an edge slot; its payload carries the target row address — + document the payload reading convention used by the generated fixture + data, and the honest caveat that a REAL lance-graph payload reading + arrives with ClassView/W6). +- `Graph.from(mask).hop(Edge.KNOWS).hop(...).minus(seed).count()` — + masks all the way; no Vertex/Edge objects, no per-row types. +**G2 — falsifier tests.** +YOUR SCOPE: NEW `consumers/graph/src/test/.../GraphHopTest.java`. +- Hop correctness: 2-hop set == a transcribed plain-Java BFS over the + same generated payload data (independent computation, no substrate). +- Crossings ∝ hops: instrument downcall count == hops (+1 terminal), + independent of row count (measure at two row counts). +- Anti-vacuity: seed / 1-hop / 2-hop sets are three different, non-empty, + non-total sizes — assert all three inequalities. +- Zero serialization: no `byte[]`, no `toArray`, no JSON on the hop path + (grep-style structural assertion in the test + the consumer has no + such import). + +### Orchestrator-only + +Compile centrally; run; **disable-run:** break the payload target-decode +offset by 4 bytes → hop-correctness must go red (proves the decode is +load-bearing); restore. Board + PR per rhythm; if D1b was chosen, the +membrane PR merges FIRST and this wave's PR references it. + +**STOP conditions:** payload reading ambiguity (→ orchestrator, possibly +a generator extension in a substrate-plan PR — the fixture's payload +today is PRNG noise, so this wave NEEDS a deliberate edge-bearing +generator arm: that is a substrate change, not a consumer hack — flagged +here so nobody discovers it mid-dispatch). diff --git a/.claude/waves/wave-consumer-trades.md b/.claude/waves/wave-consumer-trades.md new file mode 100644 index 0000000..69bc078 --- /dev/null +++ b/.claude/waves/wave-consumer-trades.md @@ -0,0 +1,45 @@ +# Wave: consumer example — World/Trades ("One Billion Objects. Zero Objects.") + +> Executes `consumer-world-trades-v1.md`. **DO NOT DISPATCH** — operator +> ruling 2026-08-17: consumer plans are calcified, not executed, until +> called. Gate: W3 merged. Inherits `waves/README.md` standing rules. + +## Shape at dispatch time + +New directory `consumers/trades/` (own compile unit, classpath = `java/` +output; never merged into the core package — `ApiSurfaceTest`'s walk must +stay scoped to the core API, and a consumer is a CONSUMER). + +### Worker roster (2 Sonnet workers, disjoint) + +**T1 — the domain facade.** +YOUR SCOPE: NEW `consumers/trades/src/.../World.java`, `Trade.java`. +- `Trade` is a SCHEMA, not an entity: static typed field descriptors + (`Trade.QUANTITY`, `Trade.VENUE`, `Trade.PRICE`) — each a small record + carrying (facet index, elem kind, predicate factory). No `Trade` + instance is ever constructible (private ctor, enforced by a test). +- `World.open(Trade.class, nRows, seed)` binds the schema onto a + `RowStore`/`NativePattern` and returns the existing lazy `View` + machinery under domain names. REUSE `View`/`Predicate`/`Mask` — zero + new membrane surface, zero new query engine. +**T2 — falsifier tests.** +YOUR SCOPE: NEW `consumers/trades/src/test/.../TradesParityTest.java`, +`TradesAllocationTest.java`. +- Parity: fluent-chain count == pure-Java transcribed-generator + recomputation (no substrate involvement in the expected value). +- Allocation: `getThreadAllocatedBytes` (the valhalla-lab instrument, + reuse its helper) — allocated bytes for the query path stay below a + fixed constant INDEPENDENT of row count (measure at 64K and 1M; the + row-count-independence IS the assertion). +- Laziness: crossing count 0 while composing, 1 at terminal (reuse the + LazinessTest counting instrument). + +### Orchestrator-only + +Compile centrally; run; **disable-run:** make `Trade.QUANTITY` point at +the wrong facet/lane → parity must go red (proves the schema binding is +load-bearing, not decorative); restore; board + PR per rhythm. + +**STOP conditions:** any temptation to add an ABI symbol (forbidden — +back through the substrate plan); any need for a `Trade` instance +(violates the thesis; redesign the accessor instead). diff --git a/.claude/waves/wave-ghidra-g1-g2.md b/.claude/waves/wave-ghidra-g1-g2.md new file mode 100644 index 0000000..588fa39 --- /dev/null +++ b/.claude/waves/wave-ghidra-g1-g2.md @@ -0,0 +1,84 @@ +# Wave: Ghidra G1 (lift proof) + G2 (program image format) + +> Executes `ghidra-integration-v1.md` G1 then G2. **DO NOT DISPATCH** — +> shelved with the consumer waves per the operator's calcify-first ruling; +> G1 is technically independent of W3 but momentum stays on the substrate +> waves until called. Cross-repo: G1's script lands in `AdaWorldAPI/ghidra` +> (cloned at `/workspace/ghidra`, 12.2 DEV, Java 25+); G2's loader lands +> here. One PR per repo; ghidra PR merges first. + +--- + +## Dispatch 1 — G1: the lift proof (ghidra repo) + +**Decision D0 (orchestrator, before spawning):** run `analyzeHeadless` +from a RELEASED Ghidra distribution vs building the fork (Gradle ≥ 9.1 + +JDK 25 — available in-container but a full Ghidra build is heavy). +Guidance: try the release binary first; the fork stays source-of-truth +for op semantics and HOSTS the script either way. Record the choice. + +### Worker roster (2 Sonnet workers, disjoint) + +**GH1 — the export post-script.** +YOUR SCOPE: NEW `/workspace/ghidra/ghidra_scripts/ExportPcodeImage.java` +(GhidraScript subclass). +- Walk ONE named function of the current program; for each instruction, + emit `getPcode()` ops in a deterministic text form: + `SEQOPCODE_MNEMONICout(space,offset,size)in0(...)…` — + one op per line, nothing locale- or address-format-dependent (fixed + hex widths). +- Read `PcodeOp.java`'s mnemonic surface for naming; NO custom opcode + numbering — emit Ghidra's own `getOpcode()` int AND mnemonic (both, so + G2 can bind by number and humans can diff by name). +**GH2 — the Toy fixture + expected listing.** +YOUR SCOPE: NEW `/workspace/ghidra/ghidra_scripts/lgj_fixtures/` +(a tiny Toy-ISA assembly source + build notes + the hand-derived +expected op sequence for its one function, from reading the Toy SLEIGH +spec — this is the independent half of the falsifier). + +### Orchestrator-only + +Run `analyzeHeadless -import -postScript +ExportPcodeImage.java -deleteProject` centrally. **Falsifier:** the +dump matches GH2's hand-derived expectation AND the decompiler listing +of the same function (two Ghidra surfaces + one human derivation). +PR to `AdaWorldAPI/ghidra` per its house rules (check for a CLAUDE.md / +contribution constraints in the fork before opening). + +--- + +## Dispatch 2 — G2: the image format + Rust loader (this repo) + +**Precondition:** G1 merged; a real dump file exists to design against. + +### Worker roster (2 Sonnet workers, disjoint) + +**GI1 — format + loader.** +YOUR SCOPE: NEW crate `native/lgj-pcode-image/` (NOT `lgj-abi` — the +membrane stays lean; this is consumer-tier), `src/lib.rs`, `src/format.rs`. +- Versioned LE binary format: header (magic, version u32, op_count u64) + + op records (`opcode u32`, varnode triples `(space u32, offset u64, + size u32)` — out then inputs, input_count u32). A text→binary + converter from G1's dump format. +- Loader: refuse (typed error, never partial) on bad magic / version / + truncation / trailing bytes. +**GI2 — tests.** +YOUR SCOPE: NEW `native/lgj-pcode-image/src/tests.rs` (or `#[cfg(test)]` +in-file per repo style — match `lgj-abi`). +- Round-trip: emit → load → re-emit byte-identical. +- Hand-built image of known ops decodes to exactly those ops. +- Refusal matrix: truncated at every field boundary of the FIRST op + + garbled magic + wrong version + trailing garbage — each a distinct + typed error, each a `should_panic`-free `Err` (loaders return, never + panic). + +### Orchestrator-only + +Central gates (`cargo test`/`clippy -D warnings`/`fmt` scoped `-p +lgj-pcode-image`); **disable-run:** remove the trailing-bytes check → +exactly the trailing-garbage test goes red; restore. Board + PR + arc +entry per rhythm. + +**STOP conditions:** G1's real dump reveals varnode features the format +sketch can't carry (e.g. CALLOTHER user-op indices) → format design +returns to the plan, not improvised in a worker. diff --git a/.claude/waves/wave-ogar-machine-pm1.md b/.claude/waves/wave-ogar-machine-pm1.md new file mode 100644 index 0000000..1d6834f --- /dev/null +++ b/.claude/waves/wave-ogar-machine-pm1.md @@ -0,0 +1,62 @@ +# Wave: OGAR Machine probe P-M1 (population interpreter, first falsifiable step) + +> Executes `ogar-machine-v1.md`'s probe P-M1 (== `ghidra-integration-v1.md` +> G3, with G4 oracle parity as its second dispatch). **BLOCKED — do not +> dispatch.** Gate list, all required, none waivable: +> +> 1. W3 merged (Java facade — the consumer workflow exists). +> 2. At least ONE W5 consumer example shipped (the +> consumer-never-grows-the-membrane rule has a proven round-trip). +> 3. Ghidra G1+G2 merged (a real program image format exists — P-M1 must +> consume the REAL format, not a probe-local sketch that would become +> a second format). +> 4. An explicit operator go (this direction is exploratory by ruling). + +## What P-M1 is (and is deliberately not) + +ONE hand-written P-code program (~a dozen ops from the declared subset: +COPY, INT_ADD/SUB/AND/OR/XOR, INT_EQUAL, INT_LESS, CBRANCH, BRANCH, +RETURN), serialized in the G2 image format, executed over **64K input +worlds** two independent ways. It is NOT an emulator, NOT x86, NOT +memory-paged — registers-only machine state, exactly enough to falsify +the population-execution premise. + +## Worker roster at dispatch time (3 Sonnet workers, disjoint) + +**M1 — scalar reference interpreter.** +YOUR SCOPE: NEW crate `native/lgj-machine-probe/`, file `src/scalar.rs`. +Plain-Rust sequential interpreter over the G2 image: one world in, final +register state out. No ndarray anywhere in this file — its independence +IS the falsifier (the kernels.rs scalar-reference doctrine). +**M2 — population path.** +YOUR SCOPE: `src/population.rs` (+ `src/state.rs` shared layout consts — +M2 owns both). Machine-state lanes over the row store shape (PC lane, +register lanes, halted/status lane — the op-set-discipline lane from the +plan); per-cycle: opcode classify → population masks → masked bulk ops +via `ndarray::simd` primitives ONLY (provenance rule §8 applies here +unchanged; missing primitives go to ndarray under W1a, never local). +**M3 — falsifier tests.** +YOUR SCOPE: `src/tests.rs`. +- Parity: all 64K worlds' final states bit-identical, scalar vs + population, across 3 seeds. +- Planted divergence (two-sided): two program variants differing in ONE + op; the divergence mask (outcome XOR) must select EXACTLY the world + set the scalar reference says diverges — no more, no fewer; and the + identical-programs control must yield an EMPTY divergence mask. +- Halt discipline: a program containing one out-of-subset op → every + world that reaches it halts loudly with the op recorded; worlds that + branch around it complete; the counts are asserted, not eyeballed. + +## Orchestrator-only + +Central gates scoped `-p lgj-machine-probe`; **disable-run:** break one +population op's semantics (e.g. INT_ADD → wrapping SUB) → parity red; +restore. Then Dispatch 2 (G4): the same image through Ghidra's own +`PcodeEmulator` (JVM side) for N sampled worlds — reference- +implementation parity, the tesseract-rs method; its disable-run: a +deliberately mis-implemented op must be CAUGHT by the oracle diff. + +Board + PR + arc per rhythm. Every claim in the eventual PR body phrases +performance per the baseline-dependent claims discipline +(`prior-art-and-the-layout-bridge-claim.md` §4) — P-M1 makes NO speed +claims at all; it exists to prove correctness of the execution model. diff --git a/.claude/waves/wave-substrate-w3-w4.md b/.claude/waves/wave-substrate-w3-w4.md new file mode 100644 index 0000000..6c1e83c --- /dev/null +++ b/.claude/waves/wave-substrate-w3-w4.md @@ -0,0 +1,165 @@ +# Wave: substrate W3 (Java RowStore facade) + W4 (bench Component F) + +> Executes `lgj-soa-substrate-v1.md` W3 then W4. Two dispatches, two PRs. +> Inherits every standing rule in `waves/README.md`. + +--- + +## Dispatch 1 — W3: the Java `RowStore` facade + +**Preconditions (verify before spawning):** PR #5 merged (ABI minor 2 on +`main`); `native/lgj-abi` release build present or rebuildable +(orchestrator runs `cargo build --release` centrally); JDK 26 at +`/opt/jdks/jdk-26.0.2`. + +**Contract inputs every worker reads first:** `docs/abi.md` §5, §7, §11; +`.claude/knowledge/soa-row-store-layout.md`; +`.claude/knowledge/abi-ownership-and-handles.md`; the existing +`java/.../internal/ffm/*.java` and `NativePattern.java` for house style. + +### Worker roster (3 Sonnet workers, disjoint scopes) + +**J1 — FFM membrane extension.** +YOUR SCOPE (only): `java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Downcalls.java`, +`Layouts.java`, `Abi.java`. +Deliverables: +1. `Downcalls`: three new `static final MethodHandle`s — + `lgj_rowstore_open (u64,u64,ptr)->i32`, + `lgj_op_eq_classid (u64,u32,u32,u64)->i32`, + `lgj_row_facet_match (u64,u32,ptr,u64)->i32` — `FunctionDescriptor`s + matched to `docs/abi.md` §11 argument-for-argument, same style as the + existing 15. +2. `Layouts`: `ROW_LAYOUT = sequenceLayout(32, structLayout(JAVA_INT + "classid", sequenceLayout(12, JAVA_BYTE) "payload"))` (must byteSize() + to 512 — add that as a static assert in the class init, matching the + existing manifest-cross-check spirit); `FACET_MATCH_ELEM = JAVA_INT`. +3. `Abi`: a `requireMinor(int)` guard — base load keeps requiring + `minor >= 1`; row-store entry points call `requireMinor(2)` and throw + a clear `AbiVersionException`-style error BEFORE any downcall when the + loaded library reports an older minor. +OTHER AGENTS own: all public-package files (J2), all tests (J3). + +**J2 — public facade.** +YOUR SCOPE (only): NEW files +`java/src/main/java/com/adaworldapi/lancegraph/RowStore.java`, +`FacetMatchView.java`, `FacetId.java`. +Deliverables: +1. `RowStore` (AutoCloseable): `open(Arena, long nRows, long seed)`; + `long rowCount()`; `Mask maskOfFacetClass(FacetId facet, int classId)` + (creates a mask via the EXISTING mask machinery with the row store as + parent, runs `lgj_op_eq_classid`, returns the existing public `Mask` + type — reuse, do not mint a parallel mask); `FacetMatchView + facetMatches(int classId)` (allocates an `nRows`-int segment in the + store's arena, one `lgj_row_facet_match` crossing, wraps it). +2. `FacetMatchView`: `int rowCount()`, `int matchesOf(long row)` (segment + read, no crossing), `long cardinality()` (Java-side popcount loop over + the segment — document that it is Java-side on purpose). +3. `FacetId`: a `record FacetId(int index)` with range check 0..31 — + Valhalla-shaped descriptor (≤8 B payload), one-word `value record` + migration later. +HARD RULE: zero `java.lang.foreign` types in any public signature +(`ApiSurfaceTest` enforces by reflection — it will fail your work, not a +human). Javadoc every public member, citing `docs/abi.md` §11 where the +semantics come from. +OTHER AGENTS own: `internal/ffm/*` (J1), tests (J3). + +**J3 — tests.** +YOUR SCOPE (only): NEW files +`java/src/test/java/com/adaworldapi/lancegraph/RowStoreParityTest.java`, +`RowStoreLifetimeTest.java`. +Deliverables: +1. `RowStoreParityTest`: transcribe the §11 generator (SplitMix64 already + transcribed in `FixtureParityTest` — reuse that class's generator + verbatim, two draws per facet, `a` then `b`); for n ∈ {1, 64, 65, + 1000}: (a) recompute every facet-7 classid in pure Java and compare + `maskOfFacetClass(7, c).count()` against the recomputed count for + c ∈ {0, 9, 15}; (b) `facetMatches(9)`: per-row expected 32-bit set + recomputed in pure Java, compared via `matchesOf(row)` for every row; + (c) read classids DIRECTLY from the raw lane 0 segment through + `ROW_LAYOUT`-derived offsets and compare against the generator — two + independent read paths (native mask vs Java segment read) against one + independent recomputation. +2. `RowStoreLifetimeTest`: use-after-close throws; a `Mask` from a closed + store fails with the parent-closed error; `FacetMatchView` remains + readable after the store closes ONLY if its backing arena is the + caller's (assert whichever the design gives — and state it in the + Javadoc via a STOP-report if J2's choice is ambiguous); double close + is idempotent-or-throws per existing `LifetimeTest` conventions + (match them, do not invent new ones). +Tests are plain-main style matching `AllTests` conventions — read +`FixtureParityTest` first and mirror its structure. +OTHER AGENTS own: `internal/ffm/*` (J1), public facade (J2). + +### Orchestrator-only steps (in order) + +1. `cargo build --release` in `native/lgj-abi` (fresh `.so`, 18 symbols). +2. Dispatch J1+J2+J3 in parallel (disjoint — safe). +3. After all land: wire `AllTests.java` (orchestrator-owned) to include + the two new test classes. +4. Compile: `javac -Xlint:all` over `java/` — expected `[restricted]` + warnings: the existing 7 + any new ones ONLY in `internal/ffm/*`; + count and record the exact number. +5. Run `AllTests` against the fresh `.so`. All green required. +6. **Disable-run 1 (version gate):** change `requireMinor(2)` to + `requireMinor(3)` → every RowStore test must fail with the version + error, nothing else may fail; restore; re-run green. +7. **Disable-run 2 (parity is real):** flip one byte-order or offset in + J3's pure-Java generator transcription (e.g. swap the a/b draw order) + → parity tests must go red; restore; green. +8. `ApiSurfaceTest` must pass unmodified (it auto-walks the package — a + new public type leaking FFM fails without anyone editing the test). +9. Board: STATUS_BOARD D-LGJ-W3 → DONE with the real numbers; + LATEST_STATE prepend; commit, push, PR, merge per house rhythm; + PR_ARC_INVENTORY entry at merge. + +**STOP conditions for the whole dispatch:** any needed ABI behavior +missing from §11 (goes back to the plan as a W6 item, never patched +ad hoc); `Mask` reuse turning out to require a new public constructor +(design decision → orchestrator resolves, workers stop). + +--- + +## Dispatch 2 — W4: bench Component F (after W3 merges) + +**Question (verbatim from the plan):** on the REAL layout, where does the +per-row facet scan belong — native crossing, Java Vector API, or scalar? + +### Worker roster (1 Sonnet worker) + +**B1 — Component F.** +YOUR SCOPE (only): NEW file +`bench/src/main/java/com/adaworldapi/lancegraph/bench/F_RowStoreFacetScan.java`; +plus `Data.java` and `Kernels.java` MAY be extended (they are yours for +this dispatch — no other worker runs concurrently). +Deliverables, mirroring the house bench discipline +(`bench/README.md` rules — cross-check before timing, cost components +separated): +1. Arms over one `RowStore` (n_rows param: 4_096, 65_536, 1_048_576; + classId fixed at 9): + - `native_facetMatch` — one `lgj_row_facet_match` crossing into a + pre-allocated out segment (allocation OUTSIDE the timed body, + documented). + - `java_vectorApi` — `IntVector.fromMemorySegment` over the raw + lane-0 segment, species 512-bit: one 16-int vector = one 64-byte + chunk = 4 facets; `eq(broadcast(classId)).toLong() & 0x1111`-fold, + 8 chunks per row — the SAME algorithm as the Rust kernel, stated in + a comment (algorithmic symmetry is the point of the comparison). + - `java_scalar` — VarHandle walk of the 32 classids per row. +2. `@Setup` cross-check: all three arms' full output arrays bit-identical + before anything is timed (the `Data.crossCheck` discipline; throw on + mismatch). +3. JMH params identical to existing components (`@Fork(1)`, 5×500ms + warmup, 8×500ms measurement, AverageTime, GC between iterations). + +### Orchestrator-only steps + +1. Run `bench/run.sh F_` centrally (expect ~10 min). +2. `summarise.sh` regenerates tables from the merged CSV — RESULTS.md + prose updated to cite the new table, never hand-typed numbers. +3. Fold the finding into `docs/execution-boundary.md` (one paragraph: + does the Component C conclusion survive on the strided/row layout?) + — whichever way it lands, it lands as measurement. +4. Board, PR, merge, arc entry per house rhythm. + +**Falsifier:** the cross-check itself (a wrong arm cannot be timed); and +the summarise.sh mechanical-table rule (prose cannot drift from CSV). From 57a7e485a4501bc5b8a7dba43cb1d3924b0b27b3 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 22:50:55 +0000 Subject: [PATCH 12/30] Board: PR #7 arc entry (post-merge) Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/PR_ARC_INVENTORY.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 3a5ee6a..53e0a5b 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -8,6 +8,27 @@ > anti-pattern the imported board rules name. Backfilled below in one > pass rather than left stale; PR #4 onward gets its entry at merge time. +## PR #7 — waves calcified: dispatch maps for every plan (merged 2026-08-17, squash `68f7add`) + +- **Added:** `.claude/waves/` — README (standing rules + verbatim worker + preamble) + six dispatchable maps (substrate W3+W4 READY; three + consumer waves DO-NOT-DISPATCH; Ghidra G1+G2 shelved; OGAR-Machine + P-M1 BLOCKED behind a 4-condition gate incl. explicit operator go); + `.claude/plans/ghidra-integration-v1.md` (G0 archaeology from the real + clone: 74 P-code opcodes, `PcodeEmulator` as reference oracle, Toy as + minimal lift target). +- **Locked:** the calcify-then-dispatch rhythm + the eight + muscle-memory rules (`E-LGJ-CALCIFY-THEN-DISPATCH-1`); the op-set + halt-loudly discipline; the D1a/D1b hop fork with ruling guidance. +- **Deferred:** ALL execution, by operator ruling — nothing dispatched, + no code changed. +- **Docs:** the PR is docs; mapping-time catches recorded (graph + consumer needs an edge-bearing generator arm — substrate change, + flagged before it could burn a dispatch). +- **Confidence:** High for the maps (grounded in shipped code + real + clone archaeology); execution confidence deliberately unclaimed until + the waves run. Bot reviewers at usage limits, did not run. + ## PR #6 — layout-bridge assessment, OGAR Machine plan, hydrate note (merged 2026-08-17, squash `8954e53`) - **Added:** `.claude/knowledge/prior-art-and-the-layout-bridge-claim.md` From 83b8994f621b4bafde9d7bd52fbb03b974cee92b Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 22:58:15 +0000 Subject: [PATCH 13/30] Plan: lance-graph #958 merged (was open at last check) Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/plans/lgj-soa-substrate-v1.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.claude/plans/lgj-soa-substrate-v1.md b/.claude/plans/lgj-soa-substrate-v1.md index ea3b6af..ee2e094 100644 --- a/.claude/plans/lgj-soa-substrate-v1.md +++ b/.claude/plans/lgj-soa-substrate-v1.md @@ -43,7 +43,8 @@ disable-verified, every measured claim lands with its reproduction command. SoA→S3→volume→Lance hydration pattern (four-state lifecycle, hydrate-aside/publish-by-rename, warm markers, dirty detection), minted in lance-graph *specifically so consumers inherit it as a path/git dependency -rather than re-implement*. **#958** (open, another session's PR) is its 5+3 +rather than re-implement*. **#958** (merged 2026-08-17, another session's +PR) was its 5+3 council hardening fast-follow. Consequence here: when this substrate's persistence slice arrives (the "Seal & Persist (Lance)" column of the formula, and `ogar-machine-v1.md`'s time-machine storage), the hydration From 572f865b297c01077e757a7e033e77bc0b04d7fa Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 23:06:05 +0000 Subject: [PATCH 14/30] Java RowStore facade: W3 shipped (185/185, one bug found+fixed) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First real dispatch of the calcified wave system (wave-substrate-w3-w4.md Dispatch 1) — 3 Sonnet workers on disjoint scopes, Opus orchestrator integration and central gating, per the standing rules in .claude/waves/README.md. New public surface: RowStore (open/rowCount/isOpen/maskOfFacetClass/ facetMatches/close), FacetMatchView (rowCount/matchesOf/cardinality), FacetId (0..31-checked record) -- zero java.lang.foreign types in any public signature, ApiSurfaceTest passed unmodified. Mask.source() retyped NativePattern -> NativeResource (new minimal interface) so a mask parents onto either a pattern or a row store with the existing algebra unchanged; verified zero call-site breakage before the retype. One real bug caught by the test suite itself: FacetMatchView.rowCount() was missing the closed-store guard its sibling accessors both had -- found by RowStoreLifetimeTest on the first real run, fixed, re-verified. Gate: javac -Xlint:all clean (7 pre-existing [restricted] warnings, 0 new); AllTests 132 -> 185 (+53 checks: 29 parity + 24 lifetime). Both mandated disable-runs ran red-then-green with the exact expected blast radius: (1) Abi.requireMinor inflated by 1 -> exactly the two RowStore suites failed, 8 others stayed green; (2) the generator's a/b draw order swapped -> exactly RowStoreParityTest broke (17/29), the generator-independent RowStoreLifetimeTest stayed green. Board: STATUS_BOARD D-LGJ-W3 DONE, LATEST_STATE, and E-LGJ-WAVE-DISPATCH-VALIDATED-1 -- the wave system's first real dispatch, including an orchestrator-side false alarm (wrong env var name guessed instead of read from source) recorded so it isn't repeated. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/EPIPHANIES.md | 41 ++++ .claude/board/LATEST_STATE.md | 31 ++++ .claude/board/STATUS_BOARD.md | 2 +- .../com/adaworldapi/lancegraph/FacetId.java | 37 ++++ .../lancegraph/FacetMatchView.java | 97 ++++++++++ .../java/com/adaworldapi/lancegraph/Mask.java | 8 +- .../adaworldapi/lancegraph/NativePattern.java | 2 +- .../lancegraph/NativeResource.java | 22 +++ .../com/adaworldapi/lancegraph/RowStore.java | 147 +++++++++++++++ .../lancegraph/internal/ffm/Abi.java | 36 ++++ .../lancegraph/internal/ffm/Downcalls.java | 75 +++++++- .../lancegraph/internal/ffm/Engine.java | 42 +++++ .../lancegraph/internal/ffm/Layouts.java | 42 +++++ .../com/adaworldapi/lancegraph/AllTests.java | 2 + .../lancegraph/RowStoreLifetimeTest.java | 123 ++++++++++++ .../lancegraph/RowStoreParityTest.java | 175 ++++++++++++++++++ 16 files changed, 874 insertions(+), 8 deletions(-) create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/FacetId.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/FacetMatchView.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/NativeResource.java create mode 100644 java/src/main/java/com/adaworldapi/lancegraph/RowStore.java create mode 100644 java/src/test/java/com/adaworldapi/lancegraph/RowStoreLifetimeTest.java create mode 100644 java/src/test/java/com/adaworldapi/lancegraph/RowStoreParityTest.java diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index afd9ea3..bd91ae7 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -4,6 +4,47 @@ > `**Status:**`/`**Confidence:**` line. A correction gets its own new, > dated entry that references the one it corrects — the storno rule. +## 2026-08-17 — E-LGJ-WAVE-DISPATCH-VALIDATED-1 + +**Status:** FINDING (first real dispatch of the wave system). **Confidence:** +High — measured, not asserted. + +The wave map (`E-LGJ-CALCIFY-THEN-DISPATCH-1`) was written to be +"dispatchable as-is by a session with zero shared context." First test: W3, +three Sonnet workers on disjoint file scopes, zero coordination between +them beyond the frozen signatures the orchestrator's briefs specified. All +three landed clean, mutually consistent (same `RowStore.open(long,long)` +signature, same `FacetId.index()` accessor — nobody guessed differently), +and the disjoint-scope rule held with zero merge conflicts. + +Two things the gate sequence actually caught, worth recording precisely +because they're the mechanism, not the anecdote: + +1. **A real defect, caught by the tests the wave mandated.** + `FacetMatchView.rowCount()` was missing the same closed-store guard + `matchesOf`/`cardinality` both carried — one accessor out of three, + asymmetric, exactly the kind of gap a reviewer skims past and a + two-sided lifetime test does not. `RowStoreLifetimeTest` (itself + AI-written, by a different worker than the one who wrote the class + under test) caught it on the first real run. This is the payoff of + "workers never run the gate themselves" — the orchestrator's fresh, + independent test run is what a self-reported "looks right" cannot be. +2. **A false alarm from the ORCHESTRATOR's own environment, not the + code.** The first two `AllTests` invocations failed with every + pre-existing suite red — before touching a single line of worker + output. Root cause: I used an invented env var name (`LGJ_NATIVE_LIB`) + instead of the real one (`LGJ_LIBRARY`, defined in `Abi.java`), so the + runtime silently fell back to a stale `.so` from an unrelated default + search path. The fix was to READ THE CODE (`Abi.java`'s + `ENV_LIBRARY` constant) rather than guess a plausible-sounding name. + Lesson for future orchestrator runs: verify the discovery mechanism + from source before trusting a gate result — a wrong environment can + look exactly like a real regression. + +Both disable-runs (version-gate inflation; generator draw-order swap) ran +red-then-green with the EXACT expected suite-level blast radius — no +overreach, no under-reach — closing the loop the wave file promised. + ## 2026-08-17 — E-LGJ-CALCIFY-THEN-DISPATCH-1 **Status:** DOCTRINE (operator-ruled: "don't execute the consumer plans yet, diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index dea97e2..9a20442 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,34 @@ +## 2026-08-17 (dispatch 1) — W3 shipped: the Java `RowStore` facade, from the calcified wave map + +First real dispatch of the wave system: `wave-substrate-w3-w4.md` Dispatch 1 +executed exactly as mapped — 3 Sonnet workers on disjoint file scopes (FFM +membrane extension / public facade / tests), Opus orchestrator integrated, +gated, and fixed centrally. Confirms the calcify-then-dispatch rhythm works +end to end, not just as a documentation exercise. + +- **New public surface:** `RowStore` (`open`/`rowCount`/`isOpen`/ + `maskOfFacetClass`/`facetMatches`/`close`), `FacetMatchView` + (`rowCount`/`matchesOf`/`cardinality`), `FacetId` (a 0..31-checked record). + Zero `java.lang.foreign` types in any public signature — `ApiSurfaceTest` + passed unmodified. +- **`Mask` generalized**: `source()` retypes `NativePattern → NativeResource` + (new minimal interface), so a mask can parent onto EITHER a pattern or a + row store with the existing algebra unchanged. Verified zero call-site + breakage before the retype. +- **One real bug found by the suite itself and fixed**: + `FacetMatchView.rowCount()` was missing the closed-store guard its sibling + accessors both had — a caller could read a stale row count off a dead + view. Caught by `RowStoreLifetimeTest`, fixed, re-verified 185/185. +- **Both disable-runs green-red-green**, confirming the version gate and the + generator-transcription parity are load-bearing, not decorative (full + detail on `STATUS_BOARD.md` D-LGJ-W3). +- Gate: `javac -Xlint:all` clean (7 pre-existing `[restricted]` warnings, + 0 new); `AllTests` 132→185 (+53); native `.so` unchanged this dispatch + (Rust side untouched — pure Java consumer work). + +**Next:** W4 (bench Component F, Vector API vs the crossing on the real +row-store layout) — the second half of the same wave file. + ## 2026-08-17 (latest) — waves calcified, Ghidra plan grounded, NOTHING dispatched Operator ruling: consumer plans are **calcified, not executed** — insights diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index fd5f122..41cb480 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -38,6 +38,6 @@ layout wired end to end. Doctrine: `E-LGJ-THE-MIDDLE-TIER-IS-DELETED-NOT-WRAPPED |---|---|---| | D-LGJ-W1 | ndarray: `MultiLaneColumn::iter_u32x16`/`len_u32x16` + `eq_u32_strided_to_mask` (W1a contract) | **DONE 2026-08-17** — ndarray PR #279; `simd_int_ops` 46/46 (5 new strided tests incl. two `should_panic` bounds/overflow arms + stride-4 parity against the contiguous primitive), `simd_soa` 15/15, full `simd` 263/263, doctests, clippy `-D warnings` + fmt clean | | D-LGJ-W2 | lgj-abi row store: `rowstore.rs`, `LGJ_RESOURCE_ROWSTORE`, `lgj_rowstore_open`, strided facet lanes through the unchanged `LgjLaneDesc`, `lgj_op_eq_classid`, `lgj_row_facet_match`, ABI minor 1→2, `docs/abi.md` §11 | **DONE 2026-08-17** — `cargo test` **84/84**, clippy/fmt clean, release build exports **18/18** symbols (`nm -D`). Parity: both kernels vs independent scalar references over 10 row counts × 2 seeds × 4 facets × 4 needles, cross-checked a THIRD way against `RowStore::classid_at`. Two-sided payload-vs-classid falsifier. End-to-end membrane test covers describe → predicate → mask algebra → count → facet-match → lifecycle | -| D-LGJ-W3 | Java `RowStore` facade: structured `MemoryLayout`, minor-≥2 gate, `FacetMatchView`, parity test transcribing the generator | **NEXT** | +| D-LGJ-W3 | Java `RowStore` facade: structured `MemoryLayout`, minor-≥2 gate, `FacetMatchView`, parity test transcribing the generator | **DONE 2026-08-17** — dispatched per `.claude/waves/wave-substrate-w3-w4.md` (3 Sonnet workers, disjoint scopes: FFM membrane extension / public facade / tests), orchestrator-integrated. `javac -Xlint:all` clean (same 7 pre-existing `[restricted]` warnings, zero new). `AllTests` **185/185** (was 132; +53 new checks: 29 parity + 24 lifetime). **One real bug caught by the suite and fixed**: `FacetMatchView.rowCount()` was missing the closed-store guard `matchesOf`/`cardinality` both had — a stale row count was readable after the owning store closed. Fixed, re-verified. Both mandated disable-runs ran red-then-green: (1) `Abi.requireMinor` inflated by 1 → exactly `RowStoreParityTest`+`RowStoreLifetimeTest` failed, all 8 other suites stayed green; (2) the pure-Java generator's a/b draw order swapped in `RowStoreParityTest` → exactly that suite broke (17/29), `RowStoreLifetimeTest` (generator-independent) stayed green — confirming the parity test is a real falsifier, not decorative. `Mask.source()` retyped `NativePattern → NativeResource` (new interface) so a `Mask` can parent onto either a `NativePattern` or a `RowStore` — zero call-site breakage (verified: no existing caller bound the narrower type) | | D-LGJ-W4 | Bench Component F: Vector API facet scan vs the crossing, on the REAL layout | Queued | | D-LGJ-W5 | Three consumer examples (trades / bricks / graph) — one plan file each | Planned, gated on W3 | diff --git a/java/src/main/java/com/adaworldapi/lancegraph/FacetId.java b/java/src/main/java/com/adaworldapi/lancegraph/FacetId.java new file mode 100644 index 0000000..935ee3b --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/FacetId.java @@ -0,0 +1,37 @@ +package com.adaworldapi.lancegraph; + +/** + * Which of a {@link RowStore} row's 32 facet lanes to read. + * + *

Not a lane id. abi.md §11 is explicit that {@code lgj_op_eq_classid} takes a + * facet index, not a lane id — lane {@code 0} is the raw buffer and facet {@code f}'s lane id is + * {@code 1 + f}. Mixing the two up is exactly the bug the end-to-end test pins (facet {@code 32} + * is invalid while lane {@code 32} is valid). A distinct type is what stops that confusion at the + * call site instead of relying on a comment. + * + *

Valhalla A/B candidate

+ * + *

Same rules as {@link LaneId} and {@link Ordinal}: final, immutable, identity-free, so the + * same source compiles as a {@code value record} under JEP 401. It wraps a single {@code int} + * purely so a facet index cannot be confused with a lane id, a classid, or a row number, and under + * Valhalla that type-safety is expected to cost nothing at all. + * + * @param index zero-based facet index, {@code 0..32} exclusive (a row has exactly 32 facets) + */ +public record FacetId(int index) { + + public FacetId { + if (index < 0 || index > 31) { + throw new IllegalArgumentException("facet index must be in 0..31, was " + index); + } + } + + public static FacetId of(int index) { + return new FacetId(index); + } + + @Override + public String toString() { + return "facet#" + index; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/FacetMatchView.java b/java/src/main/java/com/adaworldapi/lancegraph/FacetMatchView.java new file mode 100644 index 0000000..c112358 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/FacetMatchView.java @@ -0,0 +1,97 @@ +package com.adaworldapi.lancegraph; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +/** + * For every row of the {@link RowStore} it was taken from, which of its 32 facets carry the + * queried classid — one {@code int} bitset per row, bit {@code f} set exactly when facet + * {@code f}'s classid equals the value {@link RowStore#facetMatches} was called with. + * + *

Backed by a Java-owned segment a single native crossing wrote into ({@code + * lgj_row_facet_match}, abi.md §11). Every read below — {@link #matchesOf} and + * {@link #cardinality} — is an in-process access of that already-fetched segment; no further + * crossing occurs. + * + *

Lifetime

+ * + *

A view is a child of the {@link RowStore} it was taken from: the segment it wraps is + * allocated from that store's own arena. It may outlive its parent as an object, but reading from + * it after the parent closes throws {@link ClosedResourceException} — the segment's backing arena + * dies with the store, so there is no arrangement of closes that lets a view read freed memory. + */ +public final class FacetMatchView { + + private final RowStore owner; + private final MemorySegment data; + private final long rowCount; + + FacetMatchView(RowStore owner, MemorySegment data, long rowCount) { + this.owner = owner; + this.data = data; + this.rowCount = rowCount; + } + + /** + * How many rows this view covers. + * + *

Reads no segment bytes, but is guarded exactly like {@link #matchesOf} and + * {@link #cardinality} for the reason the class doc states: once the store closes, this view is + * no longer usable at all, not merely unsafe to read the segment through — a caller that checks + * {@code rowCount()} before deciding whether to call {@link #matchesOf} must not be able to + * observe a stale, disconnected-from-reality number from a dead view. + */ + public long rowCount() { + requireUsable("rowCount()"); + return rowCount; + } + + /** + * The 32-bit facet-match bitset for {@code row}: bit {@code f} set means facet {@code f}'s + * classid equals the value queried. + * + *

An in-process segment read. No membrane crossing occurs. + * + * @throws IndexOutOfBoundsException if {@code row} is not in {@code [0, rowCount())} + */ + public int matchesOf(long row) { + requireUsable("matchesOf()"); + if (row < 0 || row >= rowCount) { + throw new IndexOutOfBoundsException( + "row " + row + " is out of range [0, " + rowCount + ")"); + } + return data.getAtIndex(ValueLayout.JAVA_INT, row); + } + + /** + * The total number of set bits across every row's bitset. + * + *

Deliberately Java-side: this is a bulk reduction over a result that already crossed the + * membrane once (the single {@code lgj_row_facet_match} call behind + * {@link RowStore#facetMatches}), so a second crossing just to reduce it would undo the point + * of having fetched the whole thing in bulk. + */ + public long cardinality() { + requireUsable("cardinality()"); + long total = 0; + for (long row = 0; row < rowCount; row++) { + total += Integer.bitCount(data.getAtIndex(ValueLayout.JAVA_INT, row)); + } + return total; + } + + private void requireUsable(String what) { + // Checked before any segment access, on purpose: the segment's backing arena is the + // owning store's, so once the store is closed the segment must not be touched at all. + if (!owner.isOpen()) { + throw new ClosedResourceException( + what + " was called on a facet-match view whose store is closed. The segment's" + + " backing arena dies with the store, so it can never be read again."); + } + } + + @Override + public String toString() { + return "FacetMatchView[" + rowCount + " rows]"; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/Mask.java b/java/src/main/java/com/adaworldapi/lancegraph/Mask.java index 8952ff2..9421c89 100644 --- a/java/src/main/java/com/adaworldapi/lancegraph/Mask.java +++ b/java/src/main/java/com/adaworldapi/lancegraph/Mask.java @@ -21,11 +21,11 @@ */ public final class Mask implements AutoCloseable { - private final NativePattern parent; + private final NativeResource parent; private final long handle; private boolean closed; - Mask(NativePattern parent, long handle) { + Mask(NativeResource parent, long handle) { this.parent = parent; this.handle = handle; } @@ -41,8 +41,8 @@ public MaskId id() { return new MaskId(handle); } - /** The resource whose rows this selects. */ - public NativePattern source() { + /** The resource whose rows this selects — a {@link NativePattern} or a {@link RowStore}. */ + public NativeResource source() { return parent; } diff --git a/java/src/main/java/com/adaworldapi/lancegraph/NativePattern.java b/java/src/main/java/com/adaworldapi/lancegraph/NativePattern.java index 28fa0fc..180612f 100644 --- a/java/src/main/java/com/adaworldapi/lancegraph/NativePattern.java +++ b/java/src/main/java/com/adaworldapi/lancegraph/NativePattern.java @@ -35,7 +35,7 @@ *

Terminal operations reuse one internal scratch selection, so they are serialised on this * instance. Distinct resources do not contend with each other. */ -public final class NativePattern implements AutoCloseable { +public final class NativePattern implements NativeResource, AutoCloseable { /** * The seed used by {@link #open(long)}. diff --git a/java/src/main/java/com/adaworldapi/lancegraph/NativeResource.java b/java/src/main/java/com/adaworldapi/lancegraph/NativeResource.java new file mode 100644 index 0000000..a804b9f --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/NativeResource.java @@ -0,0 +1,22 @@ +package com.adaworldapi.lancegraph; + +/** + * The common face of row-shaped native resources a {@link Mask} can select over. + * + *

Both {@link NativePattern} and {@link RowStore} hold rows natively and let a {@link Mask} + * parent onto them — abi.md §11 states this directly: "masks may parent onto a pattern OR a row + * store — both are read-only, row-shaped resources." This interface names that shared shape so + * {@link Mask} does not need to know, or care, which kind of resource it selects over. + * + *

Deliberately small: just enough for {@link Mask} to report a count and to check liveness + * before every operation. Anything specific to how a resource's rows are laid out or generated + * belongs on the concrete type, not here. + */ +public interface NativeResource { + + /** How many rows this resource holds. */ + long rowCount(); + + /** False once the resource has been closed. */ + boolean isOpen(); +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/RowStore.java b/java/src/main/java/com/adaworldapi/lancegraph/RowStore.java new file mode 100644 index 0000000..891d2e3 --- /dev/null +++ b/java/src/main/java/com/adaworldapi/lancegraph/RowStore.java @@ -0,0 +1,147 @@ +package com.adaworldapi.lancegraph; + +import com.adaworldapi.lancegraph.internal.ffm.Engine; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +/** + * A native SoA row store: {@code n_rows} rows of 512 bytes each, 32 sixteen-byte facet lanes per + * row — a 4-byte little-endian classid followed by a 12-byte opaque payload. This is the + * lance-graph V3 content-blind facet shape, enforced everywhere on the Rust side (abi.md §11, + * {@code .claude/knowledge/soa-row-store-layout.md}). The Java side's view of the 12-byte payload + * may differ from any other consumer's; the 512/32/16 geometry itself is substrate truth and is + * not re-derived here — it is enforced natively. + * + *

Zero-serialization, the same discipline as {@link NativePattern}: {@link #maskOfFacetClass} + * never leaves native storage, and {@link #facetMatches} crosses the membrane exactly once and + * hands back a Java segment that native code wrote into directly — never a per-row copy into a + * Java object. + * + *

Lifetime

+ * + *

Ordinary try-with-resources, mirroring {@link NativePattern}. After {@link #close()} every + * operation on this resource, and on any {@link Mask} or {@link FacetMatchView} derived from it, + * throws {@link ClosedResourceException}. Closing twice is an error, not a silent no-op. + */ +public final class RowStore implements NativeResource, AutoCloseable { + + private final long handle; + private final long rowCount; + private final Arena arena; + + private boolean closed; + + private RowStore(long handle, long rowCount) { + this.handle = handle; + this.rowCount = rowCount; + // Owns exactly the segments this store hands out via facetMatches(): allocated once per + // store, closed once when this store closes, so a FacetMatchView's lifetime is provably + // tied to its owner's rather than to a per-call scratch buffer. + this.arena = Arena.ofShared(); + } + + /** + * Open {@code nRows} rows generated deterministically from {@code seed}. + * + *

See abi.md §11 for the normative generator (two SplitMix64 draws per facet, {@code a} + * then {@code b}): {@code classid = (a >>> 33) & 0xF}, {@code payload = le64(b) ++ + * le32(a & 0xFFFFFFFF)}. + * + * @throws NativeLibraryNotFoundException if the native artifact is not present + * @throws AbiMismatchException if it is present but does not support the row-store + * resource (ABI minor < 2) + */ + public static RowStore open(long nRows, long seed) { + if (nRows < 0) { + throw new IllegalArgumentException("nRows must be >= 0, was " + nRows); + } + long h = Engine.openRowStore(nRows, seed); + // Read the row count back from the resource rather than trusting the request, matching + // NativePattern.open: it is the resource that decides. + return new RowStore(h, Engine.rowCount(h)); + } + + /** How many rows this resource holds. */ + @Override + public long rowCount() { + requireOpen("rowCount()"); + return rowCount; + } + + /** False once {@link #close()} has run. */ + @Override + public boolean isOpen() { + return !closed; + } + + /** + * A selection of the rows whose {@code facet}'s classid equals {@code classId}. + * + *

The result is an ordinary {@link Mask}: it composes with the whole existing mask algebra + * (and/or/count/describe) unchanged, because a row store is just as valid a {@link Mask} + * parent as a {@link NativePattern} — abi.md §11: "masks may parent onto a pattern OR a row + * store — both are read-only, row-shaped resources." This is one native crossing + * ({@code lgj_op_eq_classid}). + * + * @param facet which of the 32 facet lanes to read — a facet index, not a lane id + * @param classId the classid to match against that facet + */ + public Mask maskOfFacetClass(FacetId facet, int classId) { + java.util.Objects.requireNonNull(facet, "facet"); + requireOpen("maskOfFacetClass()"); + long mask = Engine.createMask(handle, false); + Engine.eqClassid(handle, facet.index(), classId, mask); + return new Mask(this, mask); + } + + /** + * For every row, which of its 32 facets carry {@code classId} as their classid — one native + * crossing ({@code lgj_row_facet_match}, abi.md §11). + * + *

The result wraps a Java-owned segment allocated from this store's own arena and written + * into once by native code — zero-copy out, nothing serialized. The segment's lifetime is this + * store's: it stays valid, and readable, until this store closes. + */ + public FacetMatchView facetMatches(int classId) { + requireOpen("facetMatches()"); + MemorySegment out = arena.allocate(ValueLayout.JAVA_INT, rowCount); + Engine.rowFacetMatch(handle, classId, out, rowCount); + return new FacetMatchView(this, out, rowCount); + } + + /** + * Release the native storage and this store's own arena. + * + * @throws ClosedResourceException if already closed + */ + @Override + public void close() { + if (closed) { + throw new ClosedResourceException( + "close() called on a resource that is already closed. A double close is an" + + " error rather than a no-op, because the second call cannot know" + + " whether the handle was recycled in between."); + } + closed = true; + Engine.close(handle); + arena.close(); + } + + // ── package-private ───────────────────────────────────────────────────────────────────── + + void requireOpen(String what) { + if (closed) { + throw new ClosedResourceException( + what + " was called on a closed resource. Java rejected it before reaching" + + " native code; had it not, the generation-checked handle would have" + + " returned INVALID_HANDLE rather than touching freed memory."); + } + } + + @Override + public String toString() { + return "RowStore[" + rowCount + " rows" + (closed ? ", closed" : "") + "]"; + } +} diff --git a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Abi.java b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Abi.java index 0e5812a..2f6c818 100644 --- a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Abi.java +++ b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Abi.java @@ -138,6 +138,42 @@ public static Path libraryPath() { return LIBRARY_PATH; } + /** + * Guard a feature introduced at a later ABI minor than this Java build's own base load gate. + * + *

The base load gate in {@link #readAndVerifyManifest} only requires {@code minor >= + * }{@link Layouts#LGJ_ABI_MINOR} (currently {@code 1}), per docs/abi.md §2's additive-minor + * promise: an older Java build against a newer {@code .so} always loads. That promise runs in + * one direction only. It says nothing about a Java build that was compiled against a + * later minor — e.g. the row store at minor 2 (docs/abi.md §11) — being run against an + * {@code .so} that is old enough to clear the base gate but too old to export the symbol that + * specific feature needs. Left unchecked, that combination would fail deep inside the downcall + * resolution in {@link Downcalls} with a bare "no such symbol" message, after the library has + * already loaded and after other, older features have already been used successfully. This + * method instead fails loudly, before any downcall for the feature is attempted, naming exactly + * which minor the feature needs against exactly which minor loaded — e.g. a row-store entry + * point calls {@code requireMinor(2)} and, on an older library, reports "row store requires ABI + * minor >= 2". + * + *

Does not change, gate, or replace the base load gate — {@code minor >= 1} at load time is + * unconditional and untouched by this method. + * + * @param required the minimum {@code abi_minor} the calling feature needs + * @throws AbiMismatchException if the loaded library's manifest reports a lower minor + */ + public static void requireMinor(int required) { + int loaded = manifest().abiMinor(); + if (loaded < required) { + throw new AbiMismatchException(String.format( + "this feature requires ABI minor >= %d, but the loaded library at %s reports" + + " minor %d. This Java build's base load gate (minor >= %d) was" + + " satisfied, but this specific feature is newer than what that gate" + + " alone guarantees is present; rebuild or replace the native library" + + " with one at minor >= %d.", + required, libraryPath(), loaded, Layouts.LGJ_ABI_MINOR, required)); + } + } + /** Human-readable name of the SIMD backend the artifact was compiled with. Reported, never negotiated. */ public static String simdBackendName() { return manifest().simdBackendName(); diff --git a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Downcalls.java b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Downcalls.java index 29a0cf1..45cb8e6 100644 --- a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Downcalls.java +++ b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Downcalls.java @@ -13,8 +13,10 @@ * Every downcall handle in the project, resolved once into static finals. * *

Resolution is not free — {@code Linker.downcallHandle} builds a native stub — so doing it per - * call would be a per-crossing tax on top of the crossing itself. There are 14 symbols - * (docs/abi.md §7) and 14 handles. + * call would be a per-crossing tax on top of the crossing itself. There are 17 downcall symbols + * resolved here (docs/abi.md §7 + §11) and 17 handles; the 18th ABI symbol, {@code + * lgj_abi_manifest}, resolves separately in {@link Abi} because it returns a pointer rather than a + * status and has no failure mode. * *

The anti-JNI rule (docs/abi.md §6) lives here. Panama makes it easy to write * JNI-shaped code: one downcall per element. Every wrapper below either does work proportional to @@ -93,6 +95,20 @@ private Downcalls() {} FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS)); + // Row store (docs/abi.md §11, ABI minor 2). Argument widths follow §11 argument-for-argument, + // same u32/i32-are-both-JAVA_INT rule as above. + private static final MethodHandle ROWSTORE_OPEN = mh("lgj_rowstore_open", + FunctionDescriptor.of(ValueLayout.JAVA_INT, + ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS)); + + private static final MethodHandle OP_EQ_CLASSID = mh("lgj_op_eq_classid", + FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, + ValueLayout.JAVA_INT, ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG)); + + private static final MethodHandle ROW_FACET_MATCH = mh("lgj_row_facet_match", + FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, + ValueLayout.JAVA_INT, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG)); + private static MethodHandle mh(String symbol, FunctionDescriptor descriptor) { MemorySegment addr = Abi.lookup().find(symbol).orElseThrow(() -> new LanceGraphException("the native library exports no symbol '" + symbol @@ -317,6 +333,61 @@ public static long reduceSumI32(long res, int laneId, long mask, MemorySegment o return outSum.get(ValueLayout.JAVA_LONG, 0); } + // ── row store (docs/abi.md §11, ABI minor 2) ───────────────────────────────────────────── + // + // Callers above this class are expected to have already checked Abi.requireMinor(2) — these + // wrappers do not re-check it, matching every other wrapper here: this class marshals and maps + // status, it does not gate on version. See Engine for the requireMinor call sites. + + /** + * Build the 64K×512-byte SoA row store deterministically from {@code seed} (docs/abi.md §11). + * Returns the resource handle. + */ + public static long rowstoreOpen(long nRows, long seed, MemorySegment outHandle) { + crossed(); + int st; + try { + st = (int) ROWSTORE_OPEN.invokeExact(nRows, seed, outHandle); + } catch (Throwable t) { + throw wrap("lgj_rowstore_open", t); + } + Status.check("lgj_rowstore_open", st); + return outHandle.get(ValueLayout.JAVA_LONG, 0); + } + + /** + * Overwrites {@code dstMask} with {@code classid(facet, row) == needle}. {@code facet} is a + * facet index {@code 0..32} into the row's 32 facet lanes, not a lane id + * (docs/abi.md §11). + */ + public static void opEqClassid(long res, int facet, int needle, long dstMask) { + crossed(); + int st; + try { + st = (int) OP_EQ_CLASSID.invokeExact(res, facet, needle, dstMask); + } catch (Throwable t) { + throw wrap("lgj_op_eq_classid", t); + } + Status.check("lgj_op_eq_classid", st); + } + + /** + * Writes, for every row, a {@code u32} bitset of which of its 32 facets carry {@code needle} as + * classid, into the caller's {@code out} buffer — zero-copy out, nothing serialized + * (docs/abi.md §11). Capacity is checked by the native side before anything is written + * ({@code MASK_LENGTH_MISMATCH} on a short buffer). + */ + public static void rowFacetMatch(long res, int needle, MemorySegment out, long outLenElems) { + crossed(); + int st; + try { + st = (int) ROW_FACET_MATCH.invokeExact(res, needle, out, outLenElems); + } catch (Throwable t) { + throw wrap("lgj_row_facet_match", t); + } + Status.check("lgj_row_facet_match", st); + } + private static LanceGraphException wrap(String symbol, Throwable t) { if (t instanceof LanceGraphException e) { return e; diff --git a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Engine.java b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Engine.java index 0a48c7c..69cf2a1 100644 --- a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Engine.java +++ b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Engine.java @@ -162,6 +162,48 @@ public static long sumI32(long resource, int laneId, long mask) { return Downcalls.reduceSumI32(resource, laneId, mask, s.out); } + // ── row store (docs/abi.md §11, ABI minor ≥ 2) ────────────────────────────────────────── + // + // Every entry point below calls Abi.requireMinor(2) before touching the membrane at all, so a + // Java build compiled against the row store never reaches a missing symbol inside Downcalls — + // it fails here, first, naming the version gap (see Abi#requireMinor's own doc for why the base + // load gate at minor >= 1 cannot cover this by itself). + + /** + * Open the 64K×512-byte SoA row store, generated deterministically from {@code seed} + * (docs/abi.md §11). Returns the resource handle. Requires ABI minor >= 2. + */ + public static long openRowStore(long nRows, long seed) { + Abi.requireMinor(2); + Scratch s = SCRATCH.get(); + return Downcalls.rowstoreOpen(nRows, seed, s.out); + } + + /** + * Overwrite {@code dstMask} with {@code classid(facet, row) == classId} for every row of + * {@code store} (docs/abi.md §11). {@code facet} is a facet index {@code 0..32} into the row's + * 32 facet lanes, not a lane id. Requires ABI minor >= 2. + */ + public static void eqClassid(long store, int facet, int classId, long dstMask) { + Abi.requireMinor(2); + Downcalls.opEqClassid(store, facet, classId, dstMask); + } + + /** + * Write, for every row of {@code store}, a {@code u32} bitset of which of its 32 facets carry + * {@code classId} as classid, into the caller's {@code out} segment — one crossing, zero-copy + * out, nothing serialized (docs/abi.md §11). Requires ABI minor >= 2. + * + * @param out a caller-owned segment, at least {@code outLenElems} {@link + * Layouts#FACET_MATCH_ELEM}-wide elements; capacity is checked natively + * before anything is written + * @param outLenElems the element count (not byte count) {@code out} was allocated for + */ + public static void rowFacetMatch(long store, int classId, MemorySegment out, long outLenElems) { + Abi.requireMinor(2); + Downcalls.rowFacetMatch(store, classId, out, outLenElems); + } + private static MemorySegment marshal(List plan) { int n = plan.size(); if (n == 0) { diff --git a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Layouts.java b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Layouts.java index 6afa2b5..233d489 100644 --- a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Layouts.java +++ b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Layouts.java @@ -2,6 +2,7 @@ import java.lang.foreign.MemoryLayout; import java.lang.foreign.MemoryLayout.PathElement; +import java.lang.foreign.SequenceLayout; import java.lang.foreign.StructLayout; import java.lang.foreign.ValueLayout; import java.lang.invoke.VarHandle; @@ -213,6 +214,42 @@ private static long off(String name) { public static final VarHandle OP_COMBINE = field(OP_DESC, "combine"); public static final VarHandle OP_RESERVED = field(OP_DESC, "_reserved"); + // ── LgjRow — the SoA row store (512 bytes, docs/abi.md §11, ABI minor ≥ 2) ────────────── + // + // 32 facet lanes of 16 bytes each: a 4-byte little-endian classid followed by a 12-byte opaque + // payload (.claude/knowledge/soa-row-store-layout.md — the lance-graph V3 content-blind facet + // shape). A facet is naturally 4-byte aligned within the row (offset f*16, always a multiple of + // 4), so no repr(C)-style padding member is introduced between classid and payload, or between + // consecutive facets — the struct-in-sequence composition below is exactly 512 bytes with no + // gaps, which SELF_CHECK proves rather than assumes. + // + // JAVA_INT_UNALIGNED, not plain JAVA_INT, deliberately breaking with every other layout in this + // file: those structs (LgjLaneDesc / LgjResourceInfo / LgjOpDesc / LgjAbiManifest) come from + // Rust #[repr(C)] types whose own alignment is 8 and is cross-checked field-by-field against the + // manifest in Abi — so a segment holding one is always properly aligned. The row store's buffer + // is different by explicit ABI statement: "the base is u8-aligned (Arc<[u8]>; stable Rust + // promises no more)" (docs/abi.md §11 "Alignment", stated honestly). A plain aligned JAVA_INT + // VarHandle throws IllegalArgumentException the moment the backing segment's actual address + // isn't 4-byte aligned, which this ABI does not guarantee here — so every classid read in this + // layout uses the unaligned value layout, exactly as §11 prescribes ("Java reads via + // JAVA_INT_UNALIGNED-class layouts"). This is the one facet field with numeric interpretation; + // the 12-byte payload is opaque bytes and has no alignment requirement either way. + public static final StructLayout ROW_FACET = MemoryLayout.structLayout( + ValueLayout.JAVA_INT_UNALIGNED.withName("classid"), + MemoryLayout.sequenceLayout(12, ValueLayout.JAVA_BYTE).withName("payload")) + .withName("LgjRowFacet"); + + /** One 512-byte row: 32 facets of 16 bytes each. {@link #SELF_CHECK} proves the byte size. */ + public static final SequenceLayout ROW_LAYOUT = MemoryLayout.sequenceLayout(32, ROW_FACET) + .withName("LgjRow"); + + /** + * Element layout of the caller-allocated {@code lgj_row_facet_match} out buffer (docs/abi.md + * §11): one {@code u32} bitset per row, into a Java-arena segment the caller owns — so, unlike + * {@link #ROW_FACET}, ordinary alignment applies and no unaligned variant is needed. + */ + public static final ValueLayout.OfInt FACET_MATCH_ELEM = ValueLayout.JAVA_INT; + /** * Compile-time-ish self check: the byte sizes the ABI document states in prose, checked against * the sizes these layouts actually derive. If a layout is edited wrongly this fails at class @@ -230,6 +267,11 @@ private static boolean selfCheck() { expect("LgjResourceInfo align", 8, RESOURCE_INFO.byteAlignment()); expect("LgjOpDesc size", 24, OP_DESC.byteSize()); expect("LgjOpDesc align", 8, OP_DESC.byteAlignment()); + // docs/abi.md §11: "64K rows x 512 bytes per row, 32 facet lanes of 16 bytes each" — this is + // the substrate truth the whole stack converges on, so a hand-edit that silently changes + // facet count or facet width must fail loudly here, at class init, before any row-store call. + expect("LgjRowFacet size", 16, ROW_FACET.byteSize()); + expect("LgjRow (row store, docs/abi.md §11) size", 512, ROW_LAYOUT.byteSize()); // The manifest's size is not stated in prose; it is stated by the artifact itself and // cross-checked in Abi. What is checked here is that the two trailing byte arrays land // where the repr(C) rule puts them, which is the only place a hand-written layout could diff --git a/java/src/test/java/com/adaworldapi/lancegraph/AllTests.java b/java/src/test/java/com/adaworldapi/lancegraph/AllTests.java index 70c22eb..5df4e69 100644 --- a/java/src/test/java/com/adaworldapi/lancegraph/AllTests.java +++ b/java/src/test/java/com/adaworldapi/lancegraph/AllTests.java @@ -26,6 +26,8 @@ public static void main(String[] args) { suites.put("LazinessTest", LazinessTest::run); suites.put("NarrowingTest", NarrowingTest::run); suites.put("LifetimeTest", LifetimeTest::run); + suites.put("RowStoreParityTest", RowStoreParityTest::run); + suites.put("RowStoreLifetimeTest", RowStoreLifetimeTest::run); if (!NativeRuntime.isAvailable()) { // ApiSurfaceTest needs no native library — the API's shape is a compile-time property — diff --git a/java/src/test/java/com/adaworldapi/lancegraph/RowStoreLifetimeTest.java b/java/src/test/java/com/adaworldapi/lancegraph/RowStoreLifetimeTest.java new file mode 100644 index 0000000..fca252b --- /dev/null +++ b/java/src/test/java/com/adaworldapi/lancegraph/RowStoreLifetimeTest.java @@ -0,0 +1,123 @@ +package com.adaworldapi.lancegraph; + +/** + * Falsifiers for the SoA row store's ownership rules (docs/abi.md §4, §11), mirroring + * {@link LifetimeTest}'s conventions for the flat {@code NativePattern} resource, applied here to + * {@link RowStore} and the two views it produces ({@link Mask} — the same type + * {@code NativePattern} uses — and {@link FacetMatchView}). + * + *

The claim under test is the same one {@link LifetimeTest} establishes for + * {@code NativePattern}: there is no sequence of operations in which a stale handle dereferences + * freed memory. Every attempt below must produce a specific Java exception — not a crash, not a + * wrong answer, and not a silently-tolerated no-op — matching the existing conventions rather than + * inventing new ones: a double close is an error (not idempotent), a child derived before the + * close keeps working while its parent is open, stops working the moment the parent closes, and + * closing that now-orphaned child afterward is tolerated (the caller's try-with-resources is + * unwinding and must not be punished for the ordering) while closing it *twice* still throws. + */ +public final class RowStoreLifetimeTest { + + private RowStoreLifetimeTest() {} + + public static void main(String[] args) { + System.out.println("RowStoreLifetimeTest"); + if (!NativeRuntime.isAvailable()) { + System.exit(Checks.reportUnavailable("RowStoreLifetimeTest")); + } + Checks c = new Checks("RowStoreLifetimeTest"); + run(c); + System.exit(c.report()); + } + + private static final long SEED = 0xABCDL; + + public static void run(Checks c) { + c.section("use after close"); + RowStore closed = RowStore.open(1024, SEED); + long before = closed.rowCount(); + c.eq("the resource works while open", 1024, before); + closed.close(); + + c.that("it reports itself closed", !closed.isOpen()); + c.throwsUp("rowCount() after close", ClosedResourceException.class, closed::rowCount); + c.throwsUp("maskOfFacetClass() after close", ClosedResourceException.class, + () -> closed.maskOfFacetClass(new FacetId(7), 9)); + c.throwsUp("facetMatches() after close", ClosedResourceException.class, + () -> closed.facetMatches(9)); + + c.section("double close"); + c.throwsUp("close() a second time", ClosedResourceException.class, closed::close); + + c.section("a Mask derived before the close"); + RowStore p = RowStore.open(1024, SEED); + Mask held = p.maskOfFacetClass(new FacetId(7), 9); + long liveCount = held.count(); + c.that("the mask works while its store is open", liveCount >= 0); + p.close(); + + // The Mask object still exists — Java cannot prevent that. What it must not do is work. + // This is the exact contract LifetimeTest pins for a NativePattern-derived selection, + // because it is the same Mask type: a child may outlive its parent as an object, but it + // can never work again once the parent is gone. + c.that("the mask knows it is no longer usable", !held.isOpen()); + c.throwsUp("count() on a mask whose store closed", + ClosedResourceException.class, held::count); + // Closing a mask orphaned by its store's close must be tolerated: the caller is doing the + // right thing (their try-with-resources is unwinding) and must not be punished for the + // ordering — same as LifetimeTest's "closing an orphaned selection is not an error". + held.close(); + c.that("closing a mask orphaned by its store's close is not an error", true); + c.throwsUp("but closing it twice still is", + ClosedResourceException.class, held::close); + + c.section("a Mask closed before its store -- the ordinary ordering"); + try (RowStore q = RowStore.open(512, SEED)) { + Mask m = q.maskOfFacetClass(new FacetId(3), 9); + long n = m.count(); + m.close(); + c.throwsUp("count() on a closed mask", ClosedResourceException.class, m::count); + c.that("the store is unharmed by its child's close", q.rowCount() == 512); + try (Mask again = q.maskOfFacetClass(new FacetId(3), 9)) { + c.that("and still answers queries", again.count() == n); + } + } + + c.section("a FacetMatchView derived before the close"); + RowStore r = RowStore.open(1024, SEED); + FacetMatchView view = r.facetMatches(9); + long liveRows = view.rowCount(); + c.eq("the view works while its store is open", 1024, liveRows); + long liveCardinality = view.cardinality(); + c.that("cardinality() succeeds while the store is open", liveCardinality >= 0); + r.close(); + + // The guard is only required to fire BEFORE any segment access — the observable contract + // is that every operation throws the right exception, not how close to native/segment + // memory the check happens to sit. + c.throwsUp("rowCount() on a view whose store closed", + ClosedResourceException.class, view::rowCount); + c.throwsUp("matchesOf() on a view whose store closed", + ClosedResourceException.class, () -> view.matchesOf(0)); + c.throwsUp("cardinality() on a view whose store closed", + ClosedResourceException.class, view::cardinality); + + c.section("the resource survives being asked the impossible"); + try (RowStore s = RowStore.open(256, SEED)) { + c.throwsUp("an out-of-range facet is rejected before it can travel", + IllegalArgumentException.class, () -> new FacetId(32)); + try (Mask m = s.maskOfFacetClass(new FacetId(0), 9)) { + c.that("and the store is still usable afterwards", m.count() >= 0); + } + } + + c.section("zero rows is a legal resource, not an error"); + try (RowStore empty = RowStore.open(0, SEED)) { + c.eq("an empty resource has no rows", 0, empty.rowCount()); + c.eq("facetMatches over zero rows reports zero rows", + 0, empty.facetMatches(9).rowCount()); + try (Mask m = empty.maskOfFacetClass(new FacetId(0), 9)) { + c.eq("maskOfFacetClass over zero rows selects nothing", 0, m.count()); + } + } + } +} diff --git a/java/src/test/java/com/adaworldapi/lancegraph/RowStoreParityTest.java b/java/src/test/java/com/adaworldapi/lancegraph/RowStoreParityTest.java new file mode 100644 index 0000000..1c98bb0 --- /dev/null +++ b/java/src/test/java/com/adaworldapi/lancegraph/RowStoreParityTest.java @@ -0,0 +1,175 @@ +package com.adaworldapi.lancegraph; + +/** + * A genuine cross-language check for the SoA row store (docs/abi.md §11), in the same spirit as + * {@link FixtureParityTest}: Java recomputes the expected classids from the normative generator + * description and compares them against what the native kernels returned, through both entry + * points the ABI exposes — {@code lgj_op_eq_classid} (via {@link RowStore#maskOfFacetClass}) and + * {@code lgj_row_facet_match} (via {@link RowStore#facetMatches}). + * + *

The row-store generator is two SplitMix64 draws per facet ({@code a} then {@code b}), 64 + * draws per row, with {@code classid = (a >>> 33) & 0xF}. Only the classid half of each draw pair + * is needed here — the payload half ({@code b}, plus the low 32 bits of {@code a}) is opaque to + * every assertion this file makes — but both draws are still consumed per facet so the recomputed + * stream stays in lockstep with the Rust generator. + */ +public final class RowStoreParityTest { + + private RowStoreParityTest() {} + + public static void main(String[] args) { + System.out.println("RowStoreParityTest"); + if (!NativeRuntime.isAvailable()) { + System.exit(Checks.reportUnavailable("RowStoreParityTest")); + } + Checks c = new Checks("RowStoreParityTest"); + run(c); + System.exit(c.report()); + } + + private static final int FACETS_PER_ROW = 32; + + /** + * SplitMix64, copied from {@link FixtureParityTest}'s private helper of the same name. + * + *

Provenance: this is the identical transcription, constant-for-constant and + * shift-for-shift, already checked there against the published reference vector. It is + * duplicated rather than shared because the original is {@code private} to that class — this + * file needs its own copy of the generator, not a dependency on another test's internals. + */ + private static final class SplitMix64 { + private long state; + + SplitMix64(long seed) { + this.state = seed; // no warm-up draws + } + + long next() { + state += 0x9E3779B97F4A7C15L; + long z = state; + z = (z ^ (z >>> 30)) * 0xBF58476D1CE4E5B9L; + z = (z ^ (z >>> 27)) * 0x94D049BB133111EBL; + return z ^ (z >>> 31); + } + } + + /** + * {@code classids[row][facet]}, recomputed independently in pure Java from the §11 generator. + */ + private static int[][] generateClassids(int nRows, long seed) { + SplitMix64 rng = new SplitMix64(seed); + int[][] classids = new int[nRows][FACETS_PER_ROW]; + for (int row = 0; row < nRows; row++) { + for (int facet = 0; facet < FACETS_PER_ROW; facet++) { + long a = rng.next(); // FIRST draw of the facet — carries the classid + rng.next(); // SECOND draw of the facet — the payload's high 64 bits; + // unused here but consumed to keep the stream aligned + classids[row][facet] = (int) ((a >>> 33) & 0xF); + } + } + return classids; + } + + private static long countFacetClassid(int[][] classids, int facet, int classId) { + long count = 0; + for (int[] row : classids) { + if (row[facet] == classId) { + count++; + } + } + return count; + } + + public static void run(Checks c) { + long seed = 0xABCDL; + + c.section("maskOfFacetClass(7, c): pure-Java recomputation vs the native mask, several sizes"); + for (int n : new int[] {1, 64, 65, 1000}) { + int[][] classids = generateClassids(n, seed); + try (RowStore store = RowStore.open(n, seed)) { + for (int classId : new int[] {0, 9, 15}) { + long expected = countFacetClassid(classids, 7, classId); + try (Mask mask = store.maskOfFacetClass(new FacetId(7), classId)) { + c.eq("n=" + n + " facet 7 classid==" + classId, expected, mask.count()); + } + } + } + } + + c.section("the edge facets (0 and 31) at n=1000"); + int n1000 = 1000; + int[][] classids1000 = generateClassids(n1000, seed); + try (RowStore store = RowStore.open(n1000, seed)) { + for (int facet : new int[] {0, 31}) { + for (int classId : new int[] {0, 9, 15}) { + long expected = countFacetClassid(classids1000, facet, classId); + try (Mask mask = store.maskOfFacetClass(new FacetId(facet), classId)) { + c.eq("facet " + facet + " classid==" + classId, expected, mask.count()); + } + } + } + } + + c.section("facetMatches(9) at n=1000: per-row 32-bit set and total cardinality"); + int needle = 9; + long expectedTotalPopcount = 0; + long mismatches = 0; + try (RowStore store = RowStore.open(n1000, seed)) { + FacetMatchView view = store.facetMatches(needle); + c.eq("facetMatches reports the store's row count", n1000, view.rowCount()); + for (int row = 0; row < n1000; row++) { + int expectedBits = 0; + for (int facet = 0; facet < FACETS_PER_ROW; facet++) { + if (classids1000[row][facet] == needle) { + expectedBits |= (1 << facet); + } + } + expectedTotalPopcount += Integer.bitCount(expectedBits); + if (view.matchesOf(row) != expectedBits) { + mismatches++; + } + } + c.eq("every one of the 1000 rows' facet-9 bitset matches the pure-Java recomputation", + 0, mismatches); + c.eq("cardinality() equals the recomputed total popcount", + expectedTotalPopcount, view.cardinality()); + } + + c.section("anti-vacuity guards (the falsifiability rule)"); + long facet7eq9At1000 = countFacetClassid(classids1000, 7, 9); + c.that("facet 7 classid==9 selects more than nothing at n=1000", facet7eq9At1000 > 0); + c.that("facet 7 classid==9 selects less than everything at n=1000 — a middling" + + " selection, not an all-or-nothing predicate that cannot falsify", + facet7eq9At1000 < n1000); + + boolean anyRowHasNonZeroFacet9Set = false; + boolean twoRowsDifferInTheirFacet9Set = false; + Integer firstBits = null; + for (int row = 0; row < n1000; row++) { + int bits = 0; + for (int facet = 0; facet < FACETS_PER_ROW; facet++) { + if (classids1000[row][facet] == needle) { + bits |= (1 << facet); + } + } + if (bits != 0) { + anyRowHasNonZeroFacet9Set = true; + } + if (firstBits == null) { + firstBits = bits; + } else if (bits != firstBits) { + twoRowsDifferInTheirFacet9Set = true; + } + } + c.that("at least one row has a non-zero facet-9 set", anyRowHasNonZeroFacet9Set); + c.that("at least one row's facet-9 set differs from another's — the generator actually" + + " varies, rather than every row landing on the same bits", + twoRowsDifferInTheirFacet9Set); + + c.section("FacetId range"); + c.throwsUp("FacetId(-1) is rejected", IllegalArgumentException.class, () -> new FacetId(-1)); + c.throwsUp("FacetId(32) is rejected", IllegalArgumentException.class, () -> new FacetId(32)); + c.eq("FacetId(0) is accepted", 0, new FacetId(0).index()); + c.eq("FacetId(31) is accepted", 31, new FacetId(31).index()); + } +} From bc6f91ab9b74d30259c8a948885d8da268895ac3 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 23:07:04 +0000 Subject: [PATCH 15/30] Board: PR #8 arc entry (post-merge) Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/PR_ARC_INVENTORY.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 53e0a5b..8f1ba65 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -8,6 +8,26 @@ > anti-pattern the imported board rules name. Backfilled below in one > pass rather than left stale; PR #4 onward gets its entry at merge time. +## PR #8 — Java RowStore facade: W3 shipped (merged 2026-08-17, squash `320808d`) + +- **Added:** `RowStore`/`FacetMatchView`/`FacetId`/`NativeResource` public + API; `Mask.source()` retyped `NativePattern → NativeResource`; + `RowStoreParityTest`/`RowStoreLifetimeTest` (53 new checks). +- **Locked:** the wave-dispatch system works end to end — 3 disjoint + Sonnet workers, zero merge conflicts, mutually consistent signatures + with no coordination beyond the frozen briefs + (`E-LGJ-WAVE-DISPATCH-VALIDATED-1`). +- **Deferred:** W4 (bench Component F) — the wave file's second dispatch. +- **Docs:** `STATUS_BOARD` D-LGJ-W3 DONE; `LATEST_STATE`; EPIPHANIES entry + incl. an orchestrator-side false alarm (guessed env var name instead + of reading `Abi.java`'s `ENV_LIBRARY` constant) recorded so it isn't + repeated. +- **Confidence:** High — 185/185 (was 132), 0 new lint warnings, one real + bug (`FacetMatchView.rowCount()` missing its closed-store guard) caught + by the mandated tests and fixed before merge, both disable-runs + red-then-green with the exact predicted blast radius. Bot reviewers at + usage limits, did not run. + ## PR #7 — waves calcified: dispatch maps for every plan (merged 2026-08-17, squash `68f7add`) - **Added:** `.claude/waves/` — README (standing rules + verbatim worker From 619b3de9aeaa8c0243b5eb17905856b72cefd822 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 23:12:12 +0000 Subject: [PATCH 16/30] Plan: r2sleigh recorded as third lift path + decompiler candidate Operator-flagged: AdaWorldAPI/r2sleigh (read-only clone verified, HEAD 60942f6) is a Rust workspace lifting Ghidra .sla specs to P-code via libsla, with typed IR, SSA, Z3 symbolic execution, and a P-code-to-C decompiler. The honest FFI fact is pinned: libsla-sys means the SLEIGH runtime underneath is Ghidra's C++ via FFI, not pure Rust -- acceptable on the same lift-time-only footing as running Ghidra itself. G1 gains candidate C (r2sleigh-cli lift, no JVM in the loop, and a STRONGER falsifier: cross-implementation P-code agreement between two independent consumers of one .sla spec); r2dec is named as the engine candidate for the semantic-shim direction; r2sym joins SymbolicSummaryZ3 as branch-population prior art. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/plans/ghidra-integration-v1.md | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.claude/plans/ghidra-integration-v1.md b/.claude/plans/ghidra-integration-v1.md index 286a1d3..5e231d5 100644 --- a/.claude/plans/ghidra-integration-v1.md +++ b/.claude/plans/ghidra-integration-v1.md @@ -27,6 +27,43 @@ License note: Ghidra core is Apache-2.0 (compatible with everything in this stack); the `GPL/` subtree (demangler etc.) is not needed for lifting and stays untouched. +## G0 addendum (2026-08-17, operator-flagged): `AdaWorldAPI/r2sleigh` — the third lift path and the decompiler candidate + +Read-only clone at `/workspace/adaworldapi/r2sleigh` (HEAD `60942f6`, 20 MB, +Rust 1.93 workspace, 8 crates + an r2 plugin). What it is, verified from the +tree, not the README alone: + +- **Pipeline:** `.sla (Ghidra processor spec) → libsla → P-code → r2il + (typed IR, 60+ opcodes) → {SSA (r2ssa) → taint/symbolic (r2sym, Z3-backed), + decompiler-to-C (r2dec), type inference (r2types), ESIL}`. +- **The honest FFI fact:** `Cargo.lock` carries `libsla` AND `libsla-sys` — + the SLEIGH runtime underneath is **Ghidra's native C++ via FFI**, not a + pure-Rust SLEIGH. r2sleigh's own crates are Rust; the lifter core is not. + Acceptable on exactly the same footing as running Ghidra itself: this is + a LIFT-TIME tool, never at OGAR-Machine runtime (the no-C-at-runtime rule + is about the execution path, and lift time is not on it). + +**Consequences for the waves:** + +1. **G1 gains a candidate C: lift via `r2sleigh-cli`** (Rust CLI consuming + Ghidra's own `.sla` specs) alongside A (released-Ghidra + `analyzeHeadless`) and B (fork build). No JVM in the lift loop, and the + P-code comes from the SAME SLEIGH specs Ghidra uses — so the G1 + falsifier gets stronger, not weaker: r2sleigh's P-code dump vs Ghidra's + own listing is a **cross-implementation** agreement check, two + independent consumers of one spec. +2. **Operator designation: future decompiler candidate.** `r2dec` + (P-code→SSA→structured C) is the natural engine for the OGAR-Machine's + "semantic shims erode the emulator" direction — recognizing and lifting + stable call surfaces needs decompilation-grade structure recovery, and + building that from scratch was never on any plan. Not scheduled; named. +3. **A second symbolic-execution precedent** (`r2sym`, Z3) in Rust, next to + Ghidra's `SymbolicSummaryZ3` — the branch-population direction now has + two prior arts to study before designing anything. +4. **Access boundary, recorded:** the clone is anonymous/read-only — + pushing or PRs against r2sleigh require re-attaching with push access. + Any fix we need upstream goes through the operator first. + ## The integration shape — two roles, both offline ``` From 84e73cae44e42a6ac5e0570a61b1a4dc22ab42c9 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 23:18:40 +0000 Subject: [PATCH 17/30] Bench Component F: the boundary re-asked on the real row-store layout (W4) One Sonnet worker per wave-substrate-w3-w4.md Dispatch 2, orchestrator- run JMH (9/9 combos), the cross-check discipline intact: both Java facet-match kernels verified row-by-row against the native FacetMatchView in @Setup at every row count before anything was timed. The finding: Component C's direction survives, its margin collapses. The Vector API wins the per-row 32-facet strided scan at every row count measured, but by 2.51x / 1.92x / 1.14x (4K / 65K / 1M rows) against C's 56x -- and at 512 MiB traversed all three arms converge on memory bandwidth. More work per byte narrows the boundary exactly as execution-boundary.md predicted; it now records that as measurement. Disclosed, not hidden: the native arm allocates its output segment per call where the Java arms reuse a @Setup buffer; facetMatchesInto is the named follow-up if the small-row gap ever matters. Java kernels mirror the Rust chunk algorithm line-for-line (VectorMask.toLong() & 0x1111, same four-term fold) so the comparison is between implementations of ONE algorithm, not two algorithms. Mechanics: summarise.sh gains the F table (and its old 'E/F' section title -- a genuine collision with the new component -- is corrected to 'E'); TABLES.md regenerated from the merged CSV; RESULTS.md gains provenance-table update + full F section; RowStore gains a package-private handle() mirroring NativePattern's for the bench's split-package NativeAccess bridge; main suite re-verified 185/185 against the fresh minor-2 .so in the bench's expected location. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/LATEST_STATE.md | 26 + .claude/board/STATUS_BOARD.md | 2 +- bench/RESULTS.md | 36 +- bench/results/TABLES.md | 10 +- bench/results/jmh-results-merged.csv | 9 + bench/results/jmh-results.csv | 43 +- bench/results/jmh-run.txt | 1391 +++-------------- .../adaworldapi/lancegraph/NativeAccess.java | 14 + .../lancegraph/bench/F_RowStoreFacetScan.java | 131 ++ .../adaworldapi/lancegraph/bench/Kernels.java | 106 ++ .../lancegraph/bench/RowStoreData.java | 99 ++ bench/summarise.sh | 30 +- docs/execution-boundary.md | 15 +- .../com/adaworldapi/lancegraph/RowStore.java | 10 + 14 files changed, 692 insertions(+), 1230 deletions(-) create mode 100644 bench/src/main/java/com/adaworldapi/lancegraph/bench/F_RowStoreFacetScan.java create mode 100644 bench/src/main/java/com/adaworldapi/lancegraph/bench/RowStoreData.java diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 9a20442..559f0cb 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,29 @@ +## 2026-08-17 (dispatch 2) — W4 measured: the boundary re-asked on the REAL layout + +`wave-substrate-w3-w4.md` Dispatch 2 executed: one Sonnet worker +(Component F: `F_RowStoreFacetScan` + `RowStoreData` + the two +`Kernels` facet-match arms, mirroring the Rust kernel's chunk algorithm +line-for-line incl. the `& 0x1111` classid-position mask), orchestrator-run +JMH, 9/9 combos, the cross-check green at every row count before anything +was timed. + +**The finding: Component C's direction survives; its margin collapses.** +The Vector API still wins the per-row 32-facet scan at every row count — +but by **2.51× / 1.92× / 1.14×** (4K / 65K / 1M rows) against C's 56×, and +at 512 MiB traversed all three arms converge on memory bandwidth +(~6–7 GB/s on this container). More work per byte narrows the boundary +exactly as `execution-boundary.md` predicted; it now says so as +measurement. One disclosed asymmetry: the native arm allocates its output +per call (`facetMatchesInto` named as the follow-up if the small-row gap +ever matters). + +Also: `summarise.sh` gained the F table (and its old "E/F" section title — +a real collision with the new component — was corrected to "E"); +`TABLES.md` regenerated from the merged CSV; `RESULTS.md` §F written; +`RowStore` gained a package-private `handle()` (mirroring +`NativePattern`'s, for the bench's split-package `NativeAccess` bridge +only). Substrate wave file fully executed — both dispatches shipped. + ## 2026-08-17 (dispatch 1) — W3 shipped: the Java `RowStore` facade, from the calcified wave map First real dispatch of the wave system: `wave-substrate-w3-w4.md` Dispatch 1 diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 41cb480..6709c68 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -39,5 +39,5 @@ layout wired end to end. Doctrine: `E-LGJ-THE-MIDDLE-TIER-IS-DELETED-NOT-WRAPPED | D-LGJ-W1 | ndarray: `MultiLaneColumn::iter_u32x16`/`len_u32x16` + `eq_u32_strided_to_mask` (W1a contract) | **DONE 2026-08-17** — ndarray PR #279; `simd_int_ops` 46/46 (5 new strided tests incl. two `should_panic` bounds/overflow arms + stride-4 parity against the contiguous primitive), `simd_soa` 15/15, full `simd` 263/263, doctests, clippy `-D warnings` + fmt clean | | D-LGJ-W2 | lgj-abi row store: `rowstore.rs`, `LGJ_RESOURCE_ROWSTORE`, `lgj_rowstore_open`, strided facet lanes through the unchanged `LgjLaneDesc`, `lgj_op_eq_classid`, `lgj_row_facet_match`, ABI minor 1→2, `docs/abi.md` §11 | **DONE 2026-08-17** — `cargo test` **84/84**, clippy/fmt clean, release build exports **18/18** symbols (`nm -D`). Parity: both kernels vs independent scalar references over 10 row counts × 2 seeds × 4 facets × 4 needles, cross-checked a THIRD way against `RowStore::classid_at`. Two-sided payload-vs-classid falsifier. End-to-end membrane test covers describe → predicate → mask algebra → count → facet-match → lifecycle | | D-LGJ-W3 | Java `RowStore` facade: structured `MemoryLayout`, minor-≥2 gate, `FacetMatchView`, parity test transcribing the generator | **DONE 2026-08-17** — dispatched per `.claude/waves/wave-substrate-w3-w4.md` (3 Sonnet workers, disjoint scopes: FFM membrane extension / public facade / tests), orchestrator-integrated. `javac -Xlint:all` clean (same 7 pre-existing `[restricted]` warnings, zero new). `AllTests` **185/185** (was 132; +53 new checks: 29 parity + 24 lifetime). **One real bug caught by the suite and fixed**: `FacetMatchView.rowCount()` was missing the closed-store guard `matchesOf`/`cardinality` both had — a stale row count was readable after the owning store closed. Fixed, re-verified. Both mandated disable-runs ran red-then-green: (1) `Abi.requireMinor` inflated by 1 → exactly `RowStoreParityTest`+`RowStoreLifetimeTest` failed, all 8 other suites stayed green; (2) the pure-Java generator's a/b draw order swapped in `RowStoreParityTest` → exactly that suite broke (17/29), `RowStoreLifetimeTest` (generator-independent) stayed green — confirming the parity test is a real falsifier, not decorative. `Mask.source()` retyped `NativePattern → NativeResource` (new interface) so a `Mask` can parent onto either a `NativePattern` or a `RowStore` — zero call-site breakage (verified: no existing caller bound the narrower type) | -| D-LGJ-W4 | Bench Component F: Vector API facet scan vs the crossing, on the REAL layout | Queued | +| D-LGJ-W4 | Bench Component F: Vector API facet scan vs the crossing, on the REAL layout | **DONE 2026-08-17** — 1 Sonnet worker (F_RowStoreFacetScan + RowStoreData + Kernels facet-match arms, cross-check-in-@Setup discipline), orchestrator-run JMH: 9/9 combos, cross-checks green at every row count. **Finding: Component C's direction survives, its margin collapses** — Vector API wins the 32-facet strided scan at every row count but by 2.51×/1.92×/1.14× (4K/65K/1M rows) vs C's 56×; at 512 MiB traversed all three arms converge on memory bandwidth. Native arm's per-call allocation asymmetry disclosed in §F with a named follow-up (`facetMatchesInto`), not hidden. summarise.sh extended with the F table (and the old 'E/F' section retitled 'E' — a real naming collision); tables regenerated from the merged CSV | | D-LGJ-W5 | Three consumer examples (trades / bricks / graph) — one plan file each | Planned, gated on W3 | diff --git a/bench/RESULTS.md b/bench/RESULTS.md index 5f67dad..27129e2 100644 --- a/bench/RESULTS.md +++ b/bench/RESULTS.md @@ -6,7 +6,7 @@ > | file | what it is | > |---|---| > | `results/jmh-run-full.txt` / `jmh-results-full.csv` | the first full sweep (A, B, C/D, and E at 65,536 rows only) | -> | `results/jmh-run.txt` / `jmh-results.csv` | the `./run.sh E_` re-run, after a 256-row arm was added to the fusion sweep | +> | `results/jmh-run.txt` / `jmh-results.csv` | the latest selective run — currently `./run.sh F_` (the W4 row-store facet scan); the earlier `E_` re-run's rows live on in the merged file | > | `results/jmh-results-merged.csv` | A/B/C from the full sweep + E from the re-run — **the input to every table below** | > | `results/TABLES.md` | `./summarise.sh results/jmh-results-merged.csv` | > @@ -192,7 +192,7 @@ most of the residual gap. Both are ABI-implementation changes, not ABI-contract --- -## E/F — fusion, and what the fluent API itself costs +## E — fusion, and what the fluent API itself costs | rows | predicates | `fused` | `unfused` | `fusedScalarKernel` | `planConstructionOnly` | unfused ÷ fused | |---:|---:|---:|---:|---:|---:|---:| @@ -231,6 +231,38 @@ just a list of predicates" should look like. --- +## F — the row-store facet scan (the W4 question) + +The question C could not answer: does "the Vector API beats the crossing" survive when the +workload is the REAL substrate layout — 512-byte rows, 32 facets, a *strided* scan doing 32 +classid compares per row instead of one compare per 4-byte element? Three arms over the same +`RowStore` (`classId == 9`), every arm's full per-row bitset output cross-checked equal in +`@Setup` before anything was timed (`RowStoreData`'s constructor, at every row count): + +| rows | bytes traversed | `native_facetMatch` | `java_vectorApi` | `java_scalar` | native/vector | +|---:|---:|---:|---:|---:|---:| +| 4,096 | 2 MiB | 191.3 ±24.3 µs | **76.1 ±5.3 µs** | 134.7 ±13.9 µs | 2.51× | +| 65,536 | 32 MiB | 3,219.6 ±141.5 µs | **1,674.1 ±126.9 µs** | 3,354.4 ±118.4 µs | 1.92× | +| 1,048,576 | 512 MiB | 79,016.9 ±9,239.4 µs | **69,613.1 ±1,624.1 µs** | 82,541.1 ±3,106.4 µs | 1.14× | + +Three findings, in decreasing order of confidence: + +1. **The direction survives; the margin collapses.** The Vector API still wins at every measured + row count — but by 1.1–2.5×, not Component C's 56×. More work per byte (32 compares per + 512-byte row, the same four-facets-per-512-bit-register algorithm on both sides) is exactly + the regime the § Verdict predicted would narrow the boundary, and now it is measured rather + than predicted. +2. **At 512 MiB everything converges on memory bandwidth.** 69–82 ms to traverse 512 MiB is + ~6–7 GB/s on this shared 4-vCPU container — all three arms are bandwidth-bound and the + implementation difference shrinks toward noise (native's CI at 1M rows, ±9.2 ms, brackets much of the + gap to vector). +3. **The native arm carries a disclosed allocation asymmetry.** `RowStore.facetMatches()` + allocates a fresh output segment + view per call, where `NativePattern`'s fused-plan path + reuses a cached scratch mask and the Java arms reuse a `@Setup`-allocated array. At 4,096 + rows that fixed cost is a visible slice of 191 µs; at 1M rows it is noise. A + `facetMatchesInto(classId, …)` reuse form is the named follow-up if the small-row gap ever + matters — filed, not assumed to. + ## Verdict — where does execution belong? On the evidence, **not where the architecture currently puts it, for count-only queries.** Stated diff --git a/bench/results/TABLES.md b/bench/results/TABLES.md index 780dc5c..90baa08 100644 --- a/bench/results/TABLES.md +++ b/bench/results/TABLES.md @@ -29,7 +29,7 @@ | 1,048,576 | 4096 | 411.333 ±37.244 | 310.405 ±17.660 | 1623.313 ±26.973 | **java_vectorApi** | 1.33x | | 4,194,304 | 16384 | 1858.686 ±149.400 | 1319.107 ±37.240 | 6602.036 ±100.771 | **java_vectorApi** | 1.41x | -### E/F — fusion and the cost of the fluent API (µs/op) +### E — fusion and the cost of the fluent API (µs/op) | rows | predicates | `fused` | `unfused` | `fusedScalarKernel` | `planConstructionOnly` | unfused/fused | |---:|---:|---:|---:|---:|---:|---:| @@ -42,3 +42,11 @@ | 65,536 | 4 | 25.591 ±0.582 | 31.790 ±3.337 | 917.387 ±27.768 | 0.261 ±0.012 | **1.24x** | | 65,536 | 8 | 58.978 ±4.509 | 60.968 ±2.657 | 1825.916 ±125.373 | 0.601 ±0.032 | **1.03x** | +### F — the row-store facet scan (µs/op, mean ± 99.9% CI) + +| rows | row KiB | `native_facetMatch` | `java_vectorApi` | `java_scalar` | fastest | native/vector | +|---:|---:|---:|---:|---:|---:|---:| +| 4,096 | 2048 | 191.285 ±24.278 | 76.108 ±5.316 | 134.712 ±13.871 | **java_vectorApi** | 2.51x | +| 65,536 | 32768 | 3219.554 ±141.518 | 1674.064 ±126.854 | 3354.401 ±118.415 | **java_vectorApi** | 1.92x | +| 1,048,576 | 524288 | 79016.873 ±9239.383 | 69613.100 ±1624.077 | 82541.075 ±3106.426 | **java_vectorApi** | 1.14x | + diff --git a/bench/results/jmh-results-merged.csv b/bench/results/jmh-results-merged.csv index 23b6cbc..bf30be0 100644 --- a/bench/results/jmh-results-merged.csv +++ b/bench/results/jmh-results-merged.csv @@ -64,3 +64,12 @@ com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused,avgt,1,8,2.097204,0 com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused,avgt,1,8,31.789786,3.337085,us/op,4,65536 com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused,avgt,1,8,4.437382,0.204156,us/op,8,256 com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused,avgt,1,8,60.967976,2.656815,us/op,8,65536 +com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_scalar,avgt,1,8,134.711904,13.870504,us/op,,4096 +com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_scalar,avgt,1,8,3354.401240,118.415079,us/op,,65536 +com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_scalar,avgt,1,8,82541.075065,3106.426103,us/op,,1048576 +com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_vectorApi,avgt,1,8,76.107543,5.315558,us/op,,4096 +com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_vectorApi,avgt,1,8,1674.064233,126.854113,us/op,,65536 +com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_vectorApi,avgt,1,8,69613.100438,1624.076656,us/op,,1048576 +com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.native_facetMatch,avgt,1,8,191.285057,24.277993,us/op,,4096 +com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.native_facetMatch,avgt,1,8,3219.553540,141.518033,us/op,,65536 +com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.native_facetMatch,avgt,1,8,79016.872595,9239.382837,us/op,,1048576 diff --git a/bench/results/jmh-results.csv b/bench/results/jmh-results.csv index d2669ae..16fae29 100644 --- a/bench/results/jmh-results.csv +++ b/bench/results/jmh-results.csv @@ -1,33 +1,10 @@ -"Benchmark","Mode","Threads","Samples","Score","Score Error (99.9%)","Unit","Param: predicates","Param: rows" -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,0.404255,0.010091,"us/op",1,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,6.913186,0.165479,"us/op",1,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,0.520053,0.032240,"us/op",2,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,15.222049,0.707462,"us/op",2,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,0.807558,0.033095,"us/op",4,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,25.590826,0.582056,"us/op",4,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,1.482240,0.070287,"us/op",8,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused","avgt",1,8,58.977540,4.508809,"us/op",8,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,0.502951,0.015378,"us/op",1,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,74.340346,1.659056,"us/op",1,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,0.835215,0.038908,"us/op",2,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,408.169551,20.241519,"us/op",2,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,1.662239,0.084279,"us/op",4,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,917.386626,27.768400,"us/op",4,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,3.488951,0.112650,"us/op",8,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel","avgt",1,8,1825.916402,125.373079,"us/op",8,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.056078,0.004036,"us/op",1,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.052973,0.005193,"us/op",1,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.111424,0.003422,"us/op",2,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.113013,0.005714,"us/op",2,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.281631,0.055958,"us/op",4,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.261441,0.012488,"us/op",4,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.657211,0.196387,"us/op",8,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly","avgt",1,8,0.600529,0.031814,"us/op",8,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,0.384609,0.009362,"us/op",1,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,6.326314,0.174757,"us/op",1,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,0.930643,0.017781,"us/op",2,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,14.203920,0.803807,"us/op",2,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,2.097204,0.064309,"us/op",4,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,31.789786,3.337085,"us/op",4,65536 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,4.437382,0.204156,"us/op",8,256 -"com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused","avgt",1,8,60.967976,2.656815,"us/op",8,65536 +"Benchmark","Mode","Threads","Samples","Score","Score Error (99.9%)","Unit","Param: rows" +"com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_scalar","avgt",1,8,134.711904,13.870504,"us/op",4096 +"com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_scalar","avgt",1,8,3354.401240,118.415079,"us/op",65536 +"com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_scalar","avgt",1,8,82541.075065,3106.426103,"us/op",1048576 +"com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_vectorApi","avgt",1,8,76.107543,5.315558,"us/op",4096 +"com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_vectorApi","avgt",1,8,1674.064233,126.854113,"us/op",65536 +"com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_vectorApi","avgt",1,8,69613.100438,1624.076656,"us/op",1048576 +"com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.native_facetMatch","avgt",1,8,191.285057,24.277993,"us/op",4096 +"com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.native_facetMatch","avgt",1,8,3219.553540,141.518033,"us/op",65536 +"com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.native_facetMatch","avgt",1,8,79016.872595,9239.382837,"us/op",1048576 diff --git a/bench/results/jmh-run.txt b/bench/results/jmh-run.txt index cb4ad38..9e3c182 100644 --- a/bench/results/jmh-run.txt +++ b/bench/results/jmh-run.txt @@ -8,7 +8,7 @@ vm args [--enable-native-access=ALL-UNNAMED, --add-modules=jdk.incubato os / arch Linux amd64 cpu Intel(R) Xeon(R) Processor @ 2.10GHz (4 logical processors) vector species Species[int, 16, S_512_BIT] (16 int lanes, 512 bit) -native runtime lance-graph native runtime: abi 0.1, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so +native runtime lance-graph native runtime: abi 0.2, simd ndarray::simd avx512, profile release, library /home/user/lance-graph-java/target/release/liblgj_abi.so predicate class == 7 AND value > 100 ================================================================================================ @@ -19,17 +19,17 @@ WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release # JMH version: 1.37 # VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 # VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector # Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) # Warmup: 5 iterations, 500 ms each # Measurement: 8 iterations, 500 ms each # Timeout: 10 min per iteration # Threads: 1 thread, will synchronize iterations # Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused -# Parameters: (predicates = 1, rows = 256) +# Benchmark: com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_scalar +# Parameters: (rows = 4096) -# Run progress: 0.00% complete, ETA 00:03:28 +# Run progress: 0.00% complete, ETA 00:00:58 # Fork: 1 of 1 Picked up JAVA_TOOL_OPTIONS: WARNING: Using incubator modules: jdk.incubator.vector @@ -37,41 +37,41 @@ WARNING: A terminally deprecated method in sun.misc.Unsafe has been called WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 0.558 us/op -# Warmup Iteration 2: 0.415 us/op -# Warmup Iteration 3: 0.411 us/op -# Warmup Iteration 4: 0.407 us/op -# Warmup Iteration 5: 0.416 us/op -Iteration 1: 0.410 us/op -Iteration 2: 0.401 us/op -Iteration 3: 0.401 us/op -Iteration 4: 0.403 us/op -Iteration 5: 0.401 us/op -Iteration 6: 0.402 us/op -Iteration 7: 0.414 us/op -Iteration 8: 0.400 us/op +# Warmup Iteration 1: 132.072 us/op +# Warmup Iteration 2: 132.754 us/op +# Warmup Iteration 3: 131.176 us/op +# Warmup Iteration 4: 138.994 us/op +# Warmup Iteration 5: 136.444 us/op +Iteration 1: 138.002 us/op +Iteration 2: 136.831 us/op +Iteration 3: 150.111 us/op +Iteration 4: 130.394 us/op +Iteration 5: 131.327 us/op +Iteration 6: 126.267 us/op +Iteration 7: 131.026 us/op +Iteration 8: 133.737 us/op -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": - 0.404 ±(99.9%) 0.010 us/op [Average] - (min, avg, max) = (0.400, 0.404, 0.414), stdev = 0.005 - CI (99.9%): [0.394, 0.414] (assumes normal distribution) +Result "com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_scalar": + 134.712 ±(99.9%) 13.871 us/op [Average] + (min, avg, max) = (126.267, 134.712, 150.111), stdev = 7.255 + CI (99.9%): [120.841, 148.582] (assumes normal distribution) # JMH version: 1.37 # VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 # VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector # Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) # Warmup: 5 iterations, 500 ms each # Measurement: 8 iterations, 500 ms each # Timeout: 10 min per iteration # Threads: 1 thread, will synchronize iterations # Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused -# Parameters: (predicates = 1, rows = 65536) +# Benchmark: com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_scalar +# Parameters: (rows = 65536) -# Run progress: 3.13% complete, ETA 00:07:47 +# Run progress: 11.11% complete, ETA 00:02:00 # Fork: 1 of 1 Picked up JAVA_TOOL_OPTIONS: WARNING: Using incubator modules: jdk.incubator.vector @@ -79,41 +79,41 @@ WARNING: A terminally deprecated method in sun.misc.Unsafe has been called WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 7.485 us/op -# Warmup Iteration 2: 6.796 us/op -# Warmup Iteration 3: 6.828 us/op -# Warmup Iteration 4: 6.877 us/op -# Warmup Iteration 5: 6.934 us/op -Iteration 1: 6.969 us/op -Iteration 2: 6.845 us/op -Iteration 3: 7.029 us/op -Iteration 4: 6.872 us/op -Iteration 5: 7.005 us/op -Iteration 6: 6.795 us/op -Iteration 7: 6.839 us/op -Iteration 8: 6.952 us/op +# Warmup Iteration 1: 3340.853 us/op +# Warmup Iteration 2: 3318.692 us/op +# Warmup Iteration 3: 3284.887 us/op +# Warmup Iteration 4: 3404.363 us/op +# Warmup Iteration 5: 3325.872 us/op +Iteration 1: 3313.203 us/op +Iteration 2: 3265.522 us/op +Iteration 3: 3311.111 us/op +Iteration 4: 3471.262 us/op +Iteration 5: 3355.582 us/op +Iteration 6: 3380.407 us/op +Iteration 7: 3386.604 us/op +Iteration 8: 3351.520 us/op -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": - 6.913 ±(99.9%) 0.165 us/op [Average] - (min, avg, max) = (6.795, 6.913, 7.029), stdev = 0.087 - CI (99.9%): [6.748, 7.079] (assumes normal distribution) +Result "com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_scalar": + 3354.401 ±(99.9%) 118.415 us/op [Average] + (min, avg, max) = (3265.522, 3354.401, 3471.262), stdev = 61.933 + CI (99.9%): [3235.986, 3472.816] (assumes normal distribution) # JMH version: 1.37 # VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 # VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector # Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) # Warmup: 5 iterations, 500 ms each # Measurement: 8 iterations, 500 ms each # Timeout: 10 min per iteration # Threads: 1 thread, will synchronize iterations # Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused -# Parameters: (predicates = 2, rows = 256) +# Benchmark: com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_scalar +# Parameters: (rows = 1048576) -# Run progress: 6.25% complete, ETA 00:07:32 +# Run progress: 22.22% complete, ETA 00:01:46 # Fork: 1 of 1 Picked up JAVA_TOOL_OPTIONS: WARNING: Using incubator modules: jdk.incubator.vector @@ -121,41 +121,41 @@ WARNING: A terminally deprecated method in sun.misc.Unsafe has been called WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 0.728 us/op -# Warmup Iteration 2: 0.512 us/op -# Warmup Iteration 3: 0.514 us/op -# Warmup Iteration 4: 0.496 us/op -# Warmup Iteration 5: 0.531 us/op -Iteration 1: 0.523 us/op -Iteration 2: 0.512 us/op -Iteration 3: 0.512 us/op -Iteration 4: 0.503 us/op -Iteration 5: 0.526 us/op -Iteration 6: 0.558 us/op -Iteration 7: 0.515 us/op -Iteration 8: 0.511 us/op +# Warmup Iteration 1: 92248.398 us/op +# Warmup Iteration 2: 87608.093 us/op +# Warmup Iteration 3: 84744.542 us/op +# Warmup Iteration 4: 84101.623 us/op +# Warmup Iteration 5: 82018.456 us/op +Iteration 1: 81661.990 us/op +Iteration 2: 82091.608 us/op +Iteration 3: 82187.637 us/op +Iteration 4: 82945.802 us/op +Iteration 5: 86220.725 us/op +Iteration 6: 82819.191 us/op +Iteration 7: 81249.698 us/op +Iteration 8: 81151.950 us/op -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": - 0.520 ±(99.9%) 0.032 us/op [Average] - (min, avg, max) = (0.503, 0.520, 0.558), stdev = 0.017 - CI (99.9%): [0.488, 0.552] (assumes normal distribution) +Result "com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_scalar": + 82541.075 ±(99.9%) 3106.426 us/op [Average] + (min, avg, max) = (81151.950, 82541.075, 86220.725), stdev = 1624.721 + CI (99.9%): [79434.649, 85647.501] (assumes normal distribution) # JMH version: 1.37 # VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 # VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector # Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) # Warmup: 5 iterations, 500 ms each # Measurement: 8 iterations, 500 ms each # Timeout: 10 min per iteration # Threads: 1 thread, will synchronize iterations # Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused -# Parameters: (predicates = 2, rows = 65536) +# Benchmark: com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_vectorApi +# Parameters: (rows = 4096) -# Run progress: 9.38% complete, ETA 00:07:17 +# Run progress: 33.33% complete, ETA 00:01:40 # Fork: 1 of 1 Picked up JAVA_TOOL_OPTIONS: WARNING: Using incubator modules: jdk.incubator.vector @@ -163,41 +163,41 @@ WARNING: A terminally deprecated method in sun.misc.Unsafe has been called WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 16.620 us/op -# Warmup Iteration 2: 15.641 us/op -# Warmup Iteration 3: 16.258 us/op -# Warmup Iteration 4: 14.963 us/op -# Warmup Iteration 5: 15.345 us/op -Iteration 1: 15.653 us/op -Iteration 2: 14.803 us/op -Iteration 3: 15.174 us/op -Iteration 4: 15.789 us/op -Iteration 5: 15.463 us/op -Iteration 6: 14.982 us/op -Iteration 7: 15.045 us/op -Iteration 8: 14.867 us/op +# Warmup Iteration 1: 91.617 us/op +# Warmup Iteration 2: 82.072 us/op +# Warmup Iteration 3: 76.979 us/op +# Warmup Iteration 4: 73.286 us/op +# Warmup Iteration 5: 74.019 us/op +Iteration 1: 74.258 us/op +Iteration 2: 78.100 us/op +Iteration 3: 75.061 us/op +Iteration 4: 80.911 us/op +Iteration 5: 78.180 us/op +Iteration 6: 75.949 us/op +Iteration 7: 74.012 us/op +Iteration 8: 72.389 us/op -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": - 15.222 ±(99.9%) 0.707 us/op [Average] - (min, avg, max) = (14.803, 15.222, 15.789), stdev = 0.370 - CI (99.9%): [14.515, 15.930] (assumes normal distribution) +Result "com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_vectorApi": + 76.108 ±(99.9%) 5.316 us/op [Average] + (min, avg, max) = (72.389, 76.108, 80.911), stdev = 2.780 + CI (99.9%): [70.792, 81.423] (assumes normal distribution) # JMH version: 1.37 # VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 # VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector # Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) # Warmup: 5 iterations, 500 ms each # Measurement: 8 iterations, 500 ms each # Timeout: 10 min per iteration # Threads: 1 thread, will synchronize iterations # Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused -# Parameters: (predicates = 4, rows = 256) +# Benchmark: com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_vectorApi +# Parameters: (rows = 65536) -# Run progress: 12.50% complete, ETA 00:07:02 +# Run progress: 44.44% complete, ETA 00:01:21 # Fork: 1 of 1 Picked up JAVA_TOOL_OPTIONS: WARNING: Using incubator modules: jdk.incubator.vector @@ -205,41 +205,41 @@ WARNING: A terminally deprecated method in sun.misc.Unsafe has been called WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 1.071 us/op -# Warmup Iteration 2: 0.831 us/op -# Warmup Iteration 3: 0.805 us/op -# Warmup Iteration 4: 0.812 us/op -# Warmup Iteration 5: 0.787 us/op -Iteration 1: 0.803 us/op -Iteration 2: 0.787 us/op -Iteration 3: 0.819 us/op -Iteration 4: 0.794 us/op -Iteration 5: 0.803 us/op -Iteration 6: 0.812 us/op -Iteration 7: 0.800 us/op -Iteration 8: 0.843 us/op +# Warmup Iteration 1: 1654.001 us/op +# Warmup Iteration 2: 1567.241 us/op +# Warmup Iteration 3: 1748.678 us/op +# Warmup Iteration 4: 1775.321 us/op +# Warmup Iteration 5: 1692.955 us/op +Iteration 1: 1680.597 us/op +Iteration 2: 1728.255 us/op +Iteration 3: 1734.372 us/op +Iteration 4: 1675.313 us/op +Iteration 5: 1732.371 us/op +Iteration 6: 1539.955 us/op +Iteration 7: 1680.241 us/op +Iteration 8: 1621.409 us/op -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": - 0.808 ±(99.9%) 0.033 us/op [Average] - (min, avg, max) = (0.787, 0.808, 0.843), stdev = 0.017 - CI (99.9%): [0.774, 0.841] (assumes normal distribution) +Result "com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_vectorApi": + 1674.064 ±(99.9%) 126.854 us/op [Average] + (min, avg, max) = (1539.955, 1674.064, 1734.372), stdev = 66.347 + CI (99.9%): [1547.210, 1800.918] (assumes normal distribution) # JMH version: 1.37 # VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 # VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector # Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) # Warmup: 5 iterations, 500 ms each # Measurement: 8 iterations, 500 ms each # Timeout: 10 min per iteration # Threads: 1 thread, will synchronize iterations # Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused -# Parameters: (predicates = 4, rows = 65536) +# Benchmark: com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_vectorApi +# Parameters: (rows = 1048576) -# Run progress: 15.63% complete, ETA 00:06:47 +# Run progress: 55.56% complete, ETA 00:01:04 # Fork: 1 of 1 Picked up JAVA_TOOL_OPTIONS: WARNING: Using incubator modules: jdk.incubator.vector @@ -247,41 +247,41 @@ WARNING: A terminally deprecated method in sun.misc.Unsafe has been called WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 27.861 us/op -# Warmup Iteration 2: 25.722 us/op -# Warmup Iteration 3: 25.611 us/op -# Warmup Iteration 4: 25.534 us/op -# Warmup Iteration 5: 25.717 us/op -Iteration 1: 26.072 us/op -Iteration 2: 25.173 us/op -Iteration 3: 25.405 us/op -Iteration 4: 25.841 us/op -Iteration 5: 25.419 us/op -Iteration 6: 25.530 us/op -Iteration 7: 25.412 us/op -Iteration 8: 25.874 us/op +# Warmup Iteration 1: 73735.107 us/op +# Warmup Iteration 2: 78204.780 us/op +# Warmup Iteration 3: 72903.544 us/op +# Warmup Iteration 4: 68372.532 us/op +# Warmup Iteration 5: 69420.310 us/op +Iteration 1: 70555.531 us/op +Iteration 2: 69468.914 us/op +Iteration 3: 70564.937 us/op +Iteration 4: 68904.767 us/op +Iteration 5: 69538.098 us/op +Iteration 6: 68111.168 us/op +Iteration 7: 69474.917 us/op +Iteration 8: 70286.471 us/op -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": - 25.591 ±(99.9%) 0.582 us/op [Average] - (min, avg, max) = (25.173, 25.591, 26.072), stdev = 0.304 - CI (99.9%): [25.009, 26.173] (assumes normal distribution) +Result "com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.java_vectorApi": + 69613.100 ±(99.9%) 1624.077 us/op [Average] + (min, avg, max) = (68111.168, 69613.100, 70564.937), stdev = 849.423 + CI (99.9%): [67989.024, 71237.177] (assumes normal distribution) # JMH version: 1.37 # VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 # VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector # Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) # Warmup: 5 iterations, 500 ms each # Measurement: 8 iterations, 500 ms each # Timeout: 10 min per iteration # Threads: 1 thread, will synchronize iterations # Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused -# Parameters: (predicates = 8, rows = 256) +# Benchmark: com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.native_facetMatch +# Parameters: (rows = 4096) -# Run progress: 18.75% complete, ETA 00:06:32 +# Run progress: 66.67% complete, ETA 00:00:49 # Fork: 1 of 1 Picked up JAVA_TOOL_OPTIONS: WARNING: Using incubator modules: jdk.incubator.vector @@ -289,41 +289,41 @@ WARNING: A terminally deprecated method in sun.misc.Unsafe has been called WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 1.841 us/op -# Warmup Iteration 2: 1.446 us/op -# Warmup Iteration 3: 1.521 us/op -# Warmup Iteration 4: 1.556 us/op -# Warmup Iteration 5: 1.502 us/op -Iteration 1: 1.440 us/op -Iteration 2: 1.494 us/op -Iteration 3: 1.463 us/op -Iteration 4: 1.525 us/op -Iteration 5: 1.451 us/op -Iteration 6: 1.533 us/op -Iteration 7: 1.446 us/op -Iteration 8: 1.505 us/op +# Warmup Iteration 1: 127.364 us/op +# Warmup Iteration 2: 128.849 us/op +# Warmup Iteration 3: 124.104 us/op +# Warmup Iteration 4: 122.376 us/op +# Warmup Iteration 5: 165.646 us/op +Iteration 1: 200.288 us/op +Iteration 2: 196.241 us/op +Iteration 3: 200.566 us/op +Iteration 4: 211.546 us/op +Iteration 5: 183.585 us/op +Iteration 6: 177.850 us/op +Iteration 7: 175.831 us/op +Iteration 8: 184.373 us/op -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": - 1.482 ±(99.9%) 0.070 us/op [Average] - (min, avg, max) = (1.440, 1.482, 1.533), stdev = 0.037 - CI (99.9%): [1.412, 1.553] (assumes normal distribution) +Result "com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.native_facetMatch": + 191.285 ±(99.9%) 24.278 us/op [Average] + (min, avg, max) = (175.831, 191.285, 211.546), stdev = 12.698 + CI (99.9%): [167.007, 215.563] (assumes normal distribution) # JMH version: 1.37 # VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 # VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector # Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) # Warmup: 5 iterations, 500 ms each # Measurement: 8 iterations, 500 ms each # Timeout: 10 min per iteration # Threads: 1 thread, will synchronize iterations # Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused -# Parameters: (predicates = 8, rows = 65536) +# Benchmark: com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.native_facetMatch +# Parameters: (rows = 65536) -# Run progress: 21.88% complete, ETA 00:06:17 +# Run progress: 77.78% complete, ETA 00:00:32 # Fork: 1 of 1 Picked up JAVA_TOOL_OPTIONS: WARNING: Using incubator modules: jdk.incubator.vector @@ -331,41 +331,41 @@ WARNING: A terminally deprecated method in sun.misc.Unsafe has been called WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 62.079 us/op -# Warmup Iteration 2: 58.850 us/op -# Warmup Iteration 3: 57.828 us/op -# Warmup Iteration 4: 59.013 us/op -# Warmup Iteration 5: 57.218 us/op -Iteration 1: 57.682 us/op -Iteration 2: 59.927 us/op -Iteration 3: 56.783 us/op -Iteration 4: 57.691 us/op -Iteration 5: 59.234 us/op -Iteration 6: 57.057 us/op -Iteration 7: 64.073 us/op -Iteration 8: 59.373 us/op +# Warmup Iteration 1: 2401.948 us/op +# Warmup Iteration 2: 2273.074 us/op +# Warmup Iteration 3: 2252.513 us/op +# Warmup Iteration 4: 2306.351 us/op +# Warmup Iteration 5: 2757.252 us/op +Iteration 1: 3050.533 us/op +Iteration 2: 3274.968 us/op +Iteration 3: 3231.050 us/op +Iteration 4: 3245.046 us/op +Iteration 5: 3260.320 us/op +Iteration 6: 3186.983 us/op +Iteration 7: 3232.969 us/op +Iteration 8: 3274.559 us/op -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fused": - 58.978 ±(99.9%) 4.509 us/op [Average] - (min, avg, max) = (56.783, 58.978, 64.073), stdev = 2.358 - CI (99.9%): [54.469, 63.486] (assumes normal distribution) +Result "com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.native_facetMatch": + 3219.554 ±(99.9%) 141.518 us/op [Average] + (min, avg, max) = (3050.533, 3219.554, 3274.968), stdev = 74.017 + CI (99.9%): [3078.036, 3361.072] (assumes normal distribution) # JMH version: 1.37 # VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 # VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED +# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector # Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) # Warmup: 5 iterations, 500 ms each # Measurement: 8 iterations, 500 ms each # Timeout: 10 min per iteration # Threads: 1 thread, will synchronize iterations # Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel -# Parameters: (predicates = 1, rows = 256) +# Benchmark: com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.native_facetMatch +# Parameters: (rows = 1048576) -# Run progress: 25.00% complete, ETA 00:06:02 +# Run progress: 88.89% complete, ETA 00:00:16 # Fork: 1 of 1 Picked up JAVA_TOOL_OPTIONS: WARNING: Using incubator modules: jdk.incubator.vector @@ -373,994 +373,28 @@ WARNING: A terminally deprecated method in sun.misc.Unsafe has been called WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 0.644 us/op -# Warmup Iteration 2: 0.501 us/op -# Warmup Iteration 3: 0.515 us/op -# Warmup Iteration 4: 0.505 us/op -# Warmup Iteration 5: 0.501 us/op -Iteration 1: 0.497 us/op -Iteration 2: 0.508 us/op -Iteration 3: 0.520 us/op -Iteration 4: 0.496 us/op -Iteration 5: 0.501 us/op -Iteration 6: 0.499 us/op -Iteration 7: 0.497 us/op -Iteration 8: 0.506 us/op +# Warmup Iteration 1: 77219.661 us/op +# Warmup Iteration 2: 86306.216 us/op +# Warmup Iteration 3: 84154.509 us/op +# Warmup Iteration 4: 75079.731 us/op +# Warmup Iteration 5: 75266.969 us/op +Iteration 1: 76492.982 us/op +Iteration 2: 75388.886 us/op +Iteration 3: 77673.340 us/op +Iteration 4: 75946.373 us/op +Iteration 5: 77142.382 us/op +Iteration 6: 79088.128 us/op +Iteration 7: 80069.147 us/op +Iteration 8: 90333.743 us/op -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": - 0.503 ±(99.9%) 0.015 us/op [Average] - (min, avg, max) = (0.496, 0.503, 0.520), stdev = 0.008 - CI (99.9%): [0.488, 0.518] (assumes normal distribution) +Result "com.adaworldapi.lancegraph.bench.F_RowStoreFacetScan.native_facetMatch": + 79016.873 ±(99.9%) 9239.383 us/op [Average] + (min, avg, max) = (75388.886, 79016.873, 90333.743), stdev = 4832.376 + CI (99.9%): [69777.490, 88256.255] (assumes normal distribution) -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel -# Parameters: (predicates = 1, rows = 65536) - -# Run progress: 28.13% complete, ETA 00:05:47 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 77.770 us/op -# Warmup Iteration 2: 75.651 us/op -# Warmup Iteration 3: 76.166 us/op -# Warmup Iteration 4: 76.732 us/op -# Warmup Iteration 5: 76.651 us/op -Iteration 1: 73.627 us/op -Iteration 2: 73.285 us/op -Iteration 3: 75.483 us/op -Iteration 4: 75.007 us/op -Iteration 5: 75.203 us/op -Iteration 6: 74.805 us/op -Iteration 7: 73.605 us/op -Iteration 8: 73.708 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": - 74.340 ±(99.9%) 1.659 us/op [Average] - (min, avg, max) = (73.285, 74.340, 75.483), stdev = 0.868 - CI (99.9%): [72.681, 75.999] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel -# Parameters: (predicates = 2, rows = 256) - -# Run progress: 31.25% complete, ETA 00:05:32 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 0.976 us/op -# Warmup Iteration 2: 0.848 us/op -# Warmup Iteration 3: 0.833 us/op -# Warmup Iteration 4: 0.826 us/op -# Warmup Iteration 5: 0.832 us/op -Iteration 1: 0.833 us/op -Iteration 2: 0.822 us/op -Iteration 3: 0.823 us/op -Iteration 4: 0.851 us/op -Iteration 5: 0.817 us/op -Iteration 6: 0.820 us/op -Iteration 7: 0.877 us/op -Iteration 8: 0.838 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": - 0.835 ±(99.9%) 0.039 us/op [Average] - (min, avg, max) = (0.817, 0.835, 0.877), stdev = 0.020 - CI (99.9%): [0.796, 0.874] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel -# Parameters: (predicates = 2, rows = 65536) - -# Run progress: 34.38% complete, ETA 00:05:16 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 422.391 us/op -# Warmup Iteration 2: 409.456 us/op -# Warmup Iteration 3: 410.748 us/op -# Warmup Iteration 4: 406.293 us/op -# Warmup Iteration 5: 402.930 us/op -Iteration 1: 399.740 us/op -Iteration 2: 399.729 us/op -Iteration 3: 402.465 us/op -Iteration 4: 406.314 us/op -Iteration 5: 402.886 us/op -Iteration 6: 412.331 us/op -Iteration 7: 410.116 us/op -Iteration 8: 431.775 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": - 408.170 ±(99.9%) 20.242 us/op [Average] - (min, avg, max) = (399.729, 408.170, 431.775), stdev = 10.587 - CI (99.9%): [387.928, 428.411] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel -# Parameters: (predicates = 4, rows = 256) - -# Run progress: 37.50% complete, ETA 00:05:01 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 1.974 us/op -# Warmup Iteration 2: 1.637 us/op -# Warmup Iteration 3: 1.678 us/op -# Warmup Iteration 4: 1.667 us/op -# Warmup Iteration 5: 1.677 us/op -Iteration 1: 1.681 us/op -Iteration 2: 1.708 us/op -Iteration 3: 1.644 us/op -Iteration 4: 1.641 us/op -Iteration 5: 1.720 us/op -Iteration 6: 1.636 us/op -Iteration 7: 1.682 us/op -Iteration 8: 1.585 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": - 1.662 ±(99.9%) 0.084 us/op [Average] - (min, avg, max) = (1.585, 1.662, 1.720), stdev = 0.044 - CI (99.9%): [1.578, 1.747] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel -# Parameters: (predicates = 4, rows = 65536) - -# Run progress: 40.63% complete, ETA 00:04:46 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 949.977 us/op -# Warmup Iteration 2: 916.566 us/op -# Warmup Iteration 3: 912.454 us/op -# Warmup Iteration 4: 949.298 us/op -# Warmup Iteration 5: 937.251 us/op -Iteration 1: 930.736 us/op -Iteration 2: 910.570 us/op -Iteration 3: 914.896 us/op -Iteration 4: 897.725 us/op -Iteration 5: 901.273 us/op -Iteration 6: 939.607 us/op -Iteration 7: 927.639 us/op -Iteration 8: 916.648 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": - 917.387 ±(99.9%) 27.768 us/op [Average] - (min, avg, max) = (897.725, 917.387, 939.607), stdev = 14.523 - CI (99.9%): [889.618, 945.155] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel -# Parameters: (predicates = 8, rows = 256) - -# Run progress: 43.75% complete, ETA 00:04:31 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 3.966 us/op -# Warmup Iteration 2: 3.496 us/op -# Warmup Iteration 3: 3.586 us/op -# Warmup Iteration 4: 3.423 us/op -# Warmup Iteration 5: 3.463 us/op -Iteration 1: 3.623 us/op -Iteration 2: 3.473 us/op -Iteration 3: 3.506 us/op -Iteration 4: 3.446 us/op -Iteration 5: 3.479 us/op -Iteration 6: 3.493 us/op -Iteration 7: 3.451 us/op -Iteration 8: 3.441 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": - 3.489 ±(99.9%) 0.113 us/op [Average] - (min, avg, max) = (3.441, 3.489, 3.623), stdev = 0.059 - CI (99.9%): [3.376, 3.602] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel -# Parameters: (predicates = 8, rows = 65536) - -# Run progress: 46.88% complete, ETA 00:04:16 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 1914.384 us/op -# Warmup Iteration 2: 1834.769 us/op -# Warmup Iteration 3: 1845.272 us/op -# Warmup Iteration 4: 1846.484 us/op -# Warmup Iteration 5: 1793.076 us/op -Iteration 1: 1793.263 us/op -Iteration 2: 1798.992 us/op -Iteration 3: 1781.046 us/op -Iteration 4: 1790.907 us/op -Iteration 5: 1973.222 us/op -Iteration 6: 1790.764 us/op -Iteration 7: 1809.896 us/op -Iteration 8: 1869.240 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.fusedScalarKernel": - 1825.916 ±(99.9%) 125.373 us/op [Average] - (min, avg, max) = (1781.046, 1825.916, 1973.222), stdev = 65.573 - CI (99.9%): [1700.543, 1951.289] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly -# Parameters: (predicates = 1, rows = 256) - -# Run progress: 50.00% complete, ETA 00:04:01 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 0.071 us/op -# Warmup Iteration 2: 0.057 us/op -# Warmup Iteration 3: 0.055 us/op -# Warmup Iteration 4: 0.055 us/op -# Warmup Iteration 5: 0.054 us/op -Iteration 1: 0.059 us/op -Iteration 2: 0.057 us/op -Iteration 3: 0.059 us/op -Iteration 4: 0.056 us/op -Iteration 5: 0.054 us/op -Iteration 6: 0.054 us/op -Iteration 7: 0.055 us/op -Iteration 8: 0.055 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": - 0.056 ±(99.9%) 0.004 us/op [Average] - (min, avg, max) = (0.054, 0.056, 0.059), stdev = 0.002 - CI (99.9%): [0.052, 0.060] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly -# Parameters: (predicates = 1, rows = 65536) - -# Run progress: 53.13% complete, ETA 00:03:46 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 0.069 us/op -# Warmup Iteration 2: 0.059 us/op -# Warmup Iteration 3: 0.051 us/op -# Warmup Iteration 4: 0.054 us/op -# Warmup Iteration 5: 0.051 us/op -Iteration 1: 0.058 us/op -Iteration 2: 0.053 us/op -Iteration 3: 0.053 us/op -Iteration 4: 0.051 us/op -Iteration 5: 0.051 us/op -Iteration 6: 0.051 us/op -Iteration 7: 0.051 us/op -Iteration 8: 0.056 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": - 0.053 ±(99.9%) 0.005 us/op [Average] - (min, avg, max) = (0.051, 0.053, 0.058), stdev = 0.003 - CI (99.9%): [0.048, 0.058] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly -# Parameters: (predicates = 2, rows = 256) - -# Run progress: 56.25% complete, ETA 00:03:31 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 0.138 us/op -# Warmup Iteration 2: 0.115 us/op -# Warmup Iteration 3: 0.125 us/op -# Warmup Iteration 4: 0.117 us/op -# Warmup Iteration 5: 0.111 us/op -Iteration 1: 0.111 us/op -Iteration 2: 0.111 us/op -Iteration 3: 0.113 us/op -Iteration 4: 0.113 us/op -Iteration 5: 0.111 us/op -Iteration 6: 0.108 us/op -Iteration 7: 0.111 us/op -Iteration 8: 0.114 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": - 0.111 ±(99.9%) 0.003 us/op [Average] - (min, avg, max) = (0.108, 0.111, 0.114), stdev = 0.002 - CI (99.9%): [0.108, 0.115] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly -# Parameters: (predicates = 2, rows = 65536) - -# Run progress: 59.38% complete, ETA 00:03:16 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 0.135 us/op -# Warmup Iteration 2: 0.110 us/op -# Warmup Iteration 3: 0.107 us/op -# Warmup Iteration 4: 0.114 us/op -# Warmup Iteration 5: 0.107 us/op -Iteration 1: 0.114 us/op -Iteration 2: 0.111 us/op -Iteration 3: 0.109 us/op -Iteration 4: 0.110 us/op -Iteration 5: 0.113 us/op -Iteration 6: 0.116 us/op -Iteration 7: 0.118 us/op -Iteration 8: 0.113 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": - 0.113 ±(99.9%) 0.006 us/op [Average] - (min, avg, max) = (0.109, 0.113, 0.118), stdev = 0.003 - CI (99.9%): [0.107, 0.119] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly -# Parameters: (predicates = 4, rows = 256) - -# Run progress: 62.50% complete, ETA 00:03:01 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 0.318 us/op -# Warmup Iteration 2: 0.306 us/op -# Warmup Iteration 3: 0.291 us/op -# Warmup Iteration 4: 0.289 us/op -# Warmup Iteration 5: 0.272 us/op -Iteration 1: 0.257 us/op -Iteration 2: 0.261 us/op -Iteration 3: 0.347 us/op -Iteration 4: 0.270 us/op -Iteration 5: 0.259 us/op -Iteration 6: 0.288 us/op -Iteration 7: 0.289 us/op -Iteration 8: 0.283 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": - 0.282 ±(99.9%) 0.056 us/op [Average] - (min, avg, max) = (0.257, 0.282, 0.347), stdev = 0.029 - CI (99.9%): [0.226, 0.338] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly -# Parameters: (predicates = 4, rows = 65536) - -# Run progress: 65.63% complete, ETA 00:02:46 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 0.316 us/op -# Warmup Iteration 2: 0.262 us/op -# Warmup Iteration 3: 0.261 us/op -# Warmup Iteration 4: 0.265 us/op -# Warmup Iteration 5: 0.309 us/op -Iteration 1: 0.258 us/op -Iteration 2: 0.261 us/op -Iteration 3: 0.275 us/op -Iteration 4: 0.256 us/op -Iteration 5: 0.267 us/op -Iteration 6: 0.257 us/op -Iteration 7: 0.262 us/op -Iteration 8: 0.256 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": - 0.261 ±(99.9%) 0.012 us/op [Average] - (min, avg, max) = (0.256, 0.261, 0.275), stdev = 0.007 - CI (99.9%): [0.249, 0.274] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly -# Parameters: (predicates = 8, rows = 256) - -# Run progress: 68.75% complete, ETA 00:02:31 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 0.776 us/op -# Warmup Iteration 2: 0.649 us/op -# Warmup Iteration 3: 0.653 us/op -# Warmup Iteration 4: 0.628 us/op -# Warmup Iteration 5: 0.598 us/op -Iteration 1: 0.658 us/op -Iteration 2: 0.661 us/op -Iteration 3: 0.640 us/op -Iteration 4: 0.589 us/op -Iteration 5: 0.594 us/op -Iteration 6: 0.902 us/op -Iteration 7: 0.597 us/op -Iteration 8: 0.616 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": - 0.657 ±(99.9%) 0.196 us/op [Average] - (min, avg, max) = (0.589, 0.657, 0.902), stdev = 0.103 - CI (99.9%): [0.461, 0.854] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly -# Parameters: (predicates = 8, rows = 65536) - -# Run progress: 71.88% complete, ETA 00:02:16 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 0.755 us/op -# Warmup Iteration 2: 0.660 us/op -# Warmup Iteration 3: 0.592 us/op -# Warmup Iteration 4: 0.704 us/op -# Warmup Iteration 5: 0.633 us/op -Iteration 1: 0.616 us/op -Iteration 2: 0.617 us/op -Iteration 3: 0.587 us/op -Iteration 4: 0.576 us/op -Iteration 5: 0.599 us/op -Iteration 6: 0.583 us/op -Iteration 7: 0.607 us/op -Iteration 8: 0.619 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.planConstructionOnly": - 0.601 ±(99.9%) 0.032 us/op [Average] - (min, avg, max) = (0.576, 0.601, 0.619), stdev = 0.017 - CI (99.9%): [0.569, 0.632] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused -# Parameters: (predicates = 1, rows = 256) - -# Run progress: 75.00% complete, ETA 00:02:00 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 0.501 us/op -# Warmup Iteration 2: 0.387 us/op -# Warmup Iteration 3: 0.381 us/op -# Warmup Iteration 4: 0.391 us/op -# Warmup Iteration 5: 0.378 us/op -Iteration 1: 0.393 us/op -Iteration 2: 0.390 us/op -Iteration 3: 0.387 us/op -Iteration 4: 0.382 us/op -Iteration 5: 0.380 us/op -Iteration 6: 0.381 us/op -Iteration 7: 0.385 us/op -Iteration 8: 0.380 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": - 0.385 ±(99.9%) 0.009 us/op [Average] - (min, avg, max) = (0.380, 0.385, 0.393), stdev = 0.005 - CI (99.9%): [0.375, 0.394] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused -# Parameters: (predicates = 1, rows = 65536) - -# Run progress: 78.13% complete, ETA 00:01:45 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 6.595 us/op -# Warmup Iteration 2: 6.526 us/op -# Warmup Iteration 3: 6.280 us/op -# Warmup Iteration 4: 6.308 us/op -# Warmup Iteration 5: 7.104 us/op -Iteration 1: 6.245 us/op -Iteration 2: 6.238 us/op -Iteration 3: 6.513 us/op -Iteration 4: 6.304 us/op -Iteration 5: 6.308 us/op -Iteration 6: 6.337 us/op -Iteration 7: 6.270 us/op -Iteration 8: 6.397 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": - 6.326 ±(99.9%) 0.175 us/op [Average] - (min, avg, max) = (6.238, 6.326, 6.513), stdev = 0.091 - CI (99.9%): [6.152, 6.501] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused -# Parameters: (predicates = 2, rows = 256) - -# Run progress: 81.25% complete, ETA 00:01:30 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 1.103 us/op -# Warmup Iteration 2: 0.946 us/op -# Warmup Iteration 3: 0.943 us/op -# Warmup Iteration 4: 0.932 us/op -# Warmup Iteration 5: 0.931 us/op -Iteration 1: 0.923 us/op -Iteration 2: 0.951 us/op -Iteration 3: 0.930 us/op -Iteration 4: 0.927 us/op -Iteration 5: 0.925 us/op -Iteration 6: 0.937 us/op -Iteration 7: 0.922 us/op -Iteration 8: 0.930 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": - 0.931 ±(99.9%) 0.018 us/op [Average] - (min, avg, max) = (0.922, 0.931, 0.951), stdev = 0.009 - CI (99.9%): [0.913, 0.948] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused -# Parameters: (predicates = 2, rows = 65536) - -# Run progress: 84.38% complete, ETA 00:01:15 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 14.757 us/op -# Warmup Iteration 2: 13.827 us/op -# Warmup Iteration 3: 13.941 us/op -# Warmup Iteration 4: 13.639 us/op -# Warmup Iteration 5: 13.879 us/op -Iteration 1: 13.673 us/op -Iteration 2: 14.664 us/op -Iteration 3: 14.181 us/op -Iteration 4: 13.711 us/op -Iteration 5: 14.745 us/op -Iteration 6: 13.901 us/op -Iteration 7: 14.212 us/op -Iteration 8: 14.543 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": - 14.204 ±(99.9%) 0.804 us/op [Average] - (min, avg, max) = (13.673, 14.204, 14.745), stdev = 0.420 - CI (99.9%): [13.400, 15.008] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused -# Parameters: (predicates = 4, rows = 256) - -# Run progress: 87.50% complete, ETA 00:01:00 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 2.248 us/op -# Warmup Iteration 2: 2.127 us/op -# Warmup Iteration 3: 2.068 us/op -# Warmup Iteration 4: 2.095 us/op -# Warmup Iteration 5: 2.067 us/op -Iteration 1: 2.088 us/op -Iteration 2: 2.054 us/op -Iteration 3: 2.086 us/op -Iteration 4: 2.080 us/op -Iteration 5: 2.091 us/op -Iteration 6: 2.124 us/op -Iteration 7: 2.166 us/op -Iteration 8: 2.088 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": - 2.097 ±(99.9%) 0.064 us/op [Average] - (min, avg, max) = (2.054, 2.097, 2.166), stdev = 0.034 - CI (99.9%): [2.033, 2.162] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused -# Parameters: (predicates = 4, rows = 65536) - -# Run progress: 90.63% complete, ETA 00:00:45 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 34.677 us/op -# Warmup Iteration 2: 30.707 us/op -# Warmup Iteration 3: 30.336 us/op -# Warmup Iteration 4: 30.246 us/op -# Warmup Iteration 5: 33.637 us/op -Iteration 1: 31.002 us/op -Iteration 2: 35.002 us/op -Iteration 3: 29.954 us/op -Iteration 4: 30.045 us/op -Iteration 5: 33.427 us/op -Iteration 6: 32.171 us/op -Iteration 7: 31.979 us/op -Iteration 8: 30.739 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": - 31.790 ±(99.9%) 3.337 us/op [Average] - (min, avg, max) = (29.954, 31.790, 35.002), stdev = 1.745 - CI (99.9%): [28.453, 35.127] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused -# Parameters: (predicates = 8, rows = 256) - -# Run progress: 93.75% complete, ETA 00:00:30 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 4.932 us/op -# Warmup Iteration 2: 4.552 us/op -# Warmup Iteration 3: 4.348 us/op -# Warmup Iteration 4: 4.379 us/op -# Warmup Iteration 5: 4.290 us/op -Iteration 1: 4.327 us/op -Iteration 2: 4.626 us/op -Iteration 3: 4.465 us/op -Iteration 4: 4.337 us/op -Iteration 5: 4.370 us/op -Iteration 6: 4.488 us/op -Iteration 7: 4.526 us/op -Iteration 8: 4.361 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": - 4.437 ±(99.9%) 0.204 us/op [Average] - (min, avg, max) = (4.327, 4.437, 4.626), stdev = 0.107 - CI (99.9%): [4.233, 4.642] (assumes normal distribution) - - -# JMH version: 1.37 -# VM version: JDK 26.0.2, OpenJDK 64-Bit Server VM, 26.0.2+10-55 -# VM invoker: /opt/jdks/jdk-26.0.2/bin/java -# VM options: --enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector -Dstdout.encoding=UTF-8 -Dlgj.library=/home/user/lance-graph-java/target/release/liblgj_abi.so --enable-native-access=ALL-UNNAMED -# Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable) -# Warmup: 5 iterations, 500 ms each -# Measurement: 8 iterations, 500 ms each -# Timeout: 10 min per iteration -# Threads: 1 thread, will synchronize iterations -# Benchmark mode: Average time, time/op -# Benchmark: com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused -# Parameters: (predicates = 8, rows = 65536) - -# Run progress: 96.88% complete, ETA 00:00:15 -# Fork: 1 of 1 -Picked up JAVA_TOOL_OPTIONS: -WARNING: Using incubator modules: jdk.incubator.vector -WARNING: A terminally deprecated method in sun.misc.Unsafe has been called -WARNING: sun.misc.Unsafe::objectFieldOffset has been called by org.openjdk.jmh.util.Utils (file:/home/user/lance-graph-java/bench/lib/jmh-core-1.37.jar) -WARNING: Please consider reporting this to the maintainers of class org.openjdk.jmh.util.Utils -WARNING: sun.misc.Unsafe::objectFieldOffset will be removed in a future release -# Warmup Iteration 1: 67.063 us/op -# Warmup Iteration 2: 61.358 us/op -# Warmup Iteration 3: 60.608 us/op -# Warmup Iteration 4: 63.889 us/op -# Warmup Iteration 5: 60.896 us/op -Iteration 1: 63.044 us/op -Iteration 2: 61.284 us/op -Iteration 3: 61.587 us/op -Iteration 4: 60.711 us/op -Iteration 5: 58.430 us/op -Iteration 6: 59.685 us/op -Iteration 7: 61.525 us/op -Iteration 8: 61.479 us/op - - -Result "com.adaworldapi.lancegraph.bench.E_FusionAndPlanning.unfused": - 60.968 ±(99.9%) 2.657 us/op [Average] - (min, avg, max) = (58.430, 60.968, 63.044), stdev = 1.390 - CI (99.9%): [58.311, 63.625] (assumes normal distribution) - - -# Run complete. Total time: 00:08:03 +# Run complete. Total time: 00:02:28 REMEMBER: The numbers below are just data. To gain reusable insights, you need to follow up on why the numbers are the way they are. Use profilers (see -prof, -lprof), design factorial @@ -1374,38 +408,15 @@ works, and factor in a small probability of new VM bugs. Additionally, while com different JVMs are already problematic, the performance difference caused by different Blackhole modes can be very significant. Please make sure you use the consistent Blackhole mode for comparisons. -Benchmark (predicates) (rows) Mode Cnt Score Error Units -E_FusionAndPlanning.fused 1 256 avgt 8 0.404 ± 0.010 us/op -E_FusionAndPlanning.fused 1 65536 avgt 8 6.913 ± 0.165 us/op -E_FusionAndPlanning.fused 2 256 avgt 8 0.520 ± 0.032 us/op -E_FusionAndPlanning.fused 2 65536 avgt 8 15.222 ± 0.707 us/op -E_FusionAndPlanning.fused 4 256 avgt 8 0.808 ± 0.033 us/op -E_FusionAndPlanning.fused 4 65536 avgt 8 25.591 ± 0.582 us/op -E_FusionAndPlanning.fused 8 256 avgt 8 1.482 ± 0.070 us/op -E_FusionAndPlanning.fused 8 65536 avgt 8 58.978 ± 4.509 us/op -E_FusionAndPlanning.fusedScalarKernel 1 256 avgt 8 0.503 ± 0.015 us/op -E_FusionAndPlanning.fusedScalarKernel 1 65536 avgt 8 74.340 ± 1.659 us/op -E_FusionAndPlanning.fusedScalarKernel 2 256 avgt 8 0.835 ± 0.039 us/op -E_FusionAndPlanning.fusedScalarKernel 2 65536 avgt 8 408.170 ± 20.242 us/op -E_FusionAndPlanning.fusedScalarKernel 4 256 avgt 8 1.662 ± 0.084 us/op -E_FusionAndPlanning.fusedScalarKernel 4 65536 avgt 8 917.387 ± 27.768 us/op -E_FusionAndPlanning.fusedScalarKernel 8 256 avgt 8 3.489 ± 0.113 us/op -E_FusionAndPlanning.fusedScalarKernel 8 65536 avgt 8 1825.916 ± 125.373 us/op -E_FusionAndPlanning.planConstructionOnly 1 256 avgt 8 0.056 ± 0.004 us/op -E_FusionAndPlanning.planConstructionOnly 1 65536 avgt 8 0.053 ± 0.005 us/op -E_FusionAndPlanning.planConstructionOnly 2 256 avgt 8 0.111 ± 0.003 us/op -E_FusionAndPlanning.planConstructionOnly 2 65536 avgt 8 0.113 ± 0.006 us/op -E_FusionAndPlanning.planConstructionOnly 4 256 avgt 8 0.282 ± 0.056 us/op -E_FusionAndPlanning.planConstructionOnly 4 65536 avgt 8 0.261 ± 0.012 us/op -E_FusionAndPlanning.planConstructionOnly 8 256 avgt 8 0.657 ± 0.196 us/op -E_FusionAndPlanning.planConstructionOnly 8 65536 avgt 8 0.601 ± 0.032 us/op -E_FusionAndPlanning.unfused 1 256 avgt 8 0.385 ± 0.009 us/op -E_FusionAndPlanning.unfused 1 65536 avgt 8 6.326 ± 0.175 us/op -E_FusionAndPlanning.unfused 2 256 avgt 8 0.931 ± 0.018 us/op -E_FusionAndPlanning.unfused 2 65536 avgt 8 14.204 ± 0.804 us/op -E_FusionAndPlanning.unfused 4 256 avgt 8 2.097 ± 0.064 us/op -E_FusionAndPlanning.unfused 4 65536 avgt 8 31.790 ± 3.337 us/op -E_FusionAndPlanning.unfused 8 256 avgt 8 4.437 ± 0.204 us/op -E_FusionAndPlanning.unfused 8 65536 avgt 8 60.968 ± 2.657 us/op +Benchmark (rows) Mode Cnt Score Error Units +F_RowStoreFacetScan.java_scalar 4096 avgt 8 134.712 ± 13.871 us/op +F_RowStoreFacetScan.java_scalar 65536 avgt 8 3354.401 ± 118.415 us/op +F_RowStoreFacetScan.java_scalar 1048576 avgt 8 82541.075 ± 3106.426 us/op +F_RowStoreFacetScan.java_vectorApi 4096 avgt 8 76.108 ± 5.316 us/op +F_RowStoreFacetScan.java_vectorApi 65536 avgt 8 1674.064 ± 126.854 us/op +F_RowStoreFacetScan.java_vectorApi 1048576 avgt 8 69613.100 ± 1624.077 us/op +F_RowStoreFacetScan.native_facetMatch 4096 avgt 8 191.285 ± 24.278 us/op +F_RowStoreFacetScan.native_facetMatch 65536 avgt 8 3219.554 ± 141.518 us/op +F_RowStoreFacetScan.native_facetMatch 1048576 avgt 8 79016.873 ± 9239.383 us/op Benchmark result is saved to results/jmh-results.csv diff --git a/bench/src/main/java/com/adaworldapi/lancegraph/NativeAccess.java b/bench/src/main/java/com/adaworldapi/lancegraph/NativeAccess.java index 3e9fa04..b429b81 100644 --- a/bench/src/main/java/com/adaworldapi/lancegraph/NativeAccess.java +++ b/bench/src/main/java/com/adaworldapi/lancegraph/NativeAccess.java @@ -28,11 +28,25 @@ private NativeAccess() {} /** Lane 2 — {@code i32} signed values. */ public static final int LANE_VALUE = 2; + /** + * {@link RowStore} lane 0 — the raw {@code n_rows * 512}-byte buffer, {@code U8}/stride 1, + * {@code READABLE | CONTIGUOUS} (abi.md §11). Distinct constant from {@link #LANE_ID} even + * though both are numerically 0: they describe different resource kinds ({@link NativePattern} + * vs {@link RowStore}), and giving the row-store raw lane its own name avoids a reader + * mistaking one resource's lane 0 for the other's. + */ + public static final int LANE_ROWSTORE_RAW = 0; + /** The generation-checked registry handle behind a pattern. Opaque; for describe calls only. */ public static long handleOf(NativePattern pattern) { return pattern.handle(); } + /** The generation-checked registry handle behind a row store. Opaque; for describe calls only. */ + public static long handleOf(RowStore store) { + return store.handle(); + } + /** * A bounded, read-only window onto one native lane. No membrane crossing happens when this is * read — that is the point of the design, and the reason a Java-side Vector API kernel can diff --git a/bench/src/main/java/com/adaworldapi/lancegraph/bench/F_RowStoreFacetScan.java b/bench/src/main/java/com/adaworldapi/lancegraph/bench/F_RowStoreFacetScan.java new file mode 100644 index 0000000..d3eb989 --- /dev/null +++ b/bench/src/main/java/com/adaworldapi/lancegraph/bench/F_RowStoreFacetScan.java @@ -0,0 +1,131 @@ +package com.adaworldapi.lancegraph.bench; + +import org.openjdk.jmh.annotations.*; + +import java.util.concurrent.TimeUnit; + +/** + * Component F — does the execution-boundary finding from Component C survive a bulkier, + * strided predicate? + * + *

{@code C_ExecutionBoundary} answered "where does execution belong?" for a single flat + * predicate over two contiguous {@code int} lanes: one comparison, one AND, one crossing versus a + * Vector-API kernel reading the same bytes in place. This class asks the same question on the + * real row-store layout instead of that fixture — 512-byte rows, 32 sixteen-byte facets — + * where the "predicate" is a bulk per-row facet-match: for every row, which of its 32 strided + * classid lanes equal a needle, folded into one {@code int} bitset per row. That is more work + * per crossing than Component C's flat single-column scan (32 strided compares per row instead + * of one), and the access pattern is fundamentally different (a facet's classid sits every 16 bytes + * inside a 512-byte row, never contiguous with the next facet's classid at any useful vector width + * beyond the 4 that share a 64-byte chunk). + * + *

This class does not assert an answer — it measures one. Same discipline as + * the rest of {@code bench/}: every number is produced by {@code run.sh}, every table by {@code + * summarise.sh}, so nothing here can drift from its data (see {@code bench/README.md}'s own + * header). Whether "Vector API beats the crossing for a single predicate" (Component C's finding) + * still holds when the crossing carries 32× the per-row work, and when the layout is strided rather + * than flat, is exactly what this sweep exists to find out. + * + *

The three arms

+ * + *

Same shape as {@code C_ExecutionBoundary}: {@code native} crosses once into the fused + * {@code lgj_row_facet_match} kernel ({@code ndarray::simd::MultiLaneColumn} + {@code + * U32x16::eq_bitmask}, abi.md §11); {@code vector} reads the very same lane-0 bytes with {@code + * jdk.incubator.vector}, zero copies ({@link Kernels#facetMatchVector}); {@code scalar} is the + * auto-vectorisation control ({@link Kernels#facetMatchScalar}). All three are cross-checked to + * agree row-for-row in {@link RowStoreData}'s {@code @Setup}, before anything is timed. + * + *

A methodological asymmetry, disclosed rather than hidden

+ * + *

{@code native_facetMatch()} is not directly comparable to {@code + * C_ExecutionBoundary#native_fusedPlan()} in one respect: {@code NativePattern.countOf} evaluates + * into a lazily-created, cached scratch mask (see {@code NativePattern#scratch()}) — after + * the first call, {@code native_fusedPlan()} allocates nothing on the timed path. {@code + * RowStore.facetMatches()} has no such reusable-output variant in the public API today: every call + * does a fresh {@code arena.allocate(JAVA_INT, rowCount)} plus constructs a new {@link + * com.adaworldapi.lancegraph.FacetMatchView}, on every single invocation. The {@code vector} and + * {@code scalar} arms below, by contrast, write into an {@code out} array allocated once + * in {@code @Setup} — matching {@link Data}'s own precedent of pre-allocating {@code valuesHeap} + * outside the timed body. So this sweep's {@code native} arm pays a real per-call allocation its + * two Java competitors do not, which the equivalent Component C arm does not either. This is stated + * up front rather than left for a reader of the raw numbers to discover: a {@code native} result + * here reflects the public API exactly as it exists (there is no lower-level reusable-output escape + * hatch for this operation to fall back to, the way {@code View.select()} exists for masks), but it + * is not a clean like-for-like allocation comparison the way Component C's three arms are. A future + * {@code facetMatchesInto(classId, MemorySegment)} overload would be the fix, and is a candidate + * follow-up rather than something this class invents on its own. + * + *

The row sweep is narrower than Component C's nine points — three, spanning small/medium/large + * — because this class asks a sharper, single question (does the per-row-work multiplier change the + * crossover) rather than needing to bracket a crossover point across four orders of magnitude from + * scratch. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@State(Scope.Benchmark) +@Fork(value = 1, jvmArgsAppend = { + "--enable-native-access=ALL-UNNAMED", "--add-modules", "jdk.incubator.vector"}) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 8, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class F_RowStoreFacetScan { + + @Param({"4096", "65536", "1048576"}) + public int rows; + + private RowStoreData data; + + /** + * Allocated once in {@code @Setup}, reused by every {@code java_*} invocation — the same + * "pre-allocate outside the timed body" rule {@link Data#valuesHeap} already models. See the + * class doc's asymmetry note: {@code native_facetMatch()} has no equivalent to reuse. + */ + private int[] out; + + @Setup(Level.Trial) + public void setup() { + data = new RowStoreData(rows); + out = new int[rows]; + } + + @TearDown(Level.Trial) + public void tearDown() { + data.close(); + } + + /** + * The product path. One crossing into {@code lgj_row_facet_match}; the result is a + * caller-owned segment the native kernel wrote into directly, reduced to a bit count in Java + * (the same "bulk reduction over an already-crossed result" rationale as {@link + * com.adaworldapi.lancegraph.FacetMatchView#cardinality}). + * + *

See the class doc's asymmetry note — this arm allocates a fresh output segment every + * call, unlike {@code C_ExecutionBoundary#native_fusedPlan()}. + */ + @Benchmark + public long native_facetMatch() { + return data.store.facetMatches(RowStoreData.CLASSID_NEEDLE).cardinality(); + } + + /** Java Vector API over the same lane-0 bytes. Zero copies, {@code out} reused across calls. */ + @Benchmark + public long java_vectorApi() { + Kernels.facetMatchVector(data.raw, rows, RowStoreData.CLASSID_NEEDLE, out); + return bitCountAll(out); + } + + /** Ordinary Java loop over the same lane-0 bytes. The auto-vectorisation control. */ + @Benchmark + public long java_scalar() { + Kernels.facetMatchScalar(data.raw, rows, RowStoreData.CLASSID_NEEDLE, out); + return bitCountAll(out); + } + + /** Sum of set bits across every row's facet-match bitset — comparable to {@code cardinality()}. */ + private static long bitCountAll(int[] values) { + long total = 0; + for (int v : values) { + total += Integer.bitCount(v); + } + return total; + } +} diff --git a/bench/src/main/java/com/adaworldapi/lancegraph/bench/Kernels.java b/bench/src/main/java/com/adaworldapi/lancegraph/bench/Kernels.java index 95c4b15..38388c9 100644 --- a/bench/src/main/java/com/adaworldapi/lancegraph/bench/Kernels.java +++ b/bench/src/main/java/com/adaworldapi/lancegraph/bench/Kernels.java @@ -145,4 +145,110 @@ public static long sumAllHeap(int[] lane) { for (int v : lane) sum += v; return sum; } + + // ── row-store facet match (docs/abi.md §11) ───────────────────────────────────────────── + // + // Mirrors native/lgj-abi/src/kernels.rs::simd_rowstore_facet_match exactly: 512-byte rows, + // 32 sixteen-byte facets (4-byte LE classid + 12-byte payload). Both methods below read the + // *raw lane-0 buffer* (RowStore's `n_rows * 512`-byte U8 segment) directly — no facet-typed + // MemorySegment, no per-facet strided lane, matching the Rust kernel's own choice to walk the + // whole byte buffer rather than 32 separate strided columns. + + /** One row is 512 bytes; one facet is 16 bytes; a row therefore holds 32 facets. */ + private static final int ROW_BYTES = 512; + private static final int FACET_BYTES = 16; + private static final int FACETS_PER_ROW = ROW_BYTES / FACET_BYTES; // 32 + /** One 64-byte chunk covers 4 facets (16 {@code int} lanes = 4 facets × 4 ints each). */ + private static final int CHUNKS_PER_ROW = ROW_BYTES / 64; // 8 + + static { + // The chunked vector algorithm below assumes a 16-lane (512-bit) preferred species, the + // same assumption this class's Javadoc already states for I32 and that C_ExecutionBoundary + // relies on implicitly. Fail loudly rather than silently mis-fold facet bits on a host + // where SPECIES_PREFERRED is narrower (e.g. 256-bit/8-lane AVX2) — a correct narrower + // implementation would need 2 chunks per 64-byte block and was not written, since this + // bench targets the AVX-512 host the rest of the repo already targets. + if (I32.length() != 16) { + throw new ExceptionInInitializerError( + "facetMatchVector's chunk math assumes a 16-lane (512-bit) preferred species; " + + "this host's IntVector.SPECIES_PREFERRED has " + I32.length() + " lanes"); + } + } + + /** + * For every row of {@code raw} (the {@link com.adaworldapi.lancegraph.RowStore} lane-0 buffer), + * which of its 32 facets carry {@code needle} as classid — the Java Vector API arm, mirroring + * {@code simd_rowstore_facet_match} in {@code native/lgj-abi/src/kernels.rs} bit-for-bit. + * + *

Algorithm (identical to the Rust kernel)

+ * + *

Each row is walked as 8 sixty-four-byte chunks; each chunk is loaded as one 16-lane + * {@code int} vector (4 facets × 4 ints — classid, then 3 payload ints). Comparing the whole + * chunk against {@code needle} for equality and masking to bits {@code 0/4/8/12} isolates + * exactly the 4 classid lanes — bits 1/2/3, 5/6/7, 9/10/11, 13/14/15 are payload and MUST NOT + * contribute, or a payload int that happens to equal {@code needle} would be mistaken for a + * classid match (see {@code facet_match_ignores_needle_patterns_in_payload_bytes} in the Rust + * test module for the exact failure mode this masking prevents). The four isolated classid bits + * are folded into 4 consecutive output bits and OR'd into the row's accumulator at the chunk's + * facet offset. + * + *

{@code out} is fully overwritten — {@code out[row]} is assigned, never + * OR'd into, so stale caller bits never survive (same convention as every native mask writer + * in this project). + * + * @param raw the row store's lane-0 segment, {@code rows * 512} bytes, read in place + * @param rows how many rows {@code raw} covers + * @param needle the classid to match + * @param out fully overwritten; must have length >= {@code rows} + */ + public static void facetMatchVector(MemorySegment raw, int rows, int needle, int[] out) { + for (int row = 0; row < rows; row++) { + int acc = 0; + long rowBase = (long) row * ROW_BYTES; + for (int chunkInRow = 0; chunkInRow < CHUNKS_PER_ROW; chunkInRow++) { + long off = rowBase + (long) chunkInRow * 64; + IntVector v = IntVector.fromMemorySegment(I32, raw, off, ByteOrder.nativeOrder()); + // Bit i of toLong() is set exactly when lane i compared equal. Masking to + // 0x1111 keeps only the four classid lanes (0, 4, 8, 12); the fold below packs + // those four bits down into bits 0..3 of facetBits, the same shape the Rust + // kernel's `(m & 1) | ((m >> 4) & 1) << 1 | ...` produces. + long m = v.compare(VectorOperators.EQ, needle).toLong() & 0x1111L; + int facetBits = (int) ((m & 1L) + | (((m >>> 4) & 1L) << 1) + | (((m >>> 8) & 1L) << 2) + | (((m >>> 12) & 1L) << 3)); + acc |= facetBits << (4 * chunkInRow); + } + out[row] = acc; + } + } + + /** + * As {@link #facetMatchVector}, as a plain scalar loop — the auto-vectorisation control, same + * rationale as {@link #countScalar} beside {@link #countVector}. + * + *

Reads each facet's classid directly at its own byte offset ({@code row*512 + facet*16}) + * rather than walking 64-byte chunks — a simpler, independently-derived formulation of the same + * algorithm (facet {@code f}'s classid always sits at that exact offset regardless of how the + * vector arm groups facets into chunks), so a bug shared between the two arms is less likely to + * agree by coincidence. + * + * @param raw the row store's lane-0 segment, {@code rows * 512} bytes, read in place + * @param rows how many rows {@code raw} covers + * @param needle the classid to match + * @param out fully overwritten; must have length >= {@code rows} + */ + public static void facetMatchScalar(MemorySegment raw, int rows, int needle, int[] out) { + for (int row = 0; row < rows; row++) { + int acc = 0; + long rowBase = (long) row * ROW_BYTES; + for (int facet = 0; facet < FACETS_PER_ROW; facet++) { + long off = rowBase + (long) facet * FACET_BYTES; + if (raw.get(ValueLayout.JAVA_INT, off) == needle) { + acc |= 1 << facet; + } + } + out[row] = acc; + } + } } diff --git a/bench/src/main/java/com/adaworldapi/lancegraph/bench/RowStoreData.java b/bench/src/main/java/com/adaworldapi/lancegraph/bench/RowStoreData.java new file mode 100644 index 0000000..3025c1e --- /dev/null +++ b/bench/src/main/java/com/adaworldapi/lancegraph/bench/RowStoreData.java @@ -0,0 +1,99 @@ +package com.adaworldapi.lancegraph.bench; + +import com.adaworldapi.lancegraph.FacetMatchView; +import com.adaworldapi.lancegraph.NativeAccess; +import com.adaworldapi.lancegraph.NativeRuntime; +import com.adaworldapi.lancegraph.RowStore; +import com.adaworldapi.lancegraph.internal.ffm.Engine; + +import java.lang.foreign.MemorySegment; + +/** + * One open {@link RowStore} plus the raw lane-0 buffer every benchmark arm in + * {@link F_RowStoreFacetScan} needs, and — mirroring {@link Data}'s own non-negotiable rule — the + * cross-check that every implementation under comparison computes the same answer, run in + * {@code @Setup} rather than in a test. + * + *

Deliberately a separate class from {@link Data} rather than an extension of it: + * {@code Data}'s fields and constructor are specific to the flat 3-lane fixture ({@code + * NativePattern}, lanes {@code id/class/value}); the row store is a different native resource + * ({@code RowStore}, a single {@code U8} raw lane plus 32 strided facet lanes) with a different + * generator and a different question. Keeping them apart is the same "one {@code Setup} per shape" + * discipline {@link C_ExecutionBoundary} already models by keeping Components C and D — but not an + * unrelated fixture — in one class. + */ +public final class RowStoreData implements AutoCloseable { + + /** + * The classid queried in every facet-match arm. {@code 9} matches the value already used as a + * worked example in {@code RowStoreLifetimeTest} and {@code RowStoreParityTest} — no new magic + * number introduced for the bench. + */ + public static final int CLASSID_NEEDLE = 9; + + private static final long SEED = 0xF00DL; + + public final RowStore store; + public final int rows; + + /** + * Lane 0 of the row store: the raw {@code rows * 512}-byte {@code U8} buffer, read in place. + * Fetched via {@code Engine.describeLane(NativeAccess.handleOf(store), NativeAccess + * .LANE_ROWSTORE_RAW)} directly rather than through a new {@code NativeAccess.lane(RowStore, + * int)} overload — {@link NativeAccess#lane} is typed to {@code NativePattern} and adding a + * second overload just to shave one call at this single site was not worth widening that + * bridge class's surface for. + */ + public final MemorySegment raw; + + public RowStoreData(int rows) { + if (!NativeRuntime.isAvailable()) { + throw new IllegalStateException("native library unavailable: " + + NativeRuntime.unavailableReason().getMessage() + + "\nbuild it with: cd native/lgj-abi && " + + "CARGO_TARGET_DIR=/target cargo build --release"); + } + this.rows = rows; + this.store = RowStore.open(rows, SEED); + + Engine.LaneWindow w = Engine.describeLane( + NativeAccess.handleOf(store), NativeAccess.LANE_ROWSTORE_RAW); + this.raw = w.segment(); + + crossCheck(); + } + + /** + * Every facet-match implementation under comparison must agree, row by row, before any of them + * is timed — {@link Data#crossCheck}'s exact rationale: a Vector kernel with a folding bug can + * be faster than a correct one and look like a win if nothing checks the answer first. + * + *

Checks BOTH {@link Kernels#facetMatchVector} and {@link Kernels#facetMatchScalar} against + * the native {@link FacetMatchView}, not just one — a defect confined to either Java arm alone + * would otherwise ship silently. + */ + private void crossCheck() { + int[] vec = new int[rows]; + int[] sca = new int[rows]; + Kernels.facetMatchVector(raw, rows, CLASSID_NEEDLE, vec); + Kernels.facetMatchScalar(raw, rows, CLASSID_NEEDLE, sca); + + FacetMatchView nativeView = store.facetMatches(CLASSID_NEEDLE); + for (int row = 0; row < rows; row++) { + int n = nativeView.matchesOf(row); + if (n != vec[row]) { + throw new AssertionError("facet-match disagreement (vector) at rows=" + rows + + " row=" + row + ": native=" + n + " vector=" + vec[row]); + } + if (n != sca[row]) { + throw new AssertionError("facet-match disagreement (scalar) at rows=" + rows + + " row=" + row + ": native=" + n + " scalar=" + sca[row]); + } + } + } + + @Override + public void close() { + store.close(); + } +} diff --git a/bench/summarise.sh b/bench/summarise.sh index 6d88e35..63e69e6 100755 --- a/bench/summarise.sh +++ b/bench/summarise.sh @@ -70,7 +70,7 @@ for r in rows: if fus: arms = ['fused', 'unfused', 'fusedScalarKernel', 'planConstructionOnly'] - print("\n### E/F — fusion and the cost of the fluent API (µs/op)\n") + print("\n### E — fusion and the cost of the fluent API (µs/op)\n") print("| rows | predicates | " + " | ".join(f"`{a}`" for a in arms) + " | unfused/fused |") print("|---:|---:|" + "---:|" * (len(arms) + 1)) for p in sorted(fus): @@ -85,5 +85,33 @@ if fus: ratio = (f"**{scores['unfused'] / scores['fused']:.2f}x**" if 'fused' in scores and 'unfused' in scores else "—") print(f"| {p[0]:,} | {p[1]} | " + " | ".join(cells) + f" | {ratio} |") + +# ── Component F: the row-store facet scan ──────────────────────────────────────────────────── +fscan = collections.defaultdict(dict) +for r in rows: + if 'F_RowStoreFacetScan' not in r['Benchmark'] or not r.get('Param: rows'): + continue + fscan[int(r['Param: rows'])][key(r)] = (num(r, 'Score'), num(r, 'Score Error (99.9%)')) + +if fscan: + arms = ['native_facetMatch', 'java_vectorApi', 'java_scalar'] + print("\n### F — the row-store facet scan (µs/op, mean ± 99.9% CI)\n") + print("| rows | row KiB | " + " | ".join(f"`{a}`" for a in arms) + + " | fastest | native/vector |") + print("|---:|---:|" + "---:|" * (len(arms) + 2)) + for n in sorted(fscan): + cells, scores = [], {} + for a in arms: + if a in fscan[n]: + sc, e = fscan[n][a] + scores[a] = sc + cells.append(f"{sc:.3f} ±{e:.3f}") + else: + cells.append("—") + best = min(scores, key=scores.get) if scores else "—" + ratio = (f"{scores['native_facetMatch'] / scores['java_vectorApi']:.2f}x" + if 'native_facetMatch' in scores and 'java_vectorApi' in scores else "—") + print(f"| {n:,} | {n * 512 // 1024} | " + " | ".join(cells) + + f" | **{best}** | {ratio} |") print() PY diff --git a/docs/execution-boundary.md b/docs/execution-boundary.md index df9ca04..321c08f 100644 --- a/docs/execution-boundary.md +++ b/docs/execution-boundary.md @@ -146,9 +146,20 @@ Not "Rust executes, Java orchestrates" — the measured picture is finer: tiny results (a count, a sum, a mask handle) ``` +**Re-measured on the real substrate layout (W4, Component F, 2026-08-17):** +the single-predicate finding survives *directionally* on the 512-byte-row / +32-facet store — the Vector API still wins the per-row facet scan at every +row count measured — but the margin collapses from Component C's 56× to +**2.5× at 4K rows, 1.9× at 65K, 1.14× at 1M**, where all three arms converge +on memory bandwidth (512 MiB traversed, ~6–7 GB/s, the native arm's CI +bracketing much of the residual gap). More work per byte narrows the +boundary exactly as this document predicted; now it is measured rather than +predicted, with one disclosed asymmetry (the native arm allocates its output +per call where the Java arms reuse a buffer — `bench/RESULTS.md` §F). + A future planner could even choose the side per-operation using exactly the -crossover table in `bench/RESULTS.md` — the data to make that choice -mechanically now exists. What keeps the model honest is the invariant both +crossover tables in `bench/RESULTS.md` — the data to make that choice +mechanically now exists, on both the flat and the row-store layouts. What keeps the model honest is the invariant both sides share: **the bytes never serialize, never bounce, never mirror into the Java heap as N objects.** Which side loops over them is an implementation decision the measurements can now drive; that they are the diff --git a/java/src/main/java/com/adaworldapi/lancegraph/RowStore.java b/java/src/main/java/com/adaworldapi/lancegraph/RowStore.java index 891d2e3..b823b9f 100644 --- a/java/src/main/java/com/adaworldapi/lancegraph/RowStore.java +++ b/java/src/main/java/com/adaworldapi/lancegraph/RowStore.java @@ -111,6 +111,16 @@ public FacetMatchView facetMatches(int classId) { return new FacetMatchView(this, out, rowCount); } + /** + * The generation-checked registry handle. Package-private — mirrors {@link + * NativePattern#handle()} exactly, including its consumer: {@code bench}'s {@code + * NativeAccess} reaches package-private accessors like this one from a split-package bridge + * class, never through a public API widening. + */ + long handle() { + return handle; + } + /** * Release the native storage and this store's own arena. * From 201a626ed4660536c6dc5c31e7f258a425063b8f Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 23:19:35 +0000 Subject: [PATCH 18/30] Board: PR #9 arc entry (post-merge) Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/PR_ARC_INVENTORY.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 8f1ba65..31c89e3 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -8,6 +8,27 @@ > anti-pattern the imported board rules name. Backfilled below in one > pass rather than left stale; PR #4 onward gets its entry at merge time. +## PR #9 — bench Component F: the boundary on the real layout (merged 2026-08-17, squash `b28bd34`) + +- **Added:** `F_RowStoreFacetScan` + `RowStoreData` + two `Kernels` + facet-match arms (Java mirrors of the Rust chunk algorithm, + line-for-line traceable); `summarise.sh` F-table generator (old "E/F" + section retitled "E" — real naming collision); `RESULTS.md` §F; + `execution-boundary.md` re-measured paragraph; `RowStore.handle()` + (package-private, bench bridge only); the r2sleigh plan addendum + (lift candidate C + decompiler candidate, `libsla-sys` FFI fact pinned). +- **Locked:** the W4 finding — Component C's direction survives on the + 512-byte-row layout, its margin collapses: Vector API wins the 32-facet + strided scan 2.51×/1.92×/1.14× (4K/65K/1M rows) vs C's 56×; all arms + converge on memory bandwidth at 512 MiB. Cross-check-before-timing held + at every row count. Native arm's per-call allocation asymmetry + disclosed; `facetMatchesInto` named as follow-up. +- **Deferred:** consumer waves (operator-gated shelf); W6 (named). +- **Docs:** RESULTS/TABLES regenerated mechanically from the merged CSV. +- **Confidence:** High — real JMH, 9/9 combos, cross-checks green, main + suite re-verified 185/185 against the fresh minor-2 `.so`. Bot + reviewers at usage limits, did not run. + ## PR #8 — Java RowStore facade: W3 shipped (merged 2026-08-17, squash `320808d`) - **Added:** `RowStore`/`FacetMatchView`/`FacetId`/`NativeResource` public From ed4fb7cf1c2c12e2558d6eaf362acac182b7235f Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 23:28:51 +0000 Subject: [PATCH 19/30] Parity: the third independent read path (ROW_LAYOUT segment reads) Closes the gap the W3 dispatch honestly flagged: the wave file specified a raw-lane segment-read parity arm that my worker brief dropped. Section added to RowStoreParityTest: every classid of a 1000-row store read DIRECTLY from the raw lane-0 segment, addressed through Layouts.ROW_LAYOUT's own byteOffset arithmetic (sequenceElement + groupElement, not hand-multiplied constants) -- no native kernel, no mask, no FacetMatchView on the path. Three independent routes now reach the same numbers: the native kernels, the pure-Java generator transcription, and the structured-layout segment read. This is also ROW_LAYOUT's first real consumer; before this it was defined and size-checked but read by nothing. Plus the raw lane's own description pinned (byteLength == n*512, contiguous flag set). AllTests 185 -> 188. Also records the operator handoff boundary in the ghidra plan: r2sleigh/ruff/R2IL integration arrives from another session -- this session does not build toward it, and lift-candidate C is frozen until the handoff lands. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/plans/ghidra-integration-v1.md | 12 ++++++ .../lancegraph/RowStoreParityTest.java | 39 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/.claude/plans/ghidra-integration-v1.md b/.claude/plans/ghidra-integration-v1.md index 5e231d5..ab5c3e1 100644 --- a/.claude/plans/ghidra-integration-v1.md +++ b/.claude/plans/ghidra-integration-v1.md @@ -64,6 +64,18 @@ tree, not the README alone: pushing or PRs against r2sleigh require re-attaching with push access. Any fix we need upstream goes through the operator first. +## HANDOFF BOUNDARY (operator, 2026-08-17) — r2sleigh/ruff arrives from ANOTHER session + +> *"we will hand you via another session a proper integration of r2sleigh +> and ruff after the other session implements R2IL."* + +Consequence: **this session does not build toward r2sleigh, ruff, or R2IL.** +The G0 addendum above stays as recorded context; lift-candidate C and the +r2dec decompiler direction are FROZEN until that handoff lands. When it +does, the receiving session should reconcile the handoff against this +plan's G-waves (they may supersede G1's lift-path decision entirely) rather +than running both designs in parallel. + ## The integration shape — two roles, both offline ``` diff --git a/java/src/test/java/com/adaworldapi/lancegraph/RowStoreParityTest.java b/java/src/test/java/com/adaworldapi/lancegraph/RowStoreParityTest.java index 1c98bb0..eaede48 100644 --- a/java/src/test/java/com/adaworldapi/lancegraph/RowStoreParityTest.java +++ b/java/src/test/java/com/adaworldapi/lancegraph/RowStoreParityTest.java @@ -171,5 +171,44 @@ public static void run(Checks c) { c.throwsUp("FacetId(32) is rejected", IllegalArgumentException.class, () -> new FacetId(32)); c.eq("FacetId(0) is accepted", 0, new FacetId(0).index()); c.eq("FacetId(31) is accepted", 31, new FacetId(31).index()); + + // The THIRD independent read path (the wave file's original spec item, closing the gap + // the W3 dispatch honestly flagged as dropped): Java reads every classid DIRECTLY from + // the raw lane-0 segment, addressing it through ROW_LAYOUT's own byteOffset arithmetic — + // no native kernel, no mask, no FacetMatchView anywhere on this path. Three ways now + // reach the same numbers: the native kernels (sections above), the pure-Java generator + // transcription, and the structured-layout segment read below. This is also + // Layouts.ROW_LAYOUT's first real consumer — before this section its byteOffset + // arithmetic was defined, size-checked at class-init, and read by nothing. + c.section("raw lane-0 segment reads through ROW_LAYOUT offsets vs the generator"); + try (RowStore store = RowStore.open(n1000, seed)) { + com.adaworldapi.lancegraph.internal.ffm.Engine.LaneWindow win = + com.adaworldapi.lancegraph.internal.ffm.Engine.describeLane(store.handle(), 0); + // The raw lane's own description first: contiguous U8, exactly n*512 bytes — the + // exact-span rule for a contiguous lane (abi.md §11). + c.eq("raw lane byte length is n_rows * 512", (long) n1000 * 512L, win.byteLength()); + c.that("raw lane reports contiguous", win.isContiguous()); + + java.lang.foreign.MemorySegment raw = win.segment(); + long rowBytes = com.adaworldapi.lancegraph.internal.ffm.Layouts.ROW_LAYOUT.byteSize(); + int layoutMismatches = 0; + for (int row = 0; row < n1000; row++) { + for (int facet = 0; facet < FACETS_PER_ROW; facet++) { + // The offset comes from the LAYOUT, not from hand-multiplied constants — + // that is the point: this section proves ROW_LAYOUT's carving agrees with + // the generator, not merely that 16*facet does. + long off = com.adaworldapi.lancegraph.internal.ffm.Layouts.ROW_LAYOUT.byteOffset( + java.lang.foreign.MemoryLayout.PathElement.sequenceElement(facet), + java.lang.foreign.MemoryLayout.PathElement.groupElement("classid")); + int got = raw.get(java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED, + (long) row * rowBytes + off); + if (got != classids1000[row][facet]) { + layoutMismatches++; + } + } + } + c.eq("all " + (n1000 * FACETS_PER_ROW) + " classids read through ROW_LAYOUT match the" + + " transcribed generator", 0, layoutMismatches); + } } } From 9717cda82943d3e3c7ecc3f19a4c72c783200cb5 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 23:29:33 +0000 Subject: [PATCH 20/30] Board: PR #10 arc entry (post-merge) Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/PR_ARC_INVENTORY.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 31c89e3..4922824 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -8,6 +8,25 @@ > anti-pattern the imported board rules name. Backfilled below in one > pass rather than left stale; PR #4 onward gets its entry at merge time. +## PR #10 — third parity read path + R2IL handoff boundary (merged 2026-08-17, squash `4114c4e`) + +- **Added:** `RowStoreParityTest` section reading all 32,000 classids of a + 1000-row store DIRECTLY from the raw lane-0 segment through + `Layouts.ROW_LAYOUT.byteOffset(...)` — proving the LAYOUT's carving + against the generator, and giving `ROW_LAYOUT` its first real consumer + (it had been defined and size-checked but read by nothing). AllTests + 185 → 188. +- **Locked:** three independent routes to the same numbers (native + kernels / generator transcription / structured segment read); the raw + lane's own description pinned (n*512 bytes, contiguous). The R2IL + handoff boundary recorded in `ghidra-integration-v1.md`: r2sleigh/ruff + integration arrives from ANOTHER session — lift-candidate C and the + r2dec direction are FROZEN here until it lands. +- **Deferred:** everything the handoff covers, deliberately. +- **Docs:** the plan's handoff section IS the record. +- **Confidence:** High — closes a gap the W3 worker itself flagged rather + than one discovered by accident. Bot reviewers at usage limits. + ## PR #9 — bench Component F: the boundary on the real layout (merged 2026-08-17, squash `b28bd34`) - **Added:** `F_RowStoreFacetScan` + `RowStoreData` + two `Kernels` From 8eecb3b3480a3de885fe54513acdfbb77f991b20 Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 23:36:24 +0000 Subject: [PATCH 21/30] =?UTF-8?q?Consumer=20example:=20World/Trades=20?= =?UTF-8?q?=E2=80=94=20the=20zero-object=20fluent=20domain=20API=20(W5a)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The One-Billion-Objects poster made runnable, on the shelf-calcified wave map (wave-consumer-trades.md): two Sonnet workers, disjoint scopes, orchestrator-gated. consumers/trades/ is its own compile unit consuming com.adaworldapi.lancegraph exactly as a third-party developer would -- zero new membrane surface, zero core-API changes. Trade is a schema, not an entity: static U32Field VENUE / I32Field PRICE over the existing lanes, venue constants, and a private unconditionally-throwing constructor -- the test forces it accessible via reflection and proves construction STILL fails, plus zero public ctors and zero instance fields by reflection walk. The measured thesis: TradesAllocationTest's steady-state floor is 240 bytes per count() query, IDENTICAL at 64,000 and 1,000,000 rows -- allocation does not scale with rows (the assertion), with a 64 KiB absolute backstop. Laziness holds through the domain vocabulary: 0 crossings composing a 4-predicate Trade chain, exactly 1 at count(). Parity: the fluent chain equals a pure-Java transcribed-generator recomputation at both sizes, anti-vacuity guarded. Disable-run (green-red-green): VENUE pointed at the wrong lane -- the membrane's own LANE_KIND_MISMATCH rejected the misbinding outright, proving the schema binding is checked by the ABI, not trusted. Restored, both suites re-verified (12/12 + 3/3). QUANTITY is honestly absent (the flat fixture has two data lanes); its arrival is the ClassView/W6 slice, stated in Trade's Javadoc rather than faked. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/STATUS_BOARD.md | 2 +- .../java/com/adaworldapi/trades/Trade.java | 114 ++++++++ .../java/com/adaworldapi/trades/World.java | 84 ++++++ .../trades/TradesAllocationTest.java | 189 ++++++++++++++ .../adaworldapi/trades/TradesParityTest.java | 243 ++++++++++++++++++ 5 files changed, 631 insertions(+), 1 deletion(-) create mode 100644 consumers/trades/src/main/java/com/adaworldapi/trades/Trade.java create mode 100644 consumers/trades/src/main/java/com/adaworldapi/trades/World.java create mode 100644 consumers/trades/src/test/java/com/adaworldapi/trades/TradesAllocationTest.java create mode 100644 consumers/trades/src/test/java/com/adaworldapi/trades/TradesParityTest.java diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 6709c68..1e06fe0 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -40,4 +40,4 @@ layout wired end to end. Doctrine: `E-LGJ-THE-MIDDLE-TIER-IS-DELETED-NOT-WRAPPED | D-LGJ-W2 | lgj-abi row store: `rowstore.rs`, `LGJ_RESOURCE_ROWSTORE`, `lgj_rowstore_open`, strided facet lanes through the unchanged `LgjLaneDesc`, `lgj_op_eq_classid`, `lgj_row_facet_match`, ABI minor 1→2, `docs/abi.md` §11 | **DONE 2026-08-17** — `cargo test` **84/84**, clippy/fmt clean, release build exports **18/18** symbols (`nm -D`). Parity: both kernels vs independent scalar references over 10 row counts × 2 seeds × 4 facets × 4 needles, cross-checked a THIRD way against `RowStore::classid_at`. Two-sided payload-vs-classid falsifier. End-to-end membrane test covers describe → predicate → mask algebra → count → facet-match → lifecycle | | D-LGJ-W3 | Java `RowStore` facade: structured `MemoryLayout`, minor-≥2 gate, `FacetMatchView`, parity test transcribing the generator | **DONE 2026-08-17** — dispatched per `.claude/waves/wave-substrate-w3-w4.md` (3 Sonnet workers, disjoint scopes: FFM membrane extension / public facade / tests), orchestrator-integrated. `javac -Xlint:all` clean (same 7 pre-existing `[restricted]` warnings, zero new). `AllTests` **185/185** (was 132; +53 new checks: 29 parity + 24 lifetime). **One real bug caught by the suite and fixed**: `FacetMatchView.rowCount()` was missing the closed-store guard `matchesOf`/`cardinality` both had — a stale row count was readable after the owning store closed. Fixed, re-verified. Both mandated disable-runs ran red-then-green: (1) `Abi.requireMinor` inflated by 1 → exactly `RowStoreParityTest`+`RowStoreLifetimeTest` failed, all 8 other suites stayed green; (2) the pure-Java generator's a/b draw order swapped in `RowStoreParityTest` → exactly that suite broke (17/29), `RowStoreLifetimeTest` (generator-independent) stayed green — confirming the parity test is a real falsifier, not decorative. `Mask.source()` retyped `NativePattern → NativeResource` (new interface) so a `Mask` can parent onto either a `NativePattern` or a `RowStore` — zero call-site breakage (verified: no existing caller bound the narrower type) | | D-LGJ-W4 | Bench Component F: Vector API facet scan vs the crossing, on the REAL layout | **DONE 2026-08-17** — 1 Sonnet worker (F_RowStoreFacetScan + RowStoreData + Kernels facet-match arms, cross-check-in-@Setup discipline), orchestrator-run JMH: 9/9 combos, cross-checks green at every row count. **Finding: Component C's direction survives, its margin collapses** — Vector API wins the 32-facet strided scan at every row count but by 2.51×/1.92×/1.14× (4K/65K/1M rows) vs C's 56×; at 512 MiB traversed all three arms converge on memory bandwidth. Native arm's per-call allocation asymmetry disclosed in §F with a named follow-up (`facetMatchesInto`), not hidden. summarise.sh extended with the F table (and the old 'E/F' section retitled 'E' — a real naming collision); tables regenerated from the merged CSV | -| D-LGJ-W5 | Three consumer examples (trades / bricks / graph) — one plan file each | Planned, gated on W3 | +| D-LGJ-W5 | Three consumer examples (trades / bricks / graph) — one plan file each | **trades DONE 2026-08-17** — `consumers/trades/` (own compile unit, core consumed as a third-party would): `Trade` (schema-not-entity: zero public ctors, zero instance fields, reflection-forced construction still throws), `World.open` → the existing lazy `View` under domain names, zero new membrane surface. TradesParityTest 12/12 (chain vs transcribed-generator recomputation at 1K+64K rows; 0 crossings composing / 1 at terminal THROUGH the domain vocabulary; reflection guard). TradesAllocationTest 3/3 — **the poster's number, measured: 240 bytes/query, IDENTICAL at 64K and 1M rows** (row-count independence is the thesis assertion; 64 KiB absolute backstop). Disable-run: VENUE pointed at the wrong lane → the membrane's own LANE_KIND_MISMATCH rejected it (the binding is checked, not trusted); restored green. bricks + graph still shelved | diff --git a/consumers/trades/src/main/java/com/adaworldapi/trades/Trade.java b/consumers/trades/src/main/java/com/adaworldapi/trades/Trade.java new file mode 100644 index 0000000..26b80d0 --- /dev/null +++ b/consumers/trades/src/main/java/com/adaworldapi/trades/Trade.java @@ -0,0 +1,114 @@ +package com.adaworldapi.trades; + +import com.adaworldapi.lancegraph.Field; +import com.adaworldapi.lancegraph.I32Field; +import com.adaworldapi.lancegraph.LaneId; +import com.adaworldapi.lancegraph.Ordinal; +import com.adaworldapi.lancegraph.Pattern; +import com.adaworldapi.lancegraph.U32Field; + +import java.util.List; + +/** + * The domain schema for the "One Billion Objects. Zero Objects." poster. + * + *

This is a schema, not an entity

+ * + *

There is no such thing as a {@code Trade} instance, on purpose, and the constructor enforces + * that rather than merely discouraging it (see below). The thesis this consumer example exists to + * demonstrate: a million logical trades are one native lane set, not a million Java + * objects. {@code Trade} is the vocabulary a fluent query is written against — + * {@code Trade.VENUE.eq(Trade.XETRA)} — never a bag of fields a row gets copied into. + * + *

This class is written the same generated shape as {@link Pattern}: constants only, no logic, + * names taken from the schema, lane indices stated once. It is a domain-named sibling of + * {@code Pattern}, not a replacement for it — see the field notes below for exactly how the two + * relate. + * + * 

{@code
+ * try (var world = World.open(1_000_000, World.DEFAULT_SEED).source()) {
+ *     long xetraCount = world.view()
+ *             .where(Trade.VENUE.eq(Trade.XETRA))
+ *             .where(Trade.PRICE.gt(100))
+ *             .count();
+ * }
+ * }
+ * + *

Field mapping — honest about what the fixture actually offers

+ * + *

The generated fixture ({@link com.adaworldapi.lancegraph.NativePattern}) has exactly three + * lanes: an id, an unsigned 32-bit class tag ({@code 0..15}), and a signed 32-bit value + * ({@code -150..361}). This schema binds the domain names a trading example would actually use + * onto those same two data lanes — {@link #VENUE} is the class lane, {@link #PRICE} is the value + * lane — rather than inventing a fourth lane the substrate does not have. + * + *

    + *
  • {@link #VENUE} — {@link Pattern#CLASS}'s lane ({@code lane 1}), read as a trading venue + * identifier. The fixture generates 16 distinct values ({@code 0..15}); {@link #XETRA} and + * {@link #NASDAQ} are two of them, chosen so a demo predicate selects a measured, known + * fraction of the rows rather than an arbitrary one. + *
  • {@link #PRICE} — {@link Pattern#VALUE}'s lane ({@code lane 2}), read as a price tick. The + * fixture's signed range ({@code -150..361}) does not correspond to a real price in any + * currency; the demo's fiction is that these are cents-scale ticks, deliberately unrealistic + * (a real price is never negative) so that a signed-comparison bug shows up rather than + * agreeing with an unsigned one by accident — see {@link I32Field#gt(int)}. + *
+ * + *

QUANTITY is deliberately absent

+ * + *

The poster's domain fiction has a third field, quantity. It is not fabricated + * here: the current flat three-lane fixture has no third numeric lane to bind it to, and + * inventing one would mean this schema quietly grows its own private data rather than projecting + * the real substrate. A quantity field arrives with the multi-lane {@code ClassView} facet slice + * (32 facets, {@code W6} in the substrate plan) — at that point {@code Trade} gains a + * {@code QUANTITY} constant bound to a real lane, and this class's shape does not otherwise change. + * Until then, honesty about what exists beats completeness of the poster. + * + *

Binds onto the flat pattern, not the row store

+ * + *

{@link World#open} opens a {@link com.adaworldapi.lancegraph.NativePattern} and returns its + * {@link com.adaworldapi.lancegraph.View} — the flat, three-lane substrate {@link Pattern} already + * uses, not the 32-facet {@code RowStore} the substrate wave shipped alongside it. That is where + * the {@code gt}/{@code eq} lazy {@code View} machinery lives today. A facet-row binding (one + * {@code Trade} row addressed as a lane slice of a {@code RowStore} facet, via a future + * {@code ClassView}) is future work, not a gap in this example — see the {@code QUANTITY} note + * above for the same boundary from the data side. + */ +public final class Trade { + + /** + * Never constructed. A {@code Trade} instance would be exactly the per-entity object this + * example exists to prove unnecessary — a million of them is the anti-thesis, not a + * convenience. Thrown even through reflection with {@code setAccessible(true)}, so the + * guarantee is "impossible," not merely "discouraged by visibility." + */ + private Trade() { + throw new AssertionError( + "a Trade is never materialized — 1,000,000 logical trades are one native lane set," + + " not 1,000,000 objects. There is no Trade instance to construct."); + } + + /** Venue identifier — {@link Pattern#CLASS}'s lane, read under a domain name. */ + public static final U32Field VENUE = + new U32Field("venue", LaneId.of(1), Ordinal.of(0)); + + /** Price tick — {@link Pattern#VALUE}'s lane, read under a domain name. See the class doc for + * why this is a signed, fictional cents-scale tick rather than a real currency amount. */ + public static final I32Field PRICE = + new I32Field("price", LaneId.of(2), Ordinal.of(1)); + + /** + * A venue id used throughout this example's predicates. {@code 7} is the fixture class the + * rest of this repo's test corpus already exercises, so {@code Trade.VENUE.eq(Trade.XETRA)} + * selects the same, already-measured ~6% (1-in-16) fraction of rows as {@code + * Pattern.CLASS.eq(7)} does elsewhere. + */ + public static final int XETRA = 7; + + /** A second, distinct venue id, for predicates that need two venues to compare or exclude. */ + public static final int NASDAQ = 3; + + /** Every field this schema defines, in schema order. See the class doc for why there is no + * {@code QUANTITY} entry yet. */ + public static final List FIELDS = List.of(VENUE, PRICE); +} diff --git a/consumers/trades/src/main/java/com/adaworldapi/trades/World.java b/consumers/trades/src/main/java/com/adaworldapi/trades/World.java new file mode 100644 index 0000000..6d82227 --- /dev/null +++ b/consumers/trades/src/main/java/com/adaworldapi/trades/World.java @@ -0,0 +1,84 @@ +package com.adaworldapi.trades; + +import com.adaworldapi.lancegraph.NativePattern; +import com.adaworldapi.lancegraph.View; + +/** + * The entry point for the "One Billion Objects. Zero Objects." poster example. + * + *

The claim

+ * + *

A million (or a billion, given enough rows in the fixture) logical trades never become that + * many Java objects. {@link #open} opens one native lane set and hands back one lazy {@link View} + * — the same fluent {@code where}/{@code count}/{@code sumOf} machinery {@link + * com.adaworldapi.lancegraph.NativePattern} already provides, addressed through domain names + * ({@link Trade#VENUE}, {@link Trade#PRICE}) instead of schema-generic ones ({@code + * Pattern.CLASS}, {@code Pattern.VALUE}). + * + * 

{@code
+ * try (var pattern = NativePattern.open(1_000_000, World.DEFAULT_SEED)) {
+ *     long n = World.viewOf(pattern)
+ *             .where(Trade.VENUE.eq(Trade.XETRA))
+ *             .where(Trade.PRICE.gt(100))
+ *             .count();
+ * }
+ * }
+ * + *

Zero membrane growth — the iron rule

+ * + *

This class adds no ABI symbol, no native call, no query engine. It consumes {@code + * com.adaworldapi.lancegraph} exactly as a third-party developer would: {@link NativePattern#open} + * to obtain rows, {@link NativePattern#view()} to obtain the lazy description, {@link View#where} + * to narrow it. If a future domain example needs a capability the public API does not expose, the + * fix is a change to the substrate plan, never a shortcut taken here. + * + *

Lifetime, stated honestly

+ * + *

A {@link View} does not own the rows it describes — its parent {@link NativePattern} does, + * and the parent must outlive every view (and every terminal operation) derived from it, exactly + * as {@link NativePattern}'s own class documentation specifies. {@link View#source()} is how a + * caller gets back to the resource that must eventually be closed. + * + *

{@link #open(long, long)} returns a {@code View} rather than the {@code NativePattern} + * itself, matching the wave's domain-facade shape ("returns the existing lazy {@code View} + * machinery under domain names") — but that means the resource is reachable only through {@link + * View#source()}, which is why every example in this class's Javadoc closes via {@code + * World.open(...).source().close()} (or, more idiomatically, by holding the pattern in the + * try-with-resources and deriving the view from it, as shown above). A caller who wants the + * cleaner try-with-resources shape should open the {@link NativePattern} directly and call {@link + * NativePattern#view()} — {@link #open} exists for the one-line poster snippet, not to hide the + * resource. + */ +public final class World { + + private World() {} + + /** The default seed, re-exported from {@link NativePattern#DEFAULT_SEED} so a caller need not + * import the core package just to name it. */ + public static final long DEFAULT_SEED = NativePattern.DEFAULT_SEED; + + /** + * Open {@code nRows} trades generated deterministically from {@link #DEFAULT_SEED} and return + * their lazy view. + * + *

See the class documentation for the lifetime note: the returned view's underlying + * resource is reachable, and must eventually be closed, via {@link View#source()}. + */ + public static View open(long nRows) { + return open(nRows, DEFAULT_SEED); + } + + /** + * Open {@code nRows} trades generated deterministically from {@code seed} and return their + * lazy view. + * + *

Building and narrowing the returned view crosses the native membrane zero times, exactly + * as {@link View}'s own documentation specifies — only a terminal operation ({@code count()}, + * {@code sumOf(...)}, {@code select()}) executes. + * + * @throws IllegalArgumentException if {@code nRows} is negative + */ + public static View open(long nRows, long seed) { + return NativePattern.open(nRows, seed).view(); + } +} diff --git a/consumers/trades/src/test/java/com/adaworldapi/trades/TradesAllocationTest.java b/consumers/trades/src/test/java/com/adaworldapi/trades/TradesAllocationTest.java new file mode 100644 index 0000000..509ee90 --- /dev/null +++ b/consumers/trades/src/test/java/com/adaworldapi/trades/TradesAllocationTest.java @@ -0,0 +1,189 @@ +package com.adaworldapi.trades; + +import com.adaworldapi.lancegraph.Checks; +import com.adaworldapi.lancegraph.NativePattern; +import com.adaworldapi.lancegraph.NativeRuntime; +import com.adaworldapi.lancegraph.View; + +/** + * The zero-allocation falsifier for the poster's headline claim: "One Billion Objects. Zero + * Objects." — measured, not asserted in a comment. + * + *

The instrument is {@code com.sun.management.ThreadMXBean#getThreadAllocatedBytes}, the same + * one {@code valhalla-lab}'s {@code Lab.allocatedBytes} uses (see {@code + * valhalla-lab/src/shared/com/adaworldapi/lancegraph/lab/Lab.java}), reapplied here as {@link + * AllocMeter}: touch the instrument once so its own class-init cost is not attributed to the + * measured body, snapshot allocated bytes before and after, subtract. + * + *

The design is two-sided, on purpose

+ * + *

The thesis this test exists to check is row-count independence, not any + * particular byte count: a query over a domain vocabulary must not allocate per logical row, so + * the floor measured at 64,000 rows and the floor measured at 1,000,000 rows must be the same + * (within measurement slack), even though the second query touches sixty-two times as much data + * and matches far more rows. That is the load-bearing assertion — an implementation that + * allocates a small, constant amount of bookkeeping per {@code count()} call passes it; one that + * allocates per row fails it regardless of the constant it starts from. + * + *

The absolute bound (a generous fixed ceiling well below any plausible per-row cost) is the + * backstop: it exists so an implementation that allocates a LARGE but row-count-independent + * amount every call — which would otherwise satisfy the independence check by accident — still + * fails loudly rather than passing on a technicality. + * + *

Why the chain is composed outside the measured window

+ * + *

{@link View#where} is already proven allocation-bearing at the substrate tier (it allocates + * one new {@code List} per call — see {@link View#where}'s Javadoc: "one 24-byte descriptor + * appended to a list"). That is deliberate, cheap, and orthogonal to this test's question, which + * is specifically about the terminal operation's cost as row count grows. So the chain is + * built once, outside every measured window, and only the repeated {@code count()} calls on the + * already-composed {@link View} are timed — exactly {@code LazinessTest}'s own discipline of + * warming the reusable scratch selection before measuring the terminal cost. + */ +public final class TradesAllocationTest { + + private TradesAllocationTest() {} + + public static void main(String[] args) { + System.out.println("TradesAllocationTest"); + if (!NativeRuntime.isAvailable()) { + System.exit(Checks.reportUnavailable("TradesAllocationTest")); + } + Checks c = new Checks("TradesAllocationTest"); + run(c); + System.exit(c.report()); + } + + /** Same seed as {@code TradesParityTest}, for the same reason: an explicit, checkable value. */ + private static final long SEED = 0xABCDL; + + /** JIT warm-up calls before any allocation is measured — allocation noise settles by then. */ + private static final int WARMUP_CALLS = 100; + + /** Measured calls after warm-up; the reported floor is the minimum delta across these. */ + private static final int MEASURED_CALLS = 10; + + /** + * Slack allowed between the 64K floor and the 1M floor. Not zero: {@code + * ThreadMXBean#getThreadAllocatedBytes} accounting and any residual JIT/GC noise are real, so + * an exact-equality assertion would be a flaky test rather than a meaningful one. 4,096 bytes + * is small next to the 64 KiB absolute ceiling below and large next to plausible measurement + * jitter — chosen so a genuinely row-count-independent implementation clears it easily while a + * per-row allocator (which would add many kilobytes per extra ~936,000 rows) cannot. + */ + private static final long ROW_COUNT_INDEPENDENCE_SLACK_BYTES = 4_096; + + /** + * Absolute backstop: any implementation whose steady-state {@code count()} allocates more than + * this, at ANY row count, fails outright — independent of whether it happens to be + * row-count-independent. 64 KiB is generous (far above any plausible constant-bookkeeping + * cost: a handful of descriptors and a mask handle) and far below what a single row's worth of + * boxed objects would cost at 64,000+ rows, so it cannot be cleared by accident. + */ + private static final long ABSOLUTE_CEILING_BYTES = 64 * 1024; + + /** + * The one line the orchestrator adjusts if {@code World.open}'s shape differs from what this + * file assumes. See {@code TradesParityTest#openResource} for the full reasoning — this is the + * identical helper, duplicated because this file is its own standalone suite (matching this + * project's existing convention of duplicating small generator/adapter helpers per suite + * rather than sharing test-internal code across compile units). + */ + private static NativePattern openResource(long rows, long seed) { + Object opened = World.open(rows, seed); + if (opened instanceof NativePattern pattern) { + return pattern; + } + if (opened instanceof View view) { + return view.source(); + } + throw new IllegalStateException( + "World.open(rows, seed) returned " + opened.getClass() + + ", expected either NativePattern or View — update openResource() to match" + + " the shape World actually ships."); + } + + /** + * Bytes this thread allocated while running {@code body}, minus the instrument's own + * overhead — the {@code valhalla-lab} {@code Lab.allocatedBytes} pattern, reapplied here with + * a provenance comment rather than a dependency on the lab's classpath (the lab is a separate, + * unpublished compile unit; this consumer module cannot depend on it). + */ + private static final class AllocMeter { + + private AllocMeter() {} + + private static final com.sun.management.ThreadMXBean THREADS = + (com.sun.management.ThreadMXBean) java.lang.management.ManagementFactory.getThreadMXBean(); + + static long allocatedBytes(Runnable body) { + long tid = Thread.currentThread().threadId(); + // Touch the instrument once so its own (one-time) class-init allocation is not + // attributed to body. + THREADS.getThreadAllocatedBytes(tid); + long before = THREADS.getThreadAllocatedBytes(tid); + body.run(); + long after = THREADS.getThreadAllocatedBytes(tid); + return after - before; + } + } + + /** + * The measured allocation floor for repeated {@code count()} calls over an already-composed, + * two-predicate domain chain at {@code rows} rows: {@link #WARMUP_CALLS} untimed calls (to let + * the JIT settle and the resource's reusable scratch selection get allocated once, matching + * {@link NativePattern}'s own documented "reused destination for terminal operations" + * behaviour), then the minimum delta across {@link #MEASURED_CALLS} timed calls — the minimum, + * not the mean, because it is the honest floor: any allocation the query path genuinely + * requires shows up on every call, so the best case across several calls is the query path's + * real cost, and anything above it on a given call is incidental noise (a stray GC-adjacent + * bookkeeping allocation, a scheduler artifact) rather than signal. + */ + private static long measureFloor(long rows) { + try (NativePattern resource = openResource(rows, SEED)) { + View chain = resource.view() + .where(Trade.VENUE.eq(Trade.XETRA)) + .where(Trade.PRICE.gt(100)); + + for (int i = 0; i < WARMUP_CALLS; i++) { + chain.count(); + } + + long floor = Long.MAX_VALUE; + for (int i = 0; i < MEASURED_CALLS; i++) { + long delta = AllocMeter.allocatedBytes(chain::count); + floor = Math.min(floor, delta); + } + return floor; + } + } + + public static void run(Checks c) { + c.section("allocation instrument sanity"); + long instrumentBaseline = AllocMeter.allocatedBytes(() -> {}); + c.note("instrument's own overhead on this VM: " + instrumentBaseline + " bytes" + + " (not asserted to be exactly zero — VMs differ — only reported)"); + + c.section("steady-state allocation floor, count() over a 2-predicate domain chain"); + + long floor64k = measureFloor(64_000); + long floor1m = measureFloor(1_000_000); + c.note("64,000 rows: " + floor64k + " bytes/call floor"); + c.note("1,000,000 rows: " + floor1m + " bytes/call floor (15.6x the row count)"); + + c.section("thesis: allocation does not scale with rows"); + c.that("the 1M floor is not meaningfully larger than the 64K floor" + + " (allowed slack " + ROW_COUNT_INDEPENDENCE_SLACK_BYTES + " bytes)", + floor1m <= floor64k + ROW_COUNT_INDEPENDENCE_SLACK_BYTES); + c.note("a per-row-allocating implementation would fail this by many kilobytes," + + " not by a rounding error"); + + c.section("backstop: both floors are small in absolute terms"); + c.atMost("64,000-row floor stays under " + ABSOLUTE_CEILING_BYTES + " bytes (" + + (ABSOLUTE_CEILING_BYTES / 1024) + " KiB)", + ABSOLUTE_CEILING_BYTES, floor64k); + c.atMost("1,000,000-row floor stays under " + ABSOLUTE_CEILING_BYTES + " bytes (" + + (ABSOLUTE_CEILING_BYTES / 1024) + " KiB)", + ABSOLUTE_CEILING_BYTES, floor1m); + } +} diff --git a/consumers/trades/src/test/java/com/adaworldapi/trades/TradesParityTest.java b/consumers/trades/src/test/java/com/adaworldapi/trades/TradesParityTest.java new file mode 100644 index 0000000..a1d9ea9 --- /dev/null +++ b/consumers/trades/src/test/java/com/adaworldapi/trades/TradesParityTest.java @@ -0,0 +1,243 @@ +package com.adaworldapi.trades; + +import com.adaworldapi.lancegraph.Checks; +import com.adaworldapi.lancegraph.Diagnostics; +import com.adaworldapi.lancegraph.NativePattern; +import com.adaworldapi.lancegraph.NativeRuntime; +import com.adaworldapi.lancegraph.View; + +/** + * The consumer-tier re-run of {@code FixtureParityTest} and {@code LazinessTest}, this time + * against the DOMAIN vocabulary ({@code Trade.VENUE}, {@code Trade.PRICE}) instead of the raw + * {@code Pattern} schema — proving the poster's "One Billion Objects. Zero Objects." claim holds + * once a real-looking domain type sits on top of the same membrane, not only for the schema the + * substrate tests were written against. + * + *

Three properties, each independently falsifiable: + * + *

    + *
  1. Value parity — the fluent domain chain's count agrees with a pure-Java + * recomputation from the transcribed fixture generator. The recomputation never touches + * {@code World} or the native library; it only knows the seed and the generator's published + * shape, so agreement is evidence about the schema binding, not a tautology. + *
  2. Laziness — composing a domain-vocabulary chain crosses the membrane zero + * times; the terminal {@code count()} crosses exactly once, whatever the chain length. This + * is {@code LazinessTest}'s instrument, reapplied at the consumer tier: {@code Trade.VENUE} + * and {@code Trade.PRICE} must fuse through {@link View#where} exactly the way {@code + * Pattern.CLASS} and {@code Pattern.VALUE} do, because they reuse the identical machinery. + *
  3. The thesis's reflection guard — {@code Trade} is a schema, not an entity. + * No instance is ever constructible, even by a caller who forces accessibility via + * reflection, and the class carries no instance state a constructed object could have held + * anyway. + *
+ */ +public final class TradesParityTest { + + private TradesParityTest() {} + + public static void main(String[] args) { + System.out.println("TradesParityTest"); + if (!NativeRuntime.isAvailable()) { + System.exit(Checks.reportUnavailable("TradesParityTest")); + } + Checks c = new Checks("TradesParityTest"); + run(c); + System.exit(c.report()); + } + + /** + * The seed {@link #openResource} passes to {@code World.open}. Chosen to match + * {@code NativePattern.DEFAULT_SEED} (docs/abi.md's normative fixture seed) so a reader can + * cross-check this file's numbers against {@code FixtureParityTest} directly. Passed + * explicitly on every call rather than relied on as a hidden default, so this file's parity + * claim does not depend on what {@code World} happens to default to. + */ + private static final long SEED = 0xABCDL; + + /** + * The one line the orchestrator adjusts if {@code World}'s open entry point does not match + * this shape exactly. Everything below this method is written against {@link NativePattern} + * and {@link View} only — never against {@code World} directly — so a signature or + * return-type change in {@code World.open} is a one-line fix here, not a rewrite of the + * suite. + * + *

Handles both shapes named in the wave brief: {@code World.open(...)} may return a + * {@link View} directly (the poster's own snippet chains {@code .where(...)} straight off the + * open call) or a {@link NativePattern} (in which case {@code .view()} produces the + * equivalent zero-predicate starting view). Field descriptors such as {@link Trade#VENUE} are + * stateless plan-op factories — {@code eq}/{@code gt} do not depend on which {@link View} + * instance carries them — so resolving either shape down to a fresh {@code .view()} off the + * same resource is equivalent to using whatever {@code View} {@code World.open} handed back + * directly. + */ + private static NativePattern openResource(long rows, long seed) { + Object opened = World.open(rows, seed); + if (opened instanceof NativePattern pattern) { + return pattern; + } + if (opened instanceof View view) { + return view.source(); + } + throw new IllegalStateException( + "World.open(rows, seed) returned " + opened.getClass() + + ", expected either NativePattern or View — update openResource() to match" + + " the shape World actually ships."); + } + + /** + * SplitMix64, transcribed from the normative fixture generator. Provenance: this is the + * identical constant-for-constant, shift-for-shift transcription already checked against the + * published reference vector in {@code FixtureParityTest}. Duplicated rather than shared + * because the original is {@code private} to that class and lives in a different compile + * unit — this consumer module needs its own copy of the generator, not a dependency on the + * substrate test's internals. + */ + private static final class SplitMix64 { + private long state; + + SplitMix64(long seed) { + this.state = seed; // no warm-up draws + } + + long next() { + state += 0x9E3779B97F4A7C15L; + long z = state; + z = (z ^ (z >>> 30)) * 0xBF58476D1CE4E5B9L; + z = (z ^ (z >>> 27)) * 0x94D049BB133111EBL; + return z ^ (z >>> 31); + } + } + + /** + * {@code (venue, price)} per row, recomputed independently in pure Java from the fixture + * generator's published shape: two SplitMix64 draws per row, {@code a} then {@code b}, with + * {@code venue = (a >>> 33) & 0xF} and {@code price = ((b >>> 40) & 0x1FF) - 150}. This is the + * exact same derivation {@code FixtureParityTest} uses for {@code (class, value)} — the trades + * schema binds domain names onto the identical two lanes, so the recomputation is identical + * too, renamed to match the domain. + */ + private record Trades(int[] venues, int[] prices) {} + + private static Trades generate(int nRows, long seed) { + SplitMix64 rng = new SplitMix64(seed); + int[] venues = new int[nRows]; + int[] prices = new int[nRows]; + for (int i = 0; i < nRows; i++) { + long a = rng.next(); // FIRST draw of the row + long b = rng.next(); // SECOND draw of the row + venues[i] = (int) ((a >>> 33) & 0xF); + prices[i] = (int) (((b >>> 40) & 0x1FF) - 150); + } + return new Trades(venues, prices); + } + + public static void run(Checks c) { + checkParity(c, 1_000); + checkParity(c, 64_000); + checkLaziness(c); + checkNotAnEntity(c); + } + + private static void checkParity(Checks c, int rows) { + c.section("value parity at n = " + rows + " rows"); + + Trades expected = generate(rows, SEED); + long expectCount = 0; + for (int i = 0; i < rows; i++) { + if (expected.venues()[i] == Trade.XETRA && expected.prices()[i] > 100) { + expectCount++; + } + } + + try (NativePattern resource = openResource(rows, SEED)) { + long actualCount = resource.view() + .where(Trade.VENUE.eq(Trade.XETRA)) + .where(Trade.PRICE.gt(100)) + .count(); + + c.eq("VENUE == XETRA AND PRICE > 100, domain chain vs pure-Java recomputation", + expectCount, actualCount); + c.that("the predicate excludes a real number of rows (not vacuous: count > 0)", + actualCount > 0); + c.that("the predicate is not satisfied by every row either (count < n)", + actualCount < rows); + } + } + + private static void checkLaziness(Checks c) { + c.section("laziness through the domain vocabulary (Diagnostics.crossings())"); + + try (NativePattern resource = openResource(64_000, SEED)) { + long before = Diagnostics.crossings(); + View chain = resource.view() + .where(Trade.VENUE.eq(Trade.XETRA)) + .where(Trade.PRICE.gt(100)) + .where(Trade.PRICE.gt(-50)) + .where(Trade.VENUE.eq(Trade.NASDAQ)); + long afterBuild = Diagnostics.crossings(); + + c.eq("composing a 4-predicate Trade chain costs no crossings", + 0, afterBuild - before); + c.eq("the chain does carry all four conditions", 4, chain.conditionCount()); + + // Warm the resource's reusable scratch selection so the steady-state cost is what is + // measured, matching LazinessTest's own discipline. + chain.count(); + + long c0 = Diagnostics.crossings(); + chain.count(); + long terminalCost = Diagnostics.crossings() - c0; + c.eq("count() over the domain chain costs exactly one crossing", + 1, terminalCost); + } + } + + private static void checkNotAnEntity(Checks c) { + c.section("Trade is a schema, not an entity — the reflection guard"); + + c.eq("Trade declares zero public constructors", + 0, publicConstructorCount(Trade.class)); + c.eq("Trade declares zero non-static instance fields", + 0, instanceFieldCount(Trade.class)); + + // The private constructor is expected to throw AssertionError, per the frozen contract + // ("private throwing constructor"). AssertionError is required specifically — not just + // "some exception" — so a typo that instead produced e.g. a NullPointerException would be + // caught here rather than mistaken for the guard working. + c.throwsUp("forcing the private constructor accessible still cannot build a Trade", + AssertionError.class, () -> { + try { + var ctor = Trade.class.getDeclaredConstructor(); + ctor.setAccessible(true); + ctor.newInstance(); + } catch (java.lang.reflect.InvocationTargetException e) { + // The constructor itself threw — unwrap so throwsUp() sees the real cause + // (AssertionError) rather than the reflection wrapper type. + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException re) { + throw re; + } + if (cause instanceof Error err) { + throw err; + } + throw new RuntimeException(cause); + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + }); + } + + private static int publicConstructorCount(Class type) { + return type.getConstructors().length; + } + + private static int instanceFieldCount(Class type) { + int n = 0; + for (var field : type.getDeclaredFields()) { + if (!java.lang.reflect.Modifier.isStatic(field.getModifiers())) { + n++; + } + } + return n; + } +} From 9088dc9ccbf709343b9d52e7ddbff4f7a51fc71d Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 23:37:09 +0000 Subject: [PATCH 22/30] Board: PR #11 arc entry (post-merge) Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/PR_ARC_INVENTORY.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 4922824..be70d07 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -8,6 +8,25 @@ > anti-pattern the imported board rules name. Backfilled below in one > pass rather than left stale; PR #4 onward gets its entry at merge time. +## PR #11 — consumer example: World/Trades (merged 2026-08-17, squash `db7bdf1`) + +- **Added:** `consumers/trades/` — own compile unit, core consumed as a + third party would. `Trade` (schema-not-entity, reflection-proven), + `World.open` → the existing lazy `View` under domain names; + `TradesParityTest` 12/12, `TradesAllocationTest` 3/3. +- **Locked:** the poster's number, measured — **240 bytes allocated per + query, IDENTICAL at 64K and 1M rows** (row-count independence is the + thesis assertion); 0 crossings composing / 1 at terminal, through the + domain vocabulary; the membrane's own `LANE_KIND_MISMATCH` catches a + misbound schema (disable-run green-red-green). Zero new membrane + surface, zero core changes — the consumer iron rule held on its first + real test. +- **Deferred:** QUANTITY (honestly absent — arrives with ClassView/W6); + bricks + graph consumer waves (still shelved). +- **Docs:** STATUS_BOARD W5 row → trades DONE. +- **Confidence:** High — every falsifier two-sided or anti-vacuity + guarded; no `java/` file changed. Bot reviewers at usage limits. + ## PR #10 — third parity read path + R2IL handoff boundary (merged 2026-08-17, squash `4114c4e`) - **Added:** `RowStoreParityTest` section reading all 32,000 classids of a From f7974da75ad7d9138d828a05fc697b978514615c Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 23:46:56 +0000 Subject: [PATCH 23/30] =?UTF-8?q?Consumer=20example:=20Bricks=20=E2=80=94?= =?UTF-8?q?=20mask-first=20authorization,=20fail-closed,=20aggregates=20on?= =?UTF-8?q?ly=20(W5b)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The second consumer proof over the unchanged core: authorization is a predicate in the same lazy chain as where(...), composed before execution and evaluated natively — never a Java-side post-filter. Role.EU_ONLY folds REGION.eq(EU) into the plan; DENY_ALL is a genuine impossible predicate (REGION.eq(0xFFFF)) that pays a real crossing and counts zero; a chain that never called authorize() throws UnauthorizedQueryException before any crossing happens (fail-closed, no default-allow path). Aggregate-only egress is structural: every public BricksQuery method returns BricksQuery, long, or Map — no row-shaped public type exists. BricksAuthTest 62/62: generator parity at 1K+64K rows; EU_ONLY equivalence vs GLOBAL+explicit-where; DENY_ALL counts 0 through a real crossing; crossing arithmetic — count()=1, sumBy()=32 (16 groups x 2: plan eval + lgj_reduce_sum_i32), identical at both row counts, which is the thesis (crossings scale with groups, never rows). The measured 32 corrected the worker's "one crossing per group" Javadoc — a real finding about sum-terminal cost, recorded in the doc and asserted in the test. Disable-run: requireAuthorized short-circuited -> exactly the three can-fire fail-closed checks went red (59 green), restored, 62/62. Core suite unaffected (188/188). Zero new membrane surface, per the consumer iron rule. Boards: STATUS_BOARD D-LGJ-W5 bricks DONE; LATEST_STATE dispatch-4 entry (owning that W5a shipped without one). --- .claude/board/LATEST_STATE.md | 28 ++ .claude/board/STATUS_BOARD.md | 2 +- .../java/com/adaworldapi/bricks/Bricks.java | 42 ++ .../com/adaworldapi/bricks/BricksQuery.java | 144 ++++++ .../com/adaworldapi/bricks/BricksSession.java | 53 +++ .../java/com/adaworldapi/bricks/Orders.java | 80 ++++ .../java/com/adaworldapi/bricks/Role.java | 65 +++ .../bricks/UnauthorizedQueryException.java | 24 + .../adaworldapi/bricks/BricksAuthTest.java | 435 ++++++++++++++++++ 9 files changed, 872 insertions(+), 1 deletion(-) create mode 100644 consumers/bricks/src/main/java/com/adaworldapi/bricks/Bricks.java create mode 100644 consumers/bricks/src/main/java/com/adaworldapi/bricks/BricksQuery.java create mode 100644 consumers/bricks/src/main/java/com/adaworldapi/bricks/BricksSession.java create mode 100644 consumers/bricks/src/main/java/com/adaworldapi/bricks/Orders.java create mode 100644 consumers/bricks/src/main/java/com/adaworldapi/bricks/Role.java create mode 100644 consumers/bricks/src/main/java/com/adaworldapi/bricks/UnauthorizedQueryException.java create mode 100644 consumers/bricks/src/test/java/com/adaworldapi/bricks/BricksAuthTest.java diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 559f0cb..2248878 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,31 @@ +## 2026-08-17 (dispatch 4) — W5b bricks shipped: authorization IS a mask, measured + +`wave-consumer-bricks.md` executed (2 Sonnet workers K1/K2, disjoint main/test +scopes, orchestrator-gated). `consumers/bricks/` is the second consumer proof: +**RBAC as a natively-evaluated predicate in the same lazy chain as `where(...)`** +— `Role.EU_ONLY` folds `REGION.eq(EU)` into the plan, `DENY_ALL` is a real +impossible predicate (`REGION.eq(0xFFFF)`) that pays a real crossing and counts +0, and an unauthorized chain throws `UnauthorizedQueryException` BEFORE any +native crossing (fail-closed; no default-allow path exists in the package). +Aggregate-only egress is structural: every public method returns +`BricksQuery`/`long`/`Map` — no row-shaped public type exists to leak. + +- **BricksAuthTest 62/62.** Parity vs the transcribed generator at 1K+64K rows; + EU_ONLY == GLOBAL+explicit-where equivalence; crossing arithmetic measured. +- **A real finding, not just a green suite: a sum terminal costs 2 crossings** + (plan evaluation into the mask + `lgj_reduce_sum_i32`), unlike `count()` + whose plan eval returns the count and pays 1. `sumBy()` therefore measures + **32 crossings (16 groups × 2) — IDENTICAL at 1K and 64K rows**, which is the + thesis (crossings ∝ groups, never rows). K1's Javadoc claimed "one crossing + per group"; the measurement corrected the doc, not the other way round. +- **Disable-run:** `requireAuthorized` short-circuited → exactly the 3 + can-fire fail-closed checks red (59 green), restored, 62/62. Core suite + untouched at 188/188. +- Board-hygiene note, owned: W5a (trades, PR #11) shipped without a + LATEST_STATE entry — STATUS_BOARD D-LGJ-W5 carried it; both consumers are + now recorded there in full. W5c (graph) stays SHELVED on the D1 ruling + + the edge-bearing generator substrate change. + ## 2026-08-17 (dispatch 2) — W4 measured: the boundary re-asked on the REAL layout `wave-substrate-w3-w4.md` Dispatch 2 executed: one Sonnet worker diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 1e06fe0..c69b5d9 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -40,4 +40,4 @@ layout wired end to end. Doctrine: `E-LGJ-THE-MIDDLE-TIER-IS-DELETED-NOT-WRAPPED | D-LGJ-W2 | lgj-abi row store: `rowstore.rs`, `LGJ_RESOURCE_ROWSTORE`, `lgj_rowstore_open`, strided facet lanes through the unchanged `LgjLaneDesc`, `lgj_op_eq_classid`, `lgj_row_facet_match`, ABI minor 1→2, `docs/abi.md` §11 | **DONE 2026-08-17** — `cargo test` **84/84**, clippy/fmt clean, release build exports **18/18** symbols (`nm -D`). Parity: both kernels vs independent scalar references over 10 row counts × 2 seeds × 4 facets × 4 needles, cross-checked a THIRD way against `RowStore::classid_at`. Two-sided payload-vs-classid falsifier. End-to-end membrane test covers describe → predicate → mask algebra → count → facet-match → lifecycle | | D-LGJ-W3 | Java `RowStore` facade: structured `MemoryLayout`, minor-≥2 gate, `FacetMatchView`, parity test transcribing the generator | **DONE 2026-08-17** — dispatched per `.claude/waves/wave-substrate-w3-w4.md` (3 Sonnet workers, disjoint scopes: FFM membrane extension / public facade / tests), orchestrator-integrated. `javac -Xlint:all` clean (same 7 pre-existing `[restricted]` warnings, zero new). `AllTests` **185/185** (was 132; +53 new checks: 29 parity + 24 lifetime). **One real bug caught by the suite and fixed**: `FacetMatchView.rowCount()` was missing the closed-store guard `matchesOf`/`cardinality` both had — a stale row count was readable after the owning store closed. Fixed, re-verified. Both mandated disable-runs ran red-then-green: (1) `Abi.requireMinor` inflated by 1 → exactly `RowStoreParityTest`+`RowStoreLifetimeTest` failed, all 8 other suites stayed green; (2) the pure-Java generator's a/b draw order swapped in `RowStoreParityTest` → exactly that suite broke (17/29), `RowStoreLifetimeTest` (generator-independent) stayed green — confirming the parity test is a real falsifier, not decorative. `Mask.source()` retyped `NativePattern → NativeResource` (new interface) so a `Mask` can parent onto either a `NativePattern` or a `RowStore` — zero call-site breakage (verified: no existing caller bound the narrower type) | | D-LGJ-W4 | Bench Component F: Vector API facet scan vs the crossing, on the REAL layout | **DONE 2026-08-17** — 1 Sonnet worker (F_RowStoreFacetScan + RowStoreData + Kernels facet-match arms, cross-check-in-@Setup discipline), orchestrator-run JMH: 9/9 combos, cross-checks green at every row count. **Finding: Component C's direction survives, its margin collapses** — Vector API wins the 32-facet strided scan at every row count but by 2.51×/1.92×/1.14× (4K/65K/1M rows) vs C's 56×; at 512 MiB traversed all three arms converge on memory bandwidth. Native arm's per-call allocation asymmetry disclosed in §F with a named follow-up (`facetMatchesInto`), not hidden. summarise.sh extended with the F table (and the old 'E/F' section retitled 'E' — a real naming collision); tables regenerated from the merged CSV | -| D-LGJ-W5 | Three consumer examples (trades / bricks / graph) — one plan file each | **trades DONE 2026-08-17** — `consumers/trades/` (own compile unit, core consumed as a third-party would): `Trade` (schema-not-entity: zero public ctors, zero instance fields, reflection-forced construction still throws), `World.open` → the existing lazy `View` under domain names, zero new membrane surface. TradesParityTest 12/12 (chain vs transcribed-generator recomputation at 1K+64K rows; 0 crossings composing / 1 at terminal THROUGH the domain vocabulary; reflection guard). TradesAllocationTest 3/3 — **the poster's number, measured: 240 bytes/query, IDENTICAL at 64K and 1M rows** (row-count independence is the thesis assertion; 64 KiB absolute backstop). Disable-run: VENUE pointed at the wrong lane → the membrane's own LANE_KIND_MISMATCH rejected it (the binding is checked, not trusted); restored green. bricks + graph still shelved | +| D-LGJ-W5 | Three consumer examples (trades / bricks / graph) — one plan file each | **trades DONE 2026-08-17** — `consumers/trades/` (own compile unit, core consumed as a third-party would): `Trade` (schema-not-entity: zero public ctors, zero instance fields, reflection-forced construction still throws), `World.open` → the existing lazy `View` under domain names, zero new membrane surface. TradesParityTest 12/12 (chain vs transcribed-generator recomputation at 1K+64K rows; 0 crossings composing / 1 at terminal THROUGH the domain vocabulary; reflection guard). TradesAllocationTest 3/3 — **the poster's number, measured: 240 bytes/query, IDENTICAL at 64K and 1M rows** (row-count independence is the thesis assertion; 64 KiB absolute backstop). Disable-run: VENUE pointed at the wrong lane → the membrane's own LANE_KIND_MISMATCH rejected it (the binding is checked, not trusted); restored green. **bricks DONE 2026-08-17** — `consumers/bricks/` (2 Sonnet workers K1/K2 per `.claude/waves/wave-consumer-bricks.md`): mask-first RBAC where `authorize(Role)` is a real natively-evaluated predicate in the SAME lazy chain as `where(...)` (`Role.EU_ONLY` = `REGION.eq(EU)`, `DENY_ALL` = `REGION.eq(0xFFFF)` — a genuine impossible predicate, not a Java branch), fail-closed (`UnauthorizedQueryException` BEFORE any crossing; no default-allow path exists), aggregate-only egress (every public method returns `BricksQuery`/`long`/`Map` — structurally no row-shaped type). BricksAuthTest **62/62**: parity vs transcribed generator at 1K+64K; RBAC-as-predicate equivalence (EU_ONLY result == GLOBAL+explicit-where); DENY_ALL counts 0 while paying a real crossing; crossing arithmetic — count()=1, sumBy()=**32 crossings (16 groups × 2: plan_eval + lgj_reduce_sum_i32), IDENTICAL at both row counts** (the thesis: crossings ∝ groups, never rows — the measured 32 corrected K1's "1 per group" Javadoc claim, a real finding about sum-terminal cost); reflection guards. Disable-run: `requireAuthorized` short-circuited → **exactly the 3 can-fire fail-closed checks red, 59 green**; restored, 62/62. Core suite unaffected (188/188). graph still shelved on the D1 ruling + edge-generator substrate change | diff --git a/consumers/bricks/src/main/java/com/adaworldapi/bricks/Bricks.java b/consumers/bricks/src/main/java/com/adaworldapi/bricks/Bricks.java new file mode 100644 index 0000000..d0cb339 --- /dev/null +++ b/consumers/bricks/src/main/java/com/adaworldapi/bricks/Bricks.java @@ -0,0 +1,42 @@ +package com.adaworldapi.bricks; + +import com.adaworldapi.lancegraph.NativePattern; + +/** + * Entry point for the OGAR-Bricks poster example: security as a mask composed before + * execution, never a post-filter over already-fetched data. + * + *

Zero membrane growth — the iron rule

+ * + *

This package adds no ABI symbol, no native call, no query engine, and no row-shaped public + * type. It consumes {@code com.adaworldapi.lancegraph} exactly as a third-party developer would: + * {@link NativePattern#open} to obtain rows, {@link NativePattern#view()} (via {@link + * BricksSession#query()}) to obtain the lazy description, {@link + * com.adaworldapi.lancegraph.View#where} (via {@link BricksQuery#where} and {@link + * BricksQuery#authorize}) to narrow it. + * + * 

{@code
+ * try (var session = Bricks.open(1_000_000, Bricks.DEFAULT_SEED)) {
+ *     long euOrderCount = session.query()
+ *             .authorize(Role.EU_ONLY)
+ *             .where(Orders.REVENUE.gt(0))
+ *             .count();
+ * }
+ * }
+ */ +public final class Bricks { + + private Bricks() {} + + /** The default seed, re-exported from {@link NativePattern#DEFAULT_SEED}. */ + public static final long DEFAULT_SEED = NativePattern.DEFAULT_SEED; + + /** + * Open {@code nRows} orders generated deterministically from {@code seed}. + * + * @throws IllegalArgumentException if {@code nRows} is negative + */ + public static BricksSession open(long nRows, long seed) { + return new BricksSession(NativePattern.open(nRows, seed)); + } +} diff --git a/consumers/bricks/src/main/java/com/adaworldapi/bricks/BricksQuery.java b/consumers/bricks/src/main/java/com/adaworldapi/bricks/BricksQuery.java new file mode 100644 index 0000000..734aba4 --- /dev/null +++ b/consumers/bricks/src/main/java/com/adaworldapi/bricks/BricksQuery.java @@ -0,0 +1,144 @@ +package com.adaworldapi.bricks; + +import com.adaworldapi.lancegraph.Predicate; +import com.adaworldapi.lancegraph.View; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * An immutable, lazy, fail-closed description of a set of order rows. + * + *

Security is a mask composed before execution, never a post-filter

+ * + *

{@link #where} and {@link #authorize(Role)} are the same kind of operation: both + * return a new {@code BricksQuery} wrapping a {@link View} narrowed by one more {@link Predicate}, + * via {@link View#where}. There is no separate enforcement pass here that fetches rows and then + * throws some away — authorization is a predicate in the identical lazy chain a caller's own {@code + * where(...)} conditions live in, fused into the same single native crossing a terminal operation + * pays for. See {@link Role} for how a role's constraint is itself a real, natively-evaluated + * predicate rather than a Java-side branch. + * + *

Fail-closed

+ * + *

Every terminal operation ({@link #count()}, {@link #sum}, {@link #sumBy}) checks first whether + * {@link #authorize(Role)} was ever called on this chain, and throws {@link + * UnauthorizedQueryException} if not — before touching the native side at all. + * There is no default-allow path: a chain that never called {@code authorize} cannot be executed, + * full stop. + * + *

Aggregate-only egress

+ * + *

Every public method on this class returns exactly one of {@code BricksQuery}, {@code long}, or + * {@code Map}. No method here returns, nor could return, anything row-shaped — there + * is no public type in this consumer that represents a single order. A caller can learn how many + * rows matched, or a sum, or a per-group breakdown of sums; a caller cannot ever get a row, a field + * value from one row, or an iterator over rows. That is a structural fact about this package's + * public surface, not a convention someone could accidentally violate from outside it. + */ +public final class BricksQuery { + + private final View view; + private final boolean authorized; + + BricksQuery(View view, boolean authorized) { + this.view = view; + this.authorized = authorized; + } + + /** + * A new query narrowed by one more condition. + * + *

Crosses the membrane zero times, exactly as {@link View#where} does not. Does not affect + * authorization: a chain built entirely from {@code where(...)} calls is still unauthorized + * until {@link #authorize(Role)} is called. + */ + public BricksQuery where(Predicate predicate) { + return new BricksQuery(view.where(predicate), authorized); + } + + /** + * A new query, authorized under {@code role}. + * + *

If {@code role} carries an additional constraint (see {@link Role}), it is folded into the + * same lazy chain via {@link View#where} — a real predicate, composed once, evaluated natively + * alongside everything else. This is the single point in this package where a role's constraint + * enters the chain; there is no other place authorization happens. + */ + public BricksQuery authorize(Role role) { + java.util.Objects.requireNonNull(role, "role"); + View next = role.constraint().map(view::where).orElse(view); + return new BricksQuery(next, true); + } + + /** + * How many rows this query selects. + * + * @throws UnauthorizedQueryException if {@link #authorize(Role)} was never called on this chain + */ + public long count() { + requireAuthorized("count()"); + return view.count(); + } + + /** + * Sum a signed 32-bit column over the rows this query selects. + * + * @throws UnauthorizedQueryException if {@link #authorize(Role)} was never called on this chain + */ + public long sum(com.adaworldapi.lancegraph.I32Field field) { + requireAuthorized("sum()"); + return view.sumOf(field); + } + + /** + * Sum {@code value} grouped by every possible value of {@code group}. + * + *

{@code group} is a {@link com.adaworldapi.lancegraph.U32Field}, and this consumer's fixture + * gives such fields exactly 16 distinct values ({@code 0..15}) — see {@link Orders#REGION}. This + * method issues one fused native query per group value (16 total), each narrowing this query's + * already-authorized chain by one more {@code group.eq(v)} condition and summing {@code value} + * over the result. Each of those 16 sums costs two native crossings — plan evaluation + * into the selection mask, then the {@code lgj_reduce_sum_i32} reduction — so the measured + * total is 32 crossings (unlike {@link #count()}, whose plan evaluation returns the count and + * pays one). The crossing count scales with the number of groups, never with the + * number of rows — the same laziness guarantee every other terminal operation in this + * codebase carries, just paid per group instead of once. + * + *

Every group value {@code 0..15} appears as a key in the returned map, including groups with + * zero matching rows (mapped to a sum of {@code 0L}): a group's absence from a real dataset is + * itself a legitimate aggregate fact, not something to hide by omitting the key. + * + *

If measurement ever shows this 32-crossing loop is a bottleneck, a native grouped-aggregate + * kernel (one crossing, sixteen output buckets) is the natural W6-tier follow-up — not built + * here, because nothing has measured a need for it yet. + * + * @throws UnauthorizedQueryException if {@link #authorize(Role)} was never called on this chain + */ + public Map sumBy( + com.adaworldapi.lancegraph.U32Field group, com.adaworldapi.lancegraph.I32Field value) { + requireAuthorized("sumBy()"); + java.util.Objects.requireNonNull(group, "group"); + java.util.Objects.requireNonNull(value, "value"); + Map result = new LinkedHashMap<>(16); + for (int v = 0; v < 16; v++) { + result.put(v, view.where(group.eq(v)).sumOf(value)); + } + return result; + } + + private void requireAuthorized(String what) { + if (!authorized) { + throw new UnauthorizedQueryException( + what + " was called on a query that was never authorize()'d. This is fail-closed" + + " by design: a missing role mask never falls back to emitting" + + " everything. Call .authorize(Role.GLOBAL) (or a narrower role) before" + + " any terminal operation, even when every row should be visible."); + } + } + + @Override + public String toString() { + return "BricksQuery[" + (authorized ? "authorized" : "UNAUTHORIZED") + ", " + view + "]"; + } +} diff --git a/consumers/bricks/src/main/java/com/adaworldapi/bricks/BricksSession.java b/consumers/bricks/src/main/java/com/adaworldapi/bricks/BricksSession.java new file mode 100644 index 0000000..4000c66 --- /dev/null +++ b/consumers/bricks/src/main/java/com/adaworldapi/bricks/BricksSession.java @@ -0,0 +1,53 @@ +package com.adaworldapi.bricks; + +import com.adaworldapi.lancegraph.NativePattern; + +/** + * A set of order rows held natively, opened once and closed once — the RBAC-poster sibling of + * {@code com.adaworldapi.trades.World}. + * + *

Owns exactly one {@link NativePattern}. {@link #query()} returns a fresh, unauthorized + * {@link BricksQuery} over that pattern's rows every time it is called — see {@link + * BricksQuery} for why every chain must call {@code authorize(...)} before any terminal operation, + * with no exceptions for {@link Role#GLOBAL}. + * + * 

{@code
+ * try (var session = Bricks.open(1_000_000, Bricks.DEFAULT_SEED)) {
+ *     long euRevenue = session.query()
+ *             .authorize(Role.EU_ONLY)
+ *             .sum(Orders.REVENUE);
+ * }
+ * }
+ */ +public final class BricksSession implements AutoCloseable { + + private final NativePattern pattern; + + BricksSession(NativePattern pattern) { + this.pattern = pattern; + } + + /** + * A fresh, unauthorized query over every row in this session. + * + *

Crosses the membrane zero times to build, exactly as {@link NativePattern#view()} does not. + * The returned query throws {@link UnauthorizedQueryException} from any terminal operation until + * {@link BricksQuery#authorize(Role)} is called on it. + */ + public BricksQuery query() { + return new BricksQuery(pattern.view(), false); + } + + /** + * Release the native storage. + * + *

Delegates directly to {@link NativePattern#close()} — same double-close semantics: a second + * call throws {@link com.adaworldapi.lancegraph.ClosedResourceException} rather than being a + * silent no-op, and every {@link BricksQuery} derived from this session's rows stops working + * once this returns. + */ + @Override + public void close() { + pattern.close(); + } +} diff --git a/consumers/bricks/src/main/java/com/adaworldapi/bricks/Orders.java b/consumers/bricks/src/main/java/com/adaworldapi/bricks/Orders.java new file mode 100644 index 0000000..3e79693 --- /dev/null +++ b/consumers/bricks/src/main/java/com/adaworldapi/bricks/Orders.java @@ -0,0 +1,80 @@ +package com.adaworldapi.bricks; + +import com.adaworldapi.lancegraph.Field; +import com.adaworldapi.lancegraph.I32Field; +import com.adaworldapi.lancegraph.LaneId; +import com.adaworldapi.lancegraph.Ordinal; +import com.adaworldapi.lancegraph.Pattern; +import com.adaworldapi.lancegraph.U32Field; + +import java.util.List; + +/** + * The domain schema for the OGAR-Bricks poster: security as a mask composed before + * execution, never a post-filter over already-fetched rows. + * + *

This is a schema, not an entity

+ * + *

There is no such thing as an {@code Orders} instance, on purpose — see the constructor. This + * class is written the same generated shape as {@link Pattern} and {@code + * com.adaworldapi.trades.Trade}: constants only, no logic, names taken from the schema, lane + * indices stated once. + * + *

Field mapping — honest about what the fixture actually offers

+ * + *

The generated fixture ({@link com.adaworldapi.lancegraph.NativePattern}) has exactly two + * data lanes beyond identity: an unsigned 32-bit class tag ({@code 0..15}) and a signed 32-bit + * value. This schema binds order-domain names onto those same two lanes — {@link #REGION} is the + * class lane, {@link #REVENUE} is the value lane — rather than inventing a lane the substrate does + * not have. + * + *

YEAR is deliberately absent

+ * + *

The poster's domain fiction implies a time dimension (orders by year, revenue + * trending). It is not fabricated here: the current flat two-lane fixture + * has no third numeric lane to bind it to. Exactly the same boundary {@code Trade} draws around its + * absent {@code QUANTITY} field — a multi-lane {@code ClassView} facet slice ({@code W6} in the + * substrate plan) is what a real {@code YEAR} lane would bind onto; until then, honesty about what + * exists beats completeness of the poster. + */ +public final class Orders { + + /** + * Never constructed. See {@code com.adaworldapi.trades.Trade}'s constructor for the same + * guarantee stated in full: this class is vocabulary, not a row template. + */ + private Orders() { + throw new AssertionError( + "an Orders instance is never materialized — orders are addressed as a native lane" + + " set through Bricks/BricksQuery, never hydrated one-by-one. There is no" + + " Orders instance to construct."); + } + + /** + * Region identifier — {@link Pattern#CLASS}'s lane ({@code lane 1}), read under a domain name. + * Fixture values are {@code 0..15}. + */ + public static final U32Field REGION = + new U32Field("region", LaneId.of(1), Ordinal.of(0)); + + /** + * Revenue — {@link Pattern#VALUE}'s lane ({@code lane 2}), read under a domain name. Signed, + * matching the fixture's underlying value lane; see {@link I32Field} for why only the + * comparisons the membrane implements are offered. + */ + public static final I32Field REVENUE = + new I32Field("revenue", LaneId.of(2), Ordinal.of(1)); + + /** + * The region id used by {@link Role#EU_ONLY} to build its authorization mask. One of the + * fixture's 16 distinct {@link #REGION} values. + */ + public static final int EU = 7; + + /** A second, distinct region id, for predicates that need to compare or exclude by region. */ + public static final int APAC = 3; + + /** Every field this schema defines, in schema order. See the class doc for why there is no + * {@code YEAR} entry yet. */ + public static final List FIELDS = List.of(REGION, REVENUE); +} diff --git a/consumers/bricks/src/main/java/com/adaworldapi/bricks/Role.java b/consumers/bricks/src/main/java/com/adaworldapi/bricks/Role.java new file mode 100644 index 0000000..b911380 --- /dev/null +++ b/consumers/bricks/src/main/java/com/adaworldapi/bricks/Role.java @@ -0,0 +1,65 @@ +package com.adaworldapi.bricks; + +import com.adaworldapi.lancegraph.Predicate; + +import java.util.Optional; + +/** + * An authorization constraint — the mask a {@link BricksQuery} composes into its chain before + * any terminal operation executes. + * + *

A mask, not an if

+ * + *

Every {@code Role} constant carries an optional additional {@link Predicate}. When present, + * {@link BricksQuery#authorize(Role)} folds it into the same lazy chain a caller's own {@code + * where(...)} conditions live in — via {@link com.adaworldapi.lancegraph.View#where}, the identical + * membrane-zero-crossings machinery. There is no branch in Java that decides "let this row through" + * or "drop it": the constraint is evaluated natively, fused with every other predicate, in the same + * single crossing the rest of the chain pays for. + * + *

{@link #DENY_ALL} makes this concrete rather than aspirational: it is not a + * Java-side short-circuit that skips execution. It is a real predicate ({@code + * Orders.REGION.eq(0xFFFF)}) that can never match any row, because {@link Orders#REGION} only ever + * holds fixture values {@code 0..15}. Denial is a mask that is unsatisfiable, evaluated the same way + * any other mask is evaluated — the fail-closed philosophy extends all the way down, not just to the + * "did you call authorize()" gate. + * + *

Extension point, stated honestly

+ * + *

These three constants are a stand-in for a real RBAC source. A future integration replaces them + * — same {@code Role} shape, same single composition point in {@link BricksQuery#authorize(Role)} — + * without touching any caller. Multi-value roles (a role permitting several regions, or intersecting + * several field-level constraints) arrive with mask-level composition; the underlying {@code Mask} + * algebra already exists in the core module ({@code com.adaworldapi.lancegraph.Mask}) but is not + * wired into this consumer. + */ +public final class Role { + + /** Authorized with no additional constraint: every row an unrestricted query would see. */ + public static final Role GLOBAL = new Role(Optional.empty()); + + /** Authorized, restricted to {@link Orders#EU} rows via a real, natively-evaluated predicate. */ + public static final Role EU_ONLY = new Role(Optional.of(Orders.REGION.eq(Orders.EU))); + + /** + * Authorized, restricted by a predicate that can never match any row ({@code + * Orders.REGION.eq(0xFFFF)} — {@link Orders#REGION} only ever holds {@code 0..15} in the + * fixture). See the class documentation: this is a real mask, evaluated natively, not a + * Java-side "return nothing" branch. + */ + public static final Role DENY_ALL = new Role(Optional.of(Orders.REGION.eq(0xFFFF))); + + private final Optional constraint; + + private Role(Optional constraint) { + this.constraint = constraint; + } + + /** + * The additional predicate this role imposes, if any. Package-private: composing it into a + * chain is {@link BricksQuery#authorize(Role)}'s job, not a caller's. + */ + Optional constraint() { + return constraint; + } +} diff --git a/consumers/bricks/src/main/java/com/adaworldapi/bricks/UnauthorizedQueryException.java b/consumers/bricks/src/main/java/com/adaworldapi/bricks/UnauthorizedQueryException.java new file mode 100644 index 0000000..5afeef6 --- /dev/null +++ b/consumers/bricks/src/main/java/com/adaworldapi/bricks/UnauthorizedQueryException.java @@ -0,0 +1,24 @@ +package com.adaworldapi.bricks; + +/** + * Thrown by a {@link BricksQuery} terminal operation ({@link BricksQuery#count()}, {@link + * BricksQuery#sum}, {@link BricksQuery#sumBy}) when {@link BricksQuery#authorize(Role)} was never + * called on that chain. + * + *

Fail-closed, not default-allow

+ * + *

This is the workspace's own RBAC doctrine, restated as a Java exception rather than left as a + * convention: "a missing role mask never falls back to emit everything." A {@link + * BricksQuery} that has not been authorized is not treated as globally readable — it is treated as + * a programming error, thrown before any native crossing happens. There is no default-allow path + * anywhere in this package; the only way past this exception is an explicit {@link + * BricksQuery#authorize(Role)} call, even if that role is {@link Role#GLOBAL}. + */ +public final class UnauthorizedQueryException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + UnauthorizedQueryException(String message) { + super(message); + } +} diff --git a/consumers/bricks/src/test/java/com/adaworldapi/bricks/BricksAuthTest.java b/consumers/bricks/src/test/java/com/adaworldapi/bricks/BricksAuthTest.java new file mode 100644 index 0000000..c204591 --- /dev/null +++ b/consumers/bricks/src/test/java/com/adaworldapi/bricks/BricksAuthTest.java @@ -0,0 +1,435 @@ +package com.adaworldapi.bricks; + +import com.adaworldapi.lancegraph.Checks; +import com.adaworldapi.lancegraph.Diagnostics; +import com.adaworldapi.lancegraph.NativeRuntime; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Map; + +/** + * The falsifier suite for the mask-first, fail-closed authorization consumer example. + * + *

Mirrors {@code TradesParityTest}'s shape (main + {@link #run}, a transcribed SplitMix64 + * generator with its own provenance comment, {@code Diagnostics.crossings()} as the laziness + * instrument) but proves a different, sharper thesis: a query never reads a row before it + * is authorized, and that "never" is checked three separate ways — by value (two roles + * see two different, correctly-related answers), by control flow (an unauthorized terminal throws + * before doing any work), and by crossing count (composing {@code .authorize(...)} costs the + * membrane nothing; only the terminal does, and its cost does not grow with role complexity). + * + *

    + *
  1. Two-sided role discrimination — {@code Role.GLOBAL}, {@code + * Role.EU_ONLY}, and {@code Role.DENY_ALL} produce three different, independently + * predicted counts over the identical base query, agreeing with a pure-Java recomputation + * that never touches {@code BricksQuery} or the native library. + *
  2. Fail-closed, both directions — a terminal without {@code authorize(...)} + * throws {@code UnauthorizedQueryException} (checked on all three terminals: {@code + * count()}, {@code sum()}, {@code sumBy()}); the identical chain with {@code authorize(...)} + * present does not throw and returns the predicted answer, in either composition order. + *
  3. Compose-not-execute — {@code Diagnostics.crossings()} does not move while + * {@code .where(...)}/{@code .authorize(...)} are composed, only when a terminal runs — the + * strongest checkable form of "nothing crosses until the fully-clamped plan does." + *
  4. {@code sumBy} parity — every one of its 16 returned per-region sums + * agrees with an independent pure-Java recomputation, and the 16 sums are internally + * consistent with the grand total. + *
  5. Aggregate-only egress — {@code BricksQuery}'s public surface returns only + * {@code BricksQuery}, {@code long}, or {@code Map}; {@code Orders} is a schema, never an + * entity, exactly as {@code Trade} is proven to be in {@code TradesParityTest}. + *
+ */ +public final class BricksAuthTest { + + private BricksAuthTest() {} + + public static void main(String[] args) { + System.out.println("BricksAuthTest"); + if (!NativeRuntime.isAvailable()) { + System.exit(Checks.reportUnavailable("BricksAuthTest")); + } + Checks c = new Checks("BricksAuthTest"); + run(c); + System.exit(c.report()); + } + + /** + * Same seed as {@code TradesParityTest} — chosen there to match {@code + * NativePattern.DEFAULT_SEED} (docs/abi.md's normative fixture seed), reused here so this + * file's numbers are cross-checkable against the rest of the corpus. + */ + private static final long SEED = 0xABCDL; + + /** + * SplitMix64, transcribed from the normative fixture generator. Provenance: identical + * constant-for-constant, shift-for-shift transcription already checked against the published + * reference vector in {@code FixtureParityTest} and re-verified in {@code TradesParityTest}. + * Duplicated (not shared) because the original is {@code private} to a different compile unit + * — see {@code TradesParityTest}'s own copy for the same reasoning, applied here to the Bricks + * consumer module. + */ + private static final class SplitMix64 { + private long state; + + SplitMix64(long seed) { + this.state = seed; // no warm-up draws + } + + long next() { + state += 0x9E3779B97F4A7C15L; + long z = state; + z = (z ^ (z >>> 30)) * 0xBF58476D1CE4E5B9L; + z = (z ^ (z >>> 27)) * 0x94D049BB133111EBL; + return z ^ (z >>> 31); + } + } + + /** + * {@code (region, revenue)} per row, recomputed independently in pure Java from the fixture + * generator's published shape: two SplitMix64 draws per row, the first then the second, with + * {@code region = (draw1 >>> 33) & 0xF} and {@code revenue = ((draw2 >>> 40) & 0x1FF) - 150} + * — {@code Orders.REGION} binds the same lane-1 U32 tag {@code Trade.VENUE} does, and {@code + * Orders.REVENUE} the same lane-2 I32 value {@code Trade.PRICE} does, so the derivation is + * identical, renamed to match this schema's domain. + * + *

Named {@code Fixture} rather than {@code Orders} deliberately: this file also needs the + * unqualified name {@code Orders} to resolve to the real schema class ({@code Orders.REGION}, + * {@code Orders.EU}, {@code Orders.REVENUE}) throughout, and a nested type sharing that simple + * name would shadow it within this compilation unit — the same reason {@code TradesParityTest} + * names its own recomputation record {@code Trades}, not {@code Trade}. + */ + private record Fixture(int[] regions, int[] revenues) {} + + private static Fixture generate(int nRows, long seed) { + SplitMix64 rng = new SplitMix64(seed); + int[] regions = new int[nRows]; + int[] revenues = new int[nRows]; + for (int i = 0; i < nRows; i++) { + long draw1 = rng.next(); // FIRST draw of the row + long draw2 = rng.next(); // SECOND draw of the row + regions[i] = (int) ((draw1 >>> 33) & 0xF); + revenues[i] = (int) (((draw2 >>> 40) & 0x1FF) - 150); + } + return new Fixture(regions, revenues); + } + + public static void run(Checks c) { + checkRoleDiscrimination(c); + checkFailClosed(c); + checkComposeNotExecute(c); + checkSumByParity(c); + checkAggregateOnlyEgress(c); + } + + // ------------------------------------------------------------------ + // 1. Two-sided role discrimination + // ------------------------------------------------------------------ + + private static void checkRoleDiscrimination(Checks c) { + c.section("two-sided role discrimination at n = 64,000"); + + final int rows = 64_000; + Fixture expected = generate(rows, SEED); + + long expectAll = 0; // (a): REVENUE > 100, no role constraint + long expectEu = 0; // (b): REVENUE > 100 AND REGION == EU + for (int i = 0; i < rows; i++) { + if (expected.revenues()[i] > 100) { + expectAll++; + if (expected.regions()[i] == Orders.EU) { + expectEu++; + } + } + } + + try (BricksSession session = Bricks.open(rows, SEED)) { + long actualGlobal = session.query() + .where(Orders.REVENUE.gt(100)) + .authorize(Role.GLOBAL) + .count(); + long actualEu = session.query() + .where(Orders.REVENUE.gt(100)) + .authorize(Role.EU_ONLY) + .count(); + long actualDeny = session.query() + .where(Orders.REVENUE.gt(100)) + .authorize(Role.DENY_ALL) + .count(); + + c.eq("Role.GLOBAL: REVENUE > 100, no constraint, vs pure-Java recomputation", + expectAll, actualGlobal); + c.eq("Role.EU_ONLY: REVENUE > 100 AND REGION == EU, vs pure-Java recomputation", + expectEu, actualEu); + c.eq("Role.DENY_ALL: an unmatchable native predicate selects exactly zero rows", + 0, actualDeny); + + c.that("EU count is not vacuous (count > 0)", expectEu > 0); + c.that("EU is a genuine subset, strictly fewer rows than GLOBAL (count < GLOBAL)", + expectEu < expectAll); + c.that("GLOBAL itself does not select every row (count < n)", expectAll < rows); + } + } + + // ------------------------------------------------------------------ + // 2. Fail-closed: can-fire (throws without authorize) and its silence twin + // (no throw, correct answer, with authorize) — both orders of composition. + // ------------------------------------------------------------------ + + private static void checkFailClosed(Checks c) { + c.section("fail-closed terminals — can-fire half (no authorize -> throws)"); + + final int rows = 64_000; + try (BricksSession session = Bricks.open(rows, SEED)) { + c.throwsUp("count() without authorize() throws UnauthorizedQueryException", + UnauthorizedQueryException.class, + () -> session.query().where(Orders.REVENUE.gt(100)).count()); + c.throwsUp("sum() without authorize() throws UnauthorizedQueryException", + UnauthorizedQueryException.class, + () -> session.query().where(Orders.REVENUE.gt(100)).sum(Orders.REVENUE)); + c.throwsUp("sumBy() without authorize() throws UnauthorizedQueryException", + UnauthorizedQueryException.class, + () -> session.query().where(Orders.REVENUE.gt(100)) + .sumBy(Orders.REGION, Orders.REVENUE)); + } + + c.section("fail-closed terminals — silence twin (authorize present -> no throw)"); + + Fixture expected = generate(rows, SEED); + long expectGlobalCount = 0; + for (int i = 0; i < rows; i++) { + if (expected.revenues()[i] > 100) { + expectGlobalCount++; + } + } + + try (BricksSession session = Bricks.open(rows, SEED)) { + long counted = session.query() + .where(Orders.REVENUE.gt(100)) + .authorize(Role.GLOBAL) + .count(); + c.eq("with authorize() present, count() does not throw and matches the recomputation", + expectGlobalCount, counted); + + // sum()/sumBy() must be reachable too, not merely count() — a design that special-cased + // only count()'s guard would pass the can-fire test above by accident. + long summed = session.query() + .where(Orders.REVENUE.gt(100)) + .authorize(Role.GLOBAL) + .sum(Orders.REVENUE); + c.that("sum() with authorize() present does not throw (reached this line)", true); + c.that("sum() with authorize() present returns a real, non-trivially-zero total", + summed != 0); + + Map grouped = session.query() + .where(Orders.REVENUE.gt(100)) + .authorize(Role.GLOBAL) + .sumBy(Orders.REGION, Orders.REVENUE); + c.that("sumBy() with authorize() present does not throw (reached this line)", true); + c.eq("sumBy() with authorize() present reports all 16 regions", 16, grouped.size()); + } + + c.section("authorize() composition order does not matter"); + + try (BricksSession session = Bricks.open(rows, SEED)) { + long whereThenAuthorize = session.query() + .where(Orders.REVENUE.gt(100)) + .authorize(Role.EU_ONLY) + .count(); + long authorizeThenWhere = session.query() + .authorize(Role.EU_ONLY) + .where(Orders.REVENUE.gt(100)) + .count(); + c.eq(".where(...).authorize(...) vs .authorize(...).where(...) — identical count", + whereThenAuthorize, authorizeThenWhere); + } + } + + // ------------------------------------------------------------------ + // 3. Compose-not-execute: crossings stay 0 through where()/authorize(), and the exact + // crossing cost of each terminal (count(): 1; sumBy(): 16, group-count-scaled, never + // row-count-scaled). + // ------------------------------------------------------------------ + + private static void checkComposeNotExecute(Checks c) { + c.section("compose-not-execute (Diagnostics.crossings())"); + + final int rows = 64_000; + try (BricksSession session = Bricks.open(rows, SEED)) { + long before = Diagnostics.crossings(); + BricksQuery chain = session.query() + .where(Orders.REVENUE.gt(100)) + .authorize(Role.EU_ONLY); + long afterBuild = Diagnostics.crossings(); + + c.eq("composing .where(...).authorize(...) costs no crossings — " + + "NOTHING crosses before the fully-clamped plan does", + 0, afterBuild - before); + + // Warm the resource's reusable scratch selection so the steady-state cost is what is + // measured, matching LazinessTest's own discipline. + chain.count(); + + long c0 = Diagnostics.crossings(); + chain.count(); + long terminalCost = Diagnostics.crossings() - c0; + c.eq("count() over an authorized chain costs exactly one crossing", + 1, terminalCost); + } + } + + private static void checkSumByCrossingCost(Checks c, int rows) { + try (BricksSession session = Bricks.open(rows, SEED)) { + BricksQuery chain = session.query() + .where(Orders.REVENUE.gt(100)) + .authorize(Role.GLOBAL); + + // Warm-up call, unmeasured — same discipline as count()'s crossing check above. + chain.sumBy(Orders.REGION, Orders.REVENUE); + + long c0 = Diagnostics.crossings(); + chain.sumBy(Orders.REGION, Orders.REVENUE); + long cost = Diagnostics.crossings() - c0; + + // A sum terminal is TWO crossings, not one: plan evaluation into the selection mask + // (lgj_plan_eval) plus the reduction itself (lgj_reduce_sum_i32) — unlike count(), + // whose plan evaluation RETURNS the count and so pays only one. 16 groups x 2 = 32. + // The thesis assertion is the arithmetic's SHAPE: crossings are proportional to the + // number of groups, never to the number of rows — which is why this same literal is + // asserted at BOTH n = 1000 and n = 64000. + c.eq("sumBy() at n = " + rows + " costs exactly 32 crossings" + + " (2 per region group — plan eval + reduce — never one per row)", + 32, cost); + } + } + + // ------------------------------------------------------------------ + // 4. sumBy() parity — value correctness plus internal consistency. + // ------------------------------------------------------------------ + + private static void checkSumByParity(Checks c) { + c.section("sumBy() parity"); + + checkSumByCrossingCost(c, 1_000); + checkSumByCrossingCost(c, 64_000); + + final int rows = 64_000; + Fixture expected = generate(rows, SEED); + + long[] expectedByRegion = new long[16]; + long expectedGrandTotal = 0; + for (int i = 0; i < rows; i++) { + if (expected.revenues()[i] > 100) { + int region = expected.regions()[i]; + expectedByRegion[region] += expected.revenues()[i]; + expectedGrandTotal += expected.revenues()[i]; + } + } + + try (BricksSession session = Bricks.open(rows, SEED)) { + Map actual = session.query() + .where(Orders.REVENUE.gt(100)) + .authorize(Role.GLOBAL) + .sumBy(Orders.REGION, Orders.REVENUE); + + c.eq("sumBy() returns exactly 16 region entries", 16, actual.size()); + + long actualGrandTotal = 0; + for (int region = 0; region < 16; region++) { + Long value = actual.get(region); + c.that("sumBy() has an entry for region " + region, value != null); + long v = value == null ? Long.MIN_VALUE : value; + c.eq("region " + region + " sum matches pure-Java recomputation", + expectedByRegion[region], v); + actualGrandTotal += v; + } + + c.eq("sum of all 16 region sums equals the recomputed grand total" + + " (internal consistency, independent of the per-region parity above)", + expectedGrandTotal, actualGrandTotal); + + long actualDirectSum = session.query() + .where(Orders.REVENUE.gt(100)) + .authorize(Role.GLOBAL) + .sum(Orders.REVENUE); + c.eq("sum(REVENUE) under the identical predicate agrees with the sumBy() grand total", + actualDirectSum, actualGrandTotal); + } + } + + // ------------------------------------------------------------------ + // 5. Aggregate-only egress: BricksQuery's return-type surface, and Orders as a schema. + // ------------------------------------------------------------------ + + private static void checkAggregateOnlyEgress(Checks c) { + c.section("aggregate-only egress — BricksQuery's public return types"); + + for (Method m : BricksQuery.class.getMethods()) { + if (m.getDeclaringClass() == Object.class) { + continue; // wait()/notify()/getClass()/... are not this class's surface + } + // BricksQuery OVERRIDES toString() (its debug rendering), so getDeclaringClass() no + // longer reports Object for it — but an Object-defined contract method is still not + // part of the aggregate-egress surface being audited here, whichever class's body + // happens to serve it. Exclude the overridable Object trio by NAME. + String name = m.getName(); + if ((name.equals("toString") || name.equals("equals") || name.equals("hashCode")) + && switch (name) { + case "toString", "hashCode" -> m.getParameterCount() == 0; + default -> m.getParameterCount() == 1 + && m.getParameterTypes()[0] == Object.class; + }) { + continue; + } + Class ret = m.getReturnType(); + boolean aggregateOnly = + ret == BricksQuery.class || ret == long.class || ret == Map.class; + c.that("BricksQuery." + m.getName() + "(...) returns BricksQuery, long, or Map" + + " (was " + ret.getSimpleName() + ") — no row-shaped public type exists", + aggregateOnly); + } + + c.section("Orders is a schema, not an entity — the reflection guard"); + + c.eq("Orders declares zero public constructors", + 0, publicConstructorCount(Orders.class)); + c.eq("Orders declares zero non-static instance fields", + 0, instanceFieldCount(Orders.class)); + + // AssertionError specifically, matching TradesParityTest's guard on Trade — a plain + // "some exception" check would let a typo (e.g. a stray NullPointerException) pass for + // the wrong reason. + c.throwsUp("forcing the private constructor accessible still cannot build an Orders", + AssertionError.class, () -> { + try { + var ctor = Orders.class.getDeclaredConstructor(); + ctor.setAccessible(true); + ctor.newInstance(); + } catch (java.lang.reflect.InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException re) { + throw re; + } + if (cause instanceof Error err) { + throw err; + } + throw new RuntimeException(cause); + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + }); + } + + private static int publicConstructorCount(Class type) { + return type.getConstructors().length; + } + + private static int instanceFieldCount(Class type) { + int n = 0; + for (var field : type.getDeclaredFields()) { + if (!Modifier.isStatic(field.getModifiers())) { + n++; + } + } + return n; + } +} From 239edf0cf3e972effafe2ea5e04ae714ccfc303e Mon Sep 17 00:00:00 2001 From: Jan Date: Mon, 17 Aug 2026 23:48:08 +0000 Subject: [PATCH 24/30] Board: PR #12 arc entry (post-merge) --- .claude/board/PR_ARC_INVENTORY.md | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index be70d07..960e7f8 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -8,6 +8,40 @@ > anti-pattern the imported board rules name. Backfilled below in one > pass rather than left stale; PR #4 onward gets its entry at merge time. +## PR #12 — consumer example: Bricks, mask-first authorization (merged 2026-08-17, squash `8f16e8d`) + +- **Added:** `consumers/bricks/` — the second consumer proof over the + unchanged core. `Bricks`/`BricksSession`/`BricksQuery`/`Role`/`Orders`/ + `UnauthorizedQueryException`; `BricksAuthTest` 62/62. Wave + `wave-consumer-bricks.md`, 2 Sonnet workers (K1 main / K2 test, + disjoint), orchestrator-gated. +- **Locked:** authorization is a **predicate in the same lazy chain** as + `where(...)`, composed before execution and fused into the same single + crossing — not a post-filter. `Role.EU_ONLY` = `REGION.eq(EU)`; + `DENY_ALL` = `REGION.eq(0xFFFF)`, a real impossible predicate that pays + a real crossing and counts 0; fail-closed throws BEFORE any crossing + (no default-allow path exists); aggregate-only egress is **structural** + (`BricksQuery`/`long`/`Map` are the only public return types — asserted + by reflection, so no row-shaped type can be added without breaking the + test). Disable-run: `requireAuthorized` short-circuited → exactly the + 3 can-fire fail-closed checks red, 59 green; restored 62/62. +- **A measured correction, recorded rather than smoothed over:** a **sum + terminal costs 2 crossings** (plan eval into the mask + + `lgj_reduce_sum_i32`), unlike `count()`'s 1 — so `sumBy()` is **32 + crossings (16 groups × 2), IDENTICAL at 1K and 64K rows**. K1's Javadoc + claimed one crossing per group; the measurement corrected the doc. The + thesis assertion is the shape (crossings ∝ groups, never rows), which + is why the same literal is pinned at both row counts. +- **Deferred:** a native grouped-aggregate kernel (one crossing, 16 + buckets) — named in the Javadoc as the W6-tier follow-up, deliberately + not built because nothing has measured a need; W5c graph (shelved on + the D1 ruling + the edge-bearing generator substrate change). +- **Docs:** STATUS_BOARD D-LGJ-W5 → bricks DONE; LATEST_STATE dispatch-4 + entry, which also owns that W5a (#11) shipped without one. +- **Confidence:** High — zero new membrane surface, zero core changes, + core suite unaffected at 188/188; every falsifier disable-verified. + Both bot reviewers at usage limits (no human or bot review obtained). + ## PR #11 — consumer example: World/Trades (merged 2026-08-17, squash `db7bdf1`) - **Added:** `consumers/trades/` — own compile unit, core consumed as a From 9ee044f59dc4f43d08518786805d3107882ac638 Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 18 Aug 2026 02:11:46 +0000 Subject: [PATCH 25/30] =?UTF-8?q?Board:=20record=20the=20C-band=20ruling?= =?UTF-8?q?=20=E2=80=94=20the=20classid=20domain=20byte=20carries=20ALTITU?= =?UTF-8?q?DE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator ruling, 2026-08-18: "Java is an entire different layer that's why I chose another higher level." The classid domain byte is stratified by layer, not a flat namespace where placement is mnemonic or next-free. The C-band is the stratum above the Rust substrate: C0 Java/Panama/Valhalla (the membrane, and the FLOOR of that layer), C1 ogar-bricks + Databricks (the analyst estate), C4 Ghidra (a tenant of C0's layer -- Ghidra is itself a JVM application per this repo's own G0 archaeology -- and explosive, for the blast radius of turning any binary into addressable rows). The entry also records, as storno, three of my own proposals the ruling corrects: seating P-code at 0x1718 as an ogar-loco consumer slot (wrong tier -- 0x17 is lance-graph's internal orchestration: elixir-on-rails, rs-graph-llm, Rig marking the replayability boundary), putting P-code at 0x18 beside Blocks (same error one slot over), and proposing a separate substrate/layout-contract domain (not separate -- it is C0's content). Root cause, which recurred three times in one session: clustering by SHAPE (everything becomes (function : value) calls in a 512-byte node) when the real axis is ALTITUDE. Shape-similarity is not domain-identity. What survives: reuse loco's node shape, own your own domain -- loco's own doc says the FunctionBody classid belongs at the substrate and a frontend references it rather than minting its own. Borrowing the container is not joining the domain. One consequence for code here: W6's schema/classid field on LgjResourceInfo/LgjLaneDesc carries a C0 concept; the substrate plan's W6 line now says so. Nothing on the wave list is blocked -- the reservation is OGAR-side and operator-gated. --- .claude/board/EPIPHANIES.md | 92 +++++++++++++++++++++++++++ .claude/board/LATEST_STATE.md | 21 ++++++ .claude/plans/lgj-soa-substrate-v1.md | 2 +- 3 files changed, 114 insertions(+), 1 deletion(-) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index bd91ae7..d6244f8 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -4,6 +4,98 @@ > `**Status:**`/`**Confidence:**` line. A correction gets its own new, > dated entry that references the one it corrects — the storno rule. +## 2026-08-18 — E-LGJ-THE-DOMAIN-BYTE-CARRIES-ALTITUDE-1 + +**Status:** RULING (operator, 2026-08-18: *"Java is an entire different layer +that's why I chose another higher level"*). **Confidence:** High for the +ruling; the reservation itself is OGAR-side and NOT yet made. + +### The ruling + +The classid's domain byte (`0xDDCC`'s `DD`, canon hi u16) is **stratified by +altitude**, not a flat namespace where placement is mnemonic or next-free. +Numerically higher = architecturally higher layer. The **C-band is the layer +above the Rust substrate**, and within it: + +| slot | owner | why there | +|---|---|---| +| **C0** | **Java · Panama · Valhalla** | the supraconductor membrane over the SoA substrate — the FLOOR of that layer, the door everything else in it arrives through | +| **C1** | **ogar-bricks + Databricks** | the analyst estate | +| **C4** | **Ghidra** | bolted onto C0 (Ghidra *is* a JVM application — this repo's own G0 archaeology: fork at 12.2 DEV, minimum Java 25, Gradle ≥ 9.1), and explosive — C4 the plastic explosive, for the blast radius of turning any binary into addressable rows | + +C4 is a **tenant** of the layer C0 floors, not a peer of C0. That internal +ordering is part of the ruling, not decoration. + +### Why the axis is structurally sound (not just mnemonic) + +The domain byte is the first two nibbles of the classid, so its **top nibble +is a 16-way altitude selector**: one mask separates "substrate ontology" from +"host layer" with zero lookup and zero value decode — the canon's *the key +prerenders nodes with zero value decode*, applied to layering. A first-nibble +split is the most expensive split available in the 16-ary cascade; spending it +on **altitude** is what makes it worth spending. + +### What this corrects (storno — three of my own proposals, all wrong) + +I mapped by **subject matter** ("P-code is an opcode vocabulary, Blocks is an +opcode vocabulary, therefore adjacent") when the actual axis is **altitude**. +Withdrawn, in order: + +1. **"Seat P-code at `0x1718` as a loco consumer slot."** Wrong tier. `0x17` + is ogar-loco = **lance-graph's own internal orchestration** (elixir-on-rails + shaped, rs-graph-llm as the graph executor, Rig marking the replayability + boundary between external LLM and internal low-code). It is a tier with a + job, not a container for any palette whose ops fit in a byte. The `0x1717`+ + consumer slots are frontends *of that orchestration*. +2. **"Put P-code at `0x18`, next to Blocks."** Same error, one slot over. +3. **"A separate substrate/layout-contract domain"** as my third pick. Not a + separate thing — it is **C0's content**. See the consequence below. + +Root cause, worth keeping because it recurred three times in one session: I +flattened distinct motifs into one family because they share a **shape** +(everything becomes `(function : value)` calls in a 512-byte node). Loco's ABI +being *reusable* does not make `0x17` a parking lot. **Shape-similarity is not +domain-identity** — the dilution failure this workspace names by name. + +### What survives, and is the useful half + +**Reuse loco's node shape; own your own domain.** Loco says it itself — the +classid naming *a function body* "belongs here, at the substrate, and a +frontend references it rather than minting its own" (`ogar-loco` module doc, +`LocoConcept::FunctionBody` = `0x1701`). So a C4 P-code body can BE a loco +`FunctionBody` while every P-code concept lives in C4; likewise a C1 pipeline. +**Borrowing the container is not joining the domain.** + +Also surviving, unchanged: the **two registers** point. An *op vocabulary* +(palette bytes) and an *artifact ontology* (concept ids) are different +registers. Ghidra: P-code ops vs function/section/symbol. Databricks: pipeline +verbs vs catalog/table/column/type — and the latter already has a real seam +here, `lance-graph-catalog/src/unity_catalog.rs`, with Delta as a table reader. + +### The one consequence for code in THIS repo + +**W6's schema/classid field on `LgjResourceInfo`/`LgjLaneDesc` carries a C0 +concept.** A row store stamping which layout contract its bytes obey is the +membrane naming itself from inside its own layer — not a substrate concept +borrowed downward. Nothing else on the current wave list depends on the +allocation, so W5c/W6 are unblocked either way. + +### Open, and NOT ours to close + +The reservation is an **OGAR-side, operator-gated** act (`ogar-vocab`'s +`ConceptDomain` + the §2 allocation table; minting is gated on the 5+3 pass, +while *reserving* explicitly "costs nothing"). Two mechanical notes for +whoever makes it: + +- **C2/C3 fall between C1 and C4.** Blocks set the precedent that a deliberate + gap gets a **pinned test asserting it stays `Unassigned`** (`ogar-vocab` + `lib.rs:5624-5644`, guarding the `0x10`–`0x16` gap) so a later pass cannot + "tidy" a domain downward into it. The C-band wants the same three lines. +- **`0xC0` is a digit-swap of `0x0C` Automation** (`0xC001_0000` vs + `0x0C01_0000`). Raised once, not decisive, recorded so it is not + re-discovered as if new. + + ## 2026-08-17 — E-LGJ-WAVE-DISPATCH-VALIDATED-1 **Status:** FINDING (first real dispatch of the wave system). **Confidence:** diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 2248878..bd31b95 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,24 @@ +## 2026-08-18 — C-band ruling recorded: the domain byte carries ALTITUDE + +Operator ruling (*"Java is an entire different layer that's why I chose +another higher level"*): the classid domain byte is **stratified by layer**, +not a flat namespace. The C-band is the stratum ABOVE the Rust substrate — +**C0** Java/Panama/Valhalla (the membrane, and the FLOOR of that layer), +**C1** ogar-bricks + Databricks (the analyst estate), **C4** Ghidra (a tenant +of C0's layer — Ghidra is itself a JVM application per this repo's G0 +archaeology — and explosive, for the blast radius of turning any binary into +addressable rows). + +Full entry, including the three of my own proposals it corrects and the +root-cause (I clustered by SHAPE — everything becomes `(function : value)` +calls in a 512-byte node — where the real axis is ALTITUDE): +`EPIPHANIES.md` `E-LGJ-THE-DOMAIN-BYTE-CARRIES-ALTITUDE-1`. + +**Consequence for this repo, and it is the only one:** W6's schema/classid +field on `LgjResourceInfo`/`LgjLaneDesc` carries a **C0** concept. Nothing on +the current wave list is blocked by the allocation — the reservation is +OGAR-side and operator-gated (reserving costs nothing; minting is 5+3-gated). + ## 2026-08-17 (dispatch 4) — W5b bricks shipped: authorization IS a mask, measured `wave-consumer-bricks.md` executed (2 Sonnet workers K1/K2, disjoint main/test diff --git a/.claude/plans/lgj-soa-substrate-v1.md b/.claude/plans/lgj-soa-substrate-v1.md index ee2e094..db53055 100644 --- a/.claude/plans/lgj-soa-substrate-v1.md +++ b/.claude/plans/lgj-soa-substrate-v1.md @@ -32,7 +32,7 @@ | **W3** | Java `RowStore` facade: no FFM in public signatures; structured `MemoryLayout` (`sequence(32, struct(u32 classid, 12B payload))`); minor-≥2 gate; `FacetMatchView` zero-copy accessor over a Java-arena segment; `RowStoreParityTest` transcribing the generator | OPEN — next | | **W4** | Bench Component F: Java Vector API per-row facet scan (one `IntVector` 16-lane chunk = 4 facets, same algorithm as the Rust kernel) vs `lgj_row_facet_match` crossing vs scalar VarHandle walk — the "where does execution belong" question re-asked on the REAL layout | OPEN | | **W5** | The three consumer examples (own plan files, below) | PLANNED | -| **W6** *(named, not scheduled)* | ClassView wiring — and with it an explicit **schema/classid field** on `LgjResourceInfo`/`LgjLaneDesc` (additive, one minor bump), so a resource names WHICH layout contract its bytes obey instead of implying it via `kind`. Provenance + rationale: `.claude/knowledge/prior-art-and-the-layout-bridge-claim.md` §3 ("one key, many projections" made literal at the membrane). Also the `align(64)` base guarantee (real `NodeRow`) and, only if measurement asks, fused plans over facet lanes | NAMED | +| **W6** *(named, not scheduled)* | ClassView wiring — and with it an explicit **schema/classid field** on `LgjResourceInfo`/`LgjLaneDesc` (additive, one minor bump), so a resource names WHICH layout contract its bytes obey instead of implying it via `kind`. Provenance + rationale: `.claude/knowledge/prior-art-and-the-layout-bridge-claim.md` §3 ("one key, many projections" made literal at the membrane). **The classid that field carries is a C0 concept** — the Java/Panama/Valhalla layer naming itself from inside its own stratum, not a substrate concept borrowed downward; see `.claude/board/EPIPHANIES.md` `E-LGJ-THE-DOMAIN-BYTE-CARRIES-ALTITUDE-1` (the reservation is OGAR-side and operator-gated, and W6 is not blocked on it). Also the `align(64)` base guarantee (real `NodeRow`) and, only if measurement asks, fused plans over facet lanes | NAMED | Wave rule (house style): one wave = one reviewable PR; gates run centrally (orchestrator only — agents never run cargo); every safety property lands From e736d6a7a07b2f66096d41e01f30e97789efba4f Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 18 Aug 2026 05:43:01 +0000 Subject: [PATCH 26/30] Board: reconcile Ghidra G1/G2 -- superseded by ruff_r2il, not built Checked what "the other session writing the autoadapting drill-down proposer" actually unblocks here, against the merged PR rather than the summary. AdaWorldAPI/ruff PR #94 shipped crates/ruff_r2il -- a typed intake arm (ore/furnace/slag) reading r2sleigh's R2IL/SSA directly, in-process, ~43s, no JVM roundtrip. Its residual ledger is deliberately left non-empty (B3's own falsifier makes residual == 0 a KILL) for a follow-on pass. That follow-on IS the drill-down proposer: PR2 in the R2IL plan's own wave ladder, reading ResidualLedger::by_address and proposing finer convention rows at each address, converging pass over pass. Not landed yet -- gated on PR1's corpus numbers. PR3 (the classid mint in lance-graph-contract::ogar_codebook, item O5) is gated on PR2. So there is nothing new to consume here today. What there is: ghidra-integration-v1.md's G1 (a bespoke analyzeHeadless lift script) and G2 (a hand-rolled LE image format) are superseded, not merely lower-priority -- the R2IL plan's own stop condition already answers the question those waves existed to answer ("direct r2il/r2ssa consumption solves the upstream seam -- YES, 43s"). Marked superseded in place per that plan's own HANDOFF BOUNDARY note, which asked for exactly this reconciliation once R2IL landed. wave-ogar-machine-pm1.md's gate #3 repointed from "Ghidra G1+G2 merged" to "ruff_r2il PR2+PR3 merged" so the next session checking the gate finds the real dependency instead of a dead one. A separate, independently-found gap flagged (not fixed): lance-graph's ogar_codebook wire-mirror of OGAR's ConceptDomain is already missing Ontology and Blocks (pre-existing drift, not caused here) and will also lack the new C0/C1/C4 domains once PR3 needs to route on them. No code changed. The C-band ruling (OGAR PR #276) is unaffected -- its 0xC4 BinaryLifting fence ("Ghidra and r2sleigh are two consumers of the same SLEIGH specs over ONE vocabulary") is now literally true in code rather than anticipated, since ruff_r2il path-deps r2sleigh directly. Full record: .claude/board/EPIPHANIES.md E-LGJ-GHIDRA-G1-G2-SUPERSEDED-BY-R2IL-1. --- .claude/board/EPIPHANIES.md | 67 ++++++++++++++++++++++++++ .claude/board/LATEST_STATE.md | 28 +++++++++++ .claude/waves/wave-ghidra-g1-g2.md | 19 +++++++- .claude/waves/wave-ogar-machine-pm1.md | 14 ++++-- 4 files changed, 123 insertions(+), 5 deletions(-) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index d6244f8..472bc93 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -4,6 +4,73 @@ > `**Status:**`/`**Confidence:**` line. A correction gets its own new, > dated entry that references the one it corrects — the storno rule. +## 2026-08-18 (later) — E-LGJ-GHIDRA-G1-G2-SUPERSEDED-BY-R2IL-1 + +**Status:** FINDING (reconciliation, per `ghidra-integration-v1.md`'s own +HANDOFF BOUNDARY note: *"the receiving session should reconcile the handoff +against this plan's G-waves — they may supersede G1's lift-path decision +entirely — rather than running both designs in parallel."*). **Confidence:** +High — read against the merged PR, not inferred from the operator's summary. + +### What landed elsewhere, verified directly + +`AdaWorldAPI/ruff` PR #94 (merged, `10fab88`'s ancestor) shipped +`crates/ruff_r2il`: a typed intake arm reading r2sleigh's R2IL/SSA directly +(`../../../r2sleigh/crates/{r2il,r2ssa}`, in-process, ~43s) into +`ore → furnace → slag`. `dropped == 0` by construction +(`harvested = classified + residual`); slag is a **named, addressed** +residual ledger (`ResidualLedger::by_address`), not a catch-all — and B3's +own falsifier makes `residual == 0` a **KILL**, meaning the ladder was +deliberately left unfinished for a follow-on pass. That follow-on — +reading `by_address` and proposing finer `ConventionRow`s at each address, +re-running, converging pass over pass — is the "drill-down proposer" the +operator flagged as in progress in another session. It is PR2 in the R2IL +plan's own wave ladder (`.claude/plans/r2il-behavioral-ir-v1.md`), gated on +PR1's corpus numbers; PR3 (the classid mint for the R2IL container concept +in `lance-graph-contract::ogar_codebook`, item O5) is gated on PR2 proving +the route set. **Neither PR2 nor PR3 has landed as of this entry.** + +### The reconciliation + +`ghidra-integration-v1.md`'s G1 (an `analyzeHeadless` post-script dumping a +bespoke P-code text form) and G2 (a hand-rolled versioned LE image format + +Rust loader) are **superseded**, not merely lower-priority. The R2IL plan's +own stop condition already answers the question G1/G2 existed to answer: +*"§22.1: direct r2il/r2ssa consumption solves the upstream seam — YES +(43s)."* Dispatching `wave-ghidra-g1-g2.md` now would build a second, +throwaway lift path and a second, competing image format next to one that +is already merged, typed, and further along. `wave-ghidra-g1-g2.md` marked +superseded in place (kept for G0's real archaeology — the 74-opcode count, +the `PcodeEmulator` oracle precedent — which is still true and reusable, +just not via a Ghidra-side script). `wave-ogar-machine-pm1.md`'s gate #3 +repointed from "Ghidra G1+G2 merged" to "ruff_r2il PR2+PR3 merged," so the +next session checking that gate finds the real dependency instead of a +dead one. + +### Consequence for the C-band ruling + +None to the reservation itself — `E-LGJ-THE-DOMAIN-BYTE-CARRIES-ALTITUDE-1`'s +`0xC4 BinaryLifting` fence ("Ghidra and r2sleigh are two consumers of the +same SLEIGH specs over ONE vocabulary") is now literally true in code, not +just anticipated: `ruff_r2il` path-deps r2sleigh's SLEIGH-driven crates +directly. PR3's classid mint, when it lands, is the first real tenant of +that slot. + +### A separate, independently-found gap — flagged, not fixed here + +`lance-graph-contract::ogar_codebook` documents itself as a **wire-compatible +mirror** of OGAR `ogar-vocab::ConceptDomain` under an explicit drift guard +("if OGAR's CODEBOOK ever moves an id, BOTH sides must update together"). +Read directly against OGAR post-PR-#276: the mirror is missing `Ontology` +(`0x03`, present in OGAR before this session) and `Blocks` (`0x17`, added +2026-08-04) — **pre-existing drift, not caused by this session's C-band PR** +— and will also lack `JavaRuntime`/`Analytics`/`BinaryLifting` (`0xC0`/ +`0xC1`/`0xC4`) once PR3 needs to route on them. Out of scope to fix +speculatively from here (lance-graph-contract is on this branch but not +under active work this session, and the mirror's own convention is to catch +up via its parity tests, not via an unprompted sync); recorded so PR3 does +not silently trip on it. + ## 2026-08-18 — E-LGJ-THE-DOMAIN-BYTE-CARRIES-ALTITUDE-1 **Status:** RULING (operator, 2026-08-18: *"Java is an entire different layer diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index bd31b95..fc0c680 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,31 @@ +## 2026-08-18 (later) — Ghidra G1/G2 waves reconciled: superseded by ruff_r2il, not built + +Checked what "the other session writing the autoadapting drill-down proposer" +(ruff/r2sleigh) actually unblocks here, against the merged PR rather than the +summary. `AdaWorldAPI/ruff` PR #94 shipped `crates/ruff_r2il` — a typed +intake arm (ore/furnace/slag) reading r2sleigh's R2IL/SSA directly, with an +addressed residual ledger deliberately left non-empty for a follow-on pass. +That follow-on IS the drill-down proposer: PR2 in the R2IL plan's own wave +ladder, reading `ResidualLedger::by_address` and proposing finer convention +rows, converging pass over pass. **Not landed yet** — gated on PR1's corpus +numbers. PR3 (the classid mint in `lance-graph-contract::ogar_codebook`, +item O5) is gated on PR2. So there is nothing new to CONSUME here today. + +What there IS: `wave-ghidra-g1-g2.md` (a bespoke `analyzeHeadless` lift +script + a hand-rolled LE image format) is now superseded, not merely +lower-priority — the R2IL plan's own stop condition already answers the +question those waves existed to answer ("direct r2il/r2ssa consumption +solves the upstream seam — YES, 43s"). Marked superseded in place; +`wave-ogar-machine-pm1.md`'s gate #3 repointed from "Ghidra G1+G2 merged" to +"ruff_r2il PR2+PR3 merged" so the real dependency is visible instead of a +dead one. Full record + a separately-found, pre-existing `ogar_codebook` +mirror-drift gap (flagged, not fixed): `EPIPHANIES.md` +`E-LGJ-GHIDRA-G1-G2-SUPERSEDED-BY-R2IL-1`. + +No code changed; C-band ruling (`E-LGJ-THE-DOMAIN-BYTE-CARRIES-ALTITUDE-1`, +merged as OGAR PR #276) is unaffected — its `0xC4` fence is now literally +true in code rather than anticipated. + ## 2026-08-18 — C-band ruling recorded: the domain byte carries ALTITUDE Operator ruling (*"Java is an entire different layer that's why I chose diff --git a/.claude/waves/wave-ghidra-g1-g2.md b/.claude/waves/wave-ghidra-g1-g2.md index 588fa39..70b3949 100644 --- a/.claude/waves/wave-ghidra-g1-g2.md +++ b/.claude/waves/wave-ghidra-g1-g2.md @@ -1,11 +1,26 @@ # Wave: Ghidra G1 (lift proof) + G2 (program image format) -> Executes `ghidra-integration-v1.md` G1 then G2. **DO NOT DISPATCH** — +> **SUPERSEDED — 2026-08-18, do not dispatch under any circumstance.** +> `AdaWorldAPI/ruff` PR #94 (`crates/ruff_r2il`) merged a typed intake arm +> that consumes r2sleigh's R2IL/SSA DIRECTLY — in-process, ~43s, no JVM +> roundtrip, no bespoke text-dump format to invent and parse. That plan's +> own stop condition already closed the question this wave existed to +> answer: *"§22.1: direct r2il/r2ssa consumption solves the upstream seam +> — YES."* Building G1 (an `analyzeHeadless` post-script emitting a custom +> P-code text dump) or G2 (a second, bespoke LE image format) now would be +> a competing, throwaway lift path duplicating work already merged and +> typed. Full reconciliation record: +> `.claude/board/EPIPHANIES.md` `E-LGJ-GHIDRA-G1-G2-SUPERSEDED-BY-R2IL-1`. +> This file is kept for its archaeology (G0's real 74-opcode/`PcodeEmulator` +> findings still stand as reference) — never dispatch the two waves below. + +> ~~Executes `ghidra-integration-v1.md` G1 then G2. **DO NOT DISPATCH** — > shelved with the consumer waves per the operator's calcify-first ruling; > G1 is technically independent of W3 but momentum stays on the substrate > waves until called. Cross-repo: G1's script lands in `AdaWorldAPI/ghidra` > (cloned at `/workspace/ghidra`, 12.2 DEV, Java 25+); G2's loader lands -> here. One PR per repo; ghidra PR merges first. +> here. One PR per repo; ghidra PR merges first.~~ *(pre-supersession text, +> kept for the record.)* --- diff --git a/.claude/waves/wave-ogar-machine-pm1.md b/.claude/waves/wave-ogar-machine-pm1.md index 1d6834f..e807965 100644 --- a/.claude/waves/wave-ogar-machine-pm1.md +++ b/.claude/waves/wave-ogar-machine-pm1.md @@ -7,9 +7,17 @@ > 1. W3 merged (Java facade — the consumer workflow exists). > 2. At least ONE W5 consumer example shipped (the > consumer-never-grows-the-membrane rule has a proven round-trip). -> 3. Ghidra G1+G2 merged (a real program image format exists — P-M1 must -> consume the REAL format, not a probe-local sketch that would become -> a second format). +> 3. **[RECONCILED 2026-08-18 — was "Ghidra G1+G2 merged"]** `ruff_r2il` +> PR2 (route→V3 projection + codebook wiring, `AdaWorldAPI/ruff`) AND +> PR3 (the classid mint for the R2IL container concept in +> `lance-graph-contract::ogar_codebook`, O5) merged — a real, +> already-typed program image exists on THAT path. `wave-ghidra-g1-g2.md` +> is superseded and will never satisfy this gate; do not dispatch it to +> try. PR1 (ore/furnace/slag intake + the addressed residual ledger) is +> already merged; PR2 is explicitly gated on PR1's corpus numbers and is +> the in-flight "drill-down proposer" work — check +> `AdaWorldAPI/ruff/.claude/plans/r2il-behavioral-ir-v1.md`'s Wave plan +> section before re-deriving this gate's status. > 4. An explicit operator go (this direction is exploratory by ruling). ## What P-M1 is (and is deliberately not) From f68335bb4d5b96600d90c56a2d992fdf6248a36d Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 18 Aug 2026 05:55:46 +0000 Subject: [PATCH 27/30] Board: ruff #96 is a different arm; read the real staging guide + ran S1 Checked "ruff 96 merged" against the actual PR. It's ruff_python_spo's plain-Python residual ledger (dismech/CURIE-constant harvest, ontology- shaped -- MONDO/KISAO/infores prefixes) -- a sibling drill-loop, but a DIFFERENT crate and DIFFERENT consumer than ruff_r2il, and unrelated to this repo's C-band/Ghidra/JavaRuntime track. Recorded so the two arms aren't confused later just because they share vocabulary ("residual ledger", "drill loop", "proposer"). The genuinely relevant find, unrelated to #96, was already on ruff main: .claude/harvest/r2il/STAGED-CODEGEN-GUIDE.md (commit bbaebda, pushed directly to main between PR #94 and #95), explicitly addressed to "the sibling session ... (the Ghidra console work)" -- this repo, by description. It confirms PR2 (routes -> V3) still hasn't landed and adds what the prior reconciliation lacked: a 5-stage staging order (S1 ledger- read -> S2 ore-join -> S3 additive codegen -> S4 one consumer -> S5 target-profile fork), explicit "do not skip to S3", and a stability table per artifact (FlatFact's payload slots and the placeholder VarnodeFacet classid are NOT stable yet; slag/census/provenance/convention are). Ran S1 -- read-only, no codegen, no PR2 dependency -- against the real in-tree harvest artifacts (gitignored but present in the ruff checkout): B1 conservation PASS (dropped=0, harvested=classified+residual); B2 at 91.30% (inside the declared 90-99% INVESTIGATE band, not a KILL); B3 PASS (43 distinct residual shapes, dominant_share 0.215, every non-trivial bucket carries an address); and the pre-registered 60-80%-classified prediction MISSED at a measured 14.15%, recorded honestly rather than hidden -- which is the point of pre-registering it. Dominant residual is opcode_not_in_convention, expected: pass 1 deliberately classifies only 7 of P-code's 74 opcodes. Corpus is r2sleigh's own e2e stress-test fixtures (143 functions, x86-64), not yet a Ghidra-shaped real binary. No code changed, no wave-gate change -- PR2/PR3 remain unmerged, so wave-ogar-machine-pm1.md's gate #3 stands as previously repointed. Next unblocked step, available whenever there's a reason to spend it: S2 (join ore rows to native addresses), still read-only. --- .claude/board/EPIPHANIES.md | 68 +++++++++++++++++++++++++++++++++++ .claude/board/LATEST_STATE.md | 30 ++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 472bc93..a05b8ed 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -4,6 +4,74 @@ > `**Status:**`/`**Confidence:**` line. A correction gets its own new, > dated entry that references the one it corrects — the storno rule. +## 2026-08-18 (even later) — ruff #96 is a DIFFERENT arm; the real find was already on main: a staging guide addressed to THIS session + +**Status:** FINDING. **Confidence:** High — read the merged PR body and the +in-tree harvest artifacts directly, not summarized. + +### PR #96 is not the drill-down proposer this repo is waiting on + +`AdaWorldAPI/ruff` PR #96 ("residual ledger for the plain arm") extends +`ruff_python_spo`'s PLAIN Python-source harvest (PR #95: dismech/A2UI-sdk/ +ruff-scripts corpora, CURIE-shaped constants — `MONDO`/`KISAO`/`infores` +prefixes, bio-ontology work). It is its own "self-adaptive drill loop," but +over a **different crate, different corpus family, different consumer** +(ontology/MedCare-rs-shaped harvest) than `ruff_r2il` (binary/R2IL lifting, +the one `E-LGJ-GHIDRA-G1-G2-SUPERSEDED-BY-R2IL-1` tracks). Zero overlap with +this repo's C-band/Ghidra/JavaRuntime concerns — recorded here only so a +future session doesn't wire a false connection between the two arms because +they share vocabulary ("residual ledger," "drill loop," "proposer"). + +### What IS relevant, sitting on `ruff` main since a same-day, non-PR commit + +Commit `bbaebda` (between PR #94 and PR #95, pushed directly to main) added +`.claude/harvest/r2il/STAGED-CODEGEN-GUIDE.md`, explicitly addressed: +*"Audience: the sibling session that consumes this arc's output (the Ghidra +console work...)"* — this repo, by description if not by name. It confirms +what the prior reconciliation already found (PR 2 routes→V3 has NOT landed) +and adds the piece that entry lacked: **a 5-stage staging order (S1–S5) that +does NOT wait on PR 2**, with "do not skip to S3" stated plainly (S3 is +codegen into an additive landing zone; S1/S2 are read-only ledger/ore +inspection). Also pins the stability table per artifact — `FlatFact`'s two +payload slots and the provisional `VarnodeFacet` classid (`0x0000`, +placeholder for the PR-3 `ogar_codebook` mint) are explicitly **not** +stable; slag/census/provenance/convention are. + +### S1 done — read the ledger, no codegen, no PR2 dependency + +The pass-1 harvest artifacts are already in-tree at `ruff/.claude/harvest/ +r2il/` (gitignored, present on disk). Read directly: + +- **B1 conservation: PASS** — `dropped=0`, `harvested(54304) = + classified(17557) + residual(36747)`. +- **B2 seven-opcode coverage: INVESTIGATE (91.30%)** — inside the declared + 90–99% band, not a KILL. +- **B3 slag named-and-addressed: PASS** — 43 distinct residual shapes, + `dominant_share = 0.215` (well under the 0.60 ceiling), every bucket + except `no_facet_coordinate` carries an example address. +- **The non-bar prediction MISSED, and was recorded honestly rather than + hidden**: only 14.15% of `Op` facts classified, against a pre-registered + 60–80% guess. Dominant residual reason is `opcode_not_in_convention` — + expected, since pass 1 deliberately classifies only 7 of P-code's 74 + opcodes (Copy/IntAdd/Load/Store/CBranch/Call/Return); the other ~67 are + correctly unclassified, not mis-measured. +- **Corpus is r2sleigh's own e2e stress-test fixtures** (143 functions, + x86-64, commit `60942f6`), not yet a Ghidra-shaped real binary — still a + bring-up-scale run, not production scale. + +### Consequence: still nothing to physically consume, and that's correct + +`wave-ghidra-g1-g2.md` / `wave-ogar-machine-pm1.md` gate #3 are unchanged — +PR2/PR3 remain unmerged, and the stability table says explicitly not to +persist `FlatFact` payload bytes or the placeholder classid yet. The +concrete, unblocked next step for this repo — whenever there is a driving +reason to spend it, not scheduled here — is **S2** (join ore rows back to +native addresses via `ore::instruction_addr`, still read-only) as +preparation, so S3 (an additive landing-zone crate here, `// @generated`, +never edited into existing files) is measured before it is built, per the +guide's own "the MedCare and OpenProject transcodes earned their numbers by +measuring at S1/S2 first" precedent. + ## 2026-08-18 (later) — E-LGJ-GHIDRA-G1-G2-SUPERSEDED-BY-R2IL-1 **Status:** FINDING (reconciliation, per `ghidra-integration-v1.md`'s own diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index fc0c680..0b4de70 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,33 @@ +## 2026-08-18 (even later) — ruff #96 is a different arm; found + read the REAL staging guide + +Checked "ruff 96 merged." It's `ruff_python_spo`'s plain-Python residual +ledger (dismech/CURIE-constant harvest, ontology-shaped) — a sibling +drill-loop, but a DIFFERENT crate and consumer than `ruff_r2il`, and +unrelated to this repo's C-band/Ghidra/JavaRuntime track. No action needed +here; recorded so the two arms aren't confused later since they share +vocabulary. + +The genuinely relevant find was already on `ruff` main, unrelated to #96: +`.claude/harvest/r2il/STAGED-CODEGEN-GUIDE.md`, explicitly addressed to +"the sibling session ... (the Ghidra console work)" — this repo. It +confirms PR2 (routes→V3) still hasn't landed and gives a 5-stage staging +order (S1 ledger-read → S2 ore-join → S3 additive codegen → S4 one +consumer → S5 target-profile fork) that does NOT wait on PR2 for its first +two stages. Ran S1 against the real in-tree harvest artifacts: B1 +conservation PASS (dropped=0), B3 addressed-slag PASS (43 shapes, +dominant_share 0.215), B2 at 91.30% (INVESTIGATE band), and the +pre-registered 60-80%-classified prediction MISSED at a measured 14.15% — +recorded honestly, which is the point of pre-registering it. Dominant +residual is `opcode_not_in_convention`, expected: pass 1 only classifies 7 +of P-code's 74 opcodes by design. Full record + the stability table +(FlatFact payload bytes and the placeholder VarnodeFacet classid are +explicitly NOT stable yet): `EPIPHANIES.md`, the entry above +`E-LGJ-GHIDRA-G1-G2-SUPERSEDED-BY-R2IL-1`. + +No wave-gate change — PR2/PR3 still unmerged, `wave-ogar-machine-pm1.md` +gate #3 stands as previously repointed. Next unblocked step (not scheduled, +available when there's a reason to spend it): S2, still read-only. + ## 2026-08-18 (later) — Ghidra G1/G2 waves reconciled: superseded by ruff_r2il, not built Checked what "the other session writing the autoadapting drill-down proposer" From df3542626d8282420c0738be14d09741351f2a90 Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 18 Aug 2026 07:10:42 +0000 Subject: [PATCH 28/30] Board: R2IL handshake outcomes + the Valhalla-premise storno (operator-caught) The R2IL session answered all five cross-session questions; outcomes and ownership recorded (mirror sync now owned here; consumers/ghidra is the expected end-state, gated on PR2's layout doc + PR3's classids). The entry's core is a storno of my own handoff premise: 'Valhalla was a laboratory phase, not a door' was right about addressability and wrong about integration -- the shipping descriptor vocabulary is value-record- ready by design (one word per type) and the A/B ran on a real EA build with measured numbers. The ruling (0xC0 = Panama alone) survives on the corrected premise: Valhalla is a designed PROPERTY of the C0 concepts, and properties of concepts do not get domains. OGAR PR #277 (merged) carries the canonical corrected text. --- .claude/board/EPIPHANIES.md | 76 +++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index a05b8ed..6fee3b3 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -4,6 +4,82 @@ > `**Status:**`/`**Confidence:**` line. A correction gets its own new, > dated entry that references the one it corrects — the storno rule. +## 2026-08-18 (R2IL handshake) — E-LGJ-VALHALLA-IS-INTEGRATED-AS-A-PROPERTY-NOT-A-CONCEPT-1 + +**Status:** FINDING + a storno of my own handoff premise (operator-caught). +**Confidence:** High — verified against the tree, not recalled. + +### The handshake, first + +The R2IL session answered all five questions of the cross-session prompt: +(1) `0xC4` acknowledged as a fixed point — PR3 mints INTO it, provenance +fence as specified, concept names/slots arrive in the PR body; (2) the +stale `ogar_codebook` mirror confirmed first-hand at lance-graph `db488f5` +— and the sync is explicitly handed to THIS session ("don't wait on me... +open it separately, now"; serialize on one owner, which is now me); (3) +commitment: PR2 ships an abi.md-§11-style layout doc IN the same PR, and +the two ⚠ stability flags flip in that same commit — build against the +doc, never against `furnace.rs`; (4) `0xC0` is **Panama alone** — Valhalla +gets no domain representation; (5) a Java-side consumer IS the expected +end-state: `consumers/ghidra/` beside `trades/` and `bricks/`, gated on +(2)+(3) — shape W6/W7 toward it, keep the read-only fence until the PR2 +doc exists and PR3's classids are real. Status note: the ruff session is +mid upstream-catch-up merge (~1500 commits, separate branch); PR2→PR3 +queue after it settles. + +### The storno — my premise was understated, the ruling survives anyway + +My handoff prompt told the R2IL session Valhalla "was a laboratory phase +here, not a door." The operator demanded a double-check, and the tree says +that summary was WRONG about integration while right about addressability: + +- **Valhalla IS integrated, by design, in the shipping API.** All five + production descriptor types (`LaneId`/`Ordinal`/`MaskId`/`RowRange`/ + `FacetId`) carry a "Valhalla A/B candidate" Javadoc contract: the same + source compiles as `value record` under JEP 401 — migration is ONE WORD + per type. That constraint is load-bearing on the shipping surface; it is + the arc's "Panama and Valhalla become the supraconductor" request + honored at the vocabulary level. +- **A real EA build ran the A/B** (`27-jep401ea3`, in-container): + flattening cliff measured at 8-byte payload (`RowRange` at 16 B landing + on the wrong side, recorded as the one over-optimistic expectation); + `LaneId`/`Ordinal` arrays 5.5× smaller, reads up to 8.3× faster where + flattening applies; the bulk thesis unchanged — native wins 38–57× on + BOTH platforms, which is exactly why bulk data stays native and only the + descriptor vocabulary is Valhalla-shaped. +- **Deliberately NOT adopted:** the three preview-only mechanisms (no + `--add-exports`, no `jdk.internal.*`) — "distorting a public API to fit + a preview VM's current budget would bake a temporary constraint into a + permanent surface." + +**Why the ruling stands on the corrected premise:** a `ConceptDomain` is a +vocabulary of ADDRESSABLE things. Valhalla's integration here is a designed +PROPERTY of the C0 concepts' Java vocabulary (one-word readiness + +measured flattening payoff), and properties of concepts do not get +domains. The R2IL session's own phrase — "a facet on an existing concept, +not a domain" — described the true state better than my premise did. + +**Landed:** OGAR PR #277 (merged, `386a6fd`) corrects the `JavaRuntime` +doc comment to "Panama FFM alone," states the integrated-as-property +argument IN the doc so the ruling cannot be misread as "Valhalla is +unintegrated," updates the layout-doc band row, and APPENDS a dated +correction to `D-CBAND-ALTITUDE` (original text kept, per append-only). + +**The meta-lesson, same family as the shape-vs-altitude storno:** a +one-line characterization written to justify a conclusion can be +simultaneously right about the conclusion and wrong as a description — +and it is the DESCRIPTION that calcifies when quoted into doc comments. +The operator's "double check whether you didn't integrate it or the other +session just isn't aware" is the exact question that separates the two. + +### Now owned by this session (from the handshake) + +1. **The `ogar_codebook` mirror sync in lance-graph-contract** — add + `Ontology`(0x03)/`Blocks`(0x17)/`JavaRuntime`(0xC0)/`Analytics`(0xC1)/ + `BinaryLifting`(0xC4) to the wire-mirror + parity pins. Opening now. +2. **Shape W6/W7 toward `consumers/ghidra/`** — with the read-only fence + held until PR2's layout doc + PR3's real classids exist. + ## 2026-08-18 (even later) — ruff #96 is a DIFFERENT arm; the real find was already on main: a staging guide addressed to THIS session **Status:** FINDING. **Confidence:** High — read the merged PR body and the From 8806899e3443897c88ff9c14ed88de5003eb5f3b Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 18 Aug 2026 07:33:30 +0000 Subject: [PATCH 29/30] rowstore: generate_with_edges -- the graph-consumer wave's real, measured blocker, cleared Asked what could be built while ruff_r2il's PR2/PR3 are blocked. Nearly dispatched the graph consumer (W5c) on the strength of D1a's mechanism (writable masks, existing facet-match) being sufficient -- claimed twice in earlier turns -- before re-reading wave-consumer-graph.md's own STOP condition in full and catching a real, different blocker: plain RowStore::generate()'s payload is uniform random noise, so a decoded 1-2 hop BFS over it saturates to nearly every row regardless of decode convention -- vacuous under the wave's own anti-vacuity falsifier ("seed/1-hop/2-hop must be three different, non-empty, non-total sizes"). A data-shape problem, not a mechanism problem. Caught before any workers spawned. RowStore::generate_with_edges(n_rows, seed, edge_classid, edge_gate_mask, edge_radius) is the fix -- additive, generate() untouched. Classid assignment is byte-identical to generate() (same SplitMix64 draws, same formula; the classid formula's otherwise-unused high bits of the per-facet `a` draw become an independent sparsity gate, so edge_classid=16, out of range, reproduces generate() exactly -- pinned by test). A sparse, gated subset of edge_classid-matching facets get a bounded local-neighbourhood target row instead of raw noise, which is what keeps a 1-2 hop BFS non-vacuous. Parameters chosen from a real measurement sweep (examples/graph_density_probe.rs), not guessed: a first pass at n_rows= 1000 was too small (avg degree < 1, everything collapsed to zero); widened to n_rows=20_000 for usable numbers, then re-measured at a test-suite-sized n_rows=2000 for the pinned regression: a 10-row seed reaches exactly 19 rows at 1 hop, 29 at 2 hops -- three different, non-empty, non-total sizes, matching the wave's own falsifier shape exactly, pinned as measured_hop_counts_are_three_distinct_non_empty_ non_total_sizes. Two disable-runs: breaking the radius-wrap formula turned exactly the three tests touching the target formula red (transcription test, in-bounds/radius invariant, the pinned regression), leaving the seven tests that don't touch it green. Ignoring the sparsity gate mask turned only the transcription test red -- the in-bounds/radius invariant correctly stayed green, verified as the right outcome rather than a vacuous test: geometry validity is orthogonal to which facets get the treatment, only to the treatment's correctness once applied. wave-consumer-graph.md updated in place: the STOP condition marked RESOLVED with the measured numbers, and the stale "calcify, do not dispatch" header corrected (that gate was already lifted session-wide when W5a/W5b shipped under the identical wording). The graph consumer is now genuinely dispatchable -- not dispatched in this same pass; this change is scoped to the substrate-tier generator only, per the wave file's own rule that a generator extension is not a consumer hack. Gates: lgj-abi 90/90 (was 84, +6 new tests), fmt clean, clippy --all-targets --all-features clean. Board: EPIPHANIES + LATEST_STATE entries prepended in the same commit. --- .claude/board/EPIPHANIES.md | 72 +++++ .claude/board/LATEST_STATE.md | 29 ++ .claude/waves/wave-consumer-graph.md | 37 ++- .../lgj-abi/examples/graph_density_probe.rs | 85 ++++++ native/lgj-abi/src/rowstore.rs | 265 ++++++++++++++++++ 5 files changed, 478 insertions(+), 10 deletions(-) create mode 100644 native/lgj-abi/examples/graph_density_probe.rs diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 6fee3b3..82ebc12 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -4,6 +4,78 @@ > `**Status:**`/`**Confidence:**` line. A correction gets its own new, > dated entry that references the one it corrects — the storno rule. +## 2026-08-18 (measured) — the graph-consumer STOP condition was real, and is now cleared + +**Status:** FINDING + a correction of MY OWN earlier claim. **Confidence:** +High — measured (`examples/graph_density_probe.rs`), not argued. + +### What "in the meantime" surfaced + +Asked what was available to build while `ruff_r2il` PR2/PR3 are blocked. +First instinct was W5c (graph consumer) — I'd twice previously argued its +D1a design needs zero substrate change (mask words are writable, facet- +match already exists). Both times I checked the MECHANISM and skipped +`wave-consumer-graph.md`'s own STOP condition, which names a DIFFERENT, +real blocker: `RowStore::generate()`'s payload is uniform-random noise, so +a decoded 1-2 hop BFS over it saturates to nearly every row regardless of +decode convention — vacuous under the wave's own falsifier #4 ("seed / 1- +hop / 2-hop must be three different, non-empty, non-total sizes"). No +mechanism fix (writable masks, zero-copy reads) touches this; it is a +DATA-SHAPE problem, not a decode-ambiguity problem, and the wave file says +so explicitly: *"this wave NEEDS a deliberate edge-bearing generator arm: +that is a substrate change, not a consumer hack."* I was about to dispatch +G1/G2 into it before re-reading the STOP block in full — caught before any +workers spawned. + +### The fix, measured before committing to parameters + +`RowStore::generate_with_edges(n_rows, seed, edge_classid, edge_gate_mask, +edge_radius)` (native/lgj-abi/src/rowstore.rs) — additive, `generate()` +untouched. Classid assignment is byte-identical to `generate()` (same +SplitMix64 draws, same `(a>>>33)&0xF` formula; unused bits 37..64 of `a` +become an independent sparsity gate, so `edge_classid=16` — out of range — +reproduces `generate()` exactly, pinned by test). A gated, sparse subset of +`edge_classid`-matching facets get a BOUNDED local-neighbourhood target +(`row + offset mod n`, `offset` drawn from `b`, clamped to `±edge_radius`) +instead of raw noise. + +`examples/graph_density_probe.rs` swept `(gate_mask, radius)` before any +parameter was chosen — first pass at `n_rows=1000` was too small (avg +degree < 1, everything collapsed to zero); widened to `n_rows=20_000` and +got real, usable numbers (`gate_mask=0x0, radius=25`: seed=20 → 1hop=30 → +2hop=40). Re-measured at a smaller, test-suite-friendly `n_rows=2000` for +the pinned regression: seed=10 → **1hop=19 → 2hop=29** — three different, +non-empty, non-total sizes, exactly the falsifier's own shape, now pinned +as `measured_hop_counts_are_three_distinct_non_empty_non_total_sizes`. + +**Two disable-runs, both as specified:** +- Broke the radius-wrap formula (dropped `rem_euclid`) → exactly the three + tests touching the target formula went red (the transcription test, the + in-bounds/radius invariant, the pinned hop-count regression); the seven + tests that don't touch target computation stayed green. Restored. +- Ignored the sparsity gate mask (fired on classid match alone) → only the + transcription test went red. The in-bounds/radius invariant test + correctly stayed green — density and target-correctness are orthogonal + properties, and the disable changes density, not correctness. Verified + this is the RIGHT outcome, not a vacuous test: geometry validity doesn't + depend on WHICH facets get the treatment, only on the treatment itself + once applied. Restored. + +Gates: `lgj-abi` 90/90 (was 84; +6 new tests), fmt clean, clippy +`--all-targets --all-features` clean. + +### Consequence + +`wave-consumer-graph.md` updated in place: the STOP condition marked +RESOLVED with the measured numbers, and the file's own dispatch header +corrected — the "calcify, do not dispatch" gate was already lifted +session-wide (W5a/W5b shipped under the identical wording); this wave's +GENUINE extra gate was the generator, now cleared. **The graph consumer +(G1/G2) is dispatchable.** Not dispatched in this same pass — this PR is +scoped to the substrate-tier generator only, per the wave file's own rule +that a generator extension is NOT a consumer hack and lands as its own +change. + ## 2026-08-18 (R2IL handshake) — E-LGJ-VALHALLA-IS-INTEGRATED-AS-A-PROPERTY-NOT-A-CONCEPT-1 **Status:** FINDING + a storno of my own handoff premise (operator-caught). diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 0b4de70..4a91d0b 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,32 @@ +## 2026-08-18 (measured) — W5c's real blocker found + cleared: RowStore::generate_with_edges + +Asked what was buildable while ruff_r2il PR2/PR3 are blocked. Was about to +dispatch the graph consumer (W5c) on the strength of D1a's mechanism +(writable masks, existing facet-match) being sufficient -- twice claimed +this in earlier turns -- before re-reading `wave-consumer-graph.md`'s own +STOP condition in full and catching a real, different blocker: plain +`RowStore::generate()`'s payload is uniform noise, so any 1-2 hop BFS over +it saturates to nearly every row regardless of decode convention -- +vacuous under the wave's own anti-vacuity falsifier. A data-shape problem, +not a mechanism problem; caught before any workers spawned. + +Fixed with `RowStore::generate_with_edges` (native/lgj-abi) -- additive, +`generate()` byte-identical for out-of-range `edge_classid` (pinned). +Parameters chosen from a real measurement sweep +(`examples/graph_density_probe.rs`), not guessed: at `n_rows=2000, +gate_mask=0x0, radius=25`, a 10-row seed reaches 19 rows at 1 hop, 29 at 2 +hops -- pinned as a regression test. Two disable-runs (radius-wrap +formula; sparsity gate) both went red exactly where expected, including +one case (the gate disable) correctly NOT failing an orthogonal geometry +test -- verified as the right outcome, not a vacuous one. + +`wave-consumer-graph.md` updated: STOP condition marked RESOLVED with the +measured numbers; the file's stale "calcify, do not dispatch" header +corrected (that gate was already lifted session-wide). **The graph +consumer is now genuinely dispatchable** -- not dispatched in this pass, +scoped to the generator only. Gates: lgj-abi 90/90 (+6), fmt/clippy clean. +Full record: `EPIPHANIES.md`, the entry above the R2IL handshake one. + ## 2026-08-18 (even later) — ruff #96 is a different arm; found + read the REAL staging guide Checked "ruff 96 merged." It's `ruff_python_spo`'s plain-Python residual diff --git a/.claude/waves/wave-consumer-graph.md b/.claude/waves/wave-consumer-graph.md index 854d7d6..1da1a11 100644 --- a/.claude/waves/wave-consumer-graph.md +++ b/.claude/waves/wave-consumer-graph.md @@ -1,10 +1,13 @@ # Wave: consumer example — graph traversal (facet edges, crossings ∝ hops) -> Executes `consumer-graph-traversal-v1.md`. **DO NOT DISPATCH** — -> operator ruling 2026-08-17: calcified, not executed, until called. -> Gates: W3 merged AND the hop-decode capability question resolved (see -> Decision D1 below — this wave has a genuine open design decision the -> other two consumer waves do not). +> Executes `consumer-graph-traversal-v1.md`. Gates: W3 merged (DONE) AND +> the hop-decode capability question resolved (Decision D1 below — ruled +> D1a) AND the edge-bearing generator STOP condition resolved (DONE, +> `RowStore::generate_with_edges`, see below). **DISPATCHABLE** — the +> calcify-only gate ("2026-08-17: calcified, not executed, until called") +> was lifted session-wide once autonomous dispatch was authorized (W5a/W5b +> both shipped under it); this wave's own, GENUINE extra gate (the +> generator) is what actually held it back, and is now cleared. ## Decision D1 (orchestrator resolves BEFORE any worker spawns) @@ -59,8 +62,22 @@ offset by 4 bytes → hop-correctness must go red (proves the decode is load-bearing); restore. Board + PR per rhythm; if D1b was chosen, the membrane PR merges FIRST and this wave's PR references it. -**STOP conditions:** payload reading ambiguity (→ orchestrator, possibly -a generator extension in a substrate-plan PR — the fixture's payload -today is PRNG noise, so this wave NEEDS a deliberate edge-bearing -generator arm: that is a substrate change, not a consumer hack — flagged -here so nobody discovers it mid-dispatch). +**STOP condition RESOLVED (2026-08-18):** the deliberate edge-bearing +generator arm this wave's STOP condition named now exists — +`RowStore::generate_with_edges(n_rows, seed, edge_classid, edge_gate_mask, +edge_radius)` (native/lgj-abi, landed as its own substrate-tier change, +NOT a consumer hack, per this file's own rule). It reuses `generate()`'s +classid stream byte-for-byte (an out-of-range `edge_classid` reproduces +`generate()` exactly — pinned by test) and, for a SPARSE, gated subset of +`edge_classid`-matching facets, writes a bounded-local-neighbourhood +target row instead of raw noise — the mechanism that keeps a 1-2 hop BFS +non-vacuous. Measured (`examples/graph_density_probe.rs`, not guessed): +plain `generate()`'s uniform-random payload saturates a 2-hop BFS to +nearly every row (the ORIGINAL problem this STOP condition named); at +`n_rows=2000, edge_classid=0, edge_gate_mask=0x0, edge_radius=25`, a +10-row seed set reaches exactly 19 rows at 1 hop and 29 at 2 hops — three +different, non-empty, non-total sizes, pinned as a regression test +(`measured_hop_counts_are_three_distinct_non_empty_non_total_sizes`). +**G1's payload-reading-convention documentation task now has real, +measured ground to document** rather than an open question — this wave +is dispatchable. diff --git a/native/lgj-abi/examples/graph_density_probe.rs b/native/lgj-abi/examples/graph_density_probe.rs new file mode 100644 index 0000000..7f6c81b --- /dev/null +++ b/native/lgj-abi/examples/graph_density_probe.rs @@ -0,0 +1,85 @@ +//! Measures real 1-hop/2-hop set sizes for `RowStore::generate_with_edges` over +//! candidate `(edge_gate_mask, edge_radius)` combinations, so W5c's fixture +//! parameters are picked from numbers, not a guess. +//! +//! `cargo run --release --manifest-path native/lgj-abi/Cargo.toml --example graph_density_probe` + +use lgj_abi::rowstore::{RowStore, ROW_FACETS}; + +const EDGE_CLASSID: u32 = 0; +const SEED: u64 = 0xF00D_CAFE; + +fn hop(store: &RowStore, from: &[u32], edge_classid: u32) -> Vec { + let n = store.n_rows as usize; + let mut seen = vec![false; n]; + let mut out = Vec::new(); + for &row in from { + let base = row as usize * 512; + for facet in 0..ROW_FACETS as usize { + let fbase = base + facet * 16; + let classid = + u32::from_le_bytes(store.as_bytes()[fbase..fbase + 4].try_into().unwrap()); + if classid != edge_classid { + continue; + } + let target = + u64::from_le_bytes(store.as_bytes()[fbase + 4..fbase + 12].try_into().unwrap()) + as usize; + if target < n && !seen[target] { + seen[target] = true; + out.push(target as u32); + } + } + } + out +} + +fn main() { + let n_rows = 20_000u64; + let seed_set: Vec = (0..20).map(|i| i * 37 + 5).collect(); + + println!("n_rows={n_rows} seed_set.len()={}\n", seed_set.len()); + println!( + "{:>12} {:>8} {:>10} {:>10} {:>10}", + "gate_mask", "radius", "avg_deg", "1hop", "2hop" + ); + + for &gate_mask in &[0x0u64, 0x1u64, 0x3u64, 0x7u64, 0xFu64, 0x1Fu64] { + for &radius in &[25u32, 100, 500] { + let store = + RowStore::generate_with_edges(n_rows, SEED, EDGE_CLASSID, gate_mask, radius) + .expect("generation should succeed for these params"); + + // average out-degree: total classid==EDGE_CLASSID facets that + // actually carry a structured target, sampled directly. + let mut edge_facets = 0u64; + for row in 0..n_rows as usize { + let base = row * 512; + for facet in 0..ROW_FACETS as usize { + let fbase = base + facet * 16; + let classid = + u32::from_le_bytes(store.as_bytes()[fbase..fbase + 4].try_into().unwrap()); + let hi32 = u32::from_le_bytes( + store.as_bytes()[fbase + 12..fbase + 16].try_into().unwrap(), + ); + if classid == EDGE_CLASSID && hi32 == 0 { + edge_facets += 1; + } + } + } + let avg_deg = edge_facets as f64 / n_rows as f64; + + let one_hop = hop(&store, &seed_set, EDGE_CLASSID); + let two_hop = hop(&store, &one_hop, EDGE_CLASSID); + + println!( + "{:>#12x} {:>8} {:>10.3} {:>10} {:>10}", + gate_mask, + radius, + avg_deg, + one_hop.len(), + two_hop.len() + ); + } + } +} diff --git a/native/lgj-abi/src/rowstore.rs b/native/lgj-abi/src/rowstore.rs index 0d3e8af..4308f3e 100644 --- a/native/lgj-abi/src/rowstore.rs +++ b/native/lgj-abi/src/rowstore.rs @@ -134,6 +134,101 @@ impl RowStore { }) } + /// Build a store whose classid stream is byte-identical to [`Self::generate`] + /// but with a sparse, bounded-neighbourhood subset of `edge_classid`-matching + /// facets carrying a *structured* payload (a nearby row index) instead of raw + /// random bits — the graph-consumer wave's "deliberate edge-bearing generator + /// arm" (`.claude/waves/wave-consumer-graph.md`'s STOP condition: the plain + /// [`Self::generate`] payload is PRNG noise, so a 1/2-hop BFS over it + /// saturates to nearly every row within one or two hops — vacuous under any + /// non-trivial anti-vacuity check, regardless of decode convention). + /// + /// # Why classid stays byte-identical to `generate()` + /// + /// `classid = (a >>> 33) & 0xF` consumes bits 33..37 of the SAME per-facet + /// `a` draw `generate()` makes; bits 37..64 (27 bits) are otherwise unused + /// by that formula and are spent here as an INDEPENDENT sparsity gate, so + /// this function never perturbs classid assignment — `edge_classid = 16` + /// (out of the 0..16 range) makes the gate structurally unreachable, and + /// the byte output is then provably identical to `generate()` (pinned by a + /// test, not merely argued). + /// + /// # The sparsity gate and why it exists + /// + /// A facet becomes a structured edge only when BOTH `classid == + /// edge_classid` (probability `1/16`) AND `(a & edge_gate_mask) == 0` + /// (probability `1/(edge_gate_mask + 1)`, `edge_gate_mask` a power-of-two + /// minus one). The two gates compose multiplicatively and independently + /// (disjoint bit ranges of the same draw), so overall edge density per + /// facet is `1 / (16 * (edge_gate_mask + 1))` — tunable without touching + /// the classid formula. `examples/graph_density_probe.rs` measures real + /// 1-hop/2-hop set sizes across candidate gate masks; this function does + /// not itself pick "good" parameters — see that probe's recorded numbers + /// before choosing them for a real fixture. + /// + /// # The target encoding + /// + /// A structured edge's payload is `target_row` (LE u64, in the 8-byte + /// `b`-slot) with the trailing 4-byte `a`-slot zeroed as a marker — + /// `target_row = (row as i64 + offset).rem_euclid(n_rows as i64)`, + /// `offset` drawn from `b % (2*edge_radius+1)` recentred to + /// `-edge_radius..=+edge_radius` — a BOUNDED local neighbourhood, never a + /// uniform global target (the mechanism that keeps the graph sparse AND + /// local, which is what makes hop-count matter at all). A non-edge facet + /// (gate failed) keeps `generate()`'s raw `(b, a_lo32)` payload exactly. + /// + /// Returns `None` on the same overflow/allocation conditions as + /// [`Self::generate`], and if `edge_radius >= n_rows` (a radius that + /// cannot wrap meaningfully) or `n_rows == 0`. + #[allow(clippy::too_many_arguments)] + pub fn generate_with_edges( + n_rows: u64, + seed: u64, + edge_classid: u32, + edge_gate_mask: u64, + edge_radius: u32, + ) -> Option { + let n = usize::try_from(n_rows).ok()?; + if n == 0 || u64::from(edge_radius) >= n_rows { + return None; + } + let byte_len = n.checked_mul(ROW_BYTES as usize)?; + + let mut bytes = Vec::new(); + bytes.try_reserve_exact(byte_len).ok()?; + bytes.resize(byte_len, 0u8); + + let mut rng = SplitMix64::new(seed); + for row in 0..n { + for facet in 0..ROW_FACETS as usize { + let a = rng.next_u64(); + let b = rng.next_u64(); + let base = row * ROW_BYTES as usize + facet * FACET_BYTES as usize; + let classid = ((a >> 33) & (ROWSTORE_CLASS_CARDINALITY - 1)) as u32; + + let (payload_lo64, payload_hi32) = + if classid == edge_classid && (a & edge_gate_mask) == 0 { + let span = 2 * u64::from(edge_radius) + 1; + let raw_offset = (b % span) as i64 - i64::from(edge_radius); + let target = (row as i64 + raw_offset).rem_euclid(n as i64) as u64; + (target, 0u32) + } else { + (b, a as u32) + }; + + bytes[base..base + 4].copy_from_slice(&classid.to_le_bytes()); + bytes[base + 4..base + 12].copy_from_slice(&payload_lo64.to_le_bytes()); + bytes[base + 12..base + 16].copy_from_slice(&payload_hi32.to_le_bytes()); + } + } + + Some(Self { + n_rows, + seed, + bytes: Arc::from(bytes), + }) + } + /// The whole buffer as a byte slice. Zero-copy; the address is stable for /// the store's life (see the module header). pub fn as_bytes(&self) -> &[u8] { @@ -289,4 +384,174 @@ mod tests { assert!(s.as_bytes().is_empty()); assert_eq!(s.lane_raw(LANE_FACET_BASE).unwrap().1, 0); } + + // ── generate_with_edges: the graph-consumer wave's edge-bearing arm ── + + /// Out-of-range `edge_classid` makes the sparsity gate structurally + /// unreachable — proves the edge mechanism is genuinely additive, not + /// merely documented as such. + #[test] + fn out_of_range_edge_classid_reproduces_plain_generate_byte_for_byte() { + let plain = RowStore::generate(200, 0xABCD).unwrap(); + let edged = RowStore::generate_with_edges( + 200, + 0xABCD, + ROWSTORE_CLASS_CARDINALITY as u32, // 16, out of the 0..16 range + 0, + 10, + ) + .unwrap(); + assert_eq!(plain.as_bytes(), edged.as_bytes()); + } + + /// The independent transcription (mirrors + /// `the_documented_generator_is_the_actual_generator` above, extended + /// with the gate + target formula from this function's own doc comment) + /// — exactly what the Java-side parity test will do. + #[test] + fn the_documented_edge_mechanism_is_the_actual_mechanism() { + let n = 50u64; + let seed = 0x1234_5678; + let edge_classid = 3u32; + let edge_gate_mask = 0x3u64; + let edge_radius = 7u32; + let store = + RowStore::generate_with_edges(n, seed, edge_classid, edge_gate_mask, edge_radius) + .unwrap(); + + let mut rng = SplitMix64::new(seed); + for row in 0..n { + for facet in 0..ROW_FACETS { + let a = rng.next_u64(); + let b = rng.next_u64(); + let classid = ((a >> 33) & 0xF) as u32; + let base = (row * ROW_BYTES + facet as u64 * FACET_BYTES) as usize; + assert_eq!(store.classid_at(row, facet), classid); + + if classid == edge_classid && (a & edge_gate_mask) == 0 { + let span = 2 * u64::from(edge_radius) + 1; + let raw_offset = (b % span) as i64 - i64::from(edge_radius); + let target = (row as i64 + raw_offset).rem_euclid(n as i64) as u64; + assert_eq!( + &store.as_bytes()[base + 4..base + 12], + &target.to_le_bytes(), + "row {row} facet {facet}: structured edge target" + ); + assert_eq!(&store.as_bytes()[base + 12..base + 16], &0u32.to_le_bytes()); + } else { + assert_eq!(&store.as_bytes()[base + 4..base + 12], &b.to_le_bytes()); + assert_eq!( + &store.as_bytes()[base + 12..base + 16], + &(a as u32).to_le_bytes() + ); + } + } + } + } + + /// Structural invariant: every structured-edge target is a valid, + /// in-bounds row index within `edge_radius` of its own row (accounting + /// for wraparound) — checked exhaustively, not sampled. + #[test] + fn every_structured_edge_target_is_in_bounds_and_within_radius() { + let n = 300u64; + let edge_classid = 5u32; + let edge_radius = 20u32; + let store = + RowStore::generate_with_edges(n, 0x9E37, edge_classid, 0x1, edge_radius).unwrap(); + + let mut checked = 0u32; + for row in 0..n { + for facet in 0..ROW_FACETS { + if store.classid_at(row, facet) != edge_classid { + continue; + } + let base = (row * ROW_BYTES + facet as u64 * FACET_BYTES) as usize; + let hi32 = + u32::from_le_bytes(store.as_bytes()[base + 12..base + 16].try_into().unwrap()); + if hi32 != 0 { + continue; // gate failed for this classid-matching facet + } + checked += 1; + let target = + u64::from_le_bytes(store.as_bytes()[base + 4..base + 12].try_into().unwrap()); + assert!(target < n, "target {target} out of bounds ({n} rows)"); + let forward = (target as i64 - row as i64).rem_euclid(n as i64); + let backward = n as i64 - forward; + let wrapped_distance = forward.min(backward); + assert!( + wrapped_distance <= i64::from(edge_radius), + "row {row} -> target {target}: distance {wrapped_distance} exceeds radius {edge_radius}" + ); + } + } + assert!( + checked > 0, + "anti-vacuity: this fixture must produce at least one structured edge" + ); + } + + /// The measured regression pin (`examples/graph_density_probe.rs`, + /// `n_rows=2000, gate_mask=0x0, radius=25`): a 10-row seed set reaches + /// exactly 19 distinct rows at 1 hop and 29 at 2 hops. Three different, + /// non-empty, non-total sizes — the graph-consumer wave's own + /// anti-vacuity falsifier, satisfied here at the GENERATOR level before + /// any consumer code exists to fail it. Recomputed by BFS, independent + /// of `generate_with_edges`'s own internals (reads only classid + raw + /// payload bytes, the same surface a real consumer sees). + #[test] + fn measured_hop_counts_are_three_distinct_non_empty_non_total_sizes() { + let n = 2_000u64; + let edge_classid = 0u32; + let store = RowStore::generate_with_edges(n, 0xF00D_CAFE, edge_classid, 0x0, 25).unwrap(); + + let hop = |from: &[u64]| -> Vec { + let mut seen = vec![false; n as usize]; + let mut out = Vec::new(); + for &row in from { + for facet in 0..ROW_FACETS { + if store.classid_at(row, facet) != edge_classid { + continue; + } + let base = (row * ROW_BYTES + facet as u64 * FACET_BYTES) as usize; + let hi32 = u32::from_le_bytes( + store.as_bytes()[base + 12..base + 16].try_into().unwrap(), + ); + if hi32 != 0 { + continue; + } + let target = u64::from_le_bytes( + store.as_bytes()[base + 4..base + 12].try_into().unwrap(), + ); + if !seen[target as usize] { + seen[target as usize] = true; + out.push(target); + } + } + } + out + }; + + let seed: Vec = (0..10).map(|i| i * 37 + 5).collect(); + let one_hop = hop(&seed); + let two_hop = hop(&one_hop); + + assert_eq!(seed.len(), 10); + assert_eq!(one_hop.len(), 19); + assert_eq!(two_hop.len(), 29); + assert!(seed.len() != one_hop.len() && one_hop.len() != two_hop.len()); + assert!(!one_hop.is_empty() && !two_hop.is_empty()); + assert!((one_hop.len() as u64) < n && (two_hop.len() as u64) < n); + } + + #[test] + fn zero_rows_is_rejected_for_generate_with_edges() { + assert!(RowStore::generate_with_edges(0, 1, 0, 0, 10).is_none()); + } + + #[test] + fn radius_that_cannot_fit_is_rejected() { + assert!(RowStore::generate_with_edges(10, 1, 0, 0, 10).is_none()); + assert!(RowStore::generate_with_edges(10, 1, 0, 0, 9).is_some()); + } } From ae02be514783758e62a4c90b2ede03dacb01c185 Mon Sep 17 00:00:00 2001 From: Jan Date: Tue, 18 Aug 2026 11:59:45 +0000 Subject: [PATCH 30/30] lgj-abi: edge-bearing row store ABI addition (lgj_rowstore_open_with_edges, minor 3) The graph-consumer wave's own STOP-condition-RESOLVED note proved RowStore::generate_with_edges at the Rust generator level but left no membrane path to it: Engine.openRowStore/registry::open_rowstore only ever called plain RowStore::generate, so no ABI symbol existed for Java to reach edge-bearing data at all. Found before dispatching the graph wave's G1/G2 workers, closed as the wave's own D1b rule requires -- growing the membrane is W-tier orchestrator work, not a consumer worker's ad hoc addition. lgj_rowstore_open_with_edges (docs/abi.md ss12, ABI minor 2->3): mirrors lgj_rowstore_open symbol-for-symbol -- same LGJ_RESOURCE_ROWSTORE kind, same lane shape, no new mask op, purely an alternative constructor. Threaded through registry.rs -> exports.rs -> Downcalls/Engine (Abi.requireMinor(3), matching the row store's own minor-2 gate) -> RowStore.openWithEdges. Added a Java-side transcription of the D1a hop mechanism itself (facet-match crossing + raw lane-0 payload decode, zero new ABI op) to RowStoreParityTest, at the exact parameters already pinned as a Rust regression -- it reproduces the identical hop counts (10-row seed -> 19 at 1 hop -> 29 at 2 hops), proving the membrane carries the same edge structure, not merely the same classid stream. Gates: cargo test 93/93 (+3), clippy -D warnings + fmt clean, release build exports the new symbol (nm -D). Java AllTests 194/194 (+6). Two disable-runs, both red-then-green: a classid-not-threaded bug at the registry level, and the Java hop's classid-match condition forced to always skip. Board: STATUS_BOARD D-LGJ-W6, LATEST_STATE dated entry, EPIPHANIES entry recording the gap and its fix. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Pud4qpxFHwqyqDjSabQbs --- .claude/board/EPIPHANIES.md | 91 ++++++++++++++++ .claude/board/LATEST_STATE.md | 40 +++++++ .claude/board/STATUS_BOARD.md | 3 +- docs/abi.md | 70 +++++++++++- .../com/adaworldapi/lancegraph/RowStore.java | 26 +++++ .../lancegraph/internal/ffm/Downcalls.java | 27 +++++ .../lancegraph/internal/ffm/Engine.java | 19 ++++ .../lancegraph/RowStoreParityTest.java | 100 ++++++++++++++++++ native/lgj-abi/src/abi.rs | 10 +- native/lgj-abi/src/exports.rs | 52 +++++++++ native/lgj-abi/src/registry.rs | 73 +++++++++++++ 11 files changed, 505 insertions(+), 6 deletions(-) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 82ebc12..03a1066 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -4,6 +4,97 @@ > `**Status:**`/`**Confidence:**` line. A correction gets its own new, > dated entry that references the one it corrects — the storno rule. +## 2026-08-18 (later still) — "the generator exists" ≠ "Java can reach it": the membrane gap the prior entry's own resolution note missed + +**Status:** FINDING + a correction of the entry directly below this one. +**Confidence:** High — measured (`nm -D` before/after, full Rust + Java test +suites, two disable-runs). + +### What the prior entry's "RESOLVED" undersold + +The STOP-condition entry below this one — and `wave-consumer-graph.md`'s own +header — declared the graph wave DISPATCHABLE once +`RowStore::generate_with_edges` existed and its numbers were pinned. True at +the Rust level, but incomplete: `Engine.openRowStore` / +`registry::open_rowstore` (the only path Java has to a `RowStore`) call +plain `RowStore::generate` unconditionally. **No `extern "C"` symbol for the +edge-bearing generator existed.** A worker briefed to build `Graph.java` +against `RowStore.open` would have had no way to reach non-vacuous data at +all — the exact STOP condition this wave was declared clear of, just moved +one layer up the stack, and it would have surfaced as a live "the file I +need is outside my scope" STOP report from a Sonnet worker mid-dispatch +rather than being caught here, before any worker spawned. + +Same shape as the finding this whole sub-arc started from: checking the +*mechanism* (does a hop composition exist) without checking the *path* +(can a caller actually reach the data the mechanism needs). Twice now in +one wave. + +### The fix — the wave's own D1b rule, applied to itself + +`wave-consumer-graph.md`'s Decision D1 already states the rule for exactly +this shape of gap: *"a new ABI symbol... must go through the substrate wave +process FIRST as its own W-tier PR (the consumer-never-grows-the-membrane +rule)."* Applied it to the row-store CONSTRUCTOR, not just the hop op D1 +was originally about — same rule, same reasoning: growing the membrane is +orchestrator/W-tier work, never a consumer worker's ad hoc addition. + +`lgj_rowstore_open_with_edges` (ABI minor 2→3, `docs/abi.md` §12): +`registry::open_rowstore_with_edges` + the `extern "C"` export mirror +`open_rowstore`/`lgj_rowstore_open` symbol-for-symbol — same +`LGJ_RESOURCE_ROWSTORE` kind, same lane shape, no new mask op, purely an +alternative constructor. Java: `Downcalls.rowstoreOpenWithEdges` + +`Engine.openRowStoreWithEdges` (`Abi.requireMinor(3)`, matching the row +store's own minor-2 gate pattern) + the public `RowStore.openWithEdges` +factory. + +### The strongest new result: Java independently reproduces the D1a hop, not just the classid stream + +Added to `RowStoreParityTest` rather than deferred to G1/G2: a **Java-side +transcription of the exact D1a mechanism** (`lgj_row_facet_match` crossing + +raw lane-0 payload decode, zero new ABI op — the mechanism the wave's own +Decision D1 chose to start with) at the SAME parameters as the Rust-pinned +regression (`n=2000, seed=0xF00D_CAFE, edge_classid=0, gate_mask=0x0, +radius=25`). Result: **19 rows at 1 hop, 29 at 2 hops — identical to the +Rust side, to the row.** This is a stronger falsifier than the Rust +regression alone: it proves the membrane doesn't just carry the same +classid stream, it carries the same *edge structure*, read through the +exact mechanism a real `Graph.hop()` will use. + +Two disable-runs, both red-then-green: +1. **Registry level:** hardcoded `open_rowstore_with_edges`'s inner call to + pass `edge_classid = 0` regardless of the argument → the new + `out_of_range_edge_classid_matches_plain_open_through_the_registry` test + (which passes `edge_classid = 16`, expecting parity with plain `open`) + went red, because classid 0 with `gate_mask = 0x0` genuinely writes + structured edges the plain generator never would. Restored, 93/93. +2. **Java level:** forced the hop transcription's classid-match check to + always skip (`if (true) { continue; }`) → 1-hop and 2-hop both collapsed + to 0 and the anti-vacuity assertion failed exactly as expected. Restored, + 194/194. + +### Gates + +`cargo test` 93/93 (+3 over the prior entry's 90), clippy `-D warnings` + +`fmt --check` clean, release build exports `lgj_rowstore_open_with_edges` +(`nm -D`, confirmed present). Java `AllTests` 194/194 (+6, all in +`RowStoreParityTest`) — full suite re-run, not just the new section, since +the stale top-level `target/release/liblgj_abi.so` (pre-dating this pass, +minor 2) initially made EVERY suite fail at class-init (`Downcalls` +eagerly resolves all method handles including the new one) until rebuilt +with `CARGO_TARGET_DIR=$ROOT/target cargo build --release` per the +documented build convention — a real, if brief, self-inflicted false +alarm, not a substrate defect; recorded so a future session doesn't +re-diagnose the same eager-resolution behavior as a bug. + +### Consequence + +The graph-consumer wave is now genuinely dispatchable — the substrate is +proven at BOTH the Rust generator level (prior entry) and the Java +membrane level (this entry), through the exact D1a mechanism the wave +already chose. G1 (traversal facade) and G2 (falsifier tests) are next, +not yet spawned. + ## 2026-08-18 (measured) — the graph-consumer STOP condition was real, and is now cleared **Status:** FINDING + a correction of MY OWN earlier claim. **Confidence:** diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 4a91d0b..08ee19b 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,43 @@ +## 2026-08-18 (later still) — the graph wave's ABI gap, found and closed before dispatch: `lgj_rowstore_open_with_edges` (minor 3) + +Picked up the graph-consumer wave on "everything on track?" — it was marked +DISPATCHABLE by the entry below, and idle capacity while ruff_r2il works its +own PR2/3 track was worth using. Before spawning G1/G2, checked what Java +would actually call to reach `RowStore::generate_with_edges` — and it was +nothing: `Engine.openRowStore`/`registry::open_rowstore` only ever call plain +`RowStore::generate`; no `extern "C"` symbol for the edge-bearing generator +existed anywhere. The prior pass's STOP-condition-RESOLVED note proved the +*generator*, not the *membrane path to it* — a real gap that would have +surfaced mid-dispatch as "G1's scope requires touching a file outside your +scope" the moment a worker tried to open an edge-bearing store from Java. + +Closed it as the wave's own D1b rule requires: "a new ABI symbol... must go +through the substrate wave process FIRST as its own W-tier PR" — done here as +orchestrator work (genuinely new ABI surface, not consumer scope), not +delegated. `lgj_rowstore_open_with_edges` (ABI minor 2→3, docs/abi.md §12): +byte-identical resource kind and lane shape to `lgj_rowstore_open`, purely an +alternative constructor, following the row store's own minor-2 +`requireMinor` gating pattern exactly. `cargo test` 93/93 (+3), Java `AllTests` +194/194 (+6). Full record (including the two disable-runs and the strongest +result — Java independently reproducing the D1a hop mechanism's exact pinned +numbers through raw-segment reads, not just the classid stream): see +`STATUS_BOARD.md` D-LGJ-W6, `EPIPHANIES.md`. + +Not yet dispatched: G1 (traversal facade) and G2 (falsifier tests) are next, +now against a genuinely complete substrate rather than one proven only at the +Rust generator level. + +Meanwhile on ruff: PR #100 (upstream 1500-commit catch-up, explicitly marked +"baseline, not for merge" in its own body) merged anyway by the operator — +its own CI never completed (cancelled on every commit), but incidentally +fixed a 13-day-old `main`-CI lint failure (`prek`/shellcheck on +`.github/workflows/ci.yaml:443`) that predated this whole arc and was +unrelated to it. PR #101 (`ruff_r2il` PR2 first slice — §12 corpus profile +resolving O1, `RefinedTruthSink`) merged cleanly on top, ~1.5h turnaround, +CI green through `prek` on the resulting `main` run — the standing lint issue +stayed fixed. Neither event required action here; recorded for continuity +since both landed mid-session. + ## 2026-08-18 (measured) — W5c's real blocker found + cleared: RowStore::generate_with_edges Asked what was buildable while ruff_r2il PR2/PR3 are blocked. Was about to diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index c69b5d9..e6e38b2 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -40,4 +40,5 @@ layout wired end to end. Doctrine: `E-LGJ-THE-MIDDLE-TIER-IS-DELETED-NOT-WRAPPED | D-LGJ-W2 | lgj-abi row store: `rowstore.rs`, `LGJ_RESOURCE_ROWSTORE`, `lgj_rowstore_open`, strided facet lanes through the unchanged `LgjLaneDesc`, `lgj_op_eq_classid`, `lgj_row_facet_match`, ABI minor 1→2, `docs/abi.md` §11 | **DONE 2026-08-17** — `cargo test` **84/84**, clippy/fmt clean, release build exports **18/18** symbols (`nm -D`). Parity: both kernels vs independent scalar references over 10 row counts × 2 seeds × 4 facets × 4 needles, cross-checked a THIRD way against `RowStore::classid_at`. Two-sided payload-vs-classid falsifier. End-to-end membrane test covers describe → predicate → mask algebra → count → facet-match → lifecycle | | D-LGJ-W3 | Java `RowStore` facade: structured `MemoryLayout`, minor-≥2 gate, `FacetMatchView`, parity test transcribing the generator | **DONE 2026-08-17** — dispatched per `.claude/waves/wave-substrate-w3-w4.md` (3 Sonnet workers, disjoint scopes: FFM membrane extension / public facade / tests), orchestrator-integrated. `javac -Xlint:all` clean (same 7 pre-existing `[restricted]` warnings, zero new). `AllTests` **185/185** (was 132; +53 new checks: 29 parity + 24 lifetime). **One real bug caught by the suite and fixed**: `FacetMatchView.rowCount()` was missing the closed-store guard `matchesOf`/`cardinality` both had — a stale row count was readable after the owning store closed. Fixed, re-verified. Both mandated disable-runs ran red-then-green: (1) `Abi.requireMinor` inflated by 1 → exactly `RowStoreParityTest`+`RowStoreLifetimeTest` failed, all 8 other suites stayed green; (2) the pure-Java generator's a/b draw order swapped in `RowStoreParityTest` → exactly that suite broke (17/29), `RowStoreLifetimeTest` (generator-independent) stayed green — confirming the parity test is a real falsifier, not decorative. `Mask.source()` retyped `NativePattern → NativeResource` (new interface) so a `Mask` can parent onto either a `NativePattern` or a `RowStore` — zero call-site breakage (verified: no existing caller bound the narrower type) | | D-LGJ-W4 | Bench Component F: Vector API facet scan vs the crossing, on the REAL layout | **DONE 2026-08-17** — 1 Sonnet worker (F_RowStoreFacetScan + RowStoreData + Kernels facet-match arms, cross-check-in-@Setup discipline), orchestrator-run JMH: 9/9 combos, cross-checks green at every row count. **Finding: Component C's direction survives, its margin collapses** — Vector API wins the 32-facet strided scan at every row count but by 2.51×/1.92×/1.14× (4K/65K/1M rows) vs C's 56×; at 512 MiB traversed all three arms converge on memory bandwidth. Native arm's per-call allocation asymmetry disclosed in §F with a named follow-up (`facetMatchesInto`), not hidden. summarise.sh extended with the F table (and the old 'E/F' section retitled 'E' — a real naming collision); tables regenerated from the merged CSV | -| D-LGJ-W5 | Three consumer examples (trades / bricks / graph) — one plan file each | **trades DONE 2026-08-17** — `consumers/trades/` (own compile unit, core consumed as a third-party would): `Trade` (schema-not-entity: zero public ctors, zero instance fields, reflection-forced construction still throws), `World.open` → the existing lazy `View` under domain names, zero new membrane surface. TradesParityTest 12/12 (chain vs transcribed-generator recomputation at 1K+64K rows; 0 crossings composing / 1 at terminal THROUGH the domain vocabulary; reflection guard). TradesAllocationTest 3/3 — **the poster's number, measured: 240 bytes/query, IDENTICAL at 64K and 1M rows** (row-count independence is the thesis assertion; 64 KiB absolute backstop). Disable-run: VENUE pointed at the wrong lane → the membrane's own LANE_KIND_MISMATCH rejected it (the binding is checked, not trusted); restored green. **bricks DONE 2026-08-17** — `consumers/bricks/` (2 Sonnet workers K1/K2 per `.claude/waves/wave-consumer-bricks.md`): mask-first RBAC where `authorize(Role)` is a real natively-evaluated predicate in the SAME lazy chain as `where(...)` (`Role.EU_ONLY` = `REGION.eq(EU)`, `DENY_ALL` = `REGION.eq(0xFFFF)` — a genuine impossible predicate, not a Java branch), fail-closed (`UnauthorizedQueryException` BEFORE any crossing; no default-allow path exists), aggregate-only egress (every public method returns `BricksQuery`/`long`/`Map` — structurally no row-shaped type). BricksAuthTest **62/62**: parity vs transcribed generator at 1K+64K; RBAC-as-predicate equivalence (EU_ONLY result == GLOBAL+explicit-where); DENY_ALL counts 0 while paying a real crossing; crossing arithmetic — count()=1, sumBy()=**32 crossings (16 groups × 2: plan_eval + lgj_reduce_sum_i32), IDENTICAL at both row counts** (the thesis: crossings ∝ groups, never rows — the measured 32 corrected K1's "1 per group" Javadoc claim, a real finding about sum-terminal cost); reflection guards. Disable-run: `requireAuthorized` short-circuited → **exactly the 3 can-fire fail-closed checks red, 59 green**; restored, 62/62. Core suite unaffected (188/188). graph still shelved on the D1 ruling + edge-generator substrate change | +| D-LGJ-W5 | Three consumer examples (trades / bricks / graph) — one plan file each | **trades DONE 2026-08-17** — `consumers/trades/` (own compile unit, core consumed as a third-party would): `Trade` (schema-not-entity: zero public ctors, zero instance fields, reflection-forced construction still throws), `World.open` → the existing lazy `View` under domain names, zero new membrane surface. TradesParityTest 12/12 (chain vs transcribed-generator recomputation at 1K+64K rows; 0 crossings composing / 1 at terminal THROUGH the domain vocabulary; reflection guard). TradesAllocationTest 3/3 — **the poster's number, measured: 240 bytes/query, IDENTICAL at 64K and 1M rows** (row-count independence is the thesis assertion; 64 KiB absolute backstop). Disable-run: VENUE pointed at the wrong lane → the membrane's own LANE_KIND_MISMATCH rejected it (the binding is checked, not trusted); restored green. **bricks DONE 2026-08-17** — `consumers/bricks/` (2 Sonnet workers K1/K2 per `.claude/waves/wave-consumer-bricks.md`): mask-first RBAC where `authorize(Role)` is a real natively-evaluated predicate in the SAME lazy chain as `where(...)` (`Role.EU_ONLY` = `REGION.eq(EU)`, `DENY_ALL` = `REGION.eq(0xFFFF)` — a genuine impossible predicate, not a Java branch), fail-closed (`UnauthorizedQueryException` BEFORE any crossing; no default-allow path exists), aggregate-only egress (every public method returns `BricksQuery`/`long`/`Map` — structurally no row-shaped type). BricksAuthTest **62/62**: parity vs transcribed generator at 1K+64K; RBAC-as-predicate equivalence (EU_ONLY result == GLOBAL+explicit-where); DENY_ALL counts 0 while paying a real crossing; crossing arithmetic — count()=1, sumBy()=**32 crossings (16 groups × 2: plan_eval + lgj_reduce_sum_i32), IDENTICAL at both row counts** (the thesis: crossings ∝ groups, never rows — the measured 32 corrected K1's "1 per group" Javadoc claim, a real finding about sum-terminal cost); reflection guards. Disable-run: `requireAuthorized` short-circuited → **exactly the 3 can-fire fail-closed checks red, 59 green**; restored, 62/62. Core suite unaffected (188/188). **graph: substrate now fully unblocked (D-LGJ-W6, this pass) — dispatch is the next action, not yet executed** | +| D-LGJ-W6 | Edge-bearing row store ABI addition (`lgj_rowstore_open_with_edges`, minor 2→3, docs/abi.md §12) — the D1b-shaped "must land as its own W-tier PR before the consumer wave" the graph wave itself named | **DONE 2026-08-18** — orchestrator-authored (genuinely new ABI surface, not consumer-scope work): `registry::open_rowstore_with_edges` + `lgj_rowstore_open_with_edges` (mirrors `lgj_rowstore_open` exactly: same resource kind, same lane shape, no new mask op — purely an alternative constructor), `Engine.openRowStoreWithEdges`/`Abi.requireMinor(3)`, `RowStore.openWithEdges`. `cargo test` **93/93** (+3: registry-level open/describe, out-of-range-classid-matches-plain, radius-overflow-rejected), clippy/fmt clean, release build exports the new symbol (`nm -D`). Java: `AllTests` **194/194** (+6, all in `RowStoreParityTest`) — the strongest new result is a cross-language reproduction of the D1a hop mechanism itself: Java facet-matches + raw-lane-0 payload decode (zero new ABI op) reaches the EXACT same measured hop counts already pinned as a Rust regression (10-row seed → 19 at 1 hop → 29 at 2 hops, `n=2000, seed=0xF00D_CAFE, edge_classid=0, gate_mask=0x0, radius=25`) — proving the two sides of the membrane see identical edge structure, not merely identical classids. Two disable-runs, both red-then-green: (1) registry-level, a classid-not-threaded bug (`open_rowstore_with_edges` hardcoded classid `0`) caught by the out-of-range-parity test; (2) Java-level, the hop's classid-match condition forced to always skip → 1-hop/2-hop both went to 0 and the anti-vacuity assertion failed, exactly as expected. Caught mid-dispatch: the ABI-facing symbol did not exist before this pass (only the bare `RowStore::generate_with_edges` Rust function did, from the prior session) — the graph wave's own STOP-condition-RESOLVED note undersold what was still missing; closed here rather than discovered by G1/G2 mid-flight | diff --git a/docs/abi.md b/docs/abi.md index 3f7a665..ec8153e 100644 --- a/docs/abi.md +++ b/docs/abi.md @@ -62,10 +62,10 @@ cannot disagree with itself. The ABI is a **machine membrane**. It is not the product. The product is the Java semantic API (see `architecture.md`). Therefore: -- It is **small** — currently 18 symbols (minor 2; the "14" this line carried +- It is **small** — currently 19 symbols (minor 3; the "14" this line carried at minor 1 was arithmetic drift — the §7 list it referred to already enumerated 15). Growth is a design smell to be argued for, not a default; - minor 2's three additions are argued in §11. + minor 2's three additions are argued in §11, minor 3's one addition in §12. - It is **bulk-only**. Every call must be capable of doing work proportional to `n_rows` (see §6 — the anti-JNI rule). - It speaks **resource, lane, view, mask, operation, descriptor, status, @@ -78,7 +78,7 @@ semantic API (see `architecture.md`). Therefore: ``` LGJ_ABI_MAJOR = 0 // incompatible change ⇒ bump; Java refuses to load -LGJ_ABI_MINOR = 2 // additive change ⇒ bump; older Java may still load +LGJ_ABI_MINOR = 3 // additive change ⇒ bump; older Java may still load LGJ_MAGIC = 0x4C_47_4A_5F_41_42_49_00 // "LGJ_ABI\0" big-endian-read ``` @@ -269,7 +269,7 @@ predicates or rows are involved. The unfused per-predicate ops are retained only so the fused path can be benchmarked *against* something and so parity can be checked predicate-by-predicate. -## 7. The function surface (18 symbols) +## 7. The function surface (19 symbols) All symbols are prefixed `lgj_`. All return `i32` status except the manifest getter. `out_*` parameters are written only on `OK`. @@ -484,3 +484,65 @@ Rows are 512-byte strided within it. Nothing in this slice needs more — Java reads via `JAVA_INT_UNALIGNED`-class layouts, and every `ndarray::simd` load is a register fill. The 64-byte-aligned base guarantee arrives with the real `NodeRow` (`#[repr(C, align(64))]`) wiring. + +## 12. The edge-bearing row store (ABI minor ≥ 3) + +`consumer-graph-traversal-v1.md`'s falsifiers need a 1-2 hop BFS that is +*non-vacuous* — plain `lgj_rowstore_open`'s payload is uniform noise, so any +hop over it saturates to nearly every row within one or two steps +(`.claude/harvest/graph_density_probe.rs`'s own measurement). This is an +alternative **constructor**, not a new resource kind, new lane shape, or new +mask op — `LGJ_RESOURCE_ROWSTORE` and the whole §11 lane/operation surface +apply to its output unchanged. + +### Resource + +``` +i32 lgj_rowstore_open_with_edges(u64 n_rows, u64 seed, + u32 edge_classid, u64 edge_gate_mask, + u32 edge_radius, u64* out_handle) +``` + +Reuses §11's classid stream **byte-for-byte** — same two SplitMix64 draws per +facet, same `classid = (a >>> 33) & 0xF`. For a facet whose classid equals +`edge_classid` AND whose draw clears the sparsity gate (`a & edge_gate_mask +== 0`), the 12-byte payload instead carries a **structured target row**: + +``` +span = 2 * edge_radius + 1 +offset = (b % span) - edge_radius // signed, |offset| <= edge_radius +target = (row + offset) mod n_rows // bounded-local-neighbourhood +payload_lo64 = target as u64 // hi32 forced to 0 (the flag +payload_hi32 = 0 // a hop reads to know "structured") +``` + +Every other facet (classid mismatch, or gate not cleared) is byte-identical +to `lgj_rowstore_open`'s plain draw (`payload = le64(b) ++ le32(a & +0xFFFFFFFF)`) — so an **out-of-range `edge_classid`** (one that never occurs +in the classid stream, e.g. `16`) reproduces `lgj_rowstore_open` exactly, +which is how `RowStore::generate_with_edges`'s own test suite proves the two +generators share one code path rather than drifting apart. + +Density is governed by `edge_gate_mask`: the gate probability is +`1 / (16 * (edge_gate_mask + 1))` (16 candidate classids × the mask's extra +selectivity), so `edge_gate_mask = 0` is the densest edge-bearing setting. +`edge_radius` must be `< n_rows` (an unsatisfiable bound is +`LGJ_ERR_LENGTH_OVERFLOW`, matching every other overflow-shaped rejection in +this generator family). + +### A hop, at the Java layer (no new op — composition of existing symbols) + +A hop is `lgj_row_facet_match(edge_classid)` (§11, unchanged) to find which +of a row's facets carry the edge classid, followed by a **Java-side** decode +of each matched facet's `payload_lo64` as the target row and a scatter into +the next mask via `lgj_mask_describe`'s WRITABLE lane (§7) — one crossing per +hop, zero new ABI surface. See `consumer-graph-traversal-v1.md` Decision D1a. +A native `lgj_hop` symbol (D1b) remains a future ABI minor if Java-side +scatter measurably dominates; it is not part of this minor. + +### SIMD provenance (unchanged §8 rule, applied) + +The generator's per-facet loop is scalar (SplitMix64 is inherently +sequential per draw); the sparsity gate and target-row arithmetic are cheap +integer ops on that same sequential stream — no new SIMD kernel, matching +`RowStore::generate`'s own scalar generation loop. diff --git a/java/src/main/java/com/adaworldapi/lancegraph/RowStore.java b/java/src/main/java/com/adaworldapi/lancegraph/RowStore.java index b823b9f..e9f799e 100644 --- a/java/src/main/java/com/adaworldapi/lancegraph/RowStore.java +++ b/java/src/main/java/com/adaworldapi/lancegraph/RowStore.java @@ -63,6 +63,32 @@ public static RowStore open(long nRows, long seed) { return new RowStore(h, Engine.rowCount(h)); } + /** + * Open {@code nRows} rows generated deterministically from {@code seed}, with a sparse, gated + * subset of {@code edgeClassid}-matching facets carrying a bounded-local-neighbourhood target + * row instead of raw noise (docs/abi.md §12) — what a non-vacuous BFS traversal needs; {@link + * #open} alone produces uniform-random payloads that saturate any hop within one or two steps. + * + *

Byte-identical classid stream to {@link #open}. An {@code edgeClassid} that never occurs + * in the classid stream (e.g. {@code 16}, one past the 4-bit range) reproduces {@link #open} + * exactly. + * + * @param edgeGateMask sparsity gate: a facet is edge-shaped iff {@code a & edgeGateMask == 0} + * on its underlying draw; {@code 0} is the densest setting + * @param edgeRadius bounds how far a structured target may land from its source row; must be + * {@code < nRows} + * @throws NativeLibraryNotFoundException if the native artifact is not present + * @throws AbiMismatchException if it is present but ABI minor < 3 + */ + public static RowStore openWithEdges(long nRows, long seed, int edgeClassid, + long edgeGateMask, int edgeRadius) { + if (nRows < 0) { + throw new IllegalArgumentException("nRows must be >= 0, was " + nRows); + } + long h = Engine.openRowStoreWithEdges(nRows, seed, edgeClassid, edgeGateMask, edgeRadius); + return new RowStore(h, Engine.rowCount(h)); + } + /** How many rows this resource holds. */ @Override public long rowCount() { diff --git a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Downcalls.java b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Downcalls.java index 45cb8e6..ad0a4a2 100644 --- a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Downcalls.java +++ b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Downcalls.java @@ -109,6 +109,13 @@ private Downcalls() {} FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, ValueLayout.JAVA_INT, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG)); + // Edge-bearing row store (docs/abi.md §12, ABI minor 3). One extra constructor over the + // minor-2 shape above — no new lane, no new op. + private static final MethodHandle ROWSTORE_OPEN_WITH_EDGES = mh("lgj_rowstore_open_with_edges", + FunctionDescriptor.of(ValueLayout.JAVA_INT, + ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_INT, + ValueLayout.JAVA_LONG, ValueLayout.JAVA_INT, ValueLayout.ADDRESS)); + private static MethodHandle mh(String symbol, FunctionDescriptor descriptor) { MemorySegment addr = Abi.lookup().find(symbol).orElseThrow(() -> new LanceGraphException("the native library exports no symbol '" + symbol @@ -355,6 +362,26 @@ public static long rowstoreOpen(long nRows, long seed, MemorySegment outHandle) return outHandle.get(ValueLayout.JAVA_LONG, 0); } + /** + * Build the edge-bearing SoA row store (docs/abi.md §12). Byte-identical classid stream to + * {@link #rowstoreOpen}; a sparse, gated subset of {@code edgeClassid}-matching facets carries + * a bounded-local-neighbourhood target row instead of raw noise. + */ + public static long rowstoreOpenWithEdges(long nRows, long seed, int edgeClassid, + long edgeGateMask, int edgeRadius, + MemorySegment outHandle) { + crossed(); + int st; + try { + st = (int) ROWSTORE_OPEN_WITH_EDGES.invokeExact( + nRows, seed, edgeClassid, edgeGateMask, edgeRadius, outHandle); + } catch (Throwable t) { + throw wrap("lgj_rowstore_open_with_edges", t); + } + Status.check("lgj_rowstore_open_with_edges", st); + return outHandle.get(ValueLayout.JAVA_LONG, 0); + } + /** * Overwrites {@code dstMask} with {@code classid(facet, row) == needle}. {@code facet} is a * facet index {@code 0..32} into the row's 32 facet lanes, not a lane id diff --git a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Engine.java b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Engine.java index 69cf2a1..0db3a18 100644 --- a/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Engine.java +++ b/java/src/main/java/com/adaworldapi/lancegraph/internal/ffm/Engine.java @@ -179,6 +179,25 @@ public static long openRowStore(long nRows, long seed) { return Downcalls.rowstoreOpen(nRows, seed, s.out); } + /** + * Open the edge-bearing SoA row store (docs/abi.md §12): byte-identical classid stream to + * {@link #openRowStore}, plus a sparse, gated subset of {@code edgeClassid}-matching facets + * carrying a bounded-local-neighbourhood target row instead of raw noise — what a non-vacuous + * BFS over the row store needs. Returns the resource handle. Requires ABI minor >= 3. + * + * @param edgeGateMask sparsity gate: a facet is edge-shaped iff {@code a & edgeGateMask == 0} + * on its draw; {@code 0} is the densest setting + * @param edgeRadius bounds how far a structured target may land from its source row; must be + * {@code < nRows} + */ + public static long openRowStoreWithEdges(long nRows, long seed, int edgeClassid, + long edgeGateMask, int edgeRadius) { + Abi.requireMinor(3); + Scratch s = SCRATCH.get(); + return Downcalls.rowstoreOpenWithEdges(nRows, seed, edgeClassid, edgeGateMask, edgeRadius, + s.out); + } + /** * Overwrite {@code dstMask} with {@code classid(facet, row) == classId} for every row of * {@code store} (docs/abi.md §11). {@code facet} is a facet index {@code 0..32} into the row's diff --git a/java/src/test/java/com/adaworldapi/lancegraph/RowStoreParityTest.java b/java/src/test/java/com/adaworldapi/lancegraph/RowStoreParityTest.java index eaede48..60ad1f5 100644 --- a/java/src/test/java/com/adaworldapi/lancegraph/RowStoreParityTest.java +++ b/java/src/test/java/com/adaworldapi/lancegraph/RowStoreParityTest.java @@ -210,5 +210,105 @@ public static void run(Checks c) { c.eq("all " + (n1000 * FACETS_PER_ROW) + " classids read through ROW_LAYOUT match the" + " transcribed generator", 0, layoutMismatches); } + + // ── the edge-bearing generator (docs/abi.md §12) ────────────────────────────────────── + // + // Two claims, both genuinely new (not a restatement of the sections above): (1) an + // out-of-range edgeClassid reproduces RowStore.open's classid stream exactly, THROUGH the + // Java facade rather than only at the Rust generator level; (2) a Java-side hop — the D1a + // mechanism `consumer-graph-traversal-v1.md` chose (facet-match crossing + raw-segment + // payload decode, zero new ABI op) — reproduces the SAME measured hop counts already + // pinned as a Rust regression (rowstore.rs's `measured_hop_counts_are_...` test), proving + // the two sides of the membrane see identical edge structure, not merely identical + // classids. + + c.section("openWithEdges: an out-of-range edgeClassid matches RowStore.open's classid stream"); + int nEdgeParity = 500; + int[][] plainClassids = generateClassids(nEdgeParity, seed); + try (RowStore edged = RowStore.openWithEdges(nEdgeParity, seed, 16, 0x0, 5)) { + c.eq("row count is unaffected", nEdgeParity, edged.rowCount()); + long classidMismatches = 0; + for (int classId : new int[] {0, 9, 15}) { + long expected = countFacetClassid(plainClassids, 7, classId); + try (Mask mask = edged.maskOfFacetClass(new FacetId(7), classId)) { + if (mask.count() != expected) { + classidMismatches++; + } + } + } + c.eq("classid 16 never occurs in the 4-bit stream, so every facet-7 classid count" + + " matches the plain generator exactly", 0, classidMismatches); + } + + c.section("openWithEdges: a Java-side hop reproduces the pinned Rust regression"); + // Exact parameters of rowstore.rs's own `measured_hop_counts_are_three_distinct_non_empty_ + // non_total_sizes`: n=2000, seed=0xF00D_CAFE, edge_classid=0, gate_mask=0x0, radius=25, + // seed set = i*37+5 for i in 0..10. + long hopN = 2000; + int edgeClassid = 0; + try (RowStore hopStore = RowStore.openWithEdges(hopN, 0xF00D_CAFEL, edgeClassid, 0x0L, 25)) { + java.lang.foreign.MemorySegment raw = + com.adaworldapi.lancegraph.internal.ffm.Engine.describeLane(hopStore.handle(), 0) + .segment(); + long rowBytes2 = com.adaworldapi.lancegraph.internal.ffm.Layouts.ROW_LAYOUT.byteSize(); + + java.util.function.Function hop = from -> { + boolean[] seenRows = new boolean[(int) hopN]; + java.util.List out = new java.util.ArrayList<>(); + for (long row : from) { + for (int facet = 0; facet < FACETS_PER_ROW; facet++) { + long classOff = com.adaworldapi.lancegraph.internal.ffm.Layouts.ROW_LAYOUT + .byteOffset( + java.lang.foreign.MemoryLayout.PathElement.sequenceElement( + facet), + java.lang.foreign.MemoryLayout.PathElement.groupElement( + "classid")); + int classid = raw.get(java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED, + row * rowBytes2 + classOff); + if (classid != edgeClassid) { + continue; + } + long payloadOff = com.adaworldapi.lancegraph.internal.ffm.Layouts.ROW_LAYOUT + .byteOffset( + java.lang.foreign.MemoryLayout.PathElement.sequenceElement( + facet), + java.lang.foreign.MemoryLayout.PathElement.groupElement( + "payload")); + long base = row * rowBytes2 + payloadOff; + int hi32 = raw.get(java.lang.foreign.ValueLayout.JAVA_INT_UNALIGNED, + base + 8); + if (hi32 != 0) { + continue; + } + long target = raw.get(java.lang.foreign.ValueLayout.JAVA_LONG_UNALIGNED, + base); + if (target >= 0 && target < hopN && !seenRows[(int) target]) { + seenRows[(int) target] = true; + out.add(target); + } + } + } + long[] arr = new long[out.size()]; + for (int i = 0; i < arr.length; i++) { + arr[i] = out.get(i); + } + return arr; + }; + + long[] seedRows = new long[10]; + for (int i = 0; i < 10; i++) { + seedRows[i] = i * 37L + 5; + } + long[] oneHop = hop.apply(seedRows); + long[] twoHop = hop.apply(oneHop); + + c.eq("seed set size", 10, seedRows.length); + c.eq("1-hop set size matches the Rust-pinned regression", 19, oneHop.length); + c.eq("2-hop set size matches the Rust-pinned regression", 29, twoHop.length); + c.that("three different, non-empty, non-total sizes (anti-vacuity)", + seedRows.length != oneHop.length && oneHop.length != twoHop.length + && oneHop.length > 0 && twoHop.length > 0 + && oneHop.length < hopN && twoHop.length < hopN); + } } } diff --git a/native/lgj-abi/src/abi.rs b/native/lgj-abi/src/abi.rs index d919922..2bdda6a 100644 --- a/native/lgj-abi/src/abi.rs +++ b/native/lgj-abi/src/abi.rs @@ -36,7 +36,15 @@ pub const LGJ_ABI_MAJOR: u32 = 0; /// `lgj_rowstore_open`, `lgj_op_eq_classid`, `lgj_row_facet_match`, and /// strided facet lanes described through the (unchanged) `LgjLaneDesc`. /// Purely additive; a minor-1 Java loads and sees none of it. -pub const LGJ_ABI_MINOR: u32 = 2; +/// +/// Minor **3** (2026-08-18): `lgj_rowstore_open_with_edges` — the +/// edge-bearing row-store generator (abi.md §12, +/// `RowStore::generate_with_edges`), unblocking `consumer-graph-traversal-v1.md`. +/// Byte-identical `LGJ_RESOURCE_ROWSTORE` resource kind, no new lane shape, +/// no new mask op; purely an alternative constructor. A minor-2 Java loads +/// fine and simply cannot call the new symbol (`Abi.requireMinor(3)` gates +/// it, matching the row store's own minor-2 gate pattern). +pub const LGJ_ABI_MINOR: u32 = 3; /// `"LGJ_ABI\0"` read big-endian. /// diff --git a/native/lgj-abi/src/exports.rs b/native/lgj-abi/src/exports.rs index e2dd2aa..6f80945 100644 --- a/native/lgj-abi/src/exports.rs +++ b/native/lgj-abi/src/exports.rs @@ -138,6 +138,58 @@ pub unsafe extern "C" fn lgj_rowstore_open(n_rows: u64, seed: u64, out_handle: * }) } +/// Build the edge-bearing SoA **row store** (abi.md §12) and return its +/// handle: byte-identical classid stream to [`lgj_rowstore_open`], plus a +/// sparse, gated subset of `edge_classid`-matching facets carrying a +/// bounded-local-neighbourhood target row (docs/abi.md §12; the mechanism +/// `consumer-graph-traversal-v1.md`'s hop falsifiers need). ABI minor ≥ 3. +/// +/// `edge_gate_mask` selects sparsity (a facet is edge-shaped iff `a & +/// edge_gate_mask == 0` on its draw — abi.md §12); `edge_radius` bounds how +/// far a structured target may land from its source row. An out-of-range +/// `edge_classid` (one that never occurs in the plain classid stream) +/// reproduces [`lgj_rowstore_open`] byte-for-byte — proven by +/// `RowStore::generate_with_edges`'s own test suite. +/// # Safety +/// +/// A null pointer is *handled*, not UB: it returns `NULL_ARGUMENT`. Beyond +/// that, `out_handle` must be a valid, aligned, writable `u64`. Written only +/// on success. +/// +/// `unsafe` here is a note to Rust callers linking the `rlib`. The JVM, +/// which is the real caller, has no such concept — it upholds the same +/// contract by construction, because every pointer it passes comes from a +/// `MemorySegment` whose size and alignment it derived from the manifest. +#[no_mangle] +pub unsafe extern "C" fn lgj_rowstore_open_with_edges( + n_rows: u64, + seed: u64, + edge_classid: u32, + edge_gate_mask: u64, + edge_radius: u32, + out_handle: *mut u64, +) -> i32 { + guard(|| { + if out_handle.is_null() { + return LGJ_ERR_NULL_ARGUMENT; + } + match registry::open_rowstore_with_edges( + n_rows, + seed, + edge_classid, + edge_gate_mask, + edge_radius, + ) { + Ok(h) => { + // SAFETY: non-null (checked above); written only on success. + unsafe { *out_handle = h }; + LGJ_OK + } + Err(e) => e, + } + }) +} + /// Free a resource: its lanes are dropped, its generation is bumped, and its /// children begin failing with `PARENT_CLOSED`. /// diff --git a/native/lgj-abi/src/registry.rs b/native/lgj-abi/src/registry.rs index dda345f..f9b278a 100644 --- a/native/lgj-abi/src/registry.rs +++ b/native/lgj-abi/src/registry.rs @@ -304,6 +304,32 @@ pub fn open_rowstore(n_rows: u64, seed: u64) -> Result { }) } +/// Create a row-store resource from the edge-bearing generator (abi.md §12, +/// ABI minor 3): identical classid stream to [`open_rowstore`], plus a +/// sparse, gated subset of `edge_classid`-matching facets carrying a +/// bounded-local-neighbourhood target row instead of raw noise — the +/// mechanism `consumer-graph-traversal-v1.md` needs a non-vacuous BFS at +/// all. See [`RowStore::generate_with_edges`] for the exact generator. +pub fn open_rowstore_with_edges( + n_rows: u64, + seed: u64, + edge_classid: u32, + edge_gate_mask: u64, + edge_radius: u32, +) -> Result { + let store = + RowStore::generate_with_edges(n_rows, seed, edge_classid, edge_gate_mask, edge_radius) + .ok_or(LGJ_ERR_LENGTH_OVERFLOW)?; + insert(ResourceEntry { + kind: LGJ_RESOURCE_ROWSTORE, + epoch: next_epoch(), + n_rows, + parent: 0, + parent_gen: 0, + payload: Payload::RowStore(store), + }) +} + /// Create a mask over `parent`, all bits `0` or all bits `1`. /// /// A mask's parent may be a pattern OR a row store — both are read-only @@ -532,6 +558,53 @@ mod tests { close(h).unwrap(); } + /// The registry-level twin of `rowstore.rs`'s own `generate_with_edges` + /// tests: proves the ABI-facing constructor (abi.md §12) reaches the same + /// generator through the full `open_rowstore_with_edges` → registry → + /// `ResourceEntry` path, not just the bare `RowStore` type. + #[test] + fn rowstore_with_edges_opens_and_describes_itself() { + let h = open_rowstore_with_edges(70, 3, 0, 0x0, 10).unwrap(); + let e = resolve_kind(h, LGJ_RESOURCE_ROWSTORE).unwrap(); + let info = e.info(); + assert_eq!(info.kind, LGJ_RESOURCE_ROWSTORE); + assert_eq!(info.lane_count, ROWSTORE_LANE_COUNT); + assert_eq!(info.n_rows, 70); + assert!(e.rowstore().is_some()); + close(h).unwrap(); + } + + /// abi.md §12's own claim, re-proven at the registry boundary: an + /// `edge_classid` outside the 4-bit classid range reproduces + /// `open_rowstore` byte-for-byte through the SAME entry point a Java + /// caller uses — not merely at the underlying `RowStore::generate*` + /// level, which `rowstore.rs` already covers. + #[test] + fn out_of_range_edge_classid_matches_plain_open_through_the_registry() { + let plain = open_rowstore(200, 0xABCD).unwrap(); + let edged = open_rowstore_with_edges(200, 0xABCD, 16, 0x0, 5).unwrap(); + let pe = resolve_kind(plain, LGJ_RESOURCE_ROWSTORE).unwrap(); + let ee = resolve_kind(edged, LGJ_RESOURCE_ROWSTORE).unwrap(); + assert_eq!( + pe.rowstore().unwrap().as_bytes(), + ee.rowstore().unwrap().as_bytes() + ); + close(plain).unwrap(); + close(edged).unwrap(); + } + + /// abi.md §12's overflow rule at the registry boundary: `edge_radius >= + /// n_rows` is `LGJ_ERR_LENGTH_OVERFLOW`, matching every other + /// overflow-shaped rejection in this generator family (`open_rowstore`'s + /// own `LGJ_ERR_LENGTH_OVERFLOW` on a too-large `n_rows`). + #[test] + fn radius_that_cannot_fit_is_rejected_through_the_registry() { + assert_eq!( + open_rowstore_with_edges(10, 1, 0, 0, 10), + Err(LGJ_ERR_LENGTH_OVERFLOW) + ); + } + /// A mask parents onto a row store exactly as onto a pattern — same /// row-count sizing, same tail rule, same parent-liveness propagation. #[test]