diff --git a/.claude/rules/feast-components.md b/.claude/rules/feast-components.md index 5c03cb0bd3d..02b1c6f4dd4 100644 --- a/.claude/rules/feast-components.md +++ b/.claude/rules/feast-components.md @@ -24,6 +24,7 @@ For testing patterns and debugging, also read `skills/feast-testing/SKILL.md`. - **Unit tests**: add or update tests in `sdk/python/tests/unit/infra//` - **Integration tests**: run `make test-python-integration-local`; add a universal test case in `sdk/python/tests/integration/` if the change affects retrieval or materialization behavior +- **SQL registry binary columns**: in `infra/registry/sql.py`, a new column that stores a serialized proto or blob metadata must use `ProtoBytes`, not `LargeBinary` directly — `LargeBinary` maps to MySQL `BLOB` (64 KB cap) and silently truncates large protos - **Protos**: if you add a field to a proto message, recompile with `make protos` and update serialization helpers in `proto_registry_utils.py` - **Both SDKs**: if the change affects online serving, check whether the Go server (`go/`) also needs updating - **Skills/Rules**: if the change introduces new patterns, interfaces, or conventions that agents should follow, update the relevant section in `skills/feast-architecture/SKILL.md` (and `skills/feast-testing/SKILL.md` if testing patterns changed) diff --git a/.codecov.yaml b/.codecov.yaml new file mode 100644 index 00000000000..2fa599642ec --- /dev/null +++ b/.codecov.yaml @@ -0,0 +1,48 @@ +codecov: + require_ci_to_pass: true + +coverage: + precision: 2 + round: down + range: "50...70" + + status: + project: + default: + informational: true + target: auto + threshold: 1% + patch: + default: + informational: true + target: 70% + +comment: + layout: "reach,diff,flags,files,footer" + behavior: default + require_changes: false + require_base: false + require_head: true + show_carryforward_flags: true + +flags: + python-unit: + paths: + - sdk/python/feast/ + carryforward: true + go-feature-server: + paths: + - go/ + carryforward: true + +ignore: + - "sdk/python/tests/**" + - "**/*_pb2.py" + - "**/*_pb2_grpc.py" + - "sdk/python/feast/protos/**" + - "sdk/python/feast/embedded_go/**" + - "protos/**" + - "docs/**" + - "ui/**" + - "java/**" + - "infra/feast-operator/test/**" diff --git a/.cursor/rules/feast-components.mdc b/.cursor/rules/feast-components.mdc index a474f00fc47..f015619020d 100644 --- a/.cursor/rules/feast-components.mdc +++ b/.cursor/rules/feast-components.mdc @@ -20,6 +20,7 @@ For testing patterns and debugging, also read `skills/feast-testing/SKILL.md`. - **Unit tests**: add or update tests in `sdk/python/tests/unit/infra//` - **Integration tests**: run `make test-python-integration-local`; add a universal test case in `sdk/python/tests/integration/` if the change affects retrieval or materialization behavior +- **SQL registry binary columns**: in `infra/registry/sql.py`, a new column that stores a serialized proto or blob metadata must use `ProtoBytes`, not `LargeBinary` directly — `LargeBinary` maps to MySQL `BLOB` (64 KB cap) and silently truncates large protos - **Protos**: if you add a field to a proto message, recompile with `make protos` and update serialization helpers in `proto_registry_utils.py` - **Both SDKs**: if the change affects online serving, check whether the Go server (`go/`) also needs updating - **Skills/Rules**: if the change introduces new patterns, interfaces, or conventions that agents should follow, update the relevant section in `skills/feast-architecture/SKILL.md` (and `skills/feast-testing/SKILL.md` if testing patterns changed) diff --git a/.cursor/rules/feast-ui.mdc b/.cursor/rules/feast-ui.mdc new file mode 100644 index 00000000000..9072cbe335f --- /dev/null +++ b/.cursor/rules/feast-ui.mdc @@ -0,0 +1,19 @@ +--- +description: Formatting and lint rules for the Feast UI (React/TypeScript) +globs: ui/src/** +alwaysApply: false +--- + +## After editing any file under `ui/src/` + +1. **Run Prettier** before considering the task complete: + ```bash + cd ui && yarn prettier --write + ``` +2. **Verify** formatting passes: + ```bash + cd ui && yarn format:check + ``` + CI runs `yarn format:check` and will reject PRs with style violations. + +3. Prettier config lives in `ui/package.json` (no separate `.prettierrc`). Do not override it. diff --git a/.github/workflows/nightly_python_sdk_release.yml b/.github/workflows/nightly_python_sdk_release.yml index e350ccd3f9f..6c758360d56 100644 --- a/.github/workflows/nightly_python_sdk_release.yml +++ b/.github/workflows/nightly_python_sdk_release.yml @@ -55,11 +55,14 @@ jobs: set -e echo "$SEMANTIC_OUTPUT" - BASE_VERSION=$(echo "$SEMANTIC_OUTPUT" | grep 'The next release version is' | sed -E 's/.* ([[:digit:].]+)$/\1/' | tail -n 1) + BASE_VERSION=$(printf '%s\n' "$SEMANTIC_OUTPUT" | sed -nE 's/.*The next release version is ([[:digit:].]+)$/\1/p' | tail -n 1) if [[ -z "$BASE_VERSION" ]]; then echo "Could not determine a semantic-release next version (exit code: ${SEMANTIC_STATUS}); falling back to next patch after latest stable tag." - source infra/scripts/setup-common-functions.sh - LATEST_TAG=$(get_tag_release -s) + LATEST_TAG=$(git tag -l 'v[0-9]*.[0-9]*.[0-9]*' --sort=-v:refname | sed -nE '/^v[0-9]+\.[0-9]+\.[0-9]+$/{p;q;}') + if [[ -z "$LATEST_TAG" ]]; then + echo "Could not determine latest stable tag." + exit 1 + fi LATEST_VERSION="${LATEST_TAG#v}" IFS=. read -r MAJOR MINOR PATCH <<< "$LATEST_VERSION" BASE_VERSION="${MAJOR}.${MINOR}.$((PATCH + 1))" diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 936b2777f1d..88d5c102250 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -4,7 +4,6 @@ on: pull_request_target: types: - opened - - synchronize - labeled concurrency: diff --git a/.github/workflows/pr_registration_integration_tests.yml b/.github/workflows/pr_registration_integration_tests.yml index 76cbe701cf4..81801b643b6 100644 --- a/.github/workflows/pr_registration_integration_tests.yml +++ b/.github/workflows/pr_registration_integration_tests.yml @@ -4,7 +4,6 @@ on: pull_request_target: types: - opened - - synchronize - labeled concurrency: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a5377982b29..ffa91034d32 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -29,14 +29,16 @@ on: token: description: 'Personal Access Token' required: true - default: "" type: string publish_ui: description: 'Publish to NPM?' required: true - default: true type: boolean +permissions: + contents: read + id-token: write + jobs: publish-python-sdk: uses: ./.github/workflows/publish_python_sdk.yml diff --git a/.github/workflows/publish_web_ui.yml b/.github/workflows/publish_web_ui.yml index f8f52f6a84c..b36165625fd 100644 --- a/.github/workflows/publish_web_ui.yml +++ b/.github/workflows/publish_web_ui.yml @@ -1,26 +1,6 @@ name: publish web ui on: - workflow_dispatch: # Allows manual trigger of the workflow - inputs: - current_version: - description: 'Current version to bump from (e.g., v1.2.3). If not provided, will auto-detect from git tags' - required: false - type: string - custom_version: # Optional input for a custom version - description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' - required: false - type: string - token: - description: 'Personal Access Token' - required: false - default: "" - type: string - publish_ui: - description: 'Publish to NPM?' - required: true - default: true - type: boolean workflow_call: # Allows trigger of the workflow from another workflow inputs: current_version: @@ -39,16 +19,17 @@ on: publish_ui: description: 'Publish to NPM?' required: true - default: true type: boolean +permissions: + contents: read + id-token: write + jobs: publish-web-ui-npm: if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest - env: - # This publish is working using an NPM automation token to bypass 2FA - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + environment: production steps: - uses: actions/checkout@v4 - name: Determine current version @@ -108,8 +89,10 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version-file: './ui/.nvmrc' + node-version: '22.14.0' registry-url: 'https://registry.npmjs.org' + - name: Update npm for trusted publishing + run: npm install --global npm@11.5.1 - name: Bump file versions (temporarily for Web UI publish) if: github.event.inputs.custom_version != '' env: @@ -137,6 +120,3 @@ jobs: working-directory: ./ui if: github.event.inputs.publish_ui != 'false' run: npm publish - env: - # This publish is working using an NPM automation token to bypass 2FA - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 70a99c0ed05..1dee7f79963 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -79,9 +79,68 @@ jobs: fi make test-python-unit + - name: Upload Python coverage to Codecov + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + with: + file: ./coverage.xml + flags: python-unit + fail_ci_if_error: false + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - name: Minimize uv cache run: uv cache prune --ci + unit-test-go: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + architecture: x64 + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y make protobuf-compiler libsqlite3-dev + - name: Install Go proto plugins + run: | + go install google.golang.org/protobuf/cmd/protoc-gen-go@latest + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest + - name: Compile Go protobufs + run: make compile-protos-go + - name: Create virtual environment + run: | + uv venv + echo "${{ github.workspace }}/.venv/bin" >> $GITHUB_PATH + - name: Install feast locally + run: make install-feast-locally + - name: Run Go tests with coverage + run: | + CGO_ENABLED=1 go test \ + -coverprofile=go/coverage.out \ + -covermode=atomic \ + -skip "TestGetOnlineFeatures|TestSqliteOnlineRead" \ + ./go/... + - name: Upload Go coverage to Codecov + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + with: + file: ./go/coverage.out + flags: go-feature-server + fail_ci_if_error: false + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + unit-test-ui: runs-on: ubuntu-latest env: @@ -103,6 +162,9 @@ jobs: - name: Build yarn rollup working-directory: ./ui run: yarn build:lib + - name: Build production UI + working-directory: ./ui + run: CI=true npm run build --omit=dev - name: Run yarn tests working-directory: ./ui run: yarn test --watchAll=false diff --git a/.secrets.baseline b/.secrets.baseline index e7126304069..57fa3236f27 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -142,7 +142,7 @@ "filename": ".github/workflows/publish.yml", "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0", "is_verified": false, - "line_number": 43 + "line_number": 45 } ], ".github/workflows/publish_python_sdk.yml": [ @@ -185,7 +185,7 @@ "filename": "docs/reference/online-stores/milvus.md", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 33 + "line_number": 41 } ], "docs/reference/registries/sql.md": [ @@ -957,7 +957,7 @@ "filename": "infra/feast-operator/api/v1/featurestore_types.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 906 + "line_number": 937 } ], "infra/feast-operator/api/v1/zz_generated.deepcopy.go": [ @@ -980,7 +980,7 @@ "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "c2028031c154bbe86fd69bef740855c74b927dcf", "is_verified": false, - "line_number": 1528 + "line_number": 1570 } ], "infra/feast-operator/api/v1alpha1/featurestore_types.go": [ @@ -989,7 +989,7 @@ "filename": "infra/feast-operator/api/v1alpha1/featurestore_types.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 649 + "line_number": 651 } ], "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go": [ @@ -1172,7 +1172,7 @@ "filename": "infra/feast-operator/internal/controller/services/services.go", "hashed_secret": "36dc326eb15c7bdd8d91a6b87905bcea20b637d1", "is_verified": false, - "line_number": 180 + "line_number": 183 } ], "infra/feast-operator/internal/controller/services/tls_test.go": [ @@ -1555,5 +1555,5 @@ } ] }, - "generated_at": "2026-06-11T15:45:28Z" + "generated_at": "2026-07-17T12:34:13Z" } diff --git a/AGENTS.md b/AGENTS.md index e8ebb031cdc..2ade5b12f7c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,7 +92,7 @@ Architecture & design intent: `docs/getting-started/architecture/` (overview, wr - Use type hints on all Python function signatures - Follow existing patterns in the module you are modifying -- PR titles must follow semantic conventions: `feat:`, `fix:`, `ci:`, `chore:`, `docs:` +- PR titles must follow conventional commit conventions with a lowercase type and a capitalized subject after the colon: `feat: Add ...`, `fix: Correct ...`, `ci: Update ...`, `chore: Refresh ...`, `docs: Add ...` - Sign off commits with `git commit -s` (DCO requirement) - Uses `ruff` for Python linting and formatting; Go uses standard `gofmt` - Recompile protos after making changes to `.proto` files (`make protos`) diff --git a/CHANGELOG.md b/CHANGELOG.md index e628e8ea7a3..38b88ae86df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,147 @@ # Changelog +# [0.65.0](https://github.com/feast-dev/feast/compare/v0.64.0...v0.65.0) (2026-07-20) + + +### Bug Fixes + +* add debug logging for FIPS mode detection fallback ([6c1b24e](https://github.com/feast-dev/feast/commit/6c1b24ee6f27c469107269828b623180882de321)) +* Build embedded UI from local source ([#6525](https://github.com/feast-dev/feast/issues/6525)) ([3500349](https://github.com/feast-dev/feast/commit/35003494862f8b4af7f2d7eea321356743df074f)) +* Bump decommissioned Snowflake Python UDF runtime from 3.9 to 3.10 ([#6606](https://github.com/feast-dev/feast/issues/6606)) ([#6608](https://github.com/feast-dev/feast/issues/6608)) ([10341e4](https://github.com/feast-dev/feast/commit/10341e4d9cc05478ef863b33c3eeee3cc8da0162)) +* configure FIPS-compliant gRPC cipher suites for offline server ([6bc80a2](https://github.com/feast-dev/feast/commit/6bc80a2474e013724b1579d6de824c76e5d77f3d)) +* Correct Flink PyArrow dependency constraints ([#6604](https://github.com/feast-dev/feast/issues/6604)) ([70a9751](https://github.com/feast-dev/feast/commit/70a97515b8dc93e992d214e11c2bf9cd9ec65aa7)) +* Fix ValueError in signal handling for Trino worker threads ([#6428](https://github.com/feast-dev/feast/issues/6428)) ([506d919](https://github.com/feast-dev/feast/commit/506d919f3aaaaccdc4ac14cd23d0870302c6b13c)) +* Fixed monitoring page issues ([7946018](https://github.com/feast-dev/feast/commit/7946018c40f482bd82efb9a1c555d47dba5d4e54)) +* Make pytest config compatible with newer pytest ([#5779](https://github.com/feast-dev/feast/issues/5779)) ([a57ea33](https://github.com/feast-dev/feast/commit/a57ea331c53bf48a08e96764ff88fe2104bdb5bc)) +* Replace comma with space in DynamoDB-incompatible label tag value ([51e3a16](https://github.com/feast-dev/feast/commit/51e3a164fabf6e1cb2a1e41ae55328a4778177c4)) +* Resolve UI build warnings ([#6529](https://github.com/feast-dev/feast/issues/6529)) ([abe92af](https://github.com/feast-dev/feast/commit/abe92af5ef31283472a5d220390ed80b35baf3aa)) +* Unblock nightly UI build ([#6570](https://github.com/feast-dev/feast/issues/6570)) ([f296d4b](https://github.com/feast-dev/feast/commit/f296d4ba14c5d512429219b2b7845673e0fe524d)) +* Use LONGBLOB for SQL registry proto columns on MySQL ([#6566](https://github.com/feast-dev/feast/issues/6566)) ([7e4beb2](https://github.com/feast-dev/feast/commit/7e4beb21fdba8ee00afd7d0176989e42c10e31a1)) + + +### Features + +* Add click-to-zoom lightbox for blog post images ([#6575](https://github.com/feast-dev/feast/issues/6575)) ([1cb23fd](https://github.com/feast-dev/feast/commit/1cb23fde61862c6d53b434cd5b3ccbffea58d2a6)) +* Add dark mode support to website and blog ([#6589](https://github.com/feast-dev/feast/issues/6589)) ([7358fb8](https://github.com/feast-dev/feast/commit/7358fb8c9a9f8543f6add0e13b8b4b06ef11916b)) +* Add OnlineStore for Aerospike ([#6532](https://github.com/feast-dev/feast/issues/6532)) ([9cd35e1](https://github.com/feast-dev/feast/commit/9cd35e140a949dc44a9915300f2724dd2e702f03)) +* Add OpenLineage Consumer to Feast - receive, store, and visualize cross-producer lineage ([#6549](https://github.com/feast-dev/feast/issues/6549)) ([a834126](https://github.com/feast-dev/feast/commit/a834126b674356ea1efeafd7006f579c9148c3a1)) +* Add registry list feature views by updated since ([#6092](https://github.com/feast-dev/feast/issues/6092)) ([#6093](https://github.com/feast-dev/feast/issues/6093)) ([006c606](https://github.com/feast-dev/feast/commit/006c606183457373d8c83b1986a9f35e1d764c9a)) +* Add ScyllaDB online store with vector search ([#6508](https://github.com/feast-dev/feast/issues/6508)) ([1669661](https://github.com/feast-dev/feast/commit/1669661e15d3ba3b5ab9a9fffd19248d9c0da211)) +* Added compute and jobs UI ([ba2c05c](https://github.com/feast-dev/feast/commit/ba2c05c731be64aff8e7af0fdeefcbe8aa397308)) +* Added Iceberg REST Catalog data source support ([e0a8573](https://github.com/feast-dev/feast/commit/e0a8573eb453dd7060343c4e474e7fca7e1378f7)) +* Bring Your Own Spark - SparkApplication ([#6550](https://github.com/feast-dev/feast/issues/6550)) ([dcd496f](https://github.com/feast-dev/feast/commit/dcd496f22e109f0f77338d41e057dd71113b67d0)) +* **cassandra:** Add multi-DC support via per-datacenter execution profiles ([#6434](https://github.com/feast-dev/feast/issues/6434)) ([0de9196](https://github.com/feast-dev/feast/commit/0de9196d75a63e1ba3860de051cab40c6eba8efc)) +* Enhanced data source creation as a visual catalog with type-specific forms ([#6557](https://github.com/feast-dev/feast/issues/6557)) ([d6acbba](https://github.com/feast-dev/feast/commit/d6acbba057cde6e1c088d068e39492448c73fea1)) +* Enhanced datasets UI functionality ([de11152](https://github.com/feast-dev/feast/commit/de111525985b542b8bfa61118e1ee949254d8703)) +* Implement RegistryServer.Proto RPC with RBAC-filtered response ([#6558](https://github.com/feast-dev/feast/issues/6558)) ([#6552](https://github.com/feast-dev/feast/issues/6552)) ([0d02614](https://github.com/feast-dev/feast/commit/0d02614edcc6fb71992cbb0b539c4b0e2a50f810)) +* New zoned timestamp feature type ([#6536](https://github.com/feast-dev/feast/issues/6536)) ([#6537](https://github.com/feast-dev/feast/issues/6537)) ([eb042f0](https://github.com/feast-dev/feast/commit/eb042f04f5d9bdd7dafbaf654d5b5ec2a2572d9f)) +* **operator:** Auto-create RBAC for spark_application batch engine ([#6597](https://github.com/feast-dev/feast/issues/6597)) ([f487b37](https://github.com/feast-dev/feast/commit/f487b37fd317c63d0d0060ccf8be5d8238d484dd)) +* **operator:** integrate cluster TLS profile for OCP 5.0 compliance ([43263a6](https://github.com/feast-dev/feast/commit/43263a658abe5e2080241b5819fdd8affb4e5fef)) +* Permissions CRUD UI and OIDC auth integration in UI ([6511da1](https://github.com/feast-dev/feast/commit/6511da1323f5634595b5b2ae4e8a5055599c7885)) +* Retrieve historical features from BigQuery without entity_df ([#6569](https://github.com/feast-dev/feast/issues/6569)) ([cd5f6bb](https://github.com/feast-dev/feast/commit/cd5f6bbbd36f11f1d2e2faf8e5e773076b7a3026)), closes [#6558](https://github.com/feast-dev/feast/issues/6558) [#6552](https://github.com/feast-dev/feast/issues/6552) +* **spark:** SparkSource query+path and pre-computed offline read for BatchFeatureView ([#6440](https://github.com/feast-dev/feast/issues/6440)) ([4dc8757](https://github.com/feast-dev/feast/commit/4dc8757626c69c833a8d8174a6bd1513b1671ad7)) + + +### BREAKING CHANGES + +* total_timeout_ms is renamed to batch_total_timeout_ms. Config files using the old name must be updated. No default value change. + +Docs updated (reference + perf-tuning guide) with a short explainer on the per-attempt vs total deadline distinction. Two new unit tests pin the policy wiring: socket_timeout_ms propagates to all three scopes, and is omitted (not injected as None) when unset. + +Signed-off-by: Valentyn Kahamlyk + +* refactor(aerospike): use MAP_KEY_ORDERED, KEY_DIGEST, and instance-scoped client + +Cheap-win cleanups flagged in review, all touching the same small patch of write-path and lifecycle code. + +* Map CDTs are now created with MAP_KEY_ORDERED. map_get_by_key / map_remove_by_key on an ordered map are O(log N) in the map size instead of O(N); matters on reads of wide feature views and on the update() background scan (which walks every record in the project's set). + +* Writes drop POLICY_KEY_SEND and rely on the client default (POLICY_KEY_DIGEST). The serialized entity key is no longer stored alongside each record, saving per-record storage the read path never consumes (batch_operate preserves request order; results are paired back by zip in online_read). + +* _client moves from a class attribute to an instance attribute (set in __init__). Previously two AerospikeOnlineStore instances could share the cached client through class state until one wrote self._client. With the instance attribute the state is always per-instance from construction. + +* Drop MongoDB references from class docstrings and comments (they referred to how the storage layout was derived rather than documenting current behavior). Also rewrite the _build_batch_writes docstring to describe the policies applied on the write path. + +Unit test assertions for the write-path record are updated: bw.policy is now None (client default applies) and map ops carry map_policy={'map_order': MAP_KEY_ORDERED}. All three docker-backed integration tests still pass end-to-end (cross-FV upsert, update() background scan, full feature-store round-trip), so the read/write shape survives the ordering and policy changes against a real server. + +Signed-off-by: Valentyn Kahamlyk + +* feat(aerospike): add per-FV namespace/set overrides and prewriting hook + +Adds three configuration knobs to AerospikeOnlineStoreConfig: + +- namespace_overrides: pin individual feature views to a different + Aerospike namespace (e.g. RAM-only vs. SSD-backed) without splitting + the project across stores. +- set_overrides: place a feature view in its own set so admin ops on + it (truncate, scan-based deletes during `feast apply`) do not touch + records of other views. +- prewriting_hook: import-string-resolved callable invoked once per + online_write_batch with the rows about to be written, returning the + rows that actually go on the wire. Resolved and cached on first use; + returning [] short-circuits the wire call. + +Read, write, update and teardown paths all honour the per-FV ns/set +resolution. update() groups dropped feature views by their resolved +(ns, set) pair and issues one background scan per group. teardown() +truncates every unique (ns, set) pair the project may have written to, +including the store-level default. + +Adds 22 unit tests for the new behaviour and updates 3 existing call +sites of _build_batch_writes for the new namespace= parameter. Adds a +sample hook module under examples/online_store/aerospike_overrides_and_hooks/ +and corresponding sections in docs/reference/online-stores/aerospike.md. + +Signed-off-by: Valentyn Kahamlyk + +* test: update aerospike image tag + +Signed-off-by: Valentyn Kahamlyk + +* chore: sync README template and secrets baseline after master merge + +Signed-off-by: Valentyn Kahamlyk + +* chore: fix secrets baseline line number for v1 operator types + +Adding aerospike to the feast-operator enum shifted the allowlisted +SecretRef entry in api/v1/featurestore_types.go by one line. + +Signed-off-by: Valentyn Kahamlyk + +* docs: update aerospike docs + +Signed-off-by: Valentyn Kahamlyk + +* fix(aerospike): wire batch max_retries and fix empty projection handling + +Copilot review feedback on PR #6532: + +- Add max_retries to the batch client policy (batch_operate/batch_write path) +- Treat empty projected feature maps as present FV slots (is not None) +- Return {} from _normalize_projected_features([]) instead of None +- Fix projection unit test mock/assertions +- Correct prewriting_hook config docstring + +Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> +Signed-off-by: Valentyn Kahamlyk + +* style(aerospike): format online_read docs assignment for ruff + +Signed-off-by: Valentyn Kahamlyk + +* chore: update pixi.lock for aerospike optional extra + +Regenerate the v6 lockfile with Pixi v0.63.1 after adding the aerospike extra to pyproject.toml. + +Signed-off-by: Valentyn Kahamlyk + +* fix(aerospike): add client init lock and batch chunking + +Guard lazy client creation with a lock to avoid connection leaks under concurrent first use, and chunk batch reads/writes by batch_max_records so large materializations stay under Aerospike server batch limits. + +Signed-off-by: Valentyn Kahamlyk + # [0.64.0](https://github.com/feast-dev/feast/compare/v0.63.0...v0.64.0) (2026-06-13) diff --git a/Makefile b/Makefile index e277295e84c..2ede6f769e5 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ install-python-dependencies-ci: ## Install Python CI dependencies using uv pip s # Install CPU-only torch first to prevent CUDA dependency issues (Linux only) @if [ "$$(uname -s)" = "Linux" ]; then \ echo "Installing dependencies with torch CPU index for Linux..."; \ - uv pip sync --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \ + uv pip sync --torch-backend cpu sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \ else \ echo "Installing dependencies from PyPI for macOS..."; \ uv pip sync sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \ @@ -150,6 +150,7 @@ lock-python-dependencies-all: ## Recompile and lock all Python dependency sets f pixi run --environment $(call get_env_name,$(ver)) --manifest-path infra/scripts/pixi/pixi.toml \ "uv pip compile -p $(ver) --no-strip-extras pyproject.toml --extra minimal-sdist-build \ --no-emit-package milvus-lite \ + --no-emit-package pymilvus \ --generate-hashes --output-file sdk/python/requirements/py$(ver)-minimal-sdist-requirements.txt" && \ pixi run --environment $(call get_env_name,$(ver)) --manifest-path infra/scripts/pixi/pixi.toml \ "uv pip install -p $(ver) pybuild-deps==0.5.0 pip==25.0.1 && \ @@ -171,6 +172,9 @@ benchmark-python-local: ## Run integration + benchmark tests for Python (local d test-python-unit: ## Run Python unit tests (use pattern= to filter tests, e.g., pattern=milvus, pattern=test_online_retrieval.py, pattern=test_online_retrieval.py::test_get_online_features_milvus) uv run python -m pytest -n 8 --color=yes $(if $(pattern),-k "$(pattern)") \ + --cov=feast \ + --cov-report=xml \ + --cov-report=term-missing \ sdk/python/tests/unit # Fast unit tests only @@ -813,13 +817,12 @@ build-helm-docs: ## Build helm docs # Note: these require node and yarn to be installed build-ui: ## Build Feast UI - cd $(ROOT_DIR)/sdk/python/feast/ui && yarn upgrade @feast-dev/feast-ui --latest && yarn install && npm run build --omit=dev - -build-ui-local: ## Build Feast UI locally cd $(ROOT_DIR)/ui && yarn install && npm run build --omit=dev rm -rf $(ROOT_DIR)/sdk/python/feast/ui/build cp -r $(ROOT_DIR)/ui/build $(ROOT_DIR)/sdk/python/feast/ui/ +build-ui-local: build-ui ## Build Feast UI locally + format-ui: ## Format Feast UI cd $(ROOT_DIR)/ui && NPM_TOKEN= yarn install && NPM_TOKEN= yarn format diff --git a/README.md b/README.md index 115bd37903f..0ea5b0a135a 100644 --- a/README.md +++ b/README.md @@ -227,6 +227,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [SingleStore](https://docs.feast.dev/reference/online-stores/singlestore) * [x] [Couchbase](https://docs.feast.dev/reference/online-stores/couchbase) * [x] [MongoDB](https://docs.feast.dev/reference/online-stores/mongodb) + * [x] [Aerospike](https://docs.feast.dev/reference/online-stores/aerospike) * [x] [Qdrant (vector store)](https://docs.feast.dev/reference/online-stores/qdrant) * [x] [Milvus (vector store)](https://docs.feast.dev/reference/online-stores/milvus) * [x] [Faiss (vector store)](https://docs.feast.dev/reference/online-stores/faiss) @@ -254,7 +255,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Offline Feature Server (alpha)](https://docs.feast.dev/reference/feature-servers/offline-feature-server) * [x] [Registry server (alpha)](https://github.com/feast-dev/feast/blob/master/docs/reference/feature-servers/registry-server.md) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** - * [x] Data profiling and validation (Great Expectations) + * [x] [Feature Quality Monitoring](https://docs.feast.dev/how-to-guides/feature-monitoring) — built-in metrics, drift detection, serving log monitoring, and UI dashboard * **Feature Discovery and Governance** * [x] Python SDK for browsing feature registry * [x] CLI for browsing feature registry diff --git a/docs/README.md b/docs/README.md index 8229ac10587..e8588be340f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -71,7 +71,7 @@ Feast helps ML platform/MLOps teams with DevOps experience productionize real-ti * **batch feature engineering**: Feast supports on-demand and streaming transformations. Feast is also investing in supporting batch transformations. * **native streaming feature integration:** Feast enables users to push streaming features, but does not pull from streaming sources or manage streaming pipelines. * **lineage:** Feast helps tie feature values to model versions, but is not a complete solution for capturing end-to-end lineage from raw data sources to model versions. Feast also has community contributed plugins with [DataHub](https://datahubproject.io/docs/generated/ingestion/sources/feast/) and [Amundsen](https://github.com/amundsen-io/amundsen/blob/4a9d60176767c4d68d1cad5b093320ea22e26a49/databuilder/databuilder/extractor/feast\_extractor.py). -* **data quality / drift detection**: Feast has experimental integrations with [Great Expectations](https://greatexpectations.io/), but is not purpose built to solve data drift / data quality issues. This requires more sophisticated monitoring across data pipelines, served feature values, labels, and model versions. +* **data quality / drift detection**: Feast includes built-in [Feature Quality Monitoring](how-to-guides/feature-monitoring.md) that computes statistical metrics (null rates, distributions, percentiles), detects drift across batch data and serving logs, and provides a monitoring UI dashboard. ## Example use cases diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index c94dd88708d..d35dbf1652b 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -57,7 +57,6 @@ * [Fraud detection on GCP](tutorials/tutorials-overview/fraud-detection.md) * [Real-time credit scoring on AWS](tutorials/tutorials-overview/real-time-credit-scoring-on-aws.md) * [Driver stats on Snowflake](tutorials/tutorials-overview/driver-stats-on-snowflake.md) -* [Validating historical features with Great Expectations](tutorials/validating-historical-features.md) * [Building streaming features](tutorials/building-streaming-features.md) * [Retrieval Augmented Generation (RAG) with Feast](tutorials/rag-with-docling.md) * [RAG Fine Tuning with Feast and Milvus](../examples/rag-retriever/README.md) @@ -164,6 +163,7 @@ * [SingleStore](reference/online-stores/singlestore.md) * [Milvus](reference/online-stores/milvus.md) * [MongoDB](reference/online-stores/mongodb.md) + * [Aerospike](reference/online-stores/aerospike.md) * [Elasticsearch](reference/online-stores/elasticsearch.md) * [Qdrant](reference/online-stores/qdrant.md) * [Faiss](reference/online-stores/faiss.md) @@ -205,7 +205,7 @@ * [\[Beta\] On demand feature view](reference/beta-on-demand-feature-view.md) * [\[Alpha\] Static Artifacts Loading](reference/alpha-static-artifacts.md) * [\[Alpha\] Vector Database](reference/alpha-vector-database.md) -* [\[Alpha\] Data quality monitoring](reference/dqm.md) +* [Data Quality Monitoring](reference/dqm.md) * [\[Alpha\] Streaming feature computation with Denormalized](reference/denormalized.md) * [\[Alpha\] Feature View Versioning](reference/alpha-feature-view-versioning.md) * [OpenLineage Integration](reference/openlineage.md) diff --git a/docs/adr/ADR-0011-data-quality-monitoring.md b/docs/adr/ADR-0011-data-quality-monitoring.md index 55df3aa1ddd..657d219c48c 100644 --- a/docs/adr/ADR-0011-data-quality-monitoring.md +++ b/docs/adr/ADR-0011-data-quality-monitoring.md @@ -2,7 +2,7 @@ ## Status -Accepted +Superseded — The original external-library-based validation has been replaced by Feast's native [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) system (`feast monitor run`). ## Context @@ -12,79 +12,51 @@ Data quality issues can significantly impact ML model performance. Several compl - **Upstream pipeline bugs**: Bugs in upstream pipelines can cause invalid values to overwrite existing valid values in an online store. - **Training/serving skew**: Distribution shift between training and serving data can decrease model performance. -Feast needed a mechanism to validate data at retrieval time to catch these issues before they affect model training or serving. +Feast needed a mechanism to validate data to catch these issues before they affect model training or serving. ## Decision -Introduce a Data Quality Monitoring (DQM) module that validates datasets against user-curated rules, initially targeting historical retrieval (training dataset generation). +Introduce a Data Quality Monitoring (DQM) module that validates datasets against user-curated rules. -### Design +### Original Design (now replaced) -The validation process uses a **reference dataset** and a **profiler** pattern: +The original validation process used a **reference dataset** and a **profiler** pattern: 1. User prepares a reference dataset (saved from a known-good historical retrieval). 2. User defines a profiler function that produces a profile (set of expectations) from a dataset. 3. Validation is performed by comparing the tested dataset against the reference profile. -### Integration with Great Expectations +This approach was limited to historical retrieval only, required additional dependencies, and offered no built-in UI or automation. -The initial implementation uses [Great Expectations](https://greatexpectations.io/) as the validation engine: +### Current Design -```python -from feast.dqm.profilers.ge_profiler import ge_profiler -from great_expectations.dataset import Dataset -from great_expectations.core.expectation_suite import ExpectationSuite +The current system (`feast monitor run`) provides: -@ge_profiler -def my_profiler(dataset: Dataset) -> ExpectationSuite: - dataset.expect_column_max_to_be_between("column", 1, 2) - dataset.expect_column_values_to_not_be_null("important_feature") - return dataset.get_expectation_suite() -``` +- Automatic metric computation (null rates, percentiles, histograms) with no external dependencies +- Monitoring across batch data and serving logs +- CLI and REST API for automation +- Built-in UI monitoring dashboard +- Support for all offline store backends via SQL push-down -### Usage - -Validation is triggered during historical feature retrieval via a `validation_reference` parameter: - -```python -from feast import FeatureStore - -store = FeatureStore(".") - -job = store.get_historical_features(...) -df = job.to_df( - validation_reference=store - .get_saved_dataset("my_reference_dataset") - .as_reference(profiler=my_profiler) -) -``` - -If validation fails, a `ValidationFailed` exception is raised with details for all expectations that didn't pass. If validation succeeds, the materialized dataset is returned normally. - -### Key Decisions - -- **Profiler-based approach**: Users define their own validation rules via profiler functions rather than Feast prescribing fixed validation rules. -- **Great Expectations integration**: Leverages an established data validation framework rather than building custom validation logic. -- **Validation at retrieval time**: Validation is performed when datasets are materialized (`.to_df()` or `.to_arrow()`), not during ingestion. -- **ValidationReference as a registry object**: Saved datasets and their validation references are stored in the Feast registry for reuse. +See [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) for full documentation. ## Consequences ### Positive - Users can detect data quality issues before they affect model training. -- Flexible profiler pattern allows custom validation rules per use case. -- Integration with Great Expectations provides a rich set of built-in expectations. -- Reference datasets provide a baseline for detecting data drift. +- Native integration requires no extra dependencies. +- Covers both batch data and serving logs. +- Built-in UI provides immediate visibility into feature health. +- Baselines computed automatically on `feast apply`. ### Negative -- Currently limited to historical retrieval; online store write/read validation is planned but not yet implemented. -- Dependency on Great Expectations adds to the install footprint (optional via `feast[ge]`). -- Automatic profiling capabilities are limited; manual expectation crafting is recommended. +- Migration required from the original profiler-based approach. ## References -- Original RFC: Feast RFC-027: Data Quality Monitoring -- Implementation: `sdk/python/feast/dqm/`, `sdk/python/feast/saved_dataset.py` +- Original RFC: Feast RFC-027: Data Quality Monitoring +- Implementation: `sdk/python/feast/monitoring/` - Documentation: [Data Quality Monitoring](../reference/dqm.md) +- [Feature Quality Monitoring guide](../how-to-guides/feature-monitoring.md) diff --git a/docs/blog/feast-0-18-adds-snowflake-support-and-data-quality-monitoring.md b/docs/blog/feast-0-18-adds-snowflake-support-and-data-quality-monitoring.md index 4b4321e3259..8c587bdde9f 100644 --- a/docs/blog/feast-0-18-adds-snowflake-support-and-data-quality-monitoring.md +++ b/docs/blog/feast-0-18-adds-snowflake-support-and-data-quality-monitoring.md @@ -6,7 +6,7 @@ We are delighted to announce the release of Feast [0.18](https://github.com/feas * Snowflake offline store, which allows you to define and use features stored in Snowflake. * [Experimental] Saved Datasets, which allow training datasets to be persisted in an offline store. -* [Experimental] Data quality monitoring, which allows you to validate your training data with Great Expectations. Future work will allow you to detect issues with upstream data pipelines and check for training-serving skew. +* [Experimental] Data quality monitoring, which allows you to validate your training data. This has since been superseded by Feast's native [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) system. * Python feature server graduation from alpha status. * Performance improvements to on demand feature views, protobuf serialization and deserialization, and the Python feature server. @@ -22,7 +22,7 @@ Training datasets generated via `get_historical_features` can now be persisted i ### [Experimental] Data quality monitoring -Feast 0.18 includes the first milestone of our data quality monitoring work. Many users have requested ways to validate their training and serving data, as well as monitor for training-serving skew. Feast 0.18 allows users to validate their training data through an integration with [Great Expectations](https://greatexpectations.io/). Users can declare one of the previously generated training datasets as a reference for this validation by persisting it as a "saved dataset" (see previous section). More details about future milestones of data quality monitoring can be found [here](https://docs.feastsite.wpenginepowered.com/v/master/reference/data-quality). There's also a [tutorial on validating historical features](https://docs.feastsite.wpenginepowered.com/v/master/how-to-guides/validation/validating-historical-features) that demonstrates all new concepts in action. +Feast 0.18 includes the first milestone of our data quality monitoring work. Many users have requested ways to validate their training and serving data, as well as monitor for training-serving skew. Feast 0.18 allows users to validate their training data by declaring previously generated training datasets as a reference for validation, persisted as "saved datasets" (see previous section). This initial integration has since been superseded by Feast's native [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) system, which provides built-in metrics computation, drift detection, serving log monitoring, and a UI dashboard. ### Performance improvements diff --git a/docs/getting-started/architecture/model-inference.md b/docs/getting-started/architecture/model-inference.md index 582657dbc43..5983cb63451 100644 --- a/docs/getting-started/architecture/model-inference.md +++ b/docs/getting-started/architecture/model-inference.md @@ -17,7 +17,7 @@ of model inference): *Note: online features can be sourced from batch, streaming, or request data sources.* -These three approaches have different tradeoffs but, in general, have significant implementation differences. +These four approaches have different tradeoffs but, in general, have significant implementation differences. ## 1. Online Model Inference with Online Features Online model inference with online features is a powerful approach to serving data-driven machine learning applications. @@ -78,7 +78,7 @@ if features.to_dict().get('user_data:model_predictions') is None: model_predictions = model_server.predict(features) store.write_to_online_store(feature_view_name="user_data", df=pd.DataFrame(model_predictions)) ``` -Note that in this case a seperate call to `write_to_online_store` is required when the underlying data changes and +Note that in this case a separate call to `write_to_online_store` is required when the underlying data changes and predictions change along with it. ```python diff --git a/docs/getting-started/components/compute-engine.md b/docs/getting-started/components/compute-engine.md index d115ec5debb..8b69b9d1a8b 100644 --- a/docs/getting-started/components/compute-engine.md +++ b/docs/getting-started/components/compute-engine.md @@ -8,7 +8,7 @@ functions (UDFs). A materialization task abstracts over specific technologies or frameworks that are used to materialize data. It allows users to use a pure local serialized approach (which is the default LocalComputeEngine), or delegates the -materialization to seperate components (e.g. AWS Lambda, as implemented by the the LambdaComputeEngine). +materialization to separate components (e.g. AWS Lambda, as implemented by the LambdaComputeEngine). If the built-in engines are not sufficient, you can create your own custom materialization engine. Please see [this guide](../../how-to-guides/customizing-feast/creating-a-custom-compute-engine.md) for more details. diff --git a/docs/getting-started/concepts/dataset.md b/docs/getting-started/concepts/dataset.md index 3fabc48a140..c86c13503ed 100644 --- a/docs/getting-started/concepts/dataset.md +++ b/docs/getting-started/concepts/dataset.md @@ -1,6 +1,6 @@ # \[Alpha] Saved dataset -Feast datasets allow for conveniently saving dataframes that include both features and entities to be subsequently used for data analysis and model training. [Data Quality Monitoring](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98) was the primary motivation for creating dataset concept. +Feast datasets allow for conveniently saving dataframes that include both features and entities to be subsequently used for data analysis and model training. Data Quality Monitoring was the original motivation for creating the dataset concept. Dataset's metadata is stored in the Feast registry and raw data (features, entities, additional input keys and timestamp) is stored in the [offline store](../components/offline-store.md). diff --git a/docs/getting-started/concepts/feast-types.md b/docs/getting-started/concepts/feast-types.md index 7d864b6a18f..62cabcd3940 100644 --- a/docs/getting-started/concepts/feast-types.md +++ b/docs/getting-started/concepts/feast-types.md @@ -8,6 +8,7 @@ Feast's type system is built on top of [protobuf](https://github.com/protocolbuf Feast supports the following categories of data types: - **Primitive types**: numerical values (`Int32`, `Int64`, `Float32`, `Float64`), `String`, `Bytes`, `Bool`, and `UnixTimestamp`. +- **Zoned timestamp type**: `ZonedTimestamp` stores a timezone-aware datetime as both the UTC instant and its originating zone, so the original wall-clock zone round-trips losslessly. This differs from `UnixTimestamp`, which is always decoded as UTC and discards the source zone. Use `ZonedTimestamp` when local time-of-day or the offset/zone itself is meaningful. It must be explicitly declared in schema (it is not inferred by any backend), and is not supported as an entity key. - **Domain-specific primitives**: `PdfBytes` (PDF binary data for RAG/document pipelines) and `ImageBytes` (image binary data for multimodal pipelines). These are semantic aliases over `Bytes` and must be explicitly declared in schema — no backend infers them. - **UUID types**: `Uuid` and `TimeUuid` for universally unique identifiers. Stored as strings at the proto level but deserialized to `uuid.UUID` objects in Python. - **Array types**: ordered lists of any primitive type, e.g. `Array(Int64)`, `Array(String)`, `Array(Uuid)`. diff --git a/docs/getting-started/concepts/feature-view.md b/docs/getting-started/concepts/feature-view.md index 5be9b287305..27ded82cb84 100644 --- a/docs/getting-started/concepts/feature-view.md +++ b/docs/getting-started/concepts/feature-view.md @@ -91,7 +91,7 @@ If the `schema` parameter is not specified in the creation of the feature view, "Entity aliases" can be specified to join `entity_dataframe` columns that do not match the column names in the source table of a FeatureView. -This could be used if a user has no control over these column names or if there are multiple entities are a subclass of a more general entity. For example, "spammer" and "reporter" could be aliases of a "user" entity, and "origin" and "destination" could be aliases of a "location" entity as shown below. +This could be used if a user has no control over these column names or if multiple entities are subclasses of a more general entity. For example, "spammer" and "reporter" could be aliases of a "user" entity, and "origin" and "destination" could be aliases of a "location" entity as shown below. It is suggested that you dynamically specify the new FeatureView name using `.with_name` and `join_key_map` override using `.with_join_key_map` instead of needing to register each new copy. @@ -322,4 +322,4 @@ def driver_hourly_stats_stream(df: DataFrame): ) ``` -See [here](https://github.com/feast-dev/streaming-tutorial) for a example of how to use stream feature views to register your own streaming data pipelines in Feast. +See [here](https://github.com/feast-dev/streaming-tutorial) for an example of how to use stream feature views to register your own streaming data pipelines in Feast. diff --git a/docs/getting-started/genai.md b/docs/getting-started/genai.md index f65aeac85e2..dfd9c41954a 100644 --- a/docs/getting-started/genai.md +++ b/docs/getting-started/genai.md @@ -15,6 +15,7 @@ Feast integrates with popular vector databases to store and retrieve embedding v * **Elasticsearch**: Scalable vector search capabilities * **Postgres with PGVector**: SQL-based vector operations * **Qdrant**: Purpose-built vector database integration +* **ScyllaDB**: Native `vector` type with HNSW ANN index, full `retrieve_online_documents_v2` support These integrations allow you to: - Store embeddings as features diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index aa56d09b1d8..a05830d73e1 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -4,7 +4,7 @@ Feast (Feature Store) is an open-source feature store designed to facilitate the management and serving of machine learning features in a way that supports both batch and real-time applications. -* *For Data Scientists*: Feast is a a tool where you can easily define, store, and retrieve your features for both model development and model deployment. By using Feast, you can focus on what you do best: build features that power your AI/ML models and maximize the value of your data. +* *For Data Scientists*: Feast is a tool where you can easily define, store, and retrieve your features for both model development and model deployment. By using Feast, you can focus on what you do best: build features that power your AI/ML models and maximize the value of your data. * *For MLOps Engineers*: Feast is a library that allows you to connect your existing infrastructure (e.g., online database, application server, microservice, analytical database, and orchestration tooling) that enables your Data Scientists to ship features for their models to production using a friendly SDK without having to be concerned with software engineering challenges that occur from serving real-time production systems. By using Feast, you can focus on maintaining a resilient system, instead of implementing features for Data Scientists. diff --git a/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md b/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md index d1ca100bf74..35a46acf942 100644 --- a/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md +++ b/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md @@ -51,7 +51,7 @@ To fully implement the interface for the offline store, you will need to impleme * `pull_latest_from_table_or_query` is invoked when running materialization (using the `feast materialize` or `feast materialize-incremental` commands, or the corresponding `FeatureStore.materialize()` method. This method pull data from the offline store, and the `FeatureStore` class takes care of writing this data into the online store. * `get_historical_features` is invoked when reading values from the offline store using the `FeatureStore.get_historical_features()` method. Typically, this method is used to retrieve features when training ML models. * (optional) `offline_write_batch` is a method that supports directly pushing a pyarrow table to a feature view. Given a feature view with a specific schema, this function should write the pyarrow table to the batch source defined. More details about the push api can be found [here](../docs/reference/data-sources/push.md). This method only needs implementation if you want to support the push api in your offline store. -* (optional) `pull_all_from_table_or_query` is a method that pulls all the data from an offline store from a specified start date to a specified end date. This method is only used for **SavedDatasets** as part of data quality monitoring validation. +* (optional) `pull_all_from_table_or_query` is a method that pulls all the data from an offline store from a specified start date to a specified end date. This method is used for **SavedDatasets** and as a fallback compute path for the [Feature Quality Monitoring](../../how-to-guides/feature-monitoring.md) system (backends without native SQL push-down). * (optional) `write_logged_features` is a method that takes a pyarrow table or a path that points to a parquet file and writes the data to a defined source defined by `LoggingSource` and `LoggingConfig`. This method is only used internally for **SavedDatasets**. {% code title="feast_custom_offline_store/file.py" %} diff --git a/docs/how-to-guides/feature-monitoring.md b/docs/how-to-guides/feature-monitoring.md index ba8652ec003..aca36167323 100644 --- a/docs/how-to-guides/feature-monitoring.md +++ b/docs/how-to-guides/feature-monitoring.md @@ -462,3 +462,11 @@ The monitoring page is always accessible in the sidebar. To see actual data: 2. Run `feast apply` — this computes baseline metrics automatically 3. Schedule `feast monitor run` (or click "Compute Metrics" in the UI) to generate daily/weekly/monthly metrics + +## Related: Operational and SOX Metrics + +Feature Quality Monitoring focuses on **data-level** metrics (distributions, null rates, drift). Feast also provides **operational metrics** for infrastructure observability: + +- **Prometheus metrics** (`feast_offline_store_*`, `feast_online_store_*`) — latency, throughput, and error rates for offline/online store operations. See [Python Feature Server — Metrics](../reference/feature-servers/python-feature-server.md). +- **SOX audit logging** (`feast.audit`) — structured audit events for compliance tracking of feature store operations. +- **OpenTelemetry integration** — distributed tracing for feature serving requests. See [OpenTelemetry Integration](../getting-started/components/open-telemetry.md). diff --git a/docs/how-to-guides/online-server-performance-tuning.md b/docs/how-to-guides/online-server-performance-tuning.md index b280fe53cff..f10d7bff8c8 100644 --- a/docs/how-to-guides/online-server-performance-tuning.md +++ b/docs/how-to-guides/online-server-performance-tuning.md @@ -278,6 +278,7 @@ The online store is the single largest factor in `get_online_features()` latency | **DynamoDB** | 2–5 ms | Yes | Serverless, auto-scaling on AWS | Pay-per-request cost; batch API limits (100 items) | | **PostgreSQL** | 3–10 ms | No (threadpool) | Teams with existing Postgres infra | Connection pooling needed at scale | | **MongoDB** | 2–5 ms | Yes | Flexible schema, async-native | Requires index tuning for large datasets | +| **Aerospike** | < 1 ms | No (threadpool) | Ultra-low latency, hybrid memory (RAM + SSD), large datasets | Namespace must be pre-configured on the cluster | | **Bigtable** | 3–8 ms | No (threadpool) | Large-scale GCP workloads | Row-key design affects read performance | | **Cassandra / ScyllaDB** | 2–5 ms | No (threadpool) | Multi-region, write-heavy | Tunable consistency; requires DC-aware routing | | **Remote** | Varies | No (threadpool) | Centralized feature server architecture | Adds an HTTP hop; tune connection pool | @@ -305,6 +306,7 @@ The feature server can read from the online store using either an **async** or * | **MongoDB** | Yes | Yes | Uses `motor` (async MongoDB driver) | | **PostgreSQL** | Implemented | No | Has `online_read_async` but does not yet advertise via `async_supported`; uses sync/threadpool path | | **Redis** | Implemented | **Yes** | `online_read_async` and `online_write_batch_async` both implemented; uses sync/threadpool path for `get_online_features` (overridden with batched single pipeline) | +| **Aerospike** | Implemented | No | Async methods wrap the blocking C client via `run_in_executor`; does not yet advertise via `async_supported`, so the server still uses the threadpool path | | All others | No | No | Fall back to sync with `run_in_threadpool()` | **When async matters most:** @@ -467,6 +469,34 @@ online_store: - **`connectTimeoutMS` / `socketTimeoutMS`**: Tighter timeouts improve p99 by failing fast on slow connections. - MongoDB is one of the stores with **full async support** (read and write), so it benefits from concurrent feature view reads via `asyncio.gather()`. +### Aerospike tuning + +Aerospike offers sub-millisecond reads thanks to its hybrid-memory architecture (primary index in RAM, data on SSD or RAM). Tune the per-call policies in the Feast config and rely on the Aerospike cluster's own tuning for everything else: + +```yaml +online_store: + type: aerospike + hosts: + - ["aerospike-1.internal", 3000] + - ["aerospike-2.internal", 3000] + namespace: feast + read_timeout_ms: 150 # hard deadline for a single-record get + write_timeout_ms: 300 # hard deadline for a single-record put/operate + batch_total_timeout_ms: 500 # hard deadline for online_read / online_write_batch + socket_timeout_ms: 50 # per-attempt deadline so max_retries can actually fire + max_retries: 2 + ttl_seconds: 86400 # record-level TTL; omit to use the namespace default + client_kwargs: # escape hatch for any client-config field not surfaced above + policies: + batch: + concurrent_nodes: 0 # 0 = parallel to every node (lowest latency on multi-node clusters) +``` + +- **`*_timeout_ms` (total)** vs **`socket_timeout_ms` (per-attempt)**: `*_timeout_ms` is the hard deadline for a whole call *including* retries; `socket_timeout_ms` is the per-attempt deadline that allows `max_retries` to actually fire within that budget. Without `socket_timeout_ms`, a single slow attempt can consume the entire total deadline and retries never run. +- **`hosts`**: List every seed node. The Aerospike client discovers the rest of the cluster automatically and opens one connection pool per node. +- **`ttl_seconds: 0`** means "never expire"; omit the key to inherit the namespace's `default-ttl`. Expiry is enforced by the server's `nsup` thread — nothing to delete on the client side. +- Co-locate the feature server in the **same availability zone / rack** as the Aerospike cluster; sub-millisecond reads are bandwidth- and RTT-sensitive. + ### Remote online store tuning The Remote online store connects to a Feast feature server over HTTP. Connection pooling is critical: @@ -700,6 +730,7 @@ This applies to every connection-oriented online store: | **DynamoDB** | `max_pool_connections` (HTTP pool) | 10 | No hard limit, but AWS SDK has per-process pool caps; monitor throttling | | **Redis** | Connection per worker | 1 | `maxclients` on the Redis server (default: 10,000) | | **MongoDB** | `maxPoolSize` (in `client_kwargs`) | 100 | Server's `net.maxIncomingConnections` | +| **Aerospike** | Driver manages pool per seed node | Auto | `proto-fd-max` (default 15000) on each Aerospike node | | **Cassandra** | Driver manages pool per node | Auto | `native_transport_max_threads` on each Cassandra node | | **Remote** | `connection_pool_size` (HTTP pool) | 50 | The target feature server's worker capacity | diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index 861c3fcb114..28d0bf0098e 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -15,6 +15,7 @@ Below are supported vector databases and implemented features: | Faiss | [ ] | [ ] | [] | [] | | SQLite | [x] | [ ] | [x] | [x] | | Qdrant | [x] | [x] | [] | [] | +| ScyllaDB | [x] | [x] | [x] | [x] | *Note: V2 Support means the SDK supports retrieval of features along with vector embeddings from vector similarity search. @@ -30,7 +31,7 @@ Beyond that, we will then have `retrieve_online_documents` and `retrieve_online_ backwards compatibility and the adopt industry standard naming conventions. {% endhint %} -**Note**: Milvus and SQLite implement the v2 `retrieve_online_documents_v2` method in the SDK. This will be the longer-term solution so that Data Scientists can easily enable vector similarity search by just flipping a flag. +**Note**: Milvus, SQLite, and ScyllaDB implement the v2 `retrieve_online_documents_v2` method in the SDK. This will be the longer-term solution so that Data Scientists can easily enable vector similarity search by just flipping a flag. ## Examples diff --git a/docs/reference/codebase-structure.md b/docs/reference/codebase-structure.md index 80608b5929a..4783773c270 100644 --- a/docs/reference/codebase-structure.md +++ b/docs/reference/codebase-structure.md @@ -28,7 +28,7 @@ The majority of Feast logic lives in these Python files: There are also several important submodules: * `infra/` contains all the infrastructure components, such as the provider, offline store, online store, batch materialization engine, and registry. -* `dqm/` covers data quality monitoring, such as the dataset profiler. +* `dqm/` covers data quality monitoring. See [`monitoring/`](../../sdk/python/feast/monitoring/) for the built-in monitoring system. * `diff/` covers the logic for determining how to apply infrastructure changes upon feature repo changes (e.g. the output of `feast plan` and `feast apply`). * `embedded_go/` covers the Go feature server. * `ui/` contains the embedded Web UI, to be launched on the `feast ui` command. diff --git a/docs/reference/compute-engine/snowflake.md b/docs/reference/compute-engine/snowflake.md index e7b0dc5bd63..f6c633a4e40 100644 --- a/docs/reference/compute-engine/snowflake.md +++ b/docs/reference/compute-engine/snowflake.md @@ -24,5 +24,10 @@ batch_engine: role: sysadmin warehouse: demo_wh database: FEAST + python_udf_runtime_version: "3.10" ``` {% endcode %} + +## Configuration + +* `python_udf_runtime_version` *(optional, default: `"3.10"`)* -- The Snowflake Python UDF `RUNTIME_VERSION` used when Feast deploys its materialization UDFs. Snowflake periodically decommissions old Python UDF runtimes (for example, the 3.9 runtime was decommissioned, requiring Feast to bump its default to 3.10 -- see [#6606](https://github.com/feast-dev/feast/issues/6606)). If Snowflake decommissions the 3.10 runtime in the future, set this field to a still-supported version (e.g. `"3.11"`) instead of waiting for a new Feast release. diff --git a/docs/reference/data-sources/README.md b/docs/reference/data-sources/README.md index 24bf18dbe86..33e47672dcc 100644 --- a/docs/reference/data-sources/README.md +++ b/docs/reference/data-sources/README.md @@ -42,6 +42,10 @@ Please see [Data Source](../../getting-started/concepts/data-ingestion.md) for a [spark.md](spark.md) {% endcontent-ref %} +{% content-ref url="iceberg.md" %} +[iceberg.md](iceberg.md) +{% endcontent-ref %} + {% content-ref url="postgres.md" %} [postgres.md](postgres.md) {% endcontent-ref %} diff --git a/docs/reference/data-sources/iceberg.md b/docs/reference/data-sources/iceberg.md new file mode 100644 index 00000000000..6402a7ec419 --- /dev/null +++ b/docs/reference/data-sources/iceberg.md @@ -0,0 +1,161 @@ +# Iceberg source (contrib) + +## Description + +Iceberg data sources are tables managed by any supported Iceberg catalog. The `IcebergSource` class provides a unified interface with a configurable `catalog_type` parameter: + +- **`"rest"`** (default): [Apache Iceberg REST Catalog specification](https://iceberg.apache.org/concepts/catalog/#decoupling-using-the-rest-catalog) — Unity Catalog, Apache Polaris, Nessie, Snowflake Open Catalog +- **`"hive"`**: Hive Metastore catalog +- **`"glue"`**: AWS Glue catalog +- **`"sql"`**: SQL-based (JDBC) catalog +- **`"dynamodb"`**: DynamoDB-based catalog + +The data source carries catalog connection details (catalog_type, endpoint, warehouse, namespace, table, authentication). When the offline store (DuckDB, Spark) encounters this source, it resolves table metadata and credentials via the configured catalog at query time. + +## Examples + +### IcebergSource (REST catalog) + +Works with any Iceberg REST Catalog: + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import IcebergSource + +my_source = IcebergSource( + catalog_type="rest", # default + endpoint="http://localhost:8081/api/2.1/unity-catalog/iceberg", + warehouse="unity", + namespace="default", + table="driver_features", + timestamp_field="event_timestamp", + token_env_var="UC_TOKEN", +) +``` + +### IcebergSource (Hive Metastore) + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import IcebergSource + +my_source = IcebergSource( + catalog_type="hive", + catalog_properties={"uri": "thrift://metastore:9083"}, + warehouse="my_warehouse", + namespace="default", + table="driver_features", + timestamp_field="event_timestamp", +) +``` + +### IcebergSource (AWS Glue) + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import IcebergSource + +my_source = IcebergSource( + catalog_type="glue", + catalog_properties={"region_name": "us-east-1"}, + warehouse="my_account", + namespace="my_database", + table="driver_features", + timestamp_field="event_timestamp", +) +``` + +### UnityCatalogSource (with governance) {#unity-catalog-source} + +Extends `IcebergSource` with Unity Catalog governance: + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import ( + UnityCatalogSource, +) + +my_uc_source = UnityCatalogSource( + warehouse="production", + namespace="ml_features", + table="driver_stats", + timestamp_field="event_timestamp", + register_as_feature_table=True, # Register in UC on feast apply + sync_lineage=True, # Record lineage in UC +) +``` + +When `endpoint` is omitted, it defaults to `{DATABRICKS_HOST}/api/2.1/unity-catalog/iceberg`. +When `token_env_var` is omitted, it defaults to `DATABRICKS_TOKEN`. + +### Full Feature View Example + +```python +from datetime import timedelta + +from feast import Entity, FeatureView, Field +from feast.types import Float64, Int64 + +from feast.infra.data_sources.contrib.iceberg_catalog import ( + UnityCatalogSource, +) + +driver = Entity(name="driver_id", join_keys=["driver_id"]) + +driver_stats_source = UnityCatalogSource( + warehouse="production", + namespace="ml_features", + table="driver_hourly_stats", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) + +driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + source=driver_stats_source, + schema=[ + Field(name="conv_rate", dtype=Float64), + Field(name="acc_rate", dtype=Float64), + Field(name="avg_daily_trips", dtype=Int64), + ], + ttl=timedelta(days=1), + online=True, +) +``` + +## Configuration Reference + +### IcebergSource + +| Parameter | Type | Description | +| :--- | :--- | :--- | +| `catalog_type` | `str` | Catalog backend: `"rest"` (default), `"hive"`, `"glue"`, `"sql"`, `"dynamodb"` | +| `endpoint` | `str` | Catalog endpoint URL (required for `"rest"`, optional for others) | +| `warehouse` | `str` | Catalog/warehouse name | +| `namespace` | `str` | Schema/namespace within the catalog | +| `table` | `str` | Table name | +| `catalog_properties` | `dict` | Additional catalog-specific properties passed to PyIceberg | +| `timestamp_field` | `str` | Event timestamp column for point-in-time joins | +| `created_timestamp_column` | `str` | Optional column indicating row creation time | +| `token_env_var` | `str` | Environment variable name holding the auth token | +| `credential_vending` | `bool` | Whether to request scoped credentials (default: `True`) | +| `field_mapping` | `dict` | Column name mapping from source to feature names | + +### UnityCatalogSource (additional parameters) + +| Parameter | Type | Description | +| :--- | :--- | :--- | +| `register_as_feature_table` | `bool` | Register as UC feature table on `feast apply` (default: `True`) | +| `sync_lineage` | `bool` | Sync lineage metadata to Unity Catalog (default: `True`) | + +## Supported Types + +| Iceberg Type | Feast Type | +| :--- | :--- | +| `boolean` | `BOOL` | +| `int` | `INT32` | +| `long` | `INT64` | +| `float` | `FLOAT` | +| `double` | `DOUBLE` | +| `string` | `STRING` | +| `binary` | `BYTES` | +| `timestamp` / `timestamptz` | `INT64` | +| `decimal` | `DOUBLE` | +| `uuid` | `STRING` | diff --git a/docs/reference/data-sources/kafka.md b/docs/reference/data-sources/kafka.md index 8794c7a1e81..dd7203a6149 100644 --- a/docs/reference/data-sources/kafka.md +++ b/docs/reference/data-sources/kafka.md @@ -72,4 +72,4 @@ def driver_hourly_stats_stream(df: DataFrame): ``` ### Ingesting data -See [here](https://github.com/feast-dev/streaming-tutorial) for a example of how to ingest data from a Kafka source into Feast. +See [here](https://github.com/feast-dev/streaming-tutorial) for an example of how to ingest data from a Kafka source into Feast. diff --git a/docs/reference/data-sources/kinesis.md b/docs/reference/data-sources/kinesis.md index f2adadfec03..09706617da9 100644 --- a/docs/reference/data-sources/kinesis.md +++ b/docs/reference/data-sources/kinesis.md @@ -71,4 +71,4 @@ def driver_hourly_stats_stream(df: DataFrame): ``` ### Ingesting data -See [here](https://github.com/feast-dev/streaming-tutorial) for a example of how to ingest data from a Kafka source into Feast. The approach used in the tutorial can be easily adapted to work for Kinesis as well. +See [here](https://github.com/feast-dev/streaming-tutorial) for an example of how to ingest data from a Kafka source into Feast. The approach used in the tutorial can be easily adapted to work for Kinesis as well. diff --git a/docs/reference/dqm.md b/docs/reference/dqm.md index 5a02413e534..47090b5dd1c 100644 --- a/docs/reference/dqm.md +++ b/docs/reference/dqm.md @@ -1,77 +1,81 @@ # Data Quality Monitoring -Data Quality Monitoring (DQM) is a Feast module aimed to help users to validate their data with the user-curated set of rules. -Validation could be applied during: -* Historical retrieval (training dataset generation) -* [planned] Writing features into an online store -* [planned] Reading features from an online store +Feast's Data Quality Monitoring (DQM) system computes, stores, and serves statistical metrics for every registered feature. It gives you visibility into feature health — distributions, null rates, percentiles, histograms — across batch data and feature serving logs. -Its goal is to address several complex data problems, namely: -* Data consistency - new training datasets can be significantly different from previous datasets. This might require a change in model architecture. -* Issues/bugs in the upstream pipeline - bugs in upstream pipelines can cause invalid values to overwrite existing valid values in an online store. -* Training/serving skew - distribution shift could significantly decrease the performance of the model. +Its goal is to address several complex data problems: -> To monitor data quality, we check that the characteristics of the tested dataset (aka the tested dataset's profile) are "equivalent" to the characteristics of the reference dataset. -> How exactly profile equivalency should be measured is up to the user. +* **Data consistency** — new training datasets can differ significantly from previous datasets, potentially requiring changes in model architecture. +* **Upstream pipeline bugs** — bugs in upstream pipelines can cause invalid values to overwrite existing valid values in an online store. +* **Training/serving skew** — distribution shift between training and serving data can decrease model performance. ### Overview -The validation process consists of the following steps: -1. User prepares reference dataset (currently only [saved datasets](../getting-started/concepts/dataset.md) from historical retrieval are supported). -2. User defines profiler function, which should produce profile by given dataset (currently only profilers based on [Great Expectations](https://docs.greatexpectations.io) are allowed). -3. Validation of tested dataset is performed with reference dataset and profiler provided as parameters. +Feast's DQM system works natively with your configured offline store — no additional infrastructure or external dependencies are required. The workflow is: -### Preparations -Feast with Great Expectations support can be installed via -```shell -pip install 'feast[ge]' +1. **Register features** — run `feast apply` to register feature views. If `auto_baseline: true` is configured, baseline metrics are computed automatically. +2. **Schedule monitoring** — run `feast monitor run` on a schedule (daily recommended) to compute metrics across multiple time windows. +3. **Read metrics** — query metrics via the REST API or view them in the Feast UI. + +### Configuration + +Enable DQM in your `feature_store.yaml`: + +```yaml +data_quality_monitoring: + auto_baseline: true ``` -### Dataset profile -Currently, Feast supports only [Great Expectation's](https://greatexpectations.io/) [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) -as dataset's profile. Hence, the user needs to define a function (profiler) that would receive a dataset and return an [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite). +### Computing Metrics -Great Expectations supports automatic profiling as well as manually specifying expectations: -```python -from great_expectations.dataset import Dataset -from great_expectations.core.expectation_suite import ExpectationSuite +**Auto mode (recommended for production):** -from feast.dqm.profilers.ge_profiler import ge_profiler +```bash +feast monitor run +``` -@ge_profiler -def automatic_profiler(dataset: Dataset) -> ExpectationSuite: - from great_expectations.profile.user_configurable_profiler import UserConfigurableProfiler +This detects the latest event timestamp in the source data and computes metrics for 5 time windows: daily, weekly, biweekly, monthly, and quarterly. - return UserConfigurableProfiler( - profile_dataset=dataset, - ignored_columns=['conv_rate'], - value_set_threshold='few' - ).build_suite() +**Target a specific feature view:** + +```bash +feast monitor run --feature-view driver_stats ``` -However, from our experience capabilities of automatic profiler are quite limited. So we would recommend crafting your own expectations: -```python -@ge_profiler -def manual_profiler(dataset: Dataset) -> ExpectationSuite: - dataset.expect_column_max_to_be_between("column", 1, 2) - return dataset.get_expectation_suite() + +**Explicit date range:** + +```bash +feast monitor run \ + --feature-view driver_stats \ + --start-date 2025-01-01 \ + --end-date 2025-01-07 \ + --granularity weekly ``` +**Set a manual baseline:** +```bash +feast monitor run \ + --feature-view driver_stats \ + --start-date 2025-01-01 \ + --end-date 2025-03-31 \ + --granularity daily \ + --set-baseline +``` + +### Monitoring Feature Serving Logs + +If your feature services have logging configured, you can compute metrics from the actual features served to models in production: -### Validating Training Dataset -During retrieval of historical features, `validation_reference` can be passed as a parameter to methods `.to_df(validation_reference=...)` or `.to_arrow(validation_reference=...)` of RetrievalJob. -If parameter is provided Feast will run validation once dataset is materialized. In case if validation successful materialized dataset is returned. -Otherwise, `feast.dqm.errors.ValidationFailed` exception would be raised. It will consist of all details for expectations that didn't pass. +```bash +feast monitor run --source-type log +``` -```python -from feast import FeatureStore +### Reading Metrics -fs = FeatureStore(".") +Metrics are accessible via the REST API: -job = fs.get_historical_features(...) -job.to_df( - validation_reference=fs - .get_saved_dataset("my_reference_dataset") - .as_reference(profiler=manual_profiler) -) ``` +GET /monitoring/metrics/features?project=my_project&feature_view_name=driver_stats&granularity=daily +``` + +See the [Feature Quality Monitoring guide](../how-to-guides/feature-monitoring.md) for full API reference, UI integration, and orchestrator examples. diff --git a/docs/reference/feature-servers/registry-server.md b/docs/reference/feature-servers/registry-server.md index 496eaa8badc..4558a10ce63 100644 --- a/docs/reference/feature-servers/registry-server.md +++ b/docs/reference/feature-servers/registry-server.md @@ -214,6 +214,7 @@ Most endpoints support these common query parameters: - `feature` (optional): Filter feature views by feature name - `feature_service` (optional): Filter feature views by feature service name - `data_source` (optional): Filter feature views by data source name + - `updated_since` (optional): Only return feature views updated at or after this ISO-8601 UTC timestamp (e.g. `2024-01-01T00:00:00Z`) - `page` (optional): Page number for pagination - `limit` (optional): Number of items per page - `sort_by` (optional): Field to sort by @@ -223,27 +224,31 @@ Most endpoints support these common query parameters: # Basic list curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project" - + # With pagination and relationships curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project&include_relationships=true&page=1&limit=5&sort_by=name" - + # Filter by entity curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project&entity=user" - + # Filter by feature curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project&feature=age" - + # Filter by data source curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project&data_source=user_profile_source" - + # Filter by feature service curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project&feature_service=user_service" - + + # Filter by last-updated timestamp + curl -H "Authorization: Bearer " \ + "http://localhost:6572/api/v1/feature_views?project=my_project&updated_since=2024-06-01T00:00:00Z" + # Multiple filters combined curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project&entity=user&feature=age" diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index 6f31993f896..39294966170 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -58,6 +58,10 @@ Please see [Online Store](../../getting-started/components/online-store.md) for [mongodb.md](mongodb.md) {% endcontent-ref %} +{% content-ref url="aerospike.md" %} +[aerospike.md](aerospike.md) +{% endcontent-ref %} + {% content-ref url="hazelcast.md" %} [hazelcast.md](hazelcast.md) {% endcontent-ref %} diff --git a/docs/reference/online-stores/aerospike.md b/docs/reference/online-stores/aerospike.md new file mode 100644 index 00000000000..e5a9754796b --- /dev/null +++ b/docs/reference/online-stores/aerospike.md @@ -0,0 +1,389 @@ +# Aerospike online store (Preview) + +## Description + +The [Aerospike](https://aerospike.com/) online store provides support for materializing feature values into an Aerospike cluster for serving online features. + +{% hint style="warning" %} +The Aerospike online store is currently in **preview**. Some functionality may be unstable, and breaking changes may occur in future releases. +{% endhint %} + +## Features + +* Supports both synchronous and asynchronous read/write paths (`online_read` / `online_read_async`, `online_write_batch` / `online_write_batch_async`). Async methods wrap the blocking client in `run_in_executor`, keeping the event loop responsive in feature-server workloads. +* Partial, server-side upserts via Aerospike Map CDT operations — writing one feature view never clobbers another feature view stored on the same entity. +* Record-level TTL controlled by a single `ttl_seconds` config option (honours the namespace default, a "never expire" sentinel, or an explicit number of seconds). +* Per-feature-view **namespace overrides** and **set overrides** — pin individual feature views to RAM-only or SSD-backed namespaces, or isolate one view in its own set, without splitting projects. +* **Prewriting hook** — a configurable, import-string-resolved callable applied to every write batch for cross-cutting concerns like PII masking, application-side encryption, or value coercion. +* Authentication and TLS options for Aerospike Enterprise Edition passed straight through to the Aerospike Python client. +* `client_kwargs` escape hatch for any advanced client-config field not surfaced on `AerospikeOnlineStoreConfig`. +* Baseline: Aerospike Server **≥ 6.0** (uses batch-write / batch-operate APIs). The store has been developed against CE 8.x. + +## Getting started + +Install the Aerospike extra (alongside the dependency for the offline store of choice): + +```bash +pip install 'feast[aerospike]' +``` + +You can start from any of the standard templates (e.g. `feast init -t local` or `feast init -t aws`) and then swap in Aerospike as the online store as shown below. + +## Examples + +### Basic configuration — local Aerospike CE + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: aerospike + hosts: + - ["127.0.0.1", 3000] + namespace: feast +``` +{% endcode %} + +### Multi-node cluster + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: aerospike + hosts: + - ["aerospike-1.internal", 3000] + - ["aerospike-2.internal", 3000] + - ["aerospike-3.internal", 3000] + namespace: feast + ttl_seconds: 86400 # 24h record-level TTL + read_timeout_ms: 150 # hard deadline for a single-record get + write_timeout_ms: 300 # hard deadline for a single-record put/operate + batch_total_timeout_ms: 500 # hard deadline for online_read / online_write_batch + batch_max_records: 1000 # chunk size for batch_write / batch_operate + socket_timeout_ms: 50 # per-attempt deadline so max_retries can fire + max_retries: 2 +``` +{% endcode %} + +> **Timeout semantics.** The Aerospike client distinguishes per-attempt +> (`socket_timeout`) from total (`total_timeout`) deadlines. `*_timeout_ms` map +> to `total_timeout` — the overall budget for a call including retries. Set +> `socket_timeout_ms` as well so each individual attempt has its own (shorter) +> deadline; without it, `max_retries` effectively never fires because the +> first attempt is allowed to consume the entire total deadline. + +> **Batch chunking.** `online_read` and `online_write_batch` split large +> requests into chunks of at most `batch_max_records` (default `1000`). +> Aerospike enforces a per-node batch limit via the server `batch-max-requests` +> setting (historically `5000`). Lower `batch_max_records` if your cluster cap +> is tighter; raise it only when the server limit and client timeouts allow. + +### Aerospike Enterprise with authentication + +> Requires Aerospike Enterprise Edition. The Community Edition server has no built-in user/security model and will reject these config keys. + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: aerospike + hosts: + - ["aerospike.internal", 3000] + namespace: feast + user: feast_user + password: ${AEROSPIKE_PASSWORD} # pragma: allowlist secret + auth_mode: internal # internal | external | pki +``` +{% endcode %} + +### Aerospike Enterprise with TLS + +> Requires Aerospike Enterprise Edition. The Community Edition server does not implement TLS, so `tls` config is effective only against EE clusters. + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: aerospike + hosts: + - ["aerospike-1.internal", 4333, "aerospike-tls"] + namespace: feast + tls: + enable: true + cafile: /etc/aerospike/certs/ca.pem + certfile: /etc/aerospike/certs/client.pem + keyfile: /etc/aerospike/certs/client.key +``` +{% endcode %} + +### Per-feature-view namespace and set overrides + +Two `Dict[str, str]` config fields — `namespace_overrides` and `set_overrides` — let you place individual feature views on a different Aerospike namespace or set without splitting your project across stores. Anything not listed in either map falls back to the store-level default (`namespace` / `set_name_template`). + +Common reasons to reach for these: + +* A **hot, latency-sensitive view** belongs on a RAM-only namespace; a **wide, cold view** belongs on an SSD-backed namespace. Same project, different storage tiers. +* You want `feast apply` deletions or `truncate` on one feature view to be O(1) without scanning records of the others — give that view its own set. + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: aerospike + hosts: + - ["aerospike.internal", 3000] + namespace: feast # default namespace + set_name_template: "{project}_{collection_suffix}" + namespace_overrides: + driver_realtime_stats: feast_ram # in-memory namespace + driver_history_lookup: feast_ssd # device-backed namespace + set_overrides: + isolated_view: my_feature_repo_isolated +``` +{% endcode %} + +> **Tradeoffs.** +> +> * Every namespace listed in `namespace_overrides` MUST already exist on the cluster — Aerospike cannot create namespaces at runtime, and a missing namespace surfaces as an opaque `AEROSPIKE_ERR_PARAM` on the first read or write. +> * Putting feature views on different sets means a multi-feature-view read for the same entity becomes one Aerospike round trip per set, not one round trip total. Only opt in when the operational isolation is worth that cost. Reads that touch a single feature view are unaffected. +> * Admin operations honour the overrides automatically: `update()` (called by `feast apply`) groups dropped feature views by their resolved `(namespace, set)` and issues one background scan per group; `teardown()` truncates every unique `(namespace, set)` pair the project may have written to (including the store-level default). + +### Prewriting hooks + +`prewriting_hook` is the import path of a callable that is invoked once per `online_write_batch` call, receives the rows about to be written, and returns the rows that actually go on the wire. Use it for cross-cutting write-side concerns that you don't want sprinkled through every materialization job — PII masking, application-side encryption, dual-write fan-out, value coercion, etc. + +Hooks are referenced by import string (rather than as a Python `Callable` value) so the config survives YAML/JSON serialisation and remote-feature-server transport. The resolved callable is cached on the store instance, so import cost is paid once per store lifetime. + +**Hook signature:** + +```python +def hook( + config: RepoConfig, + table: FeatureView, + data: list[ + tuple[ + EntityKeyProto, + dict[str, ValueProto], + datetime, + datetime | None, + ] + ], +) -> list[ + tuple[ + EntityKeyProto, + dict[str, ValueProto], + datetime, + datetime | None, + ] +]: + ... +``` + +The hook MUST return a row list with the same schema as its input. Returning `[]` short-circuits the write — same path as an empty input, no wire call is issued. Hooks that raise will fail the whole batch; there is no per-row fallback. + +**1. Drop a hook function in your project.** Any module on the `PYTHONPATH` of every process that writes through Feast will do (the materialization workers, the registry CLI host, and the feature server, if you run one). + +{% code title="my_feature_repo/hooks.py" %} +```python +"""Prewriting hooks for the Aerospike online store.""" +from __future__ import annotations + +import hashlib +import os +from datetime import datetime +from typing import Optional + +from feast import FeatureView +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import RepoConfig + +# Names of features that must never reach the online store as plaintext. +# Matched by exact feature name; tweak to your project's conventions. +_SENSITIVE_FEATURES = {"email", "phone_number", "ssn"} + + +def hash_pii_string_features( + config: RepoConfig, + table: FeatureView, + data: list[ + tuple[ + EntityKeyProto, + dict[str, ValueProto], + datetime, + Optional[datetime], + ] + ], +) -> list[ + tuple[ + EntityKeyProto, + dict[str, ValueProto], + datetime, + Optional[datetime], + ] +]: + """Replace any sensitive string feature with a salted SHA-256 hex digest. + + The hash is deterministic (same input → same digest) so downstream lookups + that hash the candidate value the same way still hit. ``FEAST_PII_SALT`` + must be set on every process that materialises features; an unset salt + raises rather than silently falling back to plaintext. + """ + salt = os.environ.get("FEAST_PII_SALT") + if salt is None: + raise RuntimeError( + "FEAST_PII_SALT is not set; refusing to write feature batches " + "without a configured PII salt." + ) + salt_bytes = salt.encode("utf-8") + + def _digest(plaintext: str) -> str: + h = hashlib.sha256() + h.update(salt_bytes) + h.update(plaintext.encode("utf-8")) + return h.hexdigest() + + transformed: list[ + tuple[ + EntityKeyProto, + dict[str, ValueProto], + datetime, + Optional[datetime], + ] + ] = [] + for entity_key, values, event_ts, created_ts in data: + new_values = dict(values) + for feature_name in _SENSITIVE_FEATURES.intersection(new_values): + v = new_values[feature_name] + if v.HasField("string_val") and v.string_val: + new_values[feature_name] = ValueProto(string_val=_digest(v.string_val)) + transformed.append((entity_key, new_values, event_ts, created_ts)) + return transformed +``` +{% endcode %} + +**2. Reference the hook from `feature_store.yaml`:** + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: aerospike + hosts: + - ["aerospike.internal", 3000] + namespace: feast + prewriting_hook: my_feature_repo.hooks.hash_pii_string_features +``` +{% endcode %} + +> **Operational notes.** +> +> * The hook is **only invoked on the write path**; reads pass through the store untouched. If your hook is one-way (e.g. hashing) you have to apply the same transformation to the candidate value at read time yourself. +> * Hooks run inside the same process as the writer — they're not RPCs and not sandboxed. They can read environment variables, open files, call out to KMS, etc. Treat them as part of your trusted code base. +> * A misconfigured `prewriting_hook` (bad import path, missing function, non-callable target) raises `ValueError` / `TypeError` on the *first* `online_write_batch` call, not on store construction. Add a smoke test that writes one row at deploy time so misconfigurations surface before a real batch. + +The full set of configuration options is available in [`AerospikeOnlineStoreConfig`](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.aerospike_online_store.aerospike.AerospikeOnlineStoreConfig). + +## Data Model + +The Aerospike online store uses a **single set per project** with entity-key collocation. Features from multiple feature views for the same entity are stored together on a single Aerospike record, analogous to the MongoDB online store's "one document per entity" layout. + +| Aerospike concept | Feast mapping | +| :---------------- | :---------------------------------------------------------------------------- | +| Namespace | `online_store.namespace` (must be pre-configured on the cluster); per-feature-view override via `online_store.namespace_overrides` | +| Set | `online_store.set_name_template` → `"{project}_{collection_suffix}"` by default; per-feature-view override via `online_store.set_overrides` | +| Key | `serialize_entity_key(entity_key)` as `bytearray` user key | +| Bin `features` | Map CDT keyed by feature-view name, each value a map of `feature → native` | +| Bin `event_ts` | Map CDT keyed by feature-view name, each value an int64 epoch-ms timestamp | +| Bin `created_ts` | Top-level int64 epoch-ms timestamp (last `feast materialize`) | + +### Example record + +For a single entity carrying features from two feature views (`driver_stats` and `pricing`): + +```text +key: (ns="feast", set="my_feature_repo_latest", user_key=) +bins: + features: + driver_stats: + rating: 4.91 + trips_last_7d: 132 + pricing: + surge_multiplier: 1.2 + event_ts: + driver_stats: 1737374400000 # 2025-01-20T12:00:00Z + pricing: 1737447000000 # 2025-01-21T08:30:00Z + created_ts: 1737460805000 # 2025-01-21T12:00:05Z +``` + +### Key design decisions + +* **Record per entity, bin per concept.** `features` and `event_ts` are Aerospike Map CDT bins, not dynamic bins, which keeps the store within the 15-byte Aerospike bin-name limit regardless of how many feature views a project has. +* **Partial upserts via Map CDT ops.** Writes use `batch_write` with `map_put_items("features", {: {...}})` and `map_put("event_ts", , )`. Concurrent writes to different feature views on the same entity never clobber each other — each write mutates only its own map keys. +* **Entity-key bytes as the Aerospike user key.** Feast's `serialize_entity_key` output is passed as a `bytearray` user key (not `bytes` — the Python client hashes only the first byte of `bytes` keys, which would collapse distinct entities). +* **Timestamps as int64 epoch milliseconds.** Aerospike has no native datetime type; tz-naive timestamps are treated as UTC per the `OnlineStore` contract. + +### TTL and expiry + +`ttl_seconds` is written as record-level metadata on every `online_write_batch` call: + +| `ttl_seconds` | Aerospike TTL | Effect | +| :------------ | :-------------------------------- | :---------------------------------------------------------- | +| not set / `null` | `TTL_NAMESPACE_DEFAULT` | Record inherits the namespace's configured `default-ttl`. | +| `0` | `TTL_NEVER_EXPIRE` | Record is kept until explicitly deleted. | +| `>0` | that many seconds | Record is evicted by the server's `nsup` thread. | + +There is no per-feature-view TTL override in this version — the setting is applied uniformly for every write made by the online store. + +### Indexes + +No secondary indexes are created. All access goes through the primary key, which is the serialized entity key. + +## Async support + +Async read/write are provided by running the Aerospike Python client's blocking calls on the default thread-pool executor (`loop.run_in_executor`). The underlying C client releases the GIL during network I/O, so `await store.online_read_async(...)` keeps the event loop responsive. A native asyncio Aerospike client is not currently used. + +Both sync and async methods are fully supported: + +* `online_read` / `online_read_async` +* `online_write_batch` / `online_write_batch_async` +* `initialize` / `close` — `initialize(config)` eagerly opens the connection so feature servers pay the TCP/handshake cost at startup; `close()` releases the cached client. + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Aerospike online store. + +| | Aerospike | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/cassandra.md b/docs/reference/online-stores/cassandra.md index 198f15ca47f..5d95e526421 100644 --- a/docs/reference/online-stores/cassandra.md +++ b/docs/reference/online-stores/cassandra.md @@ -37,6 +37,51 @@ online_store: ``` {% endcode %} +### Example (Cassandra — multi-DC) + +Use `datacenters` instead of `hosts` when your cluster spans multiple datacenters. +Each entry gets a named Cassandra **execution profile** keyed by its `name` field, +enabling per-DC routing. The default profile is determined by `load_balancing.local_dc` +(or the first datacenter entry when `load_balancing` is absent). Use the optional +`routing` block to direct reads and writes to specific datacenters. The keyspace must +already exist; Feast does not create it automatically. + +`datacenters` is mutually exclusive with `hosts` and `secure_bundle_path`. + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: cassandra + keyspace: KeyspaceName + datacenters: + - name: dc1 + hosts: + - 192.168.1.1 + - 192.168.1.2 + replication_factor: 3 # optional, informational only + replication_strategy: NetworkTopologyStrategy # optional, informational only + - name: dc2 + hosts: + - 10.0.0.1 + replication_factor: 2 # optional, informational only + routing: # optional + read_dc: dc2 # DC to use for reads (default: load_balancing.local_dc) + write_dc: dc1 # DC to use for writes (default: load_balancing.local_dc) + port: 9042 # optional + username: user # optional + password: secret # optional + protocol_version: 5 # optional + load_balancing: # optional + local_dc: 'dc1' # sets the default execution profile + load_balancing_policy: 'TokenAwarePolicy(DCAwareRoundRobinPolicy)' # optional + read_concurrency: 100 # optional + write_concurrency: 100 # optional +``` +{% endcode %} + ### Example (Astra DB) {% code title="feature_store.yaml" %} diff --git a/docs/reference/online-stores/dynamodb.md b/docs/reference/online-stores/dynamodb.md index 68d3d29ca3b..a7f6b7392c9 100644 --- a/docs/reference/online-stores/dynamodb.md +++ b/docs/reference/online-stores/dynamodb.md @@ -24,7 +24,7 @@ The full set of configuration options is available in [DynamoDBOnlineStoreConfig ## Configuration -Below is a example with performance tuning options: +Below is an example with performance tuning options: {% code title="feature_store.yaml" %} ```yaml diff --git a/docs/reference/online-stores/milvus.md b/docs/reference/online-stores/milvus.md index 014c7bd68a5..58f7dbd167a 100644 --- a/docs/reference/online-stores/milvus.md +++ b/docs/reference/online-stores/milvus.md @@ -11,6 +11,14 @@ In order to use this online store, you'll need to install the Milvus extra (alon `pip install 'feast[milvus]'` +{% hint style="warning" %} +**Upgrading to milvus-lite 3.0.0+** + +Feast supports both milvus-lite 2.x and 3.x. However, if you upgrade from milvus-lite 2.x.x to 3.0.0+, the `.db` files created by the original storage format are **not compatible** with the milvus-lite 3.0.0+ engine. You will need to re-import your data into a new database — automatic migration is not available. + +See the [milvus-lite GitHub page](https://github.com/milvus-io/milvus-lite) for more details. +{% endhint %} + You can get started by using any of the other templates (e.g. `feast init -t gcp` or `feast init -t snowflake` or `feast init -t aws`), and then swapping in Redis as the online store as seen below in the examples. ## Examples diff --git a/docs/reference/online-stores/overview.md b/docs/reference/online-stores/overview.md index 6ee076b0669..663a48836dc 100644 --- a/docs/reference/online-stores/overview.md +++ b/docs/reference/online-stores/overview.md @@ -29,26 +29,26 @@ See this [issue](https://github.com/feast-dev/feast/issues/2254) for a discussio ## Functionality Matrix There are currently five core online store implementations: `SqliteOnlineStore`, `RedisOnlineStore`, `DynamoDBOnlineStore`, `SnowflakeOnlineStore`, and `DatastoreOnlineStore`. -There are several additional implementations contributed by the Feast community (`PostgreSQLOnlineStore`, `HbaseOnlineStore` and `CassandraOnlineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. +There are several additional implementations contributed by the Feast community (`PostgreSQLOnlineStore`, `HbaseOnlineStore`, `CassandraOnlineStore` and `ScyllaDBOnlineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. Details for each specific online store, such as how to configure it in a `feature_store.yaml`, can be found [here](README.md). Below is a matrix indicating which online stores support what functionality. -| | Sqlite | Redis | DynamoDB | Snowflake | Datastore | Postgres | Hbase | [[Cassandra](https://cassandra.apache.org/_/index.html) / [Astra DB](https://www.datastax.com/products/datastax-astra?utm_source=feast)] | Milvus | -| :-------------------------------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- |:----| -| write feature values to the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| read feature values from the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| update infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| generate a plan of infrastructure changes | yes | no | no | no | no | no | no | yes | no | -| support for on-demand transforms | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| readable by Python SDK | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| readable by Java | no | yes | no | no | no | no | no | no | no | -| readable by Go | yes | yes | no | no | no | no | no | no | no | -| support for entityless feature views | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| support for concurrent writing to the same key | no | yes | no | no | no | no | no | no | yes | -| support for ttl (time to live) at retrieval | no | yes | no | no | no | no | no | no | no | -| support for deleting expired data | no | yes | no | no | no | no | no | no | no | -| collocated by feature view | yes | no | yes | yes | yes | yes | yes | yes | no | -| collocated by feature service | no | no | no | no | no | no | no | no | no | -| collocated by entity key | no | yes | no | no | no | no | no | no | yes | +| | Sqlite | Redis | DynamoDB | Snowflake | Datastore | Postgres | Hbase | [[Cassandra](https://cassandra.apache.org/_/index.html) / [Astra DB](https://www.datastax.com/products/datastax-astra?utm_source=feast)] | Milvus | ScyllaDB | +| :-------------------------------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- |:----| :-- | +| write feature values to the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| read feature values from the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| update infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| generate a plan of infrastructure changes | yes | no | no | no | no | no | no | yes | no | no | +| support for on-demand transforms | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Python SDK | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Java | no | yes | no | no | no | no | no | no | no | no | +| readable by Go | yes | yes | no | no | no | no | no | no | no | no | +| support for entityless feature views | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| support for concurrent writing to the same key | no | yes | no | no | no | no | no | no | yes | no | +| support for ttl (time to live) at retrieval | no | yes | no | no | no | no | no | no | no | yes | +| support for deleting expired data | no | yes | no | no | no | no | no | no | no | yes | +| collocated by feature view | yes | no | yes | yes | yes | yes | yes | yes | no | yes | +| collocated by feature service | no | no | no | no | no | no | no | no | no | no | +| collocated by entity key | no | yes | no | no | no | no | no | no | yes | no | diff --git a/docs/reference/online-stores/scylladb.md b/docs/reference/online-stores/scylladb.md index c8583ac101a..6c2e462877b 100644 --- a/docs/reference/online-stores/scylladb.md +++ b/docs/reference/online-stores/scylladb.md @@ -2,20 +2,15 @@ ## Description -ScyllaDB is a low-latency and high-performance Cassandra-compatible (uses CQL) database. You can use the existing Cassandra connector to use ScyllaDB as an online store in Feast. - -The [ScyllaDB](https://www.scylladb.com/) online store provides support for materializing feature values into a ScyllaDB or [ScyllaDB Cloud](https://www.scylladb.com/product/scylla-cloud/) cluster for serving online features real-time. +[ScyllaDB](https://www.scylladb.com/) is a distributed real-time NoSQL database with vector search support. +This integration uses the native **`scylla-driver`** Python driver for optimised performance and supports materializing feature values into a [ScyllaDB Cloud](https://www.scylladb.com/product/scylla-cloud/) cluster for real-time online feature serving. ## Getting started -Install Feast with Cassandra support: -```bash -pip install "feast[cassandra]" -``` +Install Feast with the `scylladb` extra, which pulls in `scylla-driver` automatically: -Create a new Feast project: ```bash -feast init REPO_NAME -t cassandra +pip install feast[scylladb] ``` ### Example (ScyllaDB) @@ -26,7 +21,7 @@ project: scylla_feature_repo registry: data/registry.db provider: local online_store: - type: cassandra + type: scylladb hosts: - 172.17.0.2 keyspace: feast @@ -43,44 +38,99 @@ project: scylla_feature_repo registry: data/registry.db provider: local online_store: - type: cassandra + type: scylladb hosts: - node-0.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud - node-1.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud - node-2.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud keyspace: feast username: scylla - password: password + password: xxxxxx + local_dc: AWS_US_EAST_1 ``` {% endcode %} - -The full set of configuration options is available in [CassandraOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.cassandra_online_store.cassandra_online_store.CassandraOnlineStoreConfig). -For a full explanation of configuration options please look at file -`sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/README.md`. +## Configuration options + +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `hosts` | list[str] | *(required)* | Contact-point host addresses. | +| `port` | int | `9042` | CQL port. | +| `keyspace` | str | `feast_keyspace` | Target ScyllaDB keyspace. | +| `username` | str | `None` | Auth username. | +| `password` | str | `None` | Auth password. | +| `local_dc` | str | `None` | Local datacenter name for DC-aware load balancing. | +| `request_timeout` | float | `None` | Driver request timeout in seconds. | +| `read_concurrency` | int | `100` | `concurrency` argument passed to the driver's `execute_concurrent_with_args` for reads. Controls how many CQL statements are in-flight at once. | +| `write_concurrency` | int | `100` | `concurrency` argument passed to the driver's `execute_concurrent_with_args` for writes. Controls how many CQL statements are in-flight at once. | +| `vector_similarity_function` | str | `COSINE` | Default similarity function for vector indexes. Supported: `COSINE`, `DOT_PRODUCT`, `EUCLIDEAN`. Can be overridden per-feature via the `similarity_function` Field tag. | Storage specifications can be found at `docs/specs/online_store_format.md`. +## Vector Search + +ScyllaDB Cloud supports approximate nearest-neighbour (ANN) vector search. +To enable it for a feature view, tag the embedding `Field` with `vector_index=true` and specify the number of dimensions: + +{% code title="feature_definitions.py" %} +```python +from feast import FeatureView, Field +from feast.types import Array, Float32, String + +documents_fv = FeatureView( + name="documents", + entities=[item], + schema=[ + Field(name="text", dtype=String), + Field( + name="embedding", + dtype=Array(Float32), + tags={ + "vector_index": "true", + "dimensions": "768", + "similarity_function": "COSINE", # COSINE | DOT_PRODUCT | EUCLIDEAN + }, + ), + ], + online=True, + source=push_source, +) +``` +{% endcode %} + +When `feast apply` runs, the store automatically creates the necessary tables and HNSW ANN index for any feature view with vector-tagged fields. + +To query the top-k most similar documents: + +```python +result = store.retrieve_online_documents_v2( + features=["documents:text", "documents:embedding"], + query=[0.1, 0.2, ...], # your query embedding + top_k=10, + distance_metric="COSINE", +) +``` + ## Functionality Matrix The set of functionality supported by online stores is described in detail [here](overview.md#functionality). -Below is a matrix indicating which functionality is supported by the Cassandra plugin. +Below is a matrix indicating which functionality is supported by the ScyllaDB online store. -| | Cassandra | +| | ScyllaDB | | :-------------------------------------------------------- | :-------- | | write feature values to the online store | yes | | read feature values from the online store | yes | | update infrastructure (e.g. tables) in the online store | yes | | teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | yes | +| generate a plan of infrastructure changes | no | | support for on-demand transforms | yes | | readable by Python SDK | yes | | readable by Java | no | | readable by Go | no | | support for entityless feature views | yes | | support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | | collocated by feature view | yes | | collocated by feature service | no | | collocated by entity key | no | @@ -89,6 +139,6 @@ To compare this set of functionality against other online stores, please see the ## Resources -* [Sample application with ScyllaDB](https://feature-store.scylladb.com/stable/) +* [ScyllaDB Vector Search documentation](https://cloud.docs.scylladb.com/stable/vector-search/) * [ScyllaDB website](https://www.scylladb.com/) * [ScyllaDB Cloud documentation](https://cloud.docs.scylladb.com/stable/) diff --git a/docs/reference/openlineage.md b/docs/reference/openlineage.md index 78438082d44..bf9e18750ed 100644 --- a/docs/reference/openlineage.md +++ b/docs/reference/openlineage.md @@ -186,6 +186,12 @@ Captures materialization run metadata: ## Lineage Visualization +### Option 1: Feast UI (Built-in) + +Feast includes a built-in OpenLineage consumer that can receive, store, and visualize lineage from **all** OpenLineage producers (Airflow, Spark, dbt, Feast itself, etc.) directly in the Feast UI. See the [OpenLineage Consumer](#openlineage-consumer) section below. + +### Option 2: Marquez + Use [Marquez](https://marquezproject.ai/) to visualize your Feast lineage: ```bash @@ -216,3 +222,257 @@ Then access the Marquez UI at http://localhost:3000 to see your feature lineage. | Entity | InputDataset | | FeatureService | OutputDataset | | Materialization | RunEvent (START/COMPLETE/FAIL) | + +--- + +## OpenLineage Consumer + +Feast can act as an **OpenLineage consumer**, receiving lineage events from any OpenLineage-compatible producer and displaying them in the Feast UI. This eliminates the need for a separate Marquez deployment when you want to visualize cross-system data lineage alongside your feature store. + +### Consumer Architecture + +``` +Producers (Airflow, Spark, dbt, Feast, Flink, …) + │ + ▼ + POST /api/v1/lineage ──→ Event Processor ──→ Lineage Store (SQL) + │ + ▼ + Feast UI + ┌──────────────────────────┐ + │ Lineage tab │ + │ ├─ OpenLineage Graph │ + │ │ (all producers) │ + │ └─ ☐ Feast Only Lineage │ + │ (registry view) │ + │ │ + │ Events tab │ + │ └─ Event browser │ + └──────────────────────────┘ +``` + +When the consumer is **not** enabled, the Feast UI shows only the original registry-based lineage view — no tabs are added. + +### Enabling the Consumer + +Add the `consumer` section under `openlineage` in your `feature_store.yaml`: + +```yaml +project: my_project +registry: + registry_type: sql + path: postgresql://user:****@host:5432/feast # pragma: allowlist secret + +openlineage: + enabled: true + namespace: my_project + consumer: + enabled: true + store_type: sql + # Optional: separate database for lineage storage. + # If omitted, the SQL registry database is reused. + # connection_string: postgresql://user:****@host:5432/feast_lineage + api_key: "change-me" # pragma: allowlist secret + namespace_mapping: + airflow_ns: my_project + spark_ns: my_project +``` + +Or via environment variables: + +```bash +export FEAST_OPENLINEAGE_CONSUMER_ENABLED=true +export FEAST_OPENLINEAGE_CONSUMER_STORE_TYPE=sql +export FEAST_OPENLINEAGE_CONSUMER_API_KEY=change-me # pragma: allowlist secret +# Optional separate DB: +# export FEAST_OPENLINEAGE_CONSUMER_CONNECTION_STRING=postgresql://... +``` + +### Consumer Configuration Options + +| Option | Default | Description | +|--------|---------|-------------| +| `consumer.enabled` | `false` | Enable the OpenLineage consumer | +| `consumer.store_type` | `sql` | Storage backend type. Currently only `sql` is supported | +| `consumer.connection_string` | - | Optional separate database connection string. If omitted, reuses the SQL registry database | +| `consumer.api_key` | - | API key that producers must provide when sending events | +| `consumer.namespace_mapping` | `{}` | Maps OpenLineage namespaces to Feast projects for RBAC scoping | + +### Consumer API Endpoints + +When the consumer is enabled, the following endpoints are available on the Feast REST registry server: + +#### Event Receiver (Producer-facing) + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/api/v1/lineage` | `POST` | Receive a single OpenLineage event (or array of events) | +| `/api/v1/lineage/batch` | `POST` | Receive a batch of OpenLineage events | + +Both endpoints require the `X-API-Key` header (or `Authorization: Bearer `) if `consumer.api_key` is configured. + +#### Admin Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/lineage/openlineage/reset` | `DELETE` | Purge all OpenLineage data. Accepts optional `?namespace=X` to delete only a specific namespace. Requires API key. | + +#### OpenLineage Query Endpoints (UI-facing) + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/lineage/openlineage/graph` | `GET` | Full lineage graph with all nodes, edges, and symlinks | +| `/lineage/openlineage/graph/{node_type}/{namespace}/{name}` | `GET` | Lineage graph centered on a specific node | +| `/lineage/openlineage/events` | `GET` | Browse stored events with filtering | +| `/lineage/openlineage/jobs` | `GET` | List all known OpenLineage jobs | +| `/lineage/openlineage/datasets` | `GET` | List all known OpenLineage datasets | +| `/lineage/openlineage/runs` | `GET` | List runs with optional `?job_namespace=X&job_name=Y` filtering | +| `/lineage/openlineage/runs/{run_id}` | `GET` | Single run detail with input/output datasets | + +#### Registry Query Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/lineage/registry` | `GET` | Feast registry lineage (entities, feature views, services) | +| `/lineage/registry/all` | `GET` | All registry objects with full metadata | +| `/lineage/objects/{object_type}/{object_name}` | `GET` | Detail for a specific registry object | +| `/lineage/complete` | `GET` | Complete registry lineage with relationships | +| `/lineage/complete/all` | `GET` | Complete registry lineage for all objects | + +### Configuring Producers to Send Events to Feast + +Configure any OpenLineage producer to send events to your Feast instance: + +#### Airflow + +```python +# In airflow.cfg or environment +OPENLINEAGE_URL = "http://feast-registry:8080/api" +OPENLINEAGE_API_KEY = "change-me" # pragma: allowlist secret +``` + +#### Spark + +```properties +spark.openlineage.transport.type=http +spark.openlineage.transport.url=http://feast-registry:8080/api +spark.openlineage.transport.endpoint=/v1/lineage +spark.openlineage.transport.auth.type=api_key +spark.openlineage.transport.auth.apiKey=change-me +``` + +#### dbt + +```yaml +# In profiles.yml or environment +OPENLINEAGE_URL: "http://feast-registry:8080/api" +OPENLINEAGE_API_KEY: "change-me" # pragma: allowlist secret +``` + +#### Feast (Self-reporting) + +When both the OpenLineage producer and consumer are enabled, Feast's own events (from `feast apply`, materialization, etc.) are automatically ingested into the local consumer store — no HTTP transport is needed. + +```yaml +# In feature_store.yaml +openlineage: + enabled: true + namespace: my_project + consumer: + enabled: true + api_key: change-me # pragma: allowlist secret +``` + +### Feast UI Lineage Views + +When the consumer is enabled, the lineage page in the Feast UI shows two tabs: + +**Lineage tab** + +- **OpenLineage Graph** (default) — shows lineage from all OpenLineage producers with cross-producer connectivity. Nodes are color-coded by producer (colors generated dynamically). The graph supports filtering by type, producer, and object name. Clicking a node opens a **detail panel** showing description, schema, tags, features, entities, data quality metrics, data source info, other facets, and **run history** (for job nodes — see [Per-Run Lineage](#per-run-lineage-run-history)). +- **Feast Only Lineage** (checkbox) — switches to the original Feast registry view (DataSource → FeatureView → FeatureService) powered entirely by the Feast registry. + +**Events tab** + +- Browse individual OpenLineage events with filtering by event type, job name, and run ID. Expand any event to inspect the full JSON payload. + +### Cross-Producer Lineage Connectivity + +The consumer automatically links datasets across different producers when they refer to the same physical data. Linking mechanisms: + +1. **Shared namespace + name** — If Airflow writes to `s3://bucket/path` and Spark reads from the same `s3://bucket/path`, the graph connects them automatically. +2. **SymlinksDatasetFacet** — Producers can declare aliases. For example, Feast can declare that its internal `driver_hourly_stats` is a symlink to the Spark output at `s3://bucket/features/driver_hourly_stats/`. +3. **dataSource URI matching** — Datasets with matching `dataSource.uri` facets are linked even if their namespace or name differ. + +Compatible producers include Airflow, Spark, dbt, Flink, Feast, and Dagster. + +### RBAC for Lineage + +The OpenLineage consumer integrates with Feast's existing RBAC: + +- **Write access** (producers sending events): Authenticated via API key in the `X-API-Key` header +- **Read access** (UI viewing lineage): Namespace-based filtering maps OpenLineage namespaces to Feast projects. Users see only lineage data for namespaces they have access to via the `namespace_mapping` configuration + +### Lineage Cleanup / Reset + +Over time the OpenLineage store accumulates historical data. Two mechanisms are provided for cleanup: + +#### Admin Reset Endpoint + +Use the `DELETE /lineage/openlineage/reset` endpoint to purge lineage data. The endpoint requires the same API key used for event ingestion. + +```bash +# Purge ALL OpenLineage data +curl -X DELETE -H "X-API-Key: your-key" \ + http://localhost:8080/api/v1/lineage/openlineage/reset + +# Purge only a specific namespace +curl -X DELETE -H "X-API-Key: your-key" \ + "http://localhost:8080/api/v1/lineage/openlineage/reset?namespace=airflow://prod-cluster" +``` + +A full purge deletes data from all seven `openlineage_*` tables. A namespace-scoped purge deletes jobs, datasets, runs, events, edges, and symlinks associated with that namespace, leaving other namespaces intact. + +#### Feast Teardown Hook + +When you run `feast teardown`, Feast automatically cleans up OpenLineage data for the project's namespace (if the consumer is configured). This ensures that tearing down a Feast project doesn't leave orphaned lineage data behind. + +```bash +# Tears down the Feast project AND its OpenLineage lineage +feast teardown +``` + +### Per-Run Lineage (Run History) + +The consumer tracks individual pipeline runs in the `openlineage_runs` table. When you click on a **job node** in the OpenLineage Graph, the detail panel shows a **Run History** section with: + +- A table of past runs: truncated run ID, status badge (COMPLETE, FAIL, RUNNING, ABORT), start time, and duration +- Click any run to expand its **inputs and outputs** — the specific datasets that run consumed and produced + +#### Run History API + +```bash +# List runs for a specific job +curl "http://localhost:8080/api/v1/lineage/openlineage/runs?job_namespace=spark://emr-cluster&job_name=feature_engineering" + +# Get a single run with its I/O datasets +curl "http://localhost:8080/api/v1/lineage/openlineage/runs/{run_id}" +``` + +The run detail response includes `inputs` and `outputs` arrays, each containing the dataset namespace, name, and any I/O facets recorded by the producer. + +### Database Schema + +The consumer creates the following tables (automatically on first startup): + +| Table | Purpose | +|-------|---------| +| `openlineage_events` | Raw event storage with JSON payloads | +| `openlineage_jobs` | Deduplicated job records with producer, description, and facets | +| `openlineage_datasets` | Deduplicated dataset records with schema, facets, and Feast mapping | +| `openlineage_runs` | Run lifecycle tracking (START/COMPLETE/FAIL) | +| `openlineage_run_io` | Input/output relationships between runs and datasets | +| `openlineage_lineage_edges` | Materialized lineage graph edges for efficient traversal | +| `openlineage_dataset_symlinks` | Cross-producer dataset linking via `SymlinksDatasetFacet` and `dataSource` URI matching | + +By default these tables are created in the **same database** as the SQL registry (hybrid storage). Set `consumer.connection_string` to store them in a separate database instead. diff --git a/docs/reference/registries/sql.md b/docs/reference/registries/sql.md index ef9993c8753..8b4ff19ab11 100644 --- a/docs/reference/registries/sql.md +++ b/docs/reference/registries/sql.md @@ -83,7 +83,84 @@ If you are running Feast in Kubernetes, set the `image.repository` and There are some things to note about how the SQL registry works: - Once instantiated, the Registry ensures the tables needed to store data exist, and creates them if they do not. - Upon tearing down the feast project, the registry ensures that the tables are dropped from the database. -- The schema for how data is laid out in tables can be found . It is intentionally simple, storing the serialized protobuf versions of each Feast object keyed by its name. +- The schema for how data is laid out in tables can be found in the table definitions in [`sdk/python/feast/infra/registry/sql.py`](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/registry/sql.py). It is intentionally simple, storing the serialized protobuf versions of each Feast object keyed by its name. + +## MySQL: serialized-proto columns use `LONGBLOB` + +The registry stores each Feast object as a serialized protobuf in a binary +column. On MySQL these columns are created as `LONGBLOB` (up to 4 GB). Earlier +versions created them as `BLOB`, which caps at 64 KB — a single `FeatureView` +proto routinely exceeds that, so MySQL would silently truncate the write and the +registry would later fail to load with a protobuf `DecodeError` (for example, +`feast serve` failing to start). Other dialects (PostgreSQL, SQLite) were never +affected. + +New deployments get the correct schema automatically — the registry creates its +tables as `LONGBLOB` on first use. When an existing MySQL/MariaDB registry still +has `BLOB` columns, the registry logs an error at startup listing the affected +columns (it does not refuse to start — a registry whose protos all fit in 64 KB +is unaffected). **Existing deployments are not migrated automatically**: the +registry only creates tables that do not already exist, and it has no +schema-migration step, so previously created `BLOB` columns remain `BLOB`. To +upgrade an existing MySQL registry, alter each serialized-proto column to +`LONGBLOB`, for example: + +> ⚠️ **Run the migration carefully on a live registry.** A `BLOB`→`LONGBLOB` +> change is a column *data-type* change, which MySQL InnoDB performs with +> `ALGORITHM=COPY` — a full table rebuild under a metadata lock that blocks +> readers and writers for the duration (potentially minutes on a large table +> such as `feature_view_version_history`). `ALGORITHM=INPLACE` is **not** +> generally supported for this change and is rejected with +> `ER_ALTER_OPERATION_NOT_SUPPORTED_REASON` on most builds — do not rely on it. +> +> **Before running any `ALTER TABLE`:** +> +> 1. **Stop all `feast apply` and materialization jobs.** This is required, not +> optional — a write of a `>64 KB` proto to a not-yet-widened `BLOB` column +> truncates silently with no error, and concurrent writes also extend the +> `ALTER`'s lock duration. +> 2. Confirm there are no active writers (e.g. `SHOW PROCESSLIST`). +> 3. Verify you have a backup of the registry database. +> +> Then, to minimize the lock window: +> +> - On large tables, or on managed MySQL (AWS RDS, Aurora) without shell access, +> use an online schema-change tool — +> [`pt-online-schema-change`](https://docs.percona.com/percona-toolkit/pt-online-schema-change.html) +> (Percona Toolkit) or [`gh-ost`](https://github.com/github/gh-ost) — which +> rebuild the table without a long-held lock. For small tables a plain +> `ALTER TABLE` in the maintenance window is fine. +> - Apply one table at a time so a failure is easy to isolate and re-run. +> - Resume jobs only after all `ALTER TABLE` statements complete successfully. +> - Rollback is safe (revert `MODIFY ... BLOB`) **only** while no stored proto +> exceeds 64 KB; otherwise a revert re-introduces truncation. + +```sql +ALTER TABLE projects MODIFY project_proto LONGBLOB NOT NULL; +ALTER TABLE entities MODIFY entity_proto LONGBLOB NOT NULL; +ALTER TABLE data_sources MODIFY data_source_proto LONGBLOB NOT NULL; +ALTER TABLE feature_views MODIFY materialized_intervals LONGBLOB, + MODIFY feature_view_proto LONGBLOB NOT NULL, + MODIFY user_metadata LONGBLOB; +ALTER TABLE stream_feature_views MODIFY feature_view_proto LONGBLOB NOT NULL, + MODIFY user_metadata LONGBLOB; +ALTER TABLE on_demand_feature_views MODIFY feature_view_proto LONGBLOB NOT NULL, + MODIFY user_metadata LONGBLOB; +ALTER TABLE label_views MODIFY feature_view_proto LONGBLOB NOT NULL, + MODIFY user_metadata LONGBLOB; +ALTER TABLE feature_services MODIFY feature_service_proto LONGBLOB NOT NULL; +ALTER TABLE saved_datasets MODIFY saved_dataset_proto LONGBLOB NOT NULL; +ALTER TABLE validation_references MODIFY validation_reference_proto LONGBLOB NOT NULL; +ALTER TABLE managed_infra MODIFY infra_proto LONGBLOB NOT NULL; +ALTER TABLE permissions MODIFY permission_proto LONGBLOB NOT NULL; +-- LARGE TABLE: one row per versioned apply — likely the slowest ALTER. Use +-- pt-online-schema-change or gh-ost if this registry has significant history. +ALTER TABLE feature_view_version_history MODIFY feature_view_proto LONGBLOB NOT NULL; +``` + +Any object whose proto already exceeded 64 KB before the upgrade may have been +stored truncated; re-run `feast apply` for those objects after altering the +columns so the full proto is rewritten. ## Example Usage: Concurrent materialization The SQL Registry should be used when materializing feature views concurrently to ensure correctness of data in the registry. This can be achieved by simply running feast materialize or feature_store.materialize multiple times using a correctly configured feature_store.yaml. This will make each materialization process talk to the registry database concurrently, and ensure the metadata updates are serialized. diff --git a/docs/reference/type-system.md b/docs/reference/type-system.md index eb483c6e769..97cc6036dc8 100644 --- a/docs/reference/type-system.md +++ b/docs/reference/type-system.md @@ -24,6 +24,7 @@ Feast supports the following data types: | `Bytes` | `bytes` | Binary data | | `Bool` | `bool` | Boolean value | | `UnixTimestamp` | `datetime` | Unix timestamp (nullable) | +| `ZonedTimestamp` | `datetime` | Timezone-aware datetime preserving its source zone (nullable) | | `Uuid` | `uuid.UUID` | UUID (any version) | | `TimeUuid` | `uuid.UUID` | Time-based UUID (version 1) | | `Decimal` | `decimal.Decimal` | Arbitrary-precision decimal number | @@ -202,7 +203,8 @@ from datetime import timedelta from feast import Entity, FeatureView, Field, FileSource from feast.types import ( Int32, Int64, Float32, Float64, String, Bytes, Bool, UnixTimestamp, - Uuid, TimeUuid, Decimal, Array, Set, Map, ScalarMap, Json, Struct + Uuid, TimeUuid, Decimal, Array, Set, Map, ScalarMap, Json, Struct, + ZonedTimestamp ) # Define a data source @@ -232,6 +234,7 @@ user_features = FeatureView( Field(name="profile_picture", dtype=Bytes), Field(name="is_active", dtype=Bool), Field(name="last_login", dtype=UnixTimestamp), + Field(name="event_time", dtype=ZonedTimestamp), Field(name="session_id", dtype=Uuid), Field(name="event_id", dtype=TimeUuid), Field(name="price", dtype=Decimal), @@ -362,6 +365,43 @@ unique_prices = {decimal.Decimal("9.99"), decimal.Decimal("19.99"), decimal.Deci `Decimal` is **not** inferred from any backend schema. You must declare it explicitly in your feature view schema. The pandas dtype for `Decimal` columns is `object` (holding `decimal.Decimal` instances), not a numeric dtype. {% endhint %} +### ZonedTimestamp Type Usage Examples + +The `ZonedTimestamp` type stores a timezone-aware `datetime` as both the UTC instant +and its originating zone, so the original wall-clock zone round-trips losslessly. +By contrast, `UnixTimestamp` always decodes to UTC and discards the source zone. + +```python +from datetime import datetime, timezone +from zoneinfo import ZoneInfo + +# A datetime in a specific zone — both the instant and "America/Los_Angeles" are kept +event_time = datetime(2026, 6, 17, 9, 0, 0, tzinfo=ZoneInfo("America/Los_Angeles")) + +# ZonedTimestamp values are returned as tz-aware datetime objects, in their own zone +response = store.get_online_features( + features=["event_features:event_time"], + entity_rows=[{"user_id": 1001}], +) +result = response.to_dict() +# result["event_time"][0] == event_time (same instant AND same zone, e.g. 09:00-07:00) + +# Two values at the same instant but different zones stay distinct +la = datetime(2026, 6, 17, 9, 0, 0, tzinfo=ZoneInfo("America/Los_Angeles")) +utc = datetime(2026, 6, 17, 16, 0, 0, tzinfo=timezone.utc) # same instant as `la` + +# A naive (tz-less) datetime is interpreted as UTC +naive = datetime(2026, 6, 17, 12, 0, 0) # stored zone is empty, decoded as UTC +``` + +{% hint style="warning" %} +`ZonedTimestamp` is **not** inferred from any backend schema — you must declare it +explicitly in your feature view schema. It is not supported as an entity key. The +zone is stored as an IANA name (e.g. `America/Los_Angeles`) when available, falling +back to a fixed-offset string; offline stores that cannot natively carry a zone may +normalize to UTC on that backend. +{% endhint %} + ### Nested Collection Type Usage Examples ```python diff --git a/docs/roadmap.md b/docs/roadmap.md index e47aa79b573..d92ffa38f24 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -62,6 +62,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [SingleStore](https://docs.feast.dev/reference/online-stores/singlestore) * [x] [Couchbase](https://docs.feast.dev/reference/online-stores/couchbase) * [x] [MongoDB](https://docs.feast.dev/reference/online-stores/mongodb) + * [x] [Aerospike](https://docs.feast.dev/reference/online-stores/aerospike) * [x] [Qdrant (vector store)](https://docs.feast.dev/reference/online-stores/qdrant) * [x] [Milvus (vector store)](https://docs.feast.dev/reference/online-stores/milvus) * [x] [Faiss (vector store)](https://docs.feast.dev/reference/online-stores/faiss) @@ -89,7 +90,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Offline Feature Server (alpha)](https://docs.feast.dev/reference/feature-servers/offline-feature-server) * [x] [Registry server (alpha)](https://github.com/feast-dev/feast/blob/master/docs/reference/feature-servers/registry-server.md) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** - * [x] Data profiling and validation (Great Expectations) + * [x] [Feature Quality Monitoring](https://docs.feast.dev/how-to-guides/feature-monitoring) — built-in metrics, drift detection, serving log monitoring, and UI dashboard * **Feature Discovery and Governance** * [x] Python SDK for browsing feature registry * [x] CLI for browsing feature registry diff --git a/docs/tutorials/validating-historical-features.md b/docs/tutorials/validating-historical-features.md deleted file mode 100644 index 1984adcdcf9..00000000000 --- a/docs/tutorials/validating-historical-features.md +++ /dev/null @@ -1,916 +0,0 @@ -# Validating historical features with Great Expectations - -In this tutorial, we will use the public dataset of Chicago taxi trips to present data validation capabilities of Feast. -- The original dataset is stored in BigQuery and consists of raw data for each taxi trip (one row per trip) since 2013. -- We will generate several training datasets (aka historical features in Feast) for different periods and evaluate expectations made on one dataset against another. - -Types of features we're ingesting and generating: -- Features that aggregate raw data with daily intervals (eg, trips per day, average fare or speed for a specific day, etc.). -- Features using SQL while pulling data from BigQuery (like total trips time or total miles travelled). -- Features calculated on the fly when requested using Feast's on-demand transformations - -Our plan: - -0. Prepare environment -1. Pull data from BigQuery (optional) -2. Declare & apply features and feature views in Feast -3. Generate reference dataset -4. Develop & test profiler function -5. Run validation on different dataset using reference dataset & profiler - - -> The original notebook and datasets for this tutorial can be found on [GitHub](https://github.com/feast-dev/dqm-tutorial). - -### 0. Setup - -Install Feast Python SDK and great expectations: - - -```python -!pip install 'feast[ge]' -``` - - -### 1. Dataset preparation (Optional) - -**You can skip this step if you don't have GCP account. Please use parquet files that are coming with this tutorial instead** - - -```python -!pip install google-cloud-bigquery -``` - - -```python -import pyarrow.parquet - -from google.cloud.bigquery import Client -``` - - -```python -bq_client = Client(project='kf-feast') -``` - -Running some basic aggregations while pulling data from BigQuery. Grouping by taxi_id and day: - - -```python -data_query = """SELECT - taxi_id, - TIMESTAMP_TRUNC(trip_start_timestamp, DAY) as day, - SUM(trip_miles) as total_miles_travelled, - SUM(trip_seconds) as total_trip_seconds, - SUM(fare) as total_earned, - COUNT(*) as trip_count -FROM `bigquery-public-data.chicago_taxi_trips.taxi_trips` -WHERE - trip_miles > 0 AND trip_seconds > 60 AND - trip_start_timestamp BETWEEN '2019-01-01' and '2020-12-31' AND - trip_total < 1000 -GROUP BY taxi_id, TIMESTAMP_TRUNC(trip_start_timestamp, DAY)""" -``` - - -```python -driver_stats_table = bq_client.query(data_query).to_arrow() - -# Storing resulting dataset into parquet file -pyarrow.parquet.write_table(driver_stats_table, "trips_stats.parquet") -``` - - -```python -def entities_query(year): - return f"""SELECT - distinct taxi_id -FROM `bigquery-public-data.chicago_taxi_trips.taxi_trips` -WHERE - trip_miles > 0 AND trip_seconds > 0 AND - trip_start_timestamp BETWEEN '{year}-01-01' and '{year}-12-31' -""" -``` - - -```python -entities_2019_table = bq_client.query(entities_query(2019)).to_arrow() - -# Storing entities (taxi ids) into parquet file -pyarrow.parquet.write_table(entities_2019_table, "entities.parquet") -``` - - -## 2. Declaring features - - -```python -import pyarrow.parquet -import pandas as pd - -from feast import FeatureView, Entity, FeatureStore, Field, BatchFeatureView -from feast.types import Float64, Int64 -from feast.value_type import ValueType -from feast.data_format import ParquetFormat -from feast.on_demand_feature_view import on_demand_feature_view -from feast.infra.offline_stores.file_source import FileSource -from feast.infra.offline_stores.file import SavedDatasetFileStorage -from datetime import timedelta - -``` - - -```python -batch_source = FileSource( - timestamp_field="day", - path="trips_stats.parquet", # using parquet file that we created on previous step - file_format=ParquetFormat() -) -``` - - -```python -taxi_entity = Entity(name='taxi', join_keys=['taxi_id']) -``` - - -```python -trips_stats_fv = BatchFeatureView( - name='trip_stats', - entities=[taxi_entity], - schema=[ - Field(name="total_miles_travelled", dtype=Float64), - Field(name="total_trip_seconds", dtype=Float64), - Field(name="total_earned", dtype=Float64), - Field(name="trip_count", dtype=Int64), - - ], - ttl=timedelta(seconds=86400), - source=batch_source, -) -``` - -*Read more about feature views in [Feast docs](https://docs.feast.dev/getting-started/concepts/feature-view)* - - -```python -@on_demand_feature_view( - sources=[ - trips_stats_fv, - ], - schema=[ - Field(name="avg_fare", dtype=Float64), - Field(name="avg_speed", dtype=Float64), - Field(name="avg_trip_seconds", dtype=Float64), - Field(name="earned_per_hour", dtype=Float64), - ] -) -def on_demand_stats(inp: pd.DataFrame) -> pd.DataFrame: - out = pd.DataFrame() - out["avg_fare"] = inp["total_earned"] / inp["trip_count"] - out["avg_speed"] = 3600 * inp["total_miles_travelled"] / inp["total_trip_seconds"] - out["avg_trip_seconds"] = inp["total_trip_seconds"] / inp["trip_count"] - out["earned_per_hour"] = 3600 * inp["total_earned"] / inp["total_trip_seconds"] - return out -``` - -*Read more about on demand feature views [here](../reference/beta-on-demand-feature-view.md)* - - -```python -store = FeatureStore(".") # using feature_store.yaml that stored in the same directory -``` - - -```python -store.apply([taxi_entity, trips_stats_fv, on_demand_stats]) # writing to the registry -``` - - -## 3. Generating training (reference) dataset - - -```python -taxi_ids = pyarrow.parquet.read_table("entities.parquet").to_pandas() -``` - -Generating range of timestamps with daily frequency: - - -```python -timestamps = pd.DataFrame() -timestamps["event_timestamp"] = pd.date_range("2019-06-01", "2019-07-01", freq='D') -``` - -Cross merge (aka relation multiplication) produces entity dataframe with each taxi_id repeated for each timestamp: - - -```python -entity_df = pd.merge(taxi_ids, timestamps, how='cross') -entity_df -``` - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
taxi_idevent_timestamp
091d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-01
191d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-02
291d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-03
391d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-04
491d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-05
.........
1569797ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-06-27
1569807ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-06-28
1569817ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-06-29
1569827ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-06-30
1569837ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-07-01
-

156984 rows × 2 columns

-
- - - -Retrieving historical features for resulting entity dataframe and persisting output as a saved dataset: - - -```python -job = store.get_historical_features( - entity_df=entity_df, - features=[ - "trip_stats:total_miles_travelled", - "trip_stats:total_trip_seconds", - "trip_stats:total_earned", - "trip_stats:trip_count", - "on_demand_stats:avg_fare", - "on_demand_stats:avg_trip_seconds", - "on_demand_stats:avg_speed", - "on_demand_stats:earned_per_hour", - ] -) - -store.create_saved_dataset( - from_=job, - name='my_training_ds', - storage=SavedDatasetFileStorage(path='my_training_ds.parquet') -) -``` - -```python -, full_feature_names = False, tags = {}, _retrieval_job = , min_event_timestamp = 2019-06-01 00:00:00, max_event_timestamp = 2019-07-01 00:00:00)> -``` - - -## 4. Developing dataset profiler - -Dataset profiler is a function that accepts dataset and generates set of its characteristics. This charasteristics will be then used to evaluate (validate) next datasets. - -**Important: datasets are not compared to each other! -Feast use a reference dataset and a profiler function to generate a reference profile. -This profile will be then used during validation of the tested dataset.** - - -```python -import numpy as np - -from feast.dqm.profilers.ge_profiler import ge_profiler - -from great_expectations.core.expectation_suite import ExpectationSuite -from great_expectations.dataset import PandasDataset -``` - - -Loading saved dataset first and exploring the data: - - -```python -ds = store.get_saved_dataset('my_training_ds') -ds.to_df() -``` - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
total_earnedavg_trip_secondstaxi_idtotal_miles_travelledtrip_countearned_per_hourevent_timestamptotal_trip_secondsavg_fareavg_speed
068.252270.00000091d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...24.702.054.1189432019-06-01 00:00:00+00:004540.034.12500019.585903
1221.00560.5000007a4a6162eaf27805aef407d25d5cb21fe779cd962922cb...54.1824.059.1436222019-06-01 00:00:00+00:0013452.09.20833314.499554
2160.501010.769231f4c9d05b215d7cbd08eca76252dae51cdb7aca9651d4ef...41.3013.043.9726032019-06-01 00:00:00+00:0013140.012.34615411.315068
3183.75697.550000c1f533318f8480a59173a9728ea0248c0d3eb187f4b897...37.3020.047.4159562019-06-01 00:00:00+00:0013951.09.1875009.625116
4217.751054.076923455b6b5cae6ca5a17cddd251485f2266d13d6a2c92f07c...69.6913.057.2064512019-06-01 00:00:00+00:0013703.016.75000018.308692
.................................
15697938.001980.0000000cccf0ec1f46d1e0beefcfdeaf5188d67e170cdff92618...14.901.069.0909092019-07-01 00:00:00+00:001980.038.00000027.090909
156980135.00551.250000beefd3462e3f5a8e854942a2796876f6db73ebbd25b435...28.4016.055.1020412019-07-01 00:00:00+00:008820.08.43750011.591837
156981NaNNaN9a3c52aa112f46cf0d129fafbd42051b0fb9b0ff8dcb0e...NaNNaNNaN2019-07-01 00:00:00+00:00NaNNaNNaN
15698263.00815.00000008308c31cd99f495dea73ca276d19a6258d7b4c9c88e43...19.964.069.5705522019-07-01 00:00:00+00:003260.015.75000022.041718
156983NaNNaN7ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...NaNNaNNaN2019-07-01 00:00:00+00:00NaNNaNNaN
-

156984 rows × 10 columns

-
- - - -Feast uses [Great Expectations](https://docs.greatexpectations.io/docs/) as a validation engine and [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) as a dataset's profile. Hence, we need to develop a function that will generate ExpectationSuite. This function will receive instance of [PandasDataset](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/dataset/index.html?highlight=pandasdataset#great_expectations.dataset.PandasDataset) (wrapper around pandas.DataFrame) so we can utilize both Pandas DataFrame API and some helper functions from PandasDataset during profiling. - - -```python -DELTA = 0.1 # controlling allowed window in fraction of the value on scale [0, 1] - -@ge_profiler -def stats_profiler(ds: PandasDataset) -> ExpectationSuite: - # simple checks on data consistency - ds.expect_column_values_to_be_between( - "avg_speed", - min_value=0, - max_value=60, - mostly=0.99 # allow some outliers - ) - - ds.expect_column_values_to_be_between( - "total_miles_travelled", - min_value=0, - max_value=500, - mostly=0.99 # allow some outliers - ) - - # expectation of means based on observed values - observed_mean = ds.trip_count.mean() - ds.expect_column_mean_to_be_between("trip_count", - min_value=observed_mean * (1 - DELTA), - max_value=observed_mean * (1 + DELTA)) - - observed_mean = ds.earned_per_hour.mean() - ds.expect_column_mean_to_be_between("earned_per_hour", - min_value=observed_mean * (1 - DELTA), - max_value=observed_mean * (1 + DELTA)) - - - # expectation of quantiles - qs = [0.5, 0.75, 0.9, 0.95] - observed_quantiles = ds.avg_fare.quantile(qs) - - ds.expect_column_quantile_values_to_be_between( - "avg_fare", - quantile_ranges={ - "quantiles": qs, - "value_ranges": [[None, max_value] for max_value in observed_quantiles] - }) - - return ds.get_expectation_suite() -``` - -Testing our profiler function: - - -```python -ds.get_profile(profiler=stats_profiler) -``` - 02/02/2022 02:43:47 PM INFO: 5 expectation(s) included in expectation_suite. result_format settings filtered. - - - - -**Verify that all expectations that we coded in our profiler are present here. Otherwise (if you can't find some expectations) it means that it failed to pass on the reference dataset (do it silently is default behavior of Great Expectations).** - -Now we can create validation reference from dataset and profiler function: - - -```python -validation_reference = ds.as_reference(name="validation_reference_dataset", profiler=stats_profiler) -``` - -and test it against our existing retrieval job - - -```python -_ = job.to_df(validation_reference=validation_reference) -``` - - 02/02/2022 02:43:52 PM INFO: 5 expectation(s) included in expectation_suite. result_format settings filtered. - 02/02/2022 02:43:53 PM INFO: Validating data_asset_name None with expectation_suite_name default - - -Validation successfully passed as no exception were raised. - - -### 5. Validating new historical retrieval - -Creating new timestamps for Dec 2020: - - -```python -from feast.dqm.errors import ValidationFailed -``` - - -```python -timestamps = pd.DataFrame() -timestamps["event_timestamp"] = pd.date_range("2020-12-01", "2020-12-07", freq='D') -``` - - -```python -entity_df = pd.merge(taxi_ids, timestamps, how='cross') -entity_df -``` - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
taxi_idevent_timestamp
091d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-01
191d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-02
291d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-03
391d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-04
491d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-05
.........
354437ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-03
354447ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-04
354457ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-05
354467ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-06
354477ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-07
-

35448 rows × 2 columns

-
- - -```python -job = store.get_historical_features( - entity_df=entity_df, - features=[ - "trip_stats:total_miles_travelled", - "trip_stats:total_trip_seconds", - "trip_stats:total_earned", - "trip_stats:trip_count", - "on_demand_stats:avg_fare", - "on_demand_stats:avg_trip_seconds", - "on_demand_stats:avg_speed", - "on_demand_stats:earned_per_hour", - ] -) -``` - -Execute retrieval job with validation reference: - - -```python -try: - df = job.to_df(validation_reference=validation_reference) -except ValidationFailed as exc: - print(exc.validation_report) -``` - - 02/02/2022 02:43:58 PM INFO: 5 expectation(s) included in expectation_suite. result_format settings filtered. - 02/02/2022 02:43:59 PM INFO: Validating data_asset_name None with expectation_suite_name default - - [ - { - "expectation_config": { - "expectation_type": "expect_column_mean_to_be_between", - "kwargs": { - "column": "trip_count", - "min_value": 10.387244591346153, - "max_value": 12.695521167200855, - "result_format": "COMPLETE" - }, - "meta": {} - }, - "meta": {}, - "result": { - "observed_value": 6.692920555429092, - "element_count": 35448, - "missing_count": 31055, - "missing_percent": 87.6071992778154 - }, - "exception_info": { - "raised_exception": false, - "exception_message": null, - "exception_traceback": null - }, - "success": false - }, - { - "expectation_config": { - "expectation_type": "expect_column_mean_to_be_between", - "kwargs": { - "column": "earned_per_hour", - "min_value": 52.320624975640214, - "max_value": 63.94743052578249, - "result_format": "COMPLETE" - }, - "meta": {} - }, - "meta": {}, - "result": { - "observed_value": 68.99268345164135, - "element_count": 35448, - "missing_count": 31055, - "missing_percent": 87.6071992778154 - }, - "exception_info": { - "raised_exception": false, - "exception_message": null, - "exception_traceback": null - }, - "success": false - }, - { - "expectation_config": { - "expectation_type": "expect_column_quantile_values_to_be_between", - "kwargs": { - "column": "avg_fare", - "quantile_ranges": { - "quantiles": [ - 0.5, - 0.75, - 0.9, - 0.95 - ], - "value_ranges": [ - [ - null, - 16.4 - ], - [ - null, - 26.229166666666668 - ], - [ - null, - 36.4375 - ], - [ - null, - 42.0 - ] - ] - }, - "result_format": "COMPLETE" - }, - "meta": {} - }, - "meta": {}, - "result": { - "observed_value": { - "quantiles": [ - 0.5, - 0.75, - 0.9, - 0.95 - ], - "values": [ - 19.5, - 28.1, - 38.0, - 44.125 - ] - }, - "element_count": 35448, - "missing_count": 31055, - "missing_percent": 87.6071992778154, - "details": { - "success_details": [ - false, - false, - false, - false - ] - } - }, - "exception_info": { - "raised_exception": false, - "exception_message": null, - "exception_traceback": null - }, - "success": false - } - ] - - -Validation failed since several expectations didn't pass: -* Trip count (mean) decreased more than 10% (which is expected when comparing Dec 2020 vs June 2019) -* Average Fare increased - all quantiles are higher than expected -* Earn per hour (mean) increased more than 10% (most probably due to increased fare) - diff --git a/examples/online_store/aerospike_overrides_and_hooks/README.md b/examples/online_store/aerospike_overrides_and_hooks/README.md new file mode 100644 index 00000000000..7143b6a3dff --- /dev/null +++ b/examples/online_store/aerospike_overrides_and_hooks/README.md @@ -0,0 +1,65 @@ +# Aerospike: per-feature-view overrides + prewriting hooks + +A short companion to [`docs/reference/online-stores/aerospike.md`](../../../docs/reference/online-stores/aerospike.md) +demonstrating three deployment patterns the Aerospike online store supports +without needing any Feast extension code: + +1. **Per-feature-view namespace overrides** — pin one view to a RAM-only + namespace and another to an SSD-backed one without splitting the project. +2. **Per-feature-view set overrides** — isolate one view in its own set so + `feast apply` deletions or admin truncates only touch that view. +3. **Prewriting hooks** — apply a project-wide write-side transformation + (PII masking in this example) without sprinkling it through every + materialization job. + +Nothing here is Aerospike-specific *infrastructure* — it's all configured +in `feature_store.yaml`. This directory only adds the hook-target Python +module the YAML references. + +## Files + +| file | purpose | +|---|---| +| [`hooks.py`](hooks.py) | A pure-Python prewriting-hook module containing `hash_pii_string_features`, the same example used in the docs. Drop into any module on the writer's `PYTHONPATH`. | +| [`feature_store.yaml`](feature_store.yaml) | Reference `online_store` block showing all three features wired together. Copy the `online_store` section into your own `feature_store.yaml` — the rest is project-specific scaffolding. | + +## Prerequisites + +* Feast installed with the Aerospike extra (`pip install 'feast[aerospike]'`). +* An Aerospike cluster reachable from your writer process. The + [Aerospike online-store reference](../../../docs/reference/online-stores/aerospike.md) + shows a minimal local CE config (`127.0.0.1:3000`); run Aerospike however + you normally would (Docker, Kubernetes, bare metal). +* On every process that calls `online_write_batch` through this store + (materialization workers, the registry CLI host, the feature server if + you run one), the `FEAST_PII_SALT` environment variable must be set + before the first write — `hash_pii_string_features` raises rather than + silently writing plaintext if the salt isn't configured. +* The two namespaces referenced by `namespace_overrides` (`feast_ram` and + `feast_ssd` in the sample YAML) must already exist on the Aerospike + cluster — Aerospike cannot create namespaces at runtime. + +## Trying it out + +1. Drop `hooks.py` into a module on your `PYTHONPATH` that the writer + process can import (e.g. inside your existing feature-repo package). + The example uses the qualified path + `examples.online_store.aerospike_overrides_and_hooks.hooks.hash_pii_string_features`. +2. Copy the `online_store:` block from `feature_store.yaml` into your + own feature repo, adjusting hosts / namespaces / the hook import path + for your project. +3. `export FEAST_PII_SALT=...` (anything random and stable across + processes — rotate by re-running materialization with a new salt). +4. `feast apply` — the new config is registered. +5. Materialize as usual — the hook runs once per `online_write_batch`, + and any feature named `email`, `phone_number` or `ssn` lands in + Aerospike as a salted SHA-256 hex digest instead of plaintext. + +## Read-side note + +Prewriting hooks are **only** invoked on the write path. If your hook is +a one-way transform (hashing, encryption-without-decryption-key) you +have to apply the same transform to the candidate value at read time +yourself. Two-way transforms (deterministic encryption, Base64) need a +matching post-read step in your serving code; the Aerospike store does +not currently expose a symmetric "postreading hook". diff --git a/examples/online_store/aerospike_overrides_and_hooks/feature_store.yaml b/examples/online_store/aerospike_overrides_and_hooks/feature_store.yaml new file mode 100644 index 00000000000..bd1061f30aa --- /dev/null +++ b/examples/online_store/aerospike_overrides_and_hooks/feature_store.yaml @@ -0,0 +1,59 @@ +# Reference feature_store.yaml demonstrating all three Aerospike +# extension points wired together. Copy the `online_store:` block into +# your own feature repo and adjust hosts / namespaces / hook import +# path for your project. +# +# Prerequisites: +# - The `feast_ram` and `feast_ssd` namespaces must already exist on +# the Aerospike cluster. Aerospike cannot create namespaces at +# runtime; a missing namespace surfaces as AEROSPIKE_ERR_PARAM on +# the first read or write touching that view. +# - `FEAST_PII_SALT` must be set in every process that calls +# online_write_batch through this store. + +project: my_feature_repo +registry: data/registry.db +provider: local + +online_store: + type: aerospike + + hosts: + - ["aerospike.internal", 3000] + + # Store-level defaults. Anything not listed in *_overrides below + # falls back to these. + namespace: feast + set_name_template: "{project}_{collection_suffix}" + + # Pin individual feature views to different namespaces -- typically + # one in-memory namespace for hot, latency-sensitive views and one + # device-backed namespace for cold, wide views. + namespace_overrides: + driver_realtime_stats: feast_ram + driver_history_lookup: feast_ssd + + # Isolate one feature view in its own set so that admin operations on + # it (truncate, scan-based deletion via `feast apply`) do not touch + # the records of other views. + set_overrides: + isolated_view: my_feature_repo_isolated + + # Project-wide write-side hook. The store dynamically imports the + # callable on first use and caches it. Adjust the import path to + # whatever module is on your writers' PYTHONPATH; the value below + # assumes you have the example folder on PYTHONPATH from the + # repository root. + prewriting_hook: examples.online_store.aerospike_overrides_and_hooks.hooks.hash_pii_string_features + + # Standard timing knobs (optional -- shown for completeness). + ttl_seconds: 86400 + read_timeout_ms: 150 + write_timeout_ms: 300 + batch_total_timeout_ms: 500 + socket_timeout_ms: 50 + max_retries: 2 + +# Offline store / entity_key_serialization_version / etc. are +# project-specific and intentionally omitted; this file is a snippet, +# not a runnable repo. diff --git a/examples/online_store/aerospike_overrides_and_hooks/hooks.py b/examples/online_store/aerospike_overrides_and_hooks/hooks.py new file mode 100644 index 00000000000..15e6f8a10c7 --- /dev/null +++ b/examples/online_store/aerospike_overrides_and_hooks/hooks.py @@ -0,0 +1,109 @@ +"""Sample prewriting hooks for the Feast Aerospike online store. + +Reference the callable from ``feature_store.yaml`` via its import string, +e.g.:: + + online_store: + type: aerospike + ... + prewriting_hook: examples.online_store.aerospike_overrides_and_hooks.hooks.hash_pii_string_features + +The Aerospike online store invokes the configured callable once per +``online_write_batch`` call, passing the rows about to be written. The +callable must return a row list with the same schema. Returning ``[]`` +short-circuits the write — same path as an empty input, no wire call is +issued. +""" + +from __future__ import annotations + +import hashlib +import os +from datetime import datetime +from typing import List, Optional, Tuple + +from feast import FeatureView +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import RepoConfig + +# Names of features that must never reach the online store as plaintext. +# Match is by exact feature name; tweak to your project's conventions +# (regex, suffix-based, FV-tag-driven, etc.). +_SENSITIVE_FEATURES = frozenset({"email", "phone_number", "ssn"}) + +# Type alias for the per-row payload Feast hands to ``online_write_batch``. +WriteRow = Tuple[ + EntityKeyProto, + dict, + datetime, + Optional[datetime], +] + + +def hash_pii_string_features( + config: RepoConfig, + table: FeatureView, + data: List[WriteRow], +) -> List[WriteRow]: + """Replace any sensitive string feature with a salted SHA-256 hex digest. + + Determinism: same plaintext + same ``FEAST_PII_SALT`` → same digest. + Downstream lookups that hash the candidate value the same way still + hit; lookups against the raw plaintext silently miss. + + Safety: an unset salt raises rather than falling back to plaintext. + Set ``FEAST_PII_SALT`` on every process that materialises features + (workers, registry CLI host, feature server). + """ + salt = os.environ.get("FEAST_PII_SALT") + if salt is None: + raise RuntimeError( + "FEAST_PII_SALT is not set; refusing to write feature batches " + "without a configured PII salt." + ) + salt_bytes = salt.encode("utf-8") + + def _digest(plaintext: str) -> str: + h = hashlib.sha256() + h.update(salt_bytes) + h.update(plaintext.encode("utf-8")) + return h.hexdigest() + + transformed: List[WriteRow] = [] + for entity_key, values, event_ts, created_ts in data: + new_values = dict(values) + for feature_name in _SENSITIVE_FEATURES.intersection(new_values): + v: ValueProto = new_values[feature_name] + if v.HasField("string_val") and v.string_val: + new_values[feature_name] = ValueProto(string_val=_digest(v.string_val)) + transformed.append((entity_key, new_values, event_ts, created_ts)) + return transformed + + +def drop_rows_with_negative_amounts( + config: RepoConfig, + table: FeatureView, + data: List[WriteRow], +) -> List[WriteRow]: + """Defensive sample hook: filter rows whose ``amount`` feature is < 0. + + Demonstrates that hooks can also *remove* rows. Returning an empty + list short-circuits the wire call entirely — useful for emergency + feature-write quarantines without a code deploy. + """ + keep: List[WriteRow] = [] + for entity_key, values, event_ts, created_ts in data: + amount: Optional[ValueProto] = values.get("amount") + if ( + amount is not None + and amount.HasField("double_val") + and amount.double_val < 0 + ): + continue + if amount is not None and amount.HasField("float_val") and amount.float_val < 0: + continue + if amount is not None and amount.HasField("int64_val") and amount.int64_val < 0: + continue + keep.append((entity_key, values, event_ts, created_ts)) + return keep diff --git a/examples/ray-llm-posttrain/.gitignore b/examples/ray-llm-posttrain/.gitignore new file mode 100644 index 00000000000..4bf1a46bf0e --- /dev/null +++ b/examples/ray-llm-posttrain/.gitignore @@ -0,0 +1,13 @@ +# Feast / Ray local artifacts +data/ +.feast/ +ray_storage/ +/tmp/ray/ +ray_results/ +.ray/ + +__pycache__/ +*.py[cod] +.pytest_cache/ +.venv/ +.env diff --git a/examples/ray-llm-posttrain/README.md b/examples/ray-llm-posttrain/README.md new file mode 100644 index 00000000000..64252a5a603 --- /dev/null +++ b/examples/ray-llm-posttrain/README.md @@ -0,0 +1,35 @@ +# How to Use Feast for SLM/LLM Post-Training (with Ray) + +| Name | Type | Fields | +|---|---|---| +| `web_documents` | FeatureView | `human`, `bot`, `human_repeat_ratio`, `bot_repeat_ratio` | +| `train_example` | OnDemandFeatureView | `cleaned_human`, `cleaned_bot`, `char_count`, `is_trainable`, `sft_text` | +| `llm_posttrain` | FeatureService | `web_documents` + `train_example` | + +Source data is **prepared parquet** (`document_id` + `event_timestamp` already present). No Feast core patches. + +## Paths + +| Flag | What happens | +|---|---| +| (default) | `to_ray_dataset()` + preprocess `sft_text` (ODFV does **not** run) | +| `--via-df` | `to_df()` so ODFV `train_example` runs | + +## Setup + +```bash +uv pip install -e "../../sdk/python[ray]" -r requirements.txt +PYTHONPATH=../../sdk/python python scripts/prepare_data.py +cd feature_repo && feast apply && cd .. +``` + +## Run (data load only) + +```bash +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run --via-df +``` + +## Blog + +[How to Use Feast for SLM/LLM Post-Training with Ray](/blog/feast-ray-llm-posttrain) diff --git a/examples/ray-llm-posttrain/feature_repo/feature_definitions.py b/examples/ray-llm-posttrain/feature_repo/feature_definitions.py new file mode 100644 index 00000000000..a85470c3729 --- /dev/null +++ b/examples/ray-llm-posttrain/feature_repo/feature_definitions.py @@ -0,0 +1,108 @@ +"""Feast feature definitions for Ray + ODFV LLM post-training. + +Pipeline (supported Feast APIs only): + scripts/prepare_data.py → parquet with document_id + event_timestamp + → RaySource (parquet) + → FeatureView web_documents + → OnDemandFeatureView train_example + → FeatureService llm_posttrain +""" + +from __future__ import annotations + +from datetime import timedelta +from pathlib import Path + +from feast import Entity, FeatureService, FeatureView, Field, ValueType +from feast.infra.offline_stores.contrib.ray_offline_store.ray_source import RaySource +from feast.on_demand_feature_view import on_demand_feature_view +from feast.types import Bool, Float64, Int64, String + +_REPO_DIR = Path(__file__).resolve().parent +_PARQUET = str(_REPO_DIR / "data" / "tiny_webtext.parquet") + +document = Entity( + name="document", + join_keys=["document_id"], + value_type=ValueType.STRING, + description="Document id (added by scripts/prepare_data.py)", +) + +# Parquet already has document_id + event_timestamp (see prepare_data.py). +# Do not rely on BatchFeatureView UDFs to invent timestamps during entity-less +# retrieval — that path is not supported without Feast core changes. +tiny_web = RaySource( + name="tiny_webtext", + reader_type="parquet", + path=_PARQUET, + timestamp_field="event_timestamp", +) + +web_documents = FeatureView( + name="web_documents", + entities=[document], + ttl=timedelta(days=365), + schema=[ + Field(name="human", dtype=String), + Field(name="bot", dtype=String), + Field(name="human_repeat_ratio", dtype=Float64), + Field(name="bot_repeat_ratio", dtype=Float64), + ], + source=tiny_web, + online=False, + description="Conversation columns from prepared parquet", + tags={"use_case": "llm_posttrain", "source": "parquet"}, +) + + +@on_demand_feature_view( + sources=[web_documents], + schema=[ + Field(name="cleaned_human", dtype=String), + Field(name="cleaned_bot", dtype=String), + Field(name="char_count", dtype=Int64), + Field(name="is_trainable", dtype=Bool), + Field(name="sft_text", dtype=String), + ], + mode="pandas", +) +def train_example(inputs): + """Quality gate + human→bot SFT formatting.""" + import pandas as pd + + min_chars = 64 + max_repeat_ratio = 0.65 + + cleaned_human = inputs["human"].fillna("").astype(str).str.strip() + cleaned_bot = inputs["bot"].fillna("").astype(str).str.strip() + char_count = cleaned_bot.str.len().astype("int64") + + human_ratio = inputs["human_repeat_ratio"].fillna(1.0).astype(float) + bot_ratio = inputs["bot_repeat_ratio"].fillna(1.0).astype(float) + is_trainable = ( + (char_count >= min_chars) + & (human_ratio <= max_repeat_ratio) + & (bot_ratio <= max_repeat_ratio) + ) + + sft_text = ( + "<|im_start|>user\n" + cleaned_human + "<|im_end|>\n" + "<|im_start|>assistant\n" + cleaned_bot + "<|im_end|>" + ) + + return pd.DataFrame( + { + "cleaned_human": cleaned_human, + "cleaned_bot": cleaned_bot, + "char_count": char_count, + "is_trainable": is_trainable, + "sft_text": sft_text, + } + ) + + +llm_posttrain = FeatureService( + name="llm_posttrain", + features=[web_documents, train_example], + tags={"use_case": "llm_posttrain", "model": "gpt2"}, +) diff --git a/examples/ray-llm-posttrain/feature_repo/feature_store.yaml b/examples/ray-llm-posttrain/feature_repo/feature_store.yaml new file mode 100644 index 00000000000..226dbdfb45c --- /dev/null +++ b/examples/ray-llm-posttrain/feature_repo/feature_store.yaml @@ -0,0 +1,25 @@ +project: ray_llm_posttrain +registry: data/registry.db +provider: local + +# Laptop-friendly Ray offline store (no KubeRay) +offline_store: + type: ray + storage_path: data/ray_storage + enable_ray_logging: false + ray_conf: + num_cpus: 2 + object_store_memory: 104857600 + _memory: 524288000 + +batch_engine: + type: ray.engine + max_workers: 2 + +online_store: + type: sqlite + path: data/online_store.db + +entity_key_serialization_version: 3 +auth: + type: no_auth diff --git a/examples/ray-llm-posttrain/requirements.txt b/examples/ray-llm-posttrain/requirements.txt new file mode 100644 index 00000000000..4a82b391c07 --- /dev/null +++ b/examples/ray-llm-posttrain/requirements.txt @@ -0,0 +1,8 @@ +# Feast + Ray offline store / compute engine +feast[ray]>=0.50.0 +datasets>=2.19.0 + +# Short GPT-2 SFT +transformers>=4.40.0 +torch>=2.1.0 +accelerate>=0.30.0 diff --git a/examples/ray-llm-posttrain/scripts/prepare_data.py b/examples/ray-llm-posttrain/scripts/prepare_data.py new file mode 100644 index 00000000000..efd1a5e032b --- /dev/null +++ b/examples/ray-llm-posttrain/scripts/prepare_data.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Prepare a small local parquet seed for the example. + +Hugging Face tiny-webtext has no document_id / event_timestamp. Feast entity-less +retrieval needs those columns on the *source* data. We synthesize them here +(outside Feast) and write parquet — no Feast core changes required. + + PYTHONPATH=../../sdk/python python scripts/prepare_data.py +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pandas as pd + +REPO_ROOT = Path(__file__).resolve().parents[1] +OUT_PATH = REPO_ROOT / "feature_repo" / "data" / "tiny_webtext.parquet" +SPLIT = "train[:2000]" +DATASET = "nampdn-ai/tiny-webtext" + + +def main() -> int: + from datasets import load_dataset + + print(f"Loading {DATASET} split={SPLIT!r}...") + ds = load_dataset(DATASET, split=SPLIT) + df = ds.to_pandas() + + demo_base_ts = pd.Timestamp("2024-06-01", tz="UTC") + demo_window_seconds = 30 * 24 * 3600 + + humans = df["human"].fillna("").astype(str) + bots = df["bot"].fillna("").astype(str) + doc_ids: list[str] = [] + timestamps: list[pd.Timestamp] = [] + for human, bot in zip(humans, bots, strict=True): + digest = hashlib.sha256(f"{human}\n{bot}".encode()).hexdigest() + doc_ids.append(digest[:16]) + offset = int(digest[:8], 16) % demo_window_seconds + timestamps.append(demo_base_ts + pd.Timedelta(seconds=offset)) + + df = df.copy() + df["document_id"] = doc_ids + df["event_timestamp"] = timestamps + + OUT_PATH.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(OUT_PATH, index=False) + print(f"Wrote {len(df)} rows → {OUT_PATH}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/ray-llm-posttrain/scripts/train_sft.py b/examples/ray-llm-posttrain/scripts/train_sft.py new file mode 100644 index 00000000000..a3f4939f855 --- /dev/null +++ b/examples/ray-llm-posttrain/scripts/train_sft.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Feast conversation features → training rows (paths match the blog). + +Paths: + A) Default: get_historical_features → to_ray_dataset() → preprocess sft_text + (ODFVs do NOT run on to_ray_dataset) + B) --via-df: get_historical_features → to_df() (ODFV train_example runs) + + PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run + PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run --via-df +""" + +from __future__ import annotations + +import argparse +import sys +from datetime import datetime, timezone +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +FEATURE_REPO = REPO_ROOT / "feature_repo" +DATA_DIR = REPO_ROOT / "data" + +# Matches the blog Option A snippet (length gate) +_MIN_CHARS = 64 + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--max-steps", type=int, default=20) + parser.add_argument("--batch-size", type=int, default=2) + parser.add_argument("--max-length", type=int, default=256) + parser.add_argument( + "--output-dir", + type=Path, + default=DATA_DIR / "gpt2-sft", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print a few training rows; skip the optional GPT-2 smoke", + ) + parser.add_argument( + "--via-df", + action="store_true", + help="Use to_df() so OnDemandFeatureView train_example runs", + ) + return parser.parse_args() + + +def _date_window() -> tuple[datetime, datetime]: + return ( + datetime(2024, 6, 1, tzinfo=timezone.utc), + datetime(2024, 7, 1, tzinfo=timezone.utc), + ) + + +def _preprocess_sft_batch(batch): + """Build sft_text from FeatureView columns (blog Option A).""" + import pandas as pd + + if not isinstance(batch, pd.DataFrame): + batch = pd.DataFrame(batch) + + human = batch["human"].fillna("").astype(str).str.strip() + bot = batch["bot"].fillna("").astype(str).str.strip() + ok = bot.str.len() >= _MIN_CHARS + sft_text = ( + "<|im_start|>user\n" + human + "<|im_end|>\n" + "<|im_start|>assistant\n" + bot + "<|im_end|>" + ) + return pd.DataFrame({"sft_text": sft_text}).loc[ok].reset_index(drop=True) + + +def retrieve_via_ray_stream(): + """Option A: to_ray_dataset() + preprocess (ODFV does not run).""" + from feast import FeatureStore + + store = FeatureStore(repo_path=str(FEATURE_REPO)) + start_date, end_date = _date_window() + + print("get_historical_features → to_ray_dataset() (preprocess sft_text on Ray)") + job = store.get_historical_features( + features=[ + "web_documents:human", + "web_documents:bot", + "web_documents:human_repeat_ratio", + "web_documents:bot_repeat_ratio", + ], + start_date=start_date, + end_date=end_date, + ) + ds = job.to_ray_dataset() + return ds.map_batches(_preprocess_sft_batch, batch_format="pandas") + + +def retrieve_via_df(): + """Option B: to_df() so ODFV train_example runs, then Ray from pandas.""" + import ray + from feast import FeatureStore + + store = FeatureStore(repo_path=str(FEATURE_REPO)) + start_date, end_date = _date_window() + + print("get_historical_features → to_df() (ODFV train_example runs)") + df = store.get_historical_features( + features=store.get_feature_service("llm_posttrain"), + start_date=start_date, + end_date=end_date, + ).to_df() + + if "is_trainable" not in df.columns or "sft_text" not in df.columns: + raise RuntimeError("Expected ODFV columns is_trainable / sft_text from to_df()") + + mask = df["is_trainable"].fillna(False).astype(bool) + mask &= df["sft_text"].fillna("").astype(str).str.len() > 0 + slim = df.loc[mask, ["sft_text"]].reset_index(drop=True) + return ray.data.from_pandas(slim) + + +def train_gpt2_optional( + ray_ds, *, max_steps: int, batch_size: int, max_length: int, output_dir: Path +) -> None: + import torch + from transformers import ( + AutoModelForCausalLM, + AutoTokenizer, + DataCollatorForLanguageModeling, + Trainer, + TrainingArguments, + ) + + rows = ray_ds.take(min(500, max(50, max_steps * batch_size * 4))) + texts = [r["sft_text"] for r in rows if r.get("sft_text")] + if not texts: + raise RuntimeError("No trainable SFT rows") + + print(f"[optional] GPT-2 smoke on {len(texts)} rows, {max_steps} steps...") + tokenizer = AutoTokenizer.from_pretrained("gpt2") + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + model = AutoModelForCausalLM.from_pretrained("gpt2") + encodings = tokenizer( + texts, + truncation=True, + max_length=max_length, + padding="max_length", + return_tensors="pt", + ) + + class _TextDataset(torch.utils.data.Dataset): + def __len__(self) -> int: + return encodings["input_ids"].shape[0] + + def __getitem__(self, idx: int) -> dict: + return { + "input_ids": encodings["input_ids"][idx], + "attention_mask": encodings["attention_mask"][idx], + "labels": encodings["input_ids"][idx].clone(), + } + + output_dir.mkdir(parents=True, exist_ok=True) + args = TrainingArguments( + output_dir=str(output_dir), + per_device_train_batch_size=batch_size, + max_steps=max_steps, + logging_steps=max(1, max_steps // 5), + save_steps=max_steps, + learning_rate=5e-5, + report_to=[], + remove_unused_columns=False, + ) + trainer = Trainer( + model=model, + args=args, + train_dataset=_TextDataset(), + data_collator=DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False), + ) + trainer.train() + trainer.save_model(str(output_dir)) + tokenizer.save_pretrained(str(output_dir)) + print(f"Saved optional checkpoint to {output_dir}") + + +def main() -> int: + args = _parse_args() + if not (FEATURE_REPO / "feature_store.yaml").exists(): + print(f"Missing feature repo at {FEATURE_REPO}", file=sys.stderr) + return 1 + + if args.via_df: + ds = retrieve_via_df() + else: + ds = retrieve_via_ray_stream() + + sample = ds.take(3) + print(f"Sample training rows: {len(sample)}") + for i, row in enumerate(sample): + preview = str(row.get("sft_text", row))[:160].replace("\n", "\\n") + print(f" [{i}] {preview}...") + + if args.dry_run: + print("Done (trainer skipped).") + return 0 + + train_gpt2_optional( + ds, + max_steps=args.max_steps, + batch_size=args.batch_size, + max_length=args.max_length, + output_dir=args.output_dir, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index fd31828aa05..f77f882a9d2 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.64.0 +version: 0.65.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index 595a9bf9ead..cd8cc475031 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.64.0` +Current chart version is `0.65.0` ## Installation @@ -42,7 +42,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"quay.io/feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.64.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.65.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 03a2f1f0b1d..f2bc97d7a2b 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: quay.io/feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.64.0 + tag: 0.65.0 logLevel: "WARNING" # Set log level DEBUG, INFO, WARNING, ERROR, and CRITICAL (case-insensitive) diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 881c595ba97..dc49ff3fb2f 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.64.0 +version: 0.65.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index afa964c7656..2a288bb48aa 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.64.0` +Feature store for machine learning Current chart version is `0.65.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.64.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.64.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.65.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.65.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index 7b6e40c3da0..b20c1778a18 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.64.0 -appVersion: v0.64.0 +version: 0.65.0 +appVersion: v0.65.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 04714b14ed2..571449d9009 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.64.0](https://img.shields.io/badge/Version-0.64.0-informational?style=flat-square) ![AppVersion: v0.64.0](https://img.shields.io/badge/AppVersion-v0.64.0-informational?style=flat-square) +![Version: 0.65.0](https://img.shields.io/badge/Version-0.65.0-informational?style=flat-square) ![AppVersion: v0.65.0](https://img.shields.io/badge/AppVersion-v0.65.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"quay.io/feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.64.0"` | Image tag | +| image.tag | string | `"0.65.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index 0051f028279..3367dd665fa 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: quay.io/feastdev/feature-server-java # image.tag -- Image tag - tag: 0.64.0 + tag: 0.65.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index ca053c674bc..9dbc3f73cb4 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.64.0 -appVersion: v0.64.0 +version: 0.65.0 +appVersion: v0.65.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index f6e39356eee..ad1dd75cd65 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.64.0](https://img.shields.io/badge/Version-0.64.0-informational?style=flat-square) ![AppVersion: v0.64.0](https://img.shields.io/badge/AppVersion-v0.64.0-informational?style=flat-square) +![Version: 0.65.0](https://img.shields.io/badge/Version-0.65.0-informational?style=flat-square) ![AppVersion: v0.65.0](https://img.shields.io/badge/AppVersion-v0.65.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"quay.io/feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.64.0"` | Image tag | +| image.tag | string | `"0.65.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index af34cfa486d..266cd4b48aa 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: quay.io/feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.64.0 + tag: 0.65.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 8f610567ded..3f29ad7dfdc 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.64.0 + version: 0.65.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.64.0 + version: 0.65.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index 5b470437397..b70608a389e 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.64.0 +VERSION ?= 0.65.0 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") diff --git a/infra/feast-operator/api/feastversion/version.go b/infra/feast-operator/api/feastversion/version.go index 742ede75fd5..deabd34ac38 100644 --- a/infra/feast-operator/api/feastversion/version.go +++ b/infra/feast-operator/api/feastversion/version.go @@ -17,4 +17,4 @@ limitations under the License. package feastversion // Feast release version. Keep on line #20, this is critical to release CI -const FeastVersion = "0.64.0" +const FeastVersion = "0.65.0" diff --git a/infra/feast-operator/api/v1/featurestore_types.go b/infra/feast-operator/api/v1/featurestore_types.go index 3dd68311245..271ae1fe6b1 100644 --- a/infra/feast-operator/api/v1/featurestore_types.go +++ b/infra/feast-operator/api/v1/featurestore_types.go @@ -111,6 +111,35 @@ type OpenLineageConfig struct { // Keys must be valid Feast OpenLineageConfig YAML field names. // +optional ExtraConfig map[string]string `json:"extraConfig,omitempty"` + // Consumer configures the OpenLineage consumer (event receiver) that enables + // Feast to receive and display lineage from external producers (Airflow, Spark, dbt, etc.). + // +optional + Consumer *OpenLineageConsumerConfig `json:"consumer,omitempty"` +} + +// OpenLineageConsumerConfig configures the OpenLineage consumer (event receiver). +// When enabled, the Feast REST server exposes POST /api/v1/lineage to receive +// OpenLineage events from any producer, storing them for visualization in the Feast UI. +type OpenLineageConsumerConfig struct { + // Enable the OpenLineage consumer. + Enabled bool `json:"enabled"` + // StoreType is the storage backend for lineage events. Currently only "sql" is supported. + // +kubebuilder:default="sql" + // +kubebuilder:validation:Enum=sql + // +optional + StoreType *string `json:"storeType,omitempty"` + // Reference to a Secret containing the key "connection_string" for a separate + // lineage database. If omitted, the SQL registry database is reused. + // +optional + ConnectionStringSecretRef *corev1.LocalObjectReference `json:"connectionStringSecretRef,omitempty"` + // Reference to a Secret containing the key "api_key" that producers must + // provide in the X-API-Key header when sending events. + // +optional + ApiKeySecretRef *corev1.LocalObjectReference `json:"apiKeySecretRef,omitempty"` + // NamespaceMapping maps OpenLineage namespaces to Feast projects for + // RBAC-based filtering of lineage data in the UI. + // +optional + NamespaceMapping map[string]string `json:"namespaceMapping,omitempty"` } // FeatureStoreSpec defines the desired state of FeatureStore @@ -597,7 +626,7 @@ type OnlineStoreFilePersistence struct { // OnlineStoreDBStorePersistence configures the DB store persistence for the online store service type OnlineStoreDBStorePersistence struct { // Type of the persistence type you want to use. - // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb + // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb;aerospike;scylladb Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -623,6 +652,8 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{ "milvus", "hybrid", "mongodb", + "aerospike", + "scylladb", } // LocalRegistryConfig configures the registry service diff --git a/infra/feast-operator/api/v1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1/zz_generated.deepcopy.go index 9402f95e34a..2a6b6a69266 100644 --- a/infra/feast-operator/api/v1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1/zz_generated.deepcopy.go @@ -1043,6 +1043,11 @@ func (in *OpenLineageConfig) DeepCopyInto(out *OpenLineageConfig) { (*out)[key] = val } } + if in.Consumer != nil { + in, out := &in.Consumer, &out.Consumer + *out = new(OpenLineageConsumerConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenLineageConfig. @@ -1055,6 +1060,43 @@ func (in *OpenLineageConfig) DeepCopy() *OpenLineageConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenLineageConsumerConfig) DeepCopyInto(out *OpenLineageConsumerConfig) { + *out = *in + if in.StoreType != nil { + in, out := &in.StoreType, &out.StoreType + *out = new(string) + **out = **in + } + if in.ConnectionStringSecretRef != nil { + in, out := &in.ConnectionStringSecretRef, &out.ConnectionStringSecretRef + *out = new(corev1.LocalObjectReference) + **out = **in + } + if in.ApiKeySecretRef != nil { + in, out := &in.ApiKeySecretRef, &out.ApiKeySecretRef + *out = new(corev1.LocalObjectReference) + **out = **in + } + if in.NamespaceMapping != nil { + in, out := &in.NamespaceMapping, &out.NamespaceMapping + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenLineageConsumerConfig. +func (in *OpenLineageConsumerConfig) DeepCopy() *OpenLineageConsumerConfig { + if in == nil { + return nil + } + out := new(OpenLineageConsumerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OptionalCtrConfigs) DeepCopyInto(out *OptionalCtrConfigs) { *out = *in diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 87d13003805..11e201dd0d6 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -373,7 +373,7 @@ type OnlineStoreFilePersistence struct { // OnlineStoreDBStorePersistence configures the DB store persistence for the online store service type OnlineStoreDBStorePersistence struct { // Type of the persistence type you want to use. - // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb + // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb;aerospike;scylladb Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -399,6 +399,8 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{ "milvus", "hybrid", "mongodb", + "aerospike", + "scylladb", } // LocalRegistryConfig configures the registry service diff --git a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml index 82e1bc57b36..19af99046e9 100644 --- a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml +++ b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml @@ -147,10 +147,10 @@ metadata: } ] capabilities: Basic Install - createdAt: "2026-06-13T11:22:06Z" + createdAt: "2026-07-20T13:27:58Z" operators.operatorframework.io/builder: operator-sdk-v1.41.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 - name: feast-operator.v0.64.0 + name: feast-operator.v0.65.0 namespace: placeholder spec: apiservicedefinitions: {} @@ -180,11 +180,11 @@ spec: resources: - configmaps - persistentvolumeclaims - - serviceaccounts - services verbs: - create - delete + - deletecollection - get - list - update @@ -193,18 +193,45 @@ spec: - "" resources: - namespaces - - pods - secrets verbs: - get - list - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - deletecollection + - get + - list + - watch - apiGroups: - "" resources: - pods/exec verbs: - create + - apiGroups: + - "" + resources: + - pods/log + verbs: + - get + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -246,6 +273,14 @@ spec: - patch - update - watch + - apiGroups: + - config.openshift.io + resources: + - apiservers + verbs: + - get + - list + - watch - apiGroups: - feast.dev resources: @@ -321,6 +356,14 @@ spec: - list - update - watch + - apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + verbs: + - create + - delete + - get - apiGroups: - authentication.k8s.io resources: @@ -364,13 +407,13 @@ spec: - /manager env: - name: RELATED_IMAGE_FEATURE_SERVER - value: quay.io/feastdev/feature-server:0.64.0 + value: quay.io/feastdev/feature-server:0.65.0 - name: RELATED_IMAGE_CRON_JOB value: quay.io/openshift/origin-cli:4.17 - name: GOMEMLIMIT value: 230MiB - name: OIDC_ISSUER_URL - image: quay.io/feastdev/feast-operator:0.64.0 + image: quay.io/feastdev/feast-operator:0.65.0 livenessProbe: httpGet: path: /healthz @@ -460,8 +503,8 @@ spec: name: Feast Community url: https://lf-aidata.atlassian.net/wiki/spaces/FEAST/ relatedImages: - - image: quay.io/feastdev/feature-server:0.64.0 + - image: quay.io/feastdev/feature-server:0.65.0 name: feature-server - image: quay.io/openshift/origin-cli:4.17 name: cron-job - version: 0.64.0 + version: 0.65.0 diff --git a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml index d8791ae8a26..338dc75bd63 100644 --- a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml +++ b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml @@ -161,8 +161,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -211,6 +212,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -286,7 +316,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -471,7 +501,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -564,8 +593,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -614,6 +644,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -689,7 +748,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -793,6 +852,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -1761,8 +1873,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1811,6 +1924,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1887,7 +2030,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2256,6 +2399,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -2274,8 +2419,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2324,6 +2470,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2400,7 +2576,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2904,8 +3080,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2955,6 +3132,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3033,8 +3240,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -4209,8 +4415,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -4259,6 +4466,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -4334,7 +4570,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -5228,9 +5464,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -5643,6 +5878,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -6160,8 +6441,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6210,6 +6492,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6286,7 +6598,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6473,7 +6785,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -6568,8 +6879,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6618,6 +6930,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6694,7 +7036,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6799,6 +7141,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -7778,8 +8173,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -7829,6 +8225,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -7907,8 +8333,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8281,6 +8706,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -8300,8 +8727,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -8351,6 +8779,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -8429,8 +8887,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8945,8 +9402,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -8997,6 +9455,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -9077,7 +9565,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -10268,8 +10755,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -10310,12 +10798,42 @@ spec: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string - fieldPath: - description: Path of the field to select - in the specified API version. + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. type: string required: - - fieldPath + - key + - path + - volumeName type: object x-kubernetes-map-type: atomic resourceFieldRef: @@ -10394,7 +10912,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -11296,9 +11814,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -11715,6 +12232,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -12313,8 +12876,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12363,6 +12927,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12438,7 +13031,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -12623,7 +13216,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -12706,8 +13298,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12756,6 +13349,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12831,7 +13453,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -13083,8 +13705,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13133,6 +13756,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13209,7 +13862,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -13578,6 +14231,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -13596,8 +14251,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13646,6 +14302,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13722,7 +14408,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -14127,8 +14813,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14178,6 +14865,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14256,8 +14973,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -14701,8 +15417,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14751,6 +15468,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14826,7 +15572,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -15720,9 +16466,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -16135,6 +16880,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -16573,8 +17364,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -16623,6 +17415,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -16699,7 +17521,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -16886,7 +17708,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -16971,8 +17792,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17021,6 +17843,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17097,7 +17949,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -17353,8 +18205,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17404,6 +18257,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17482,8 +18365,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -17856,6 +18738,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -17875,8 +18759,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17926,6 +18811,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -18004,8 +18919,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -18419,8 +19333,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -18471,6 +19386,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -18551,7 +19496,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -19006,8 +19950,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -19056,6 +20001,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -19132,7 +20107,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -20034,9 +21009,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -20453,6 +21427,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project diff --git a/infra/feast-operator/cmd/main.go b/infra/feast-operator/cmd/main.go index 0e5565cce2b..0d833f1469b 100644 --- a/infra/feast-operator/cmd/main.go +++ b/infra/feast-operator/cmd/main.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "context" "crypto/tls" "flag" "os" @@ -25,12 +26,16 @@ import ( // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" + configv1 "github.com/openshift/api/config/v1" + tlspkg "github.com/openshift/controller-runtime-common/pkg/tls" appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -61,6 +66,7 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(configv1.Install(scheme)) utilruntime.Must(routev1.AddToScheme(scheme)) utilruntime.Must(feastdevv1alpha1.AddToScheme(scheme)) utilruntime.Must(feastdevv1.AddToScheme(scheme)) @@ -95,7 +101,6 @@ func main() { var enableLeaderElection bool var probeAddr string var secureMetrics bool - var enableHTTP2 bool var featureStoreMetrics bool var tlsOpts []func(*tls.Config) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ @@ -106,8 +111,6 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") - flag.BoolVar(&enableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") flag.BoolVar(&featureStoreMetrics, "feature-store-metrics", true, "Enable Prometheus gauges exposing online/offline store and registry configuration per FeatureStore. "+ "Disable with --feature-store-metrics=false.") @@ -119,21 +122,55 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - // if the enable-http2 flag is false (the default), http/2 should be disabled - // due to its vulnerabilities. More specifically, disabling http/2 will - // prevent from being vulnerable to the HTTP/2 Stream Cancellation and - // Rapid Reset CVEs. For more information see: - // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 - // - https://github.com/advisories/GHSA-4374-p667-p6c8 - disableHTTP2 := func(c *tls.Config) { - setupLog.Info("disabling http/2") - c.NextProtos = []string{"http/1.1"} + // Fetch cluster TLS profile from apiservers.config.openshift.io/cluster + cfg := ctrl.GetConfigOrDie() + bootstrapClient, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + setupLog.Error(err, "unable to create bootstrap client for TLS profile fetch") + os.Exit(1) } - if !enableHTTP2 { - tlsOpts = append(tlsOpts, disableHTTP2) + tlsProfileFetched := false + tlsProfile, err := tlspkg.FetchAPIServerTLSProfile(context.Background(), bootstrapClient) + if err != nil { + switch { + case apimeta.IsNoMatchError(err): + setupLog.Info("TLS profile not available, using hardened defaults (non-OpenShift cluster)") + case apierrors.IsNotFound(err): + setupLog.Info("APIServer resource not found, using hardened defaults") + default: + setupLog.Error(err, "unable to read APIServer TLS profile, refusing to start with unknown TLS posture") + os.Exit(1) + } + } else { + tlsProfileFetched = true + tlsConfigFn, unsupported := tlspkg.NewTLSConfigFromProfile(tlsProfile) + if len(unsupported) > 0 { + setupLog.Info("TLS profile contains ciphers unsupported by Go", "unsupported", unsupported) + } + tlsOpts = append(tlsOpts, tlsConfigFn) + } + + tlsAdherenceFetched := false + tlsAdherence, err := tlspkg.FetchAPIServerTLSAdherencePolicy(context.Background(), bootstrapClient) + if err != nil { + switch { + case apimeta.IsNoMatchError(err): + setupLog.Info("TLS adherence policy not available (non-OpenShift cluster)") + case apierrors.IsNotFound(err): + setupLog.Info("APIServer resource not found, skipping adherence policy") + default: + setupLog.Error(err, "unable to read APIServer TLS adherence policy, refusing to start") + os.Exit(1) + } + } else { + tlsAdherenceFetched = true } + tlsOpts = append(tlsOpts, func(c *tls.Config) { + c.NextProtos = []string{"h2", "http/1.1"} + }) + webhookServer := webhook.NewServer(webhook.Options{ TLSOpts: tlsOpts, }) @@ -162,7 +199,7 @@ func main() { metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization } - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, Metrics: metricsServerOptions, WebhookServer: webhookServer, @@ -230,6 +267,32 @@ func main() { } // +kubebuilder:scaffold:builder + // Register SecurityProfileWatcher to restart on TLS profile changes + ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler()) + defer cancel() + + if tlsProfileFetched { + watcher := &tlspkg.SecurityProfileWatcher{ + Client: mgr.GetClient(), + InitialTLSProfileSpec: tlsProfile, + OnProfileChange: func(_ context.Context, _, _ configv1.TLSProfileSpec) { + setupLog.Info("TLS profile changed, initiating shutdown to reload") + cancel() + }, + } + if tlsAdherenceFetched { + watcher.InitialTLSAdherencePolicy = tlsAdherence + watcher.OnAdherencePolicyChange = func(_ context.Context, _, _ configv1.TLSAdherencePolicy) { + setupLog.Info("TLS adherence policy changed, initiating shutdown to reload") + cancel() + } + } + if err := watcher.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to set up TLS profile watcher") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) @@ -240,7 +303,7 @@ func main() { } setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + if err := mgr.Start(ctx); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) } diff --git a/infra/feast-operator/config/component_metadata.yaml b/infra/feast-operator/config/component_metadata.yaml index 129d6029155..7ee38fdb165 100644 --- a/infra/feast-operator/config/component_metadata.yaml +++ b/infra/feast-operator/config/component_metadata.yaml @@ -1,5 +1,5 @@ # This file is required to configure Feast release information for ODH/RHOAI Operator releases: - name: Feast - version: 0.64.0 + version: 0.65.0 repoUrl: https://github.com/feast-dev/feast diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 0851b9abf96..b840d46a3b1 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -161,8 +161,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -211,6 +212,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -286,7 +316,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -471,7 +501,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -564,8 +593,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -614,6 +644,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -689,7 +748,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -793,6 +852,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -1761,8 +1873,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1811,6 +1924,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1887,7 +2030,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2256,6 +2399,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -2274,8 +2419,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2324,6 +2470,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2400,7 +2576,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2904,8 +3080,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2955,6 +3132,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3033,8 +3240,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -4209,8 +4415,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -4259,6 +4466,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -4334,7 +4570,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -5228,9 +5464,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -5643,6 +5878,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -6160,8 +6441,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6210,6 +6492,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6286,7 +6598,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6473,7 +6785,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -6568,8 +6879,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6618,6 +6930,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6694,7 +7036,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6799,6 +7141,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -7778,8 +8173,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -7829,6 +8225,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -7907,8 +8333,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8281,6 +8706,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -8300,8 +8727,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -8351,6 +8779,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -8429,8 +8887,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8945,8 +9402,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -8997,6 +9455,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -9077,7 +9565,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -10268,8 +10755,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -10310,12 +10798,42 @@ spec: description: Version of the schema the FieldPath is written in terms of, defaults to "v1". type: string - fieldPath: - description: Path of the field to select - in the specified API version. + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. type: string required: - - fieldPath + - key + - path + - volumeName type: object x-kubernetes-map-type: atomic resourceFieldRef: @@ -10394,7 +10912,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -11296,9 +11814,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -11715,6 +12232,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -12313,8 +12876,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12363,6 +12927,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12438,7 +13031,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -12623,7 +13216,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -12706,8 +13298,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12756,6 +13349,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12831,7 +13453,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -13083,8 +13705,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13133,6 +13756,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13209,7 +13862,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -13578,6 +14231,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -13596,8 +14251,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13646,6 +14302,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13722,7 +14408,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -14127,8 +14813,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14178,6 +14865,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14256,8 +14973,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -14701,8 +15417,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14751,6 +15468,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14826,7 +15572,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -15720,9 +16466,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -16135,6 +16880,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -16573,8 +17364,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -16623,6 +17415,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -16699,7 +17521,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -16886,7 +17708,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -16971,8 +17792,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17021,6 +17843,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17097,7 +17949,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -17353,8 +18205,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17404,6 +18257,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17482,8 +18365,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -17856,6 +18738,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -17875,8 +18759,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17926,6 +18811,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -18004,8 +18919,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -18419,8 +19333,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -18471,6 +19386,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -18551,7 +19496,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -19006,8 +19950,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -19056,6 +20001,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -19132,7 +20107,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -20034,9 +21009,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -20453,6 +21427,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project diff --git a/infra/feast-operator/config/default/related_image_fs_patch.yaml b/infra/feast-operator/config/default/related_image_fs_patch.yaml index 4e314795de2..5ab07fd91e1 100644 --- a/infra/feast-operator/config/default/related_image_fs_patch.yaml +++ b/infra/feast-operator/config/default/related_image_fs_patch.yaml @@ -9,6 +9,6 @@ spec: - name: manager env: - name: RELATED_IMAGE_FEATURE_SERVER - value: quay.io/feastdev/feature-server:0.64.0 + value: quay.io/feastdev/feature-server:0.65.0 - name: RELATED_IMAGE_CRON_JOB value: quay.io/openshift/origin-cli:4.17 diff --git a/infra/feast-operator/config/manager/kustomization.yaml b/infra/feast-operator/config/manager/kustomization.yaml index c713f0fe470..5f3ce6cadda 100644 --- a/infra/feast-operator/config/manager/kustomization.yaml +++ b/infra/feast-operator/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: quay.io/feastdev/feast-operator - newTag: 0.64.0 + newTag: 0.65.0 diff --git a/infra/feast-operator/config/overlays/odh/params.env b/infra/feast-operator/config/overlays/odh/params.env index 49bbac59c71..b0d55d6bd70 100644 --- a/infra/feast-operator/config/overlays/odh/params.env +++ b/infra/feast-operator/config/overlays/odh/params.env @@ -1,5 +1,5 @@ -RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.64.0 -RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.64.0 +RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.65.0 +RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.65.0 RELATED_IMAGE_CRON_JOB=quay.io/openshift/origin-cli:4.17 # Set at deploy time by the Open Data Hub operator from GatewayConfig (external OIDC). OIDC_ISSUER_URL= diff --git a/infra/feast-operator/config/overlays/rhoai/params.env b/infra/feast-operator/config/overlays/rhoai/params.env index 92e02fb51b3..dabacfd458c 100644 --- a/infra/feast-operator/config/overlays/rhoai/params.env +++ b/infra/feast-operator/config/overlays/rhoai/params.env @@ -1,5 +1,5 @@ -RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.64.0 -RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.64.0 +RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.65.0 +RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.65.0 RELATED_IMAGE_CRON_JOB=registry.redhat.io/openshift4/ose-cli@sha256:bc35a9fc663baf0d6493cc57e89e77a240a36c43cf38fb78d8e61d3b87cf5cc5 # Set at deploy time by the Open Data Hub operator from GatewayConfig (external OIDC). OIDC_ISSUER_URL= \ No newline at end of file diff --git a/infra/feast-operator/config/rbac/role.yaml b/infra/feast-operator/config/rbac/role.yaml index 0c1bd7be84b..a79dca283ed 100644 --- a/infra/feast-operator/config/rbac/role.yaml +++ b/infra/feast-operator/config/rbac/role.yaml @@ -9,11 +9,11 @@ rules: resources: - configmaps - persistentvolumeclaims - - serviceaccounts - services verbs: - create - delete + - deletecollection - get - list - update @@ -22,18 +22,45 @@ rules: - "" resources: - namespaces - - pods - secrets verbs: - get - list - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - deletecollection + - get + - list + - watch - apiGroups: - "" resources: - pods/exec verbs: - create +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -75,6 +102,14 @@ rules: - patch - update - watch +- apiGroups: + - config.openshift.io + resources: + - apiservers + verbs: + - get + - list + - watch - apiGroups: - feast.dev resources: @@ -150,3 +185,11 @@ rules: - list - update - watch +- apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + verbs: + - create + - delete + - get diff --git a/infra/feast-operator/config/samples/v1_featurestore_openlineage_consumer.yaml b/infra/feast-operator/config/samples/v1_featurestore_openlineage_consumer.yaml new file mode 100644 index 00000000000..9e242896135 --- /dev/null +++ b/infra/feast-operator/config/samples/v1_featurestore_openlineage_consumer.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: Secret +metadata: + name: openlineage-producer-secret + namespace: feast +stringData: + api_key: "your-marquez-api-key" #pragma: allowlist secret +--- +apiVersion: v1 +kind: Secret +metadata: + name: openlineage-consumer-secret + namespace: feast +stringData: + api_key: "consumer-api-key-for-producers" #pragma: allowlist secret +--- +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: sample-openlineage-consumer + namespace: feast +spec: + feastProject: my_project + services: + registry: + local: + persistence: + store: + type: sql + secretRef: + name: registry-db-secret + openlineage: + enabled: true + transportType: http + transportUrl: "http://localhost:8080/api" + transportEndpoint: "v1/lineage" + apiKeySecretRef: + name: openlineage-producer-secret + extraConfig: + namespace: "my_project" + producer: "feast-operator" + emit_on_apply: "true" + emit_on_materialize: "true" + # consumer enables Feast as an OpenLineage event receiver. + # External producers (Airflow, Spark, dbt) can POST events to + # the Feast REST server at POST /api/v1/lineage. + # The Feast UI then displays lineage from all producers in + # Registry, OpenLineage, and Merged views. + consumer: + enabled: true + storeType: sql + # Optional: use a separate database for lineage storage. + # If omitted, the SQL registry database is reused. + # connectionStringSecretRef: + # name: lineage-db-secret + apiKeySecretRef: + name: openlineage-consumer-secret + # namespaceMapping maps OL namespaces to Feast projects + # for RBAC-scoped visibility in the UI. + namespaceMapping: + airflow_production: my_project + spark_etl: my_project + dbt_analytics: my_project diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index a6453c3e6a3..854d625bff5 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -169,8 +169,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -219,6 +220,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -294,7 +324,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -479,7 +509,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -572,8 +601,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -622,6 +652,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -697,7 +756,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -801,6 +860,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -1769,8 +1881,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1819,6 +1932,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1895,7 +2038,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2264,6 +2407,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -2282,8 +2427,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2332,6 +2478,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2408,7 +2584,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2912,8 +3088,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2963,6 +3140,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3041,8 +3248,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -4217,8 +4423,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -4267,6 +4474,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -4342,7 +4578,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -5236,9 +5472,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -5651,6 +5886,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -6168,8 +6449,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6218,6 +6500,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6294,7 +6606,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6481,7 +6793,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -6576,8 +6887,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6626,6 +6938,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6702,7 +7044,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6807,6 +7149,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -7786,8 +8181,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -7837,6 +8233,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -7915,8 +8341,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8289,6 +8714,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -8308,8 +8735,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -8359,6 +8787,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -8437,8 +8895,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8953,8 +9410,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -9005,20 +9463,50 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic - resourceFieldRef: + fileKeyRef: description: |- - Selects a resource of the container: only resources limits and requests - (limits.cpu, limits.memory, limits. + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. properties: - containerName: - description: 'Container name: required - for volumes, optional for env - vars' + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. type: string - divisor: - anyOf: - - type: integer - - type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string description: Specifies the output format of the exposed resources, defaults to "1" @@ -9085,7 +9573,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -10276,8 +10763,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -10326,6 +10814,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -10402,7 +10920,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -11304,9 +11822,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -11723,6 +12240,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -12321,8 +12884,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12371,6 +12935,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12446,7 +13039,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -12631,7 +13224,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -12714,8 +13306,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12764,6 +13357,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12839,7 +13461,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -13091,8 +13713,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13141,6 +13764,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13217,7 +13870,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -13586,6 +14239,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -13604,8 +14259,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13654,6 +14310,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13730,7 +14416,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -14135,8 +14821,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14186,6 +14873,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14264,8 +14981,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -14709,8 +15425,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14759,6 +15476,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14834,7 +15580,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -15728,9 +16474,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -16143,6 +16888,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -16581,8 +17372,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -16631,6 +17423,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -16707,7 +17529,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -16894,7 +17716,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -16979,8 +17800,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17029,6 +17851,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17105,7 +17957,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -17361,8 +18213,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17412,6 +18265,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17490,8 +18373,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -17864,6 +18746,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -17883,8 +18767,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17934,6 +18819,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -18012,8 +18927,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -18427,8 +19341,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -18479,6 +19394,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -18559,7 +19504,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -19014,8 +19958,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -19064,6 +20009,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -19140,7 +20115,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -20042,9 +21017,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -20461,6 +21435,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -21017,11 +22037,11 @@ rules: resources: - configmaps - persistentvolumeclaims - - serviceaccounts - services verbs: - create - delete + - deletecollection - get - list - update @@ -21030,18 +22050,45 @@ rules: - "" resources: - namespaces - - pods - secrets verbs: - get - list - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - deletecollection + - get + - list + - watch - apiGroups: - "" resources: - pods/exec verbs: - create +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -21083,6 +22130,14 @@ rules: - patch - update - watch +- apiGroups: + - config.openshift.io + resources: + - apiservers + verbs: + - get + - list + - watch - apiGroups: - feast.dev resources: @@ -21158,6 +22213,14 @@ rules: - list - update - watch +- apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + verbs: + - create + - delete + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -21293,14 +22356,14 @@ spec: - /manager env: - name: RELATED_IMAGE_FEATURE_SERVER - value: quay.io/feastdev/feature-server:0.64.0 + value: quay.io/feastdev/feature-server:0.65.0 - name: RELATED_IMAGE_CRON_JOB value: quay.io/openshift/origin-cli:4.17 - name: GOMEMLIMIT value: 230MiB - name: OIDC_ISSUER_URL value: "" - image: quay.io/feastdev/feast-operator:0.64.0 + image: quay.io/feastdev/feast-operator:0.65.0 livenessProbe: httpGet: path: /healthz diff --git a/infra/feast-operator/dist/operator-e2e-tests b/infra/feast-operator/dist/operator-e2e-tests index cb65e549ff8..0d5ff42aef8 100755 Binary files a/infra/feast-operator/dist/operator-e2e-tests and b/infra/feast-operator/dist/operator-e2e-tests differ diff --git a/infra/feast-operator/docs/api/markdown/ref.md b/infra/feast-operator/docs/api/markdown/ref.md index dd7acf55fb3..0a7782feb2a 100644 --- a/infra/feast-operator/docs/api/markdown/ref.md +++ b/infra/feast-operator/docs/api/markdown/ref.md @@ -726,6 +726,31 @@ emit_on_materialize) and transport-specific options (e.g. kafka bootstrap_servers, topic; file path). Boolean values ("true"/"false") and integer values are automatically coerced to their native YAML types. Keys must be valid Feast OpenLineageConfig YAML field names. | +| `consumer` _[OpenLineageConsumerConfig](#openlineageconsumerconfig)_ | Consumer configures the OpenLineage consumer (event receiver) that enables +Feast to receive and display lineage from external producers (Airflow, Spark, dbt, etc.). | + + +#### OpenLineageConsumerConfig + + + +OpenLineageConsumerConfig configures the OpenLineage consumer (event receiver). +When enabled, the Feast REST server exposes POST /api/v1/lineage to receive +OpenLineage events from any producer, storing them for visualization in the Feast UI. + +_Appears in:_ +- [OpenLineageConfig](#openlineageconfig) + +| Field | Description | +| --- | --- | +| `enabled` _boolean_ | Enable the OpenLineage consumer. | +| `storeType` _string_ | StoreType is the storage backend for lineage events. Currently only "sql" is supported. | +| `connectionStringSecretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core)_ | Reference to a Secret containing the key "connection_string" for a separate +lineage database. If omitted, the SQL registry database is reused. | +| `apiKeySecretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core)_ | Reference to a Secret containing the key "api_key" that producers must +provide in the X-API-Key header when sending events. | +| `namespaceMapping` _object (keys:string, values:string)_ | NamespaceMapping maps OpenLineage namespaces to Feast projects for +RBAC-based filtering of lineage data in the UI. | #### OptionalCtrConfigs diff --git a/infra/feast-operator/go.mod b/infra/feast-operator/go.mod index 021e1a1b020..ab19a1de20a 100644 --- a/infra/feast-operator/go.mod +++ b/infra/feast-operator/go.mod @@ -3,25 +3,28 @@ module github.com/feast-dev/feast/infra/feast-operator go 1.25.0 require ( - github.com/onsi/ginkgo/v2 v2.22.2 - github.com/onsi/gomega v1.36.2 - github.com/openshift/api v0.0.0-20240912201240-0a8800162826 // release-4.17 + github.com/onsi/ginkgo/v2 v2.28.1 + github.com/onsi/gomega v1.39.1 + github.com/openshift/api v0.0.0-20260317165824-54a3998d81eb // release-4.17 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.33.1 - k8s.io/apimachinery v0.33.1 - k8s.io/client-go v0.33.1 - sigs.k8s.io/controller-runtime v0.21.0 + k8s.io/api v0.35.2 + k8s.io/apimachinery v0.35.2 + k8s.io/client-go v0.35.2 + sigs.k8s.io/controller-runtime v0.23.3 ) require ( + github.com/openshift/controller-runtime-common v0.0.0-20260428152732-64ee174f5e2e github.com/prometheus-operator/prometheus-operator/pkg/client v0.83.0 - github.com/prometheus/client_golang v1.22.0 - github.com/prometheus/client_model v0.6.1 - k8s.io/utils v0.0.0-20250502105355-0f33e8f1c979 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 + k8s.io/apiextensions-apiserver v0.35.1 + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 ) require ( cel.dev/expr v0.25.1 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect @@ -31,8 +34,8 @@ require ( github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/fxamacker/cbor/v2 v2.8.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect @@ -40,41 +43,44 @@ require ( github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/google/btree v1.1.3 // indirect - github.com/google/cel-go v0.23.2 // indirect - github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/cel-go v0.26.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect + github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/openshift/library-go v0.0.0-20260213153706-03f1709971c5 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.83.0 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/spf13/cobra v1.10.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect go.opentelemetry.io/otel v1.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect - go.opentelemetry.io/proto/otlp v1.4.0 // indirect + go.opentelemetry.io/proto/otlp v1.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/mod v0.35.0 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.20.0 // indirect @@ -88,16 +94,15 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/grpc v1.79.3 // indirect google.golang.org/protobuf v1.36.10 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - k8s.io/apiextensions-apiserver v0.33.1 // indirect - k8s.io/apiserver v0.33.1 // indirect - k8s.io/component-base v0.33.1 // indirect + k8s.io/apiserver v0.35.1 // indirect + k8s.io/component-base v0.35.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/infra/feast-operator/go.sum b/infra/feast-operator/go.sum index 6c80ee96e61..b642252f7d3 100644 --- a/infra/feast-operator/go.sum +++ b/infra/feast-operator/go.sum @@ -1,5 +1,7 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -10,7 +12,7 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -23,10 +25,16 @@ github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjT github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fxamacker/cbor/v2 v2.8.0 h1:fFtUGXUzXPHTIUdne5+zzMPTfffl3RD5qYnkY40vtxU= -github.com/fxamacker/cbor/v2 v2.8.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -42,36 +50,35 @@ github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZ github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4= -github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo= -github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= -github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= +github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -82,42 +89,53 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= -github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= -github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= -github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= -github.com/openshift/api v0.0.0-20240912201240-0a8800162826 h1:A8D9SN/hJUwAbdO0rPCVTqmuBOctdgurr53gK701SYo= -github.com/openshift/api v0.0.0-20240912201240-0a8800162826/go.mod h1:OOh6Qopf21pSzqNVCB5gomomBXb8o5sGKZxG2KNpaXM= +github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= +github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= +github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= +github.com/openshift/api v0.0.0-20260317165824-54a3998d81eb h1:iwBR3mzmyE3EMFx7R3CQ9lOccTS0dNht8TW82aGITg0= +github.com/openshift/api v0.0.0-20260317165824-54a3998d81eb/go.mod h1:pyVjK0nZ4sRs4fuQVQ4rubsJdahI1PB94LnQ8sGdvxo= +github.com/openshift/controller-runtime-common v0.0.0-20260428152732-64ee174f5e2e h1:k89oIo2EjX0PRSdi1kesktCyWp50SC9WwKurvupvRGs= +github.com/openshift/controller-runtime-common v0.0.0-20260428152732-64ee174f5e2e/go.mod h1:XGabTMnNbz0M5Oa7IbscZp/jmcc7aHobvOCUWwkzKvM= +github.com/openshift/library-go v0.0.0-20260213153706-03f1709971c5 h1:9Pe6iVOMjt9CdA/vaKBNUSoEIjIe1po5Ha3ABRYXLJI= +github.com/openshift/library-go v0.0.0-20260213153706-03f1709971c5/go.mod h1:K3FoNLgNBFYbFuG+Kr8usAnQxj1w84XogyUp2M8rK8k= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.83.0 h1:j9Ce3W6X6Tzi0QnSap+YzGwpqJLJGP/7xV6P9f86jjM= github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.83.0/go.mod h1:sSxwdmprUfmRfTknPc4KIjUd2ZIc/kirw4UdXNhOauM= github.com/prometheus-operator/prometheus-operator/pkg/client v0.83.0 h1:odshP0+Jo6iUNGpK8MOFA6p5Yj0QOV4yLgiqFU5MVuI= github.com/prometheus-operator/prometheus-operator/pkg/client v0.83.0/go.mod h1:6Ndhfow0psSp7dV1qp9zK5h++CDKz4eSFWPbrHd5Iic= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.10.0 h1:a5/WeUlSDCvV5a45ljW2ZFtV0bTDpkfSAj3uqB6Sc+0= +github.com/spf13/cobra v1.10.0/go.mod h1:9dhySC7dnTtEiqzmqfkLj47BslqLCUPMXjG2lj/NgoE= +github.com/spf13/pflag v1.0.8/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -131,20 +149,26 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= @@ -153,57 +177,38 @@ go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRk go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= -go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= -go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= +go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= @@ -219,41 +224,40 @@ google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.33.1 h1:tA6Cf3bHnLIrUK4IqEgb2v++/GYUtqiu9sRVk3iBXyw= -k8s.io/api v0.33.1/go.mod h1:87esjTn9DRSRTD4fWMXamiXxJhpOIREjWOSjsW1kEHw= -k8s.io/apiextensions-apiserver v0.33.1 h1:N7ccbSlRN6I2QBcXevB73PixX2dQNIW0ZRuguEE91zI= -k8s.io/apiextensions-apiserver v0.33.1/go.mod h1:uNQ52z1A1Gu75QSa+pFK5bcXc4hq7lpOXbweZgi4dqA= -k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4= -k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= -k8s.io/apiserver v0.33.1 h1:yLgLUPDVC6tHbNcw5uE9mo1T6ELhJj7B0geifra3Qdo= -k8s.io/apiserver v0.33.1/go.mod h1:VMbE4ArWYLO01omz+k8hFjAdYfc3GVAYPrhP2tTKccs= -k8s.io/client-go v0.33.1 h1:ZZV/Ks2g92cyxWkRRnfUDsnhNn28eFpt26aGc8KbXF4= -k8s.io/client-go v0.33.1/go.mod h1:JAsUrl1ArO7uRVFWfcj6kOomSlCv+JpvIsp6usAGefA= -k8s.io/component-base v0.33.1 h1:EoJ0xA+wr77T+G8p6T3l4efT2oNwbqBVKR71E0tBIaI= -k8s.io/component-base v0.33.1/go.mod h1:guT/w/6piyPfTgq7gfvgetyXMIh10zuXA6cRRm3rDuY= +k8s.io/api v0.35.2 h1:tW7mWc2RpxW7HS4CoRXhtYHSzme1PN1UjGHJ1bdrtdw= +k8s.io/api v0.35.2/go.mod h1:7AJfqGoAZcwSFhOjcGM7WV05QxMMgUaChNfLTXDRE60= +k8s.io/apiextensions-apiserver v0.35.1 h1:p5vvALkknlOcAqARwjS20kJffgzHqwyQRM8vHLwgU7w= +k8s.io/apiextensions-apiserver v0.35.1/go.mod h1:2CN4fe1GZ3HMe4wBr25qXyJnJyZaquy4nNlNmb3R7AQ= +k8s.io/apimachinery v0.35.2 h1:NqsM/mmZA7sHW02JZ9RTtk3wInRgbVxL8MPfzSANAK8= +k8s.io/apimachinery v0.35.2/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/apiserver v0.35.1 h1:potxdhhTL4i6AYAa2QCwtlhtB1eCdWQFvJV6fXgJzxs= +k8s.io/apiserver v0.35.1/go.mod h1:BiL6Dd3A2I/0lBnteXfWmCFobHM39vt5+hJQd7Lbpi4= +k8s.io/client-go v0.35.2 h1:YUfPefdGJA4aljDdayAXkc98DnPkIetMl4PrKX97W9o= +k8s.io/client-go v0.35.2/go.mod h1:4QqEwh4oQpeK8AaefZ0jwTFJw/9kIjdQi0jpKeYvz7g= +k8s.io/component-base v0.35.1 h1:XgvpRf4srp037QWfGBLFsYMUQJkE5yMa94UsJU7pmcE= +k8s.io/component-base v0.35.1/go.mod h1:HI/6jXlwkiOL5zL9bqA3en1Ygv60F03oEpnuU1G56Bs= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= -k8s.io/utils v0.0.0-20250502105355-0f33e8f1c979 h1:jgJW5IePPXLGB8e/1wvd0Ich9QE97RvvF3a8J3fP/Lg= -k8s.io/utils v0.0.0-20250502105355-0f33e8f1c979/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= -sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/controller-runtime v0.23.3 h1:VjB/vhoPoA9l1kEKZHBMnQF33tdCLQKJtydy4iqwZ80= +sigs.k8s.io/controller-runtime v0.23.3/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v4 v4.7.0 h1:qPeWmscJcXP0snki5IYF79Z8xrl8ETFxgMd7wez1XkI= -sigs.k8s.io/structured-merge-diff/v4 v4.7.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 h1:2WOzJpHUBVrrkDjU4KBT8n5LDcj824eX0I5UKcgeRUs= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/infra/feast-operator/internal/controller/featurestore_controller.go b/infra/feast-operator/internal/controller/featurestore_controller.go index ae877447ddb..b94808c0df5 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller.go +++ b/infra/feast-operator/internal/controller/featurestore_controller.go @@ -60,14 +60,19 @@ type FeatureStoreReconciler struct { Metrics *feastmetrics.FeatureStoreMetrics } +// +kubebuilder:rbac:groups=config.openshift.io,resources=apiservers,verbs=get;list;watch // +kubebuilder:rbac:groups=feast.dev,resources=featurestores,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=feast.dev,resources=featurestores/status,verbs=get;update;patch // +kubebuilder:rbac:groups=feast.dev,resources=featurestores/finalizers,verbs=update // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;create;update;watch;delete -// +kubebuilder:rbac:groups=core,resources=services;configmaps;persistentvolumeclaims;serviceaccounts,verbs=get;list;create;update;watch;delete +// +kubebuilder:rbac:groups=core,resources=services;configmaps;persistentvolumeclaims,verbs=get;list;create;update;watch;delete;deletecollection +// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;create;update;watch;delete // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings;clusterroles;clusterrolebindings;subjectaccessreviews,verbs=get;list;create;update;watch;delete -// +kubebuilder:rbac:groups=core,resources=secrets;pods;namespaces,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=secrets;namespaces,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;delete;deletecollection // +kubebuilder:rbac:groups=core,resources=pods/exec,verbs=create +// +kubebuilder:rbac:groups=core,resources=pods/log,verbs=get +// +kubebuilder:rbac:groups=sparkoperator.k8s.io,resources=sparkapplications,verbs=create;get;delete // +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;create;update;watch;delete // +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete diff --git a/infra/feast-operator/internal/controller/services/batch_engine_rbac.go b/infra/feast-operator/internal/controller/services/batch_engine_rbac.go new file mode 100644 index 00000000000..97afcd3b48d --- /dev/null +++ b/infra/feast-operator/internal/controller/services/batch_engine_rbac.go @@ -0,0 +1,285 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "embed" + "fmt" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" +) + +const ( + BatchEngineFeastType FeastServiceType = "batch-engine" + BatchDriverFeastType FeastServiceType = "batch-driver" +) + +//go:embed rbac_templates/*.yaml +var batchEngineRBACTemplates embed.FS + +// BatchEngineRBACTemplate declares RBAC requirements for a batch compute engine. +type BatchEngineRBACTemplate struct { + EngineType string `json:"engine_type" yaml:"engine_type"` + Server *RBACRoleSpec `json:"server,omitempty" yaml:"server,omitempty"` + Driver *DriverRBACSpec `json:"driver,omitempty" yaml:"driver,omitempty"` +} + +// RBACRoleSpec defines policy rules for a Role. +type RBACRoleSpec struct { + Rules []rbacv1.PolicyRule `json:"rules" yaml:"rules"` +} + +// DriverRBACSpec defines policy rules and optional SA creation for a driver Role. +type DriverRBACSpec struct { + CreateServiceAccount bool `json:"create_service_account" yaml:"create_service_account"` + Rules []rbacv1.PolicyRule `json:"rules" yaml:"rules"` +} + +func loadBatchEngineTemplate(engineType string) (*BatchEngineRBACTemplate, error) { + data, err := batchEngineRBACTemplates.ReadFile( + "rbac_templates/" + engineType + ".yaml", + ) + if err != nil { + return nil, nil + } + var tmpl BatchEngineRBACTemplate + if err := yaml.Unmarshal(data, &tmpl); err != nil { + return nil, fmt.Errorf("failed to parse RBAC template for engine %q: %w", engineType, err) + } + return &tmpl, nil +} + +func (feast *FeastServices) reconcileBatchEngineRBAC() error { + config, ok := feast.getBatchEngineConfig() + if !ok { + return feast.deleteBatchEngineRBAC() + } + + engineType, _ := config["type"].(string) + if engineType == "" { + return feast.deleteBatchEngineRBAC() + } + + tmpl, err := loadBatchEngineTemplate(engineType) + if err != nil { + return err + } + if tmpl == nil { + return feast.deleteBatchEngineRBAC() + } + + if tmpl.Server != nil { + if err := feast.ensureBatchEngineRole(BatchEngineFeastType, tmpl.Server.Rules); err != nil { + return err + } + if err := feast.ensureBatchEngineRoleBinding(BatchEngineFeastType, feast.initFeastSA().Name); err != nil { + return err + } + } + + if tmpl.Driver != nil { + driverSAName := resolveBatchDriverSAName(feast.Handler.FeatureStore, config) + if tmpl.Driver.CreateServiceAccount { + if err := feast.ensureBatchDriverServiceAccount(driverSAName); err != nil { + return err + } + } + if err := feast.ensureBatchEngineRole(BatchDriverFeastType, tmpl.Driver.Rules); err != nil { + return err + } + if err := feast.ensureBatchEngineRoleBinding(BatchDriverFeastType, driverSAName); err != nil { + return err + } + } + + return nil +} + +// getBatchEngineConfig returns the parsed batch-engine ConfigMap data. +// ok=false means no batch engine is configured or the ConfigMap is unreadable. +func (feast *FeastServices) getBatchEngineConfig() (map[string]interface{}, bool) { + appliedSpec := feast.Handler.FeatureStore.Status.Applied + if appliedSpec.BatchEngine == nil || appliedSpec.BatchEngine.ConfigMapRef == nil { + return nil, false + } + + configMapKey := appliedSpec.BatchEngine.ConfigMapKey + if configMapKey == "" { + configMapKey = "config" + } + + cm, err := feast.getConfigMap(appliedSpec.BatchEngine.ConfigMapRef.Name) + if err != nil { + return nil, false + } + + data, found := cm.Data[configMapKey] + if !found { + return nil, false + } + + var config map[string]interface{} + if err := yaml.Unmarshal([]byte(data), &config); err != nil { + return nil, false + } + return config, true +} + +// resolveBatchDriverSAName returns the ServiceAccount name for the Spark driver. +// If batch engine config sets a non-empty service_account, that value wins. +// Otherwise defaults to feast--batch-driver (same name used for RBAC). +func resolveBatchDriverSAName(featureStore *feastdevv1.FeatureStore, config map[string]interface{}) string { + if sa, ok := config["service_account"].(string); ok && sa != "" { + return sa + } + return GetFeastServiceName(featureStore, BatchDriverFeastType) +} + +func (feast *FeastServices) ensureBatchEngineRole(feastType FeastServiceType, rules []rbacv1.PolicyRule) error { + logger := log.FromContext(feast.Handler.Context) + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: feast.GetFeastServiceName(feastType), + Namespace: feast.Handler.FeatureStore.Namespace, + }, + } + role.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("Role")) + + op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, role, func() error { + role.Labels = feast.getFeastTypeLabels(feastType) + role.Rules = rules + return controllerutil.SetControllerReference(feast.Handler.FeatureStore, role, feast.Handler.Scheme) + }) + if err != nil { + return err + } + if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "Role", role.Name, "operation", op) + } + return nil +} + +func (feast *FeastServices) ensureBatchEngineRoleBinding(feastType FeastServiceType, saName string) error { + logger := log.FromContext(feast.Handler.Context) + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: feast.GetFeastServiceName(feastType), + Namespace: feast.Handler.FeatureStore.Namespace, + }, + } + roleBinding.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("RoleBinding")) + + op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, roleBinding, func() error { + roleBinding.Labels = feast.getFeastTypeLabels(feastType) + roleBinding.Subjects = []rbacv1.Subject{{ + Kind: rbacv1.ServiceAccountKind, + Name: saName, + Namespace: feast.Handler.FeatureStore.Namespace, + }} + roleBinding.RoleRef = rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "Role", + Name: feast.GetFeastServiceName(feastType), + } + return controllerutil.SetControllerReference(feast.Handler.FeatureStore, roleBinding, feast.Handler.Scheme) + }) + if err != nil { + return err + } + if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "RoleBinding", roleBinding.Name, "operation", op) + } + return nil +} + +func (feast *FeastServices) ensureBatchDriverServiceAccount(saName string) error { + logger := log.FromContext(feast.Handler.Context) + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + Namespace: feast.Handler.FeatureStore.Namespace, + }, + } + sa.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ServiceAccount")) + + op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, sa, func() error { + if sa.Labels == nil { + sa.Labels = map[string]string{} + } + for k, v := range feast.getFeastTypeLabels(BatchDriverFeastType) { + sa.Labels[k] = v + } + return controllerutil.SetControllerReference(feast.Handler.FeatureStore, sa, feast.Handler.Scheme) + }) + if err != nil { + return err + } + if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "ServiceAccount", sa.Name, "operation", op) + } + return nil +} + +func (feast *FeastServices) deleteBatchEngineRBAC() error { + serverRoleName := feast.GetFeastServiceName(BatchEngineFeastType) + driverRoleName := feast.GetFeastServiceName(BatchDriverFeastType) + ns := feast.Handler.FeatureStore.Namespace + + serverRoleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: serverRoleName, Namespace: ns}, + } + serverRoleBinding.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("RoleBinding")) + if err := feast.Handler.DeleteOwnedFeastObj(serverRoleBinding); err != nil { + return err + } + + serverRole := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: serverRoleName, Namespace: ns}, + } + serverRole.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("Role")) + if err := feast.Handler.DeleteOwnedFeastObj(serverRole); err != nil { + return err + } + + driverRoleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: driverRoleName, Namespace: ns}, + } + driverRoleBinding.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("RoleBinding")) + if err := feast.Handler.DeleteOwnedFeastObj(driverRoleBinding); err != nil { + return err + } + + driverRole := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: driverRoleName, Namespace: ns}, + } + driverRole.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("Role")) + if err := feast.Handler.DeleteOwnedFeastObj(driverRole); err != nil { + return err + } + + driverSA := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: driverRoleName, Namespace: ns}, + } + driverSA.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ServiceAccount")) + return feast.Handler.DeleteOwnedFeastObj(driverSA) +} diff --git a/infra/feast-operator/internal/controller/services/batch_engine_rbac_test.go b/infra/feast-operator/internal/controller/services/batch_engine_rbac_test.go new file mode 100644 index 00000000000..9d792c6f923 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/batch_engine_rbac_test.go @@ -0,0 +1,195 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" +) + +var _ = Describe("Batch Engine RBAC", func() { + + Describe("loadBatchEngineTemplate", func() { + It("should load spark_application template", func() { + tmpl, err := loadBatchEngineTemplate("spark_application") + Expect(err).NotTo(HaveOccurred()) + Expect(tmpl).NotTo(BeNil()) + Expect(tmpl.EngineType).To(Equal("spark_application")) + Expect(tmpl.Server).NotTo(BeNil()) + Expect(tmpl.Server.Rules).NotTo(BeEmpty()) + Expect(tmpl.Driver).NotTo(BeNil()) + Expect(tmpl.Driver.CreateServiceAccount).To(BeTrue()) + Expect(tmpl.Driver.Rules).NotTo(BeEmpty()) + }) + + It("should return nil for unknown engine type", func() { + tmpl, err := loadBatchEngineTemplate("nonexistent_engine") + Expect(err).NotTo(HaveOccurred()) + Expect(tmpl).To(BeNil()) + }) + + It("should contain correct server rules for spark_application", func() { + tmpl, err := loadBatchEngineTemplate("spark_application") + Expect(err).NotTo(HaveOccurred()) + + serverRules := tmpl.Server.Rules + Expect(serverRules).To(HaveLen(4)) + + hasConfigMapRule := false + hasSparkAppRule := false + hasPodListRule := false + hasPodLogRule := false + + for _, rule := range serverRules { + if containsResource(rule, "configmaps") && containsVerb(rule, "create") && containsVerb(rule, "delete") { + hasConfigMapRule = true + } + if containsResource(rule, "sparkapplications") && containsVerb(rule, "create") && containsVerb(rule, "get") && containsVerb(rule, "delete") { + hasSparkAppRule = true + } + if containsResource(rule, "pods") && containsVerb(rule, "list") { + hasPodListRule = true + } + if containsResource(rule, "pods/log") && containsVerb(rule, "get") { + hasPodLogRule = true + } + } + + Expect(hasConfigMapRule).To(BeTrue(), "should have configmaps create/delete rule") + Expect(hasSparkAppRule).To(BeTrue(), "should have sparkapplications create/get/delete rule") + Expect(hasPodListRule).To(BeTrue(), "should have pods list rule") + Expect(hasPodLogRule).To(BeTrue(), "should have pods/log get rule") + }) + + It("should contain correct driver rules for spark_application", func() { + tmpl, err := loadBatchEngineTemplate("spark_application") + Expect(err).NotTo(HaveOccurred()) + + driverRules := tmpl.Driver.Rules + Expect(driverRules).To(HaveLen(2)) + + hasPodRule := false + hasResourceRule := false + for _, rule := range driverRules { + if containsResource(rule, "pods") && + containsVerb(rule, "create") && + containsVerb(rule, "deletecollection") { + hasPodRule = true + } + if containsResource(rule, "services") && + containsResource(rule, "configmaps") && + containsResource(rule, "persistentvolumeclaims") && + containsVerb(rule, "deletecollection") { + hasResourceRule = true + } + } + Expect(hasPodRule).To(BeTrue(), "should have pods CRUD + deletecollection rule") + Expect(hasResourceRule).To(BeTrue(), "should have services/configmaps/PVCs CRUD + deletecollection rule") + }) + }) + + Describe("BatchEngineRBACTemplate YAML parsing", func() { + It("should correctly unmarshal a template", func() { + yamlData := ` +engine_type: test_engine +server: + rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] +driver: + create_service_account: true + rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] +` + var tmpl BatchEngineRBACTemplate + err := yaml.Unmarshal([]byte(yamlData), &tmpl) + Expect(err).NotTo(HaveOccurred()) + Expect(tmpl.EngineType).To(Equal("test_engine")) + Expect(tmpl.Server).NotTo(BeNil()) + Expect(tmpl.Server.Rules).To(HaveLen(1)) + Expect(tmpl.Driver).NotTo(BeNil()) + Expect(tmpl.Driver.CreateServiceAccount).To(BeTrue()) + Expect(tmpl.Driver.Rules).To(HaveLen(1)) + }) + + It("should handle server-only template (no driver)", func() { + yamlData := ` +engine_type: server_only +server: + rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "delete"] +` + var tmpl BatchEngineRBACTemplate + err := yaml.Unmarshal([]byte(yamlData), &tmpl) + Expect(err).NotTo(HaveOccurred()) + Expect(tmpl.Server).NotTo(BeNil()) + Expect(tmpl.Driver).To(BeNil()) + }) + }) +}) + +var _ = Describe("resolveBatchDriverSAName", func() { + It("defaults to feast--batch-driver when service_account is omitted", func() { + fs := &feastdevv1.FeatureStore{ObjectMeta: metav1.ObjectMeta{Name: "spark-pg-e2e", Namespace: "feast-spark"}} + Expect(resolveBatchDriverSAName(fs, map[string]interface{}{ + "type": "spark_application", + "image": "quay.io/example/driver:v1", + })).To(Equal("feast-spark-pg-e2e-batch-driver")) + }) + + It("defaults when service_account is empty string", func() { + fs := &feastdevv1.FeatureStore{ObjectMeta: metav1.ObjectMeta{Name: "spark-pg-e2e"}} + Expect(resolveBatchDriverSAName(fs, map[string]interface{}{ + "service_account": "", + })).To(Equal("feast-spark-pg-e2e-batch-driver")) + }) + + It("keeps an explicit service_account override", func() { + fs := &feastdevv1.FeatureStore{ObjectMeta: metav1.ObjectMeta{Name: "spark-pg-e2e"}} + Expect(resolveBatchDriverSAName(fs, map[string]interface{}{ + "service_account": "my-custom-driver", + })).To(Equal("my-custom-driver")) + }) +}) + +func containsResource(rule rbacv1.PolicyRule, resource string) bool { + for _, r := range rule.Resources { + if r == resource { + return true + } + } + return false +} + +func containsVerb(rule rbacv1.PolicyRule, verb string) bool { + for _, v := range rule.Verbs { + if v == verb { + return true + } + } + return false +} diff --git a/infra/feast-operator/internal/controller/services/rbac_templates/spark_application.yaml b/infra/feast-operator/internal/controller/services/rbac_templates/spark_application.yaml new file mode 100644 index 00000000000..c03e0f3db57 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/rbac_templates/spark_application.yaml @@ -0,0 +1,26 @@ +engine_type: spark_application + +server: + rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "delete"] + - apiGroups: ["sparkoperator.k8s.io"] + resources: ["sparkapplications"] + verbs: ["create", "get", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + +driver: + create_service_account: true + rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "watch", "delete", "deletecollection"] + - apiGroups: [""] + resources: ["services", "configmaps", "persistentvolumeclaims"] + verbs: ["create", "get", "list", "watch", "delete", "deletecollection"] diff --git a/infra/feast-operator/internal/controller/services/repo_config.go b/infra/feast-operator/internal/controller/services/repo_config.go index 454dd5b234a..5df6bba3fbc 100644 --- a/infra/feast-operator/internal/controller/services/repo_config.go +++ b/infra/feast-operator/internal/controller/services/repo_config.go @@ -85,7 +85,7 @@ func getServiceRepoConfig( } if appliedSpec.BatchEngine != nil { - err := setRepoConfigBatchEngine(appliedSpec.BatchEngine, configMapExtractionFunc, &repoConfig) + err := setRepoConfigBatchEngine(featureStore, appliedSpec.BatchEngine, configMapExtractionFunc, &repoConfig) if err != nil { return repoConfig, err } @@ -342,6 +342,7 @@ func setRepoConfigOffline(services *feastdevv1.FeatureStoreServices, secretExtra } func setRepoConfigBatchEngine( + featureStore *feastdevv1.FeatureStore, batchEngineConfig *feastdevv1.BatchEngineConfig, configMapExtractionFunc func(configMapRef string, configMapKey string) (map[string]interface{}, error), repoConfig *RepoConfig) error { @@ -362,6 +363,12 @@ func setRepoConfigBatchEngine( return fmt.Errorf("batch engine config must contain 'type' field") } delete(config, "type") + // Inject service_account only for spark_application so baked feature_store.yaml + // matches the SA/RoleBinding created by reconcileBatchEngineRBAC. + // Other batch engines are left unchanged. + if engineType == "spark_application" { + config["service_account"] = resolveBatchDriverSAName(featureStore, config) + } repoConfig.BatchEngine = &ComputeEngineConfig{ Type: engineType, Parameters: config, @@ -466,6 +473,54 @@ func setRepoConfigOpenLineage( yamlCfg.ApiKey = &apiKeyStr } + if ol.Consumer != nil { + consumerCfg := &OpenLineageConsumerYamlConfig{ + Enabled: ol.Consumer.Enabled, + StoreType: ol.Consumer.StoreType, + NamespaceMapping: ol.Consumer.NamespaceMapping, + } + + if ol.Consumer.ConnectionStringSecretRef != nil { + params, err := secretExtractionFunc("", ol.Consumer.ConnectionStringSecretRef.Name, "") + if err != nil { + return fmt.Errorf("failed to read consumer connection string from secret %s: %w", + ol.Consumer.ConnectionStringSecretRef.Name, err) + } + connStr, exists := params["connection_string"] + if !exists { + return fmt.Errorf("secret %q does not contain the required key \"connection_string\"", + ol.Consumer.ConnectionStringSecretRef.Name) + } + connStrStr, ok := connStr.(string) + if !ok { + return fmt.Errorf("key \"connection_string\" in secret %q must be a string, got %T", + ol.Consumer.ConnectionStringSecretRef.Name, connStr) + } + consumerCfg.ConnectionString = &connStrStr + } + + if ol.Consumer.ApiKeySecretRef != nil { + params, err := secretExtractionFunc("", ol.Consumer.ApiKeySecretRef.Name, "") + if err != nil { + return fmt.Errorf("failed to read consumer API key from secret %s: %w", + ol.Consumer.ApiKeySecretRef.Name, err) + } + apiKey, exists := params["api_key"] + if !exists { + return fmt.Errorf("secret %q does not contain the required key \"api_key\"", + ol.Consumer.ApiKeySecretRef.Name) + } + apiKeyStr, ok := apiKey.(string) + if !ok { + return fmt.Errorf("key \"api_key\" in secret %q must be a string, got %T", + ol.Consumer.ApiKeySecretRef.Name, apiKey) + } + consumerCfg.ApiKey = &apiKeyStr + } + + yamlCfg.Consumer = consumerCfg + } + repoConfig.OpenLineage = yamlCfg return nil } diff --git a/infra/feast-operator/internal/controller/services/repo_config_test.go b/infra/feast-operator/internal/controller/services/repo_config_test.go index 5ae22a795f6..89941c6beb5 100644 --- a/infra/feast-operator/internal/controller/services/repo_config_test.go +++ b/infra/feast-operator/internal/controller/services/repo_config_test.go @@ -673,6 +673,72 @@ var _ = Describe("Repo Config", func() { Expect(repoConfig.Materialization).To(BeNil()) Expect(repoConfig.OpenLineage).To(BeNil()) }) + + It("should inject default batch_engine.service_account when ConfigMap omits it", func() { + featureStore := minimalFeatureStore() + featureStore.Name = "spark-pg-e2e" + featureStore.Spec.BatchEngine = &feastdevv1.BatchEngineConfig{ + ConfigMapRef: &corev1.LocalObjectReference{Name: "spark-pg-batch-engine"}, + } + ApplyDefaultsToStatus(featureStore) + + extractCM := func(configMapRef string, configMapKey string) (map[string]interface{}, error) { + return map[string]interface{}{ + "type": "spark_application", + "image": "quay.io/example/feast-spark-driver:v6", + // service_account intentionally omitted + }, nil + } + + repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret, extractCM, false) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.BatchEngine).NotTo(BeNil()) + Expect(repoConfig.BatchEngine.Type).To(Equal("spark_application")) + Expect(repoConfig.BatchEngine.Parameters["service_account"]).To(Equal("feast-spark-pg-e2e-batch-driver")) + }) + + It("should preserve explicit batch_engine.service_account from ConfigMap", func() { + featureStore := minimalFeatureStore() + featureStore.Name = "spark-pg-e2e" + featureStore.Spec.BatchEngine = &feastdevv1.BatchEngineConfig{ + ConfigMapRef: &corev1.LocalObjectReference{Name: "spark-pg-batch-engine"}, + } + ApplyDefaultsToStatus(featureStore) + + extractCM := func(configMapRef string, configMapKey string) (map[string]interface{}, error) { + return map[string]interface{}{ + "type": "spark_application", + "service_account": "my-custom-driver", + }, nil + } + + repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret, extractCM, false) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.BatchEngine.Parameters["service_account"]).To(Equal("my-custom-driver")) + }) + + It("should not inject service_account for non-spark_application batch engines", func() { + featureStore := minimalFeatureStore() + featureStore.Name = "spark-pg-e2e" + featureStore.Spec.BatchEngine = &feastdevv1.BatchEngineConfig{ + ConfigMapRef: &corev1.LocalObjectReference{Name: "other-batch-engine"}, + } + ApplyDefaultsToStatus(featureStore) + + extractCM := func(configMapRef string, configMapKey string) (map[string]interface{}, error) { + return map[string]interface{}{ + "type": "spark", + // no service_account — must stay omitted for non-spark_application + }, nil + } + + repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret, extractCM, false) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.BatchEngine).NotTo(BeNil()) + Expect(repoConfig.BatchEngine.Type).To(Equal("spark")) + _, hasSA := repoConfig.BatchEngine.Parameters["service_account"] + Expect(hasSA).To(BeFalse()) + }) }) It("should fail to create the repo configs", func() { featureStore := minimalFeatureStore() diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index c6271c28d3a..4d5429bbbde 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -79,6 +79,9 @@ func (feast *FeastServices) Deploy() error { if err := feast.createServiceAccount(); err != nil { return err } + if err := feast.reconcileBatchEngineRBAC(); err != nil { + return err + } if err := feast.createDeployment(); err != nil { return err } diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go index 366f0c8d765..098362af96b 100644 --- a/infra/feast-operator/internal/controller/services/services_types.go +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -362,12 +362,22 @@ type MaterializationYamlConfig struct { // emit_on_apply, emit_on_materialize, transport-specific options, etc.) appear at // the same YAML level as the typed connection fields. type OpenLineageYamlConfig struct { - Enabled bool `yaml:"enabled"` - TransportType *string `yaml:"transport_type,omitempty"` - TransportUrl *string `yaml:"transport_url,omitempty"` - TransportEndpoint *string `yaml:"transport_endpoint,omitempty"` - ApiKey *string `yaml:"api_key,omitempty"` - ExtraConfig map[string]interface{} `yaml:",inline,omitempty"` + Enabled bool `yaml:"enabled"` + TransportType *string `yaml:"transport_type,omitempty"` + TransportUrl *string `yaml:"transport_url,omitempty"` + TransportEndpoint *string `yaml:"transport_endpoint,omitempty"` + ApiKey *string `yaml:"api_key,omitempty"` + ExtraConfig map[string]interface{} `yaml:",inline,omitempty"` + Consumer *OpenLineageConsumerYamlConfig `yaml:"consumer,omitempty"` +} + +// OpenLineageConsumerYamlConfig maps to the openlineage.consumer section of feature_store.yaml. +type OpenLineageConsumerYamlConfig struct { + Enabled bool `yaml:"enabled"` + StoreType *string `yaml:"store_type,omitempty"` + ConnectionString *string `yaml:"connection_string,omitempty"` + ApiKey *string `yaml:"api_key,omitempty"` + NamespaceMapping map[string]string `yaml:"namespace_mapping,omitempty"` } // OfflineStoreConfig is the configuration that relates to reading from and writing to the Feast offline store. diff --git a/infra/website/docs/blog/feast-data-quality-monitoring.md b/infra/website/docs/blog/feast-data-quality-monitoring.md new file mode 100644 index 00000000000..2c918c83a63 --- /dev/null +++ b/infra/website/docs/blog/feast-data-quality-monitoring.md @@ -0,0 +1,224 @@ +--- +title: Data Quality Monitoring in Feast 0.64 +description: Feast 0.64 adds native data quality monitoring with baseline metrics, batch and serving-log analysis, REST APIs, CLI workflows, and a built-in monitoring UI. +date: 2026-06-26 +authors: ["Jitendra Yejare", "Nikhil Kathole", "Francisco Javier Arceo"] +--- + +
+ Feast Data Quality Monitoring +
+ +# Data Quality Monitoring in Feast 0.64 + +Serving ML models in production is extremely hard. + +The reason is simple: production models depend on data from many different places. Every source system has some probability of operational error: a delayed pipeline, a schema change, a column that starts producing nulls, a categorical value that changes meaning, a late partition, a silent backfill, or a service that behaves differently under production traffic. + +The more data sources a model depends on, the more chances there are for one of those systems to drift, fail, or change underneath you. That creates a basic tension in ML systems. Models are data hungry and often benefit from orthogonal features from many upstream systems, but every additional upstream dependency increases operational risk. What ML wants for predictive power can conflict with what engineering wants for reliability. + +The only way to manage that tension is to monitor what is actually happening in production. Feature quality problems rarely arrive as neat exceptions. A model may keep serving predictions while one upstream table starts producing nulls, a batch pipeline shifts a numeric distribution, or production requests drift away from the training baseline. By the time these issues show up in model metrics, the debugging path usually crosses feature definitions, data sources, materialization jobs, and serving logs. + +Feast 0.64 adds a native data quality monitoring system that brings those signals directly into the feature store. Instead of relying on a separate validation framework, Feast can now compute, store, serve, and visualize feature-level statistics across batch data and logged serving data. + +The biggest change is that monitoring is now a first-class Feast workflow: + +- `feast apply` can compute baseline metrics for registered feature views +- `feast monitor run` can compute scheduled daily, weekly, biweekly, monthly, and quarterly metrics +- REST endpoints expose monitoring jobs, per-feature metrics, aggregate feature-view and feature-service metrics, baselines, and time series +- the Feast UI includes a Monitoring page with filters, summary tabs, feature drilldowns, histograms, and time-series charts +- compute is pushed into supported offline stores where possible, with a Python fallback for other backends + +## From validation to monitoring + +Feast previously supported an external-library-based validation path for historical retrievals. That integration was useful, but it lived outside the normal feature store workflow: users had to install extra dependencies, write profiler code, and run validation against saved datasets. + +That original integration proved the need for data quality inside Feast. It helped answer an important question: after generating a training dataset, does this dataset satisfy the expectations we care about? + +But production feature quality problems usually happen after the training dataset is generated. A pipeline may keep running while an upstream producer changes a column, shifts a distribution, starts sending nulls, or changes the meaning of a categorical value. In those cases, the feature code may be perfectly correct while the data feeding it has changed. + +Feast needed monitoring that was closer to the system that actually computes and serves features. By coupling DQM to Feast's compute engines and offline stores, Feast can compute quality metrics where the data already lives, reuse feature metadata, compare batch and serving-log distributions, and expose the results through the same CLI, REST API, and UI used to operate the feature store. + +This also helps when teams maintain multiple feature execution paths. For example, a feature may be generated one way for training and another way for low-latency serving or streaming. DQM is not a formal proof that two implementations are equivalent, but distribution metrics, baselines, and serving-log comparisons provide an early warning when those paths start producing meaningfully different values. + +The new system is broader and more operational. It automatically computes statistical profiles for registered features, stores them in monitoring tables, and makes them available to the CLI, REST API, and UI. This gives teams the kind of feature health view they need after features are already in production, not only during one historical retrieval. + +For each feature, Feast can track: + +| Metric family | Examples | +|---|---| +| completeness | row count, null count, null rate | +| numeric profile | mean, standard deviation, min, max | +| percentiles | p50, p75, p90, p95, p99 | +| distributions | numeric histograms or categorical top values | +| aggregate health | feature-view and feature-service summaries | + +## Baselines start at registration + +The simplest way to turn on monitoring is to enable DQM in `feature_store.yaml`: + +```yaml +data_quality_monitoring: + auto_baseline: true +``` + +When `auto_baseline` is enabled, `feast apply` computes baseline metrics for feature views that do not already have one. The baseline is marked as the reference distribution and can be compared with later scheduled metrics. + +That matters because the baseline lives next to the feature definitions. When a feature view is registered, Feast can also capture what "normal" looked like at registration time. Later monitoring runs can answer whether the current data still resembles that baseline. + +For Feast Operator deployments, the same setting is available on the `FeatureStore` custom resource: + +```yaml +apiVersion: feast.dev/v1 +kind: FeatureStore +spec: + feastProject: my_project + dataQualityMonitoring: + autoBaseline: true +``` + +## Scheduled monitoring with the CLI + +For ongoing monitoring, schedule: + +```bash +feast monitor run +``` + +In auto mode, Feast detects the latest event timestamp in the source data and computes metrics across the supported granularities: daily, weekly, biweekly, monthly, and quarterly. + +You can also scope monitoring to a specific feature view: + +```bash +feast monitor run --feature-view driver_stats +``` + +Or compute a specific window and mark it as a baseline: + +```bash +feast monitor run \ + --feature-view driver_stats \ + --start-date 2025-01-01 \ + --end-date 2025-03-31 \ + --granularity daily \ + --set-baseline +``` + +This makes the CLI easy to wire into Airflow, Kubeflow Pipelines, cron, or any scheduler that already runs Feast materialization jobs. + +## Monitoring serving logs + +Batch data tells you whether source features look healthy. Serving logs tell you what your models actually received. + +If a `FeatureService` has logging configured, Feast can compute monitoring metrics from the logged online features: + +```bash +feast monitor run --source-type log +``` + +You can also run batch and log monitoring together: + +```bash +feast monitor run --source-type all +``` + +Log metrics are stored with `data_source_type="log"` alongside batch metrics. Feast normalizes logged feature names back to the feature view and feature name, which lets the UI and API compare batch and serving distributions without forcing users to maintain a separate mapping. + +## The new Monitoring UI + +The most visible 0.64 improvement is the Monitoring page in the Feast UI. It turns DQM from a background job into something feature owners can inspect without leaving Feast. + +The page includes three main tabs: + +| Tab | What it shows | +|---|---| +| Features | per-feature metrics such as null rate, row count, freshness, and health | +| Feature Views | aggregate quality summaries per feature view | +| Feature Services | aggregate quality summaries for model-facing feature services | + +At the top of the page, users can filter by feature view, granularity, source type, and date range. Baseline is treated as its own view because it represents all baseline data rather than a normal date window. The page also includes a Compute Metrics action that triggers DQM computation from the UI, plus Refresh for reloading already computed results. + +
+ Feast DQM Monitoring dashboard showing feature metrics, filters, histograms, and health status +
+ +Clicking a feature opens a detail page with: + +- a distribution chart for numeric histograms or categorical values +- a statistics panel with null rate, mean, standard deviation, min, max, and percentiles +- a granularity selector that can switch between computed windows and baseline +- time-series charts for metric drift, including aggregate statistics and null-rate trends + +
+ Feast DQM numeric feature detail page with distribution chart, statistics, and time-series analysis +
+ +
+ Feast DQM categorical feature detail page with category distribution and statistics +
+ +This is the workflow we wanted: feature owners can start from a table of health signals, filter down to the part of the feature store they care about, and then drill into the exact feature whose distribution changed. + +## How compute engines fit in + +DQM is intentionally tied to Feast's compute and offline-store architecture. The goal is to compute metrics where the data already lives whenever possible, then store the results in backend-specific monitoring tables. + +Supported backends push computation into the underlying system: + +| Backend | Compute path | Storage path | +|---|---|---| +| PostgreSQL | SQL push-down | `INSERT ON CONFLICT` | +| Snowflake | SQL push-down | `MERGE` with JSON metrics | +| BigQuery | SQL push-down | BigQuery `MERGE` | +| Redshift | SQL push-down | Data API-backed writes | +| Spark | SparkSQL push-down | Parquet-backed tables | +| Oracle | SQL through Ibis | `MERGE` | +| DuckDB | in-memory SQL | Parquet files | +| Dask | PyArrow compute | Parquet files | + +For backends without native monitoring support, Feast falls back to pulling data through the offline store and computing metrics with PyArrow and NumPy. That fallback keeps the API consistent while still allowing mature warehouse and distributed engines to do the heavy lifting. + +This design is especially important for larger feature stores. A null-rate or histogram job should not require exporting a warehouse table into a separate monitoring system. If the feature data already lives in Snowflake, BigQuery, Spark, Redshift, or another supported backend, Feast can push the computation closer to that data. + +Feast 0.64 also adds the Apache Flink compute engine, continuing the broader move toward a unified compute-engine model. DQM follows the same direction: feature quality checks should be part of the feature platform's execution model, not a sidecar that every team wires up differently. + +## REST APIs for automation + +The UI and CLI are built on top of monitoring APIs that can also be used by external systems: + +| Method | Endpoint | Use | +|---|---|---| +| `POST` | `/monitoring/compute` | submit a batch DQM job | +| `POST` | `/monitoring/auto_compute` | auto-detect dates and compute all granularities | +| `POST` | `/monitoring/compute/transient` | compute ad hoc metrics without storing them | +| `POST` | `/monitoring/compute/log` | compute metrics from serving logs | +| `POST` | `/monitoring/auto_compute/log` | auto-compute log metrics | +| `GET` | `/monitoring/jobs/{job_id}` | read DQM job status | +| `GET` | `/monitoring/metrics/features` | read per-feature metrics | +| `GET` | `/monitoring/metrics/feature_views` | read feature-view summaries | +| `GET` | `/monitoring/metrics/feature_services` | read feature-service summaries | +| `GET` | `/monitoring/metrics/baseline` | read baseline metrics | +| `GET` | `/monitoring/metrics/timeseries` | read trend data for charts and alerts | + +The transient compute endpoint is useful for exploration. If someone wants to inspect a very specific date range, Feast can compute fresh metrics and return them directly without storing them as part of the scheduled monitoring history. + +## Production shape + +A typical production setup now looks like this: + +1. Add `data_quality_monitoring.auto_baseline: true` to `feature_store.yaml` +2. Run `feast apply` to register features and compute baseline metrics +3. Schedule `feast monitor run` for batch metrics +4. Enable feature-service logging and schedule `feast monitor run --source-type log` for production serving metrics +5. Use the UI to investigate feature health and distribution changes +6. Use REST APIs to connect monitoring results to alerting, orchestration, or custom dashboards + +Monitoring also respects Feast's existing authorization model. Compute operations require update permissions, while reads and transient exploration require describe permissions. That keeps the new DQM surface aligned with the rest of the registry and feature-store API. + +## What's next + +Feast 0.64 makes DQM part of the feature store instead of an integration around it. The release adds the backend compute path, the CLI, the REST API, and the UI surface in one coherent workflow. + +The next step for users is simple: enable baselines, run monitoring jobs on the same cadence as your data pipelines, and use the UI to make feature quality visible to the teams that own production models. + +For setup details, see the [Feature Quality Monitoring guide](/docs/how-to-guides/feature-monitoring) and the [0.64.0 changelog](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md#0640-2026-06-13). diff --git a/infra/website/docs/blog/feast-mlflow-kubeflow.md b/infra/website/docs/blog/feast-mlflow-kubeflow.md index 3e15cbda26a..d0b89ac7138 100644 --- a/infra/website/docs/blog/feast-mlflow-kubeflow.md +++ b/infra/website/docs/blog/feast-mlflow-kubeflow.md @@ -199,43 +199,25 @@ For cross-system lineage that extends beyond Feast into upstream data pipelines ### Data quality monitoring -Feast integrates with data quality frameworks like [Great Expectations](https://greatexpectations.io/) to detect feature drift, stale data, and schema violations before they silently degrade model performance. The workflow centers on Feast's `SavedDataset` and `ValidationReference` APIs: you save a profiled dataset during training, define a profiler using Great Expectations, and then validate new feature data against that reference in subsequent runs. +Feast's native data quality monitoring system automatically computes statistical metrics — null rates, distributions, percentiles, histograms — for every registered feature across both batch data and serving logs. It detects drift by comparing current metrics against baselines computed during `feast apply`. -```python -from feast import FeatureStore -from feast.dqm.profilers.ge_profiler import ge_profiler -from great_expectations.core import ExpectationSuite -from great_expectations.dataset import PandasDataset - -store = FeatureStore(repo_path=".") - -@ge_profiler -def my_profiler(dataset: PandasDataset) -> ExpectationSuite: - dataset.expect_column_values_to_be_between("conv_rate", min_value=0, max_value=1) - dataset.expect_column_values_to_be_between("acc_rate", min_value=0, max_value=1) - return dataset.get_expectation_suite() - -reference_job = store.get_historical_features( - entity_df=entity_df, - features=["driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate"], -) - -dataset = store.create_saved_dataset( - from_=reference_job, - name="driver_stats_validation", - storage=storage, -) +```yaml +# feature_store.yaml +data_quality_monitoring: + auto_baseline: true +``` -reference = dataset.as_reference(name="driver_stats_ref", profiler=my_profiler) +```bash +# Compute metrics across all granularities (daily, weekly, monthly, quarterly) +feast monitor run -new_job = store.get_historical_features( - entity_df=new_entity_df, - features=["driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate"], -) -new_job.to_df(validation_reference=reference) +# Monitor serving logs +feast monitor run --source-type log ``` -If validation fails, Feast raises a `ValidationFailed` exception with details on which expectations were violated. Monitoring feature distributions over time — and comparing them to the distributions seen during training — allows you to detect training–serving skew early, before it causes silent model degradation in production. +The monitoring UI dashboard (accessible from the sidebar) provides per-feature health status, distribution histograms, time-series drift charts, and configurable filters. Metrics are also available via REST API endpoints for integration with external alerting systems. + +For details, see the [Feature Quality Monitoring guide](/docs/how-to-guides/feature-monitoring). ### Feast Feature Registry vs. MLflow Model Registry diff --git a/infra/website/docs/blog/feast-ray-llm-posttrain.md b/infra/website/docs/blog/feast-ray-llm-posttrain.md new file mode 100644 index 00000000000..4b5eac30214 --- /dev/null +++ b/infra/website/docs/blog/feast-ray-llm-posttrain.md @@ -0,0 +1,304 @@ +--- +title: "How to Use Feast for SLM/LLM Post-Training with Ray" +description: "Keep conversation features in Feast, retrieve them for training, then stream into your trainer with Ray." +date: 2026-07-14 +authors: ["Chaitanya Patel"] +--- + +# How to Use Feast for SLM/LLM Post-Training with Ray + +Your support bot answers a lot of tickets. It’s fine—but it sounds generic. The team wants a smaller model that talks more like *your* agents: your refund wording, your product names, your tone. + +So someone says: **fine-tune on our real chats.** + +That part sounds easy. The messy part is the data—exports, notebook cleaning, and prompt formatting scattered across training scripts. + +This post walks through the [ray-llm-posttrain example](https://github.com/feast-dev/feast/tree/master/examples/ray-llm-posttrain): + +1. Put conversation features in Feast +2. Retrieve them with `get_historical_features` (entity-less date range) +3. Get rows into your trainer — stream with Ray **or** materialize with `.to_df()` + +You bring your own trainer. GPT-2 in the script is optional smoke only. + +## What’s in the example + +| Name | Type | What it holds | +|---|---|---| +| `web_documents` | [FeatureView](https://docs.feast.dev/getting-started/concepts/feature-view) | `human`, `bot`, `human_repeat_ratio`, `bot_repeat_ratio` | +| `train_example` | [OnDemandFeatureView](https://docs.feast.dev/reference/beta-on-demand-feature-view) | `cleaned_human`, `cleaned_bot`, `char_count`, `is_trainable`, `sft_text` | +| `llm_posttrain` | FeatureService | Bundles `web_documents` + `train_example` | + +Full definitions live in [feature_definitions.py](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/feature_repo/feature_definitions.py). Ray is the [offline store](https://docs.feast.dev/reference/offline-stores/ray) and one way to stream rows out—not a separate feature catalog. + +This example stays on **supported Feast APIs only** (no core patches). Conversation rows already include `document_id` and `event_timestamp` before Feast reads them. + +## Step 1: Point Feast at conversation data + +### Ray offline store (local) + +From the example [feature_store.yaml](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/feature_repo/feature_store.yaml). Cap Ray resources on a laptop—see [Ray offline store: resource management](https://docs.feast.dev/reference/offline-stores/ray#important-resource-management): + +```yaml +project: ray_llm_posttrain +registry: data/registry.db +provider: local + +offline_store: + type: ray + storage_path: data/ray_storage + enable_ray_logging: false + ray_conf: + num_cpus: 2 + object_store_memory: 104857600 + _memory: 524288000 + +batch_engine: + type: ray.engine + max_workers: 2 + +online_store: + type: sqlite + path: data/online_store.db + +entity_key_serialization_version: 3 +auth: + type: no_auth +``` + +You can also start from the built-in template: + +```bash +feast init -t ray my_ray_project +``` + +See the [Ray template / offline store docs](https://docs.feast.dev/reference/offline-stores/ray#quick-start-with-ray-template) and the related blog [Scaling ML with Feast and Ray](/blog/feast-ray-distributed-processing). + +### Demo seed: prepare parquet, then `RaySource` + +[RaySource](https://docs.feast.dev/reference/data-sources/ray) tells Feast how to load data through Ray. Hugging Face is only used in a **prepare script**—not as a live Feast source that invents timestamps at retrieval time. + +`nampdn-ai/tiny-webtext` has no `document_id` / `event_timestamp`. Entity-less retrieval needs those columns on the source. We add them **outside Feast**, write parquet, then point Feast at that file (supported path): + +```bash +PYTHONPATH=../../sdk/python python scripts/prepare_data.py +# → feature_repo/data/tiny_webtext.parquet +``` + +```python +from feast.infra.offline_stores.contrib.ray_offline_store.ray_source import RaySource + +tiny_web = RaySource( + name="tiny_webtext", + reader_type="parquet", + path="data/tiny_webtext.parquet", + timestamp_field="event_timestamp", +) +``` + +In production you’d skip the HF prepare step and register your real conversation store (warehouse / lake / parquet) that already has join keys and timestamps. + +More reader types are in the [Ray data source reference](https://docs.feast.dev/reference/data-sources/ray#supported-reader_type-values). + +### Feature view + +```python +web_documents = FeatureView( + name="web_documents", + entities=[document], + ttl=timedelta(days=365), + schema=[ + Field(name="human", dtype=String), + Field(name="bot", dtype=String), + Field(name="human_repeat_ratio", dtype=Float64), + Field(name="bot_repeat_ratio", dtype=Float64), + ], + source=tiny_web, + online=False, +) +``` + +### Optional: OnDemandFeatureView for derived training features + +If you want Feast to own `sft_text` / quality gates (same idea as in the [ODFV docs](https://docs.feast.dev/reference/beta-on-demand-feature-view)): + +```python +@on_demand_feature_view( + sources=[web_documents], + schema=[ + Field(name="cleaned_human", dtype=String), + Field(name="cleaned_bot", dtype=String), + Field(name="char_count", dtype=Int64), + Field(name="is_trainable", dtype=Bool), + Field(name="sft_text", dtype=String), + ], + mode="pandas", +) +def train_example(inputs): + cleaned_human = inputs["human"].fillna("").astype(str).str.strip() + cleaned_bot = inputs["bot"].fillna("").astype(str).str.strip() + # ... length + repeat-ratio gate ... + sft_text = ( + "<|im_start|>user\n" + cleaned_human + "<|im_end|>\n" + "<|im_start|>assistant\n" + cleaned_bot + "<|im_end|>" + ) + return pd.DataFrame({...}) +``` + +```python +llm_posttrain = FeatureService( + name="llm_posttrain", + features=[web_documents, train_example], +) +``` + +Apply: + +```bash +cd examples/ray-llm-posttrain/feature_repo +feast apply +``` + +## Step 2: Retrieve for training (entity-less) + +No `entity_df`—just a date window. That pattern is covered in [Historical Features Without Entity IDs](/blog/entity-less-historical-features-retrieval) and the [FAQ](https://docs.feast.dev/getting-started/faq#how-do-i-run-get_historical_features-without-providing-an-entity-dataframe): + +```python +from datetime import datetime, timezone +from feast import FeatureStore + +store = FeatureStore(repo_path="feature_repo") + +job = store.get_historical_features( + features=[ + "web_documents:human", + "web_documents:bot", + "web_documents:human_repeat_ratio", + "web_documents:bot_repeat_ratio", + ], + start_date=datetime(2024, 6, 1, tzinfo=timezone.utc), + end_date=datetime(2024, 7, 1, tzinfo=timezone.utc), +) +``` + +Then choose how you turn that job into training rows. + +## Step 3: Two ways into the trainer + +| Path | ODFV runs? | When to use | +|---|---|---| +| `job.to_ray_dataset()` then preprocess | **No** | Stream FeatureView columns; shape `sft_text` yourself | +| `job.to_df()` / `to_arrow()` | **Yes** | Want `train_example` outputs from Feast | + +Pick **Option A** when you want full control over text formatting or need custom preprocessing (e.g., multi-turn chat templates, tokenization-aware truncation). Pick **Option B** when you want Feast to enforce quality gates consistently across training and serving. + +### Option A — Stream with Ray, preprocess yourself + +`to_ray_dataset()` returns a Ray Dataset of retrieved FeatureView columns. It does **not** apply OnDemandFeatureViews. Build training text with Ray `map_batches` (as in [train_sft.py](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/scripts/train_sft.py)): + +```python +ds = job.to_ray_dataset() + +def preprocess_sft(batch): + import pandas as pd + + if not isinstance(batch, pd.DataFrame): + batch = pd.DataFrame(batch) + human = batch["human"].fillna("").astype(str).str.strip() + bot = batch["bot"].fillna("").astype(str).str.strip() + ok = bot.str.len() >= 64 + sft_text = ( + "<|im_start|>user\n" + human + "<|im_end|>\n" + "<|im_start|>assistant\n" + bot + "<|im_end|>" + ) + return pd.DataFrame({"sft_text": sft_text}).loc[ok].reset_index(drop=True) + +train_ds = ds.map_batches(preprocess_sft, batch_format="pandas") +# → hand train_ds to your SLM/LLM trainer +``` + +Run the example default path: + +```bash +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run +``` + +### Option B — Use the ODFV, then train + +Materialize with `.to_df()` so `train_example` runs (same retrieval/serving idea as in the [ODFV overview](https://docs.feast.dev/reference/beta-on-demand-feature-view#why-use-on-demand-feature-views)): + +```python +df = store.get_historical_features( + features=store.get_feature_service("llm_posttrain"), + start_date=datetime(2024, 6, 1, tzinfo=timezone.utc), + end_date=datetime(2024, 7, 1, tzinfo=timezone.utc), +).to_df() + +trainable = df[df["is_trainable"] & df["sft_text"].astype(str).str.len().gt(0)] +# trainable["sft_text"] → your trainer +# or: import ray; ray.data.from_pandas(trainable[["sft_text"]]) +``` + +```bash +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run --via-df +``` + +### The same ODFV at serving time + +The `train_example` ODFV runs identically during online serving — the quality gate and formatting logic stay in one place: + +```python +# At inference time, the same ODFV runs on the fly +features = store.get_online_features( + features=["train_example:sft_text", "train_example:is_trainable"], + entity_rows=[{"document_id": "doc_42"}], +).to_dict() +# features["sft_text"], features["is_trainable"] — same logic as training +``` + +## Try the full example + +```bash +cd examples/ray-llm-posttrain +uv pip install -e "../../sdk/python[ray]" -r requirements.txt +PYTHONPATH=../../sdk/python python scripts/prepare_data.py +cd feature_repo && feast apply && cd .. + +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run --via-df + +# optional GPT-2 smoke +PYTHONPATH=../../sdk/python python scripts/train_sft.py --max-steps 20 +``` + +Details: [ray-llm-posttrain README](https://github.com/feast-dev/feast/tree/master/examples/ray-llm-posttrain). + +## Takeaways + +1. **Keep conversation features in Feast** — this example’s `web_documents`. +2. **Stream with Ray** — `to_ray_dataset()`, then preprocess training text yourself. +3. **Want ODFVs** — `.to_df()` / `.to_arrow()` to materialize, then train. +4. **Bring your own trainer** — GPT-2 in the example is optional. + +## References + +**This example** + +- [ray-llm-posttrain example](https://github.com/feast-dev/feast/tree/master/examples/ray-llm-posttrain) +- [feature_definitions.py](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/feature_repo/feature_definitions.py) +- [train_sft.py](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/scripts/train_sft.py) + +**Docs** + +- [Ray offline store](https://docs.feast.dev/reference/offline-stores/ray) +- [Ray data source](https://docs.feast.dev/reference/data-sources/ray) +- [Ray compute engine](https://docs.feast.dev/reference/compute-engine/ray) +- [On demand feature views](https://docs.feast.dev/reference/beta-on-demand-feature-view) +- [Feature retrieval](https://docs.feast.dev/getting-started/concepts/feature-retrieval) +- [FAQ: historical features without entity dataframe](https://docs.feast.dev/getting-started/faq#how-do-i-run-get_historical_features-without-providing-an-entity-dataframe) + +**Related blogs & tutorials** + +- [Historical Features Without Entity IDs](/blog/entity-less-historical-features-retrieval) +- [Scaling ML with Feast and Ray](/blog/feast-ray-distributed-processing) +- [Validating historical features](https://docs.feast.dev/tutorials/validating-historical-features) diff --git a/infra/website/docs/blog/feast-unity-catalog-integration.md b/infra/website/docs/blog/feast-unity-catalog-integration.md new file mode 100644 index 00000000000..1ee977a71eb --- /dev/null +++ b/infra/website/docs/blog/feast-unity-catalog-integration.md @@ -0,0 +1,342 @@ +--- +title: "Feast Gets Native Apache Iceberg Support" +description: "Feast now reads features from any Iceberg catalog — REST, SQL, Hive, Glue, DynamoDB. Connect to Unity Catalog, Apache Polaris, Nessie, or your own PyIceberg catalog. Full support for get_historical_features, materialize, and online serving." +date: 2026-07-18 +authors: ["Nikhil Kathole"] +--- + +# Feast Gets Native Apache Iceberg Support + +Apache Iceberg has become the open table format. Your data lake is probably already on it — whether through Databricks, Snowflake, AWS, or self-managed infrastructure. But until now, connecting Feast to Iceberg tables meant either going through Spark (heavyweight, slow to start) or copying data into Feast-managed Parquet files (data duplication, governance gap). + +Feast now ships a native `IcebergSource` that reads directly from any Iceberg catalog. No data copies. Your feature tables live where they already live — in your Iceberg catalog — and Feast reads from them via PyIceberg. With the DuckDB offline store, you don't even need a Spark cluster — reads happen entirely in-process. + +## Why This Matters + +Before this, the path from "data in Iceberg" to "features in Feast" looked like this: + +1. Data engineers build Iceberg tables in their catalog (UC, Glue, Hive) +2. ML engineers copy data to Feast-managed Parquet files, or configure a SparkSource that couples them to a specific compute engine +3. Two copies of the data. Two metadata systems. No connection between them. + +Now the path is: + +1. Data engineers build Iceberg tables in their catalog +2. ML engineers point `IcebergSource` at the table +3. Done. Feast reads directly from the catalog via PyIceberg. One copy. One source of truth. Choose DuckDB for lightweight local reads or Spark when you need distributed compute — the data source definition stays the same either way. + +## What You Get + +### Any Iceberg Catalog + +`IcebergSource` supports every catalog backend that PyIceberg supports: + +| `catalog_type` | Backend | Example Use Case | +|---|---|---| +| `"rest"` | Iceberg REST Catalog | Databricks Unity Catalog, Apache Polaris, Project Nessie, Snowflake Open Catalog | +| `"sql"` | SQL-backed catalog | Local dev with SQLite, CI/CD, PostgreSQL-backed catalogs | +| `"hive"` | Hive Metastore | On-premise Hadoop, EMR | +| `"glue"` | AWS Glue Data Catalog | AWS-native lakehouse | +| `"dynamodb"` | DynamoDB catalog | Serverless AWS | + +### Both Offline Stores + +| Operation | DuckDB | Spark | +|---|---|---| +| `feast apply` | Yes | Yes | +| `get_historical_features` | Yes | Yes | +| `materialize` / `materialize-incremental` | Yes | Yes | +| `get_online_features` | Yes | Yes | + +Both offline stores use PyIceberg for the actual Iceberg table scan — the difference is what happens after. DuckDB processes the Arrow table in-process (no JVM, no cluster), making it ideal for local development and moderate-scale workloads. Spark is there when you need distributed compute over large datasets. The same `IcebergSource` definition works with either offline store — just change `offline_store.type` in your YAML. + +### Full Iceberg Semantics + +Every read goes through PyIceberg's `table.scan().to_arrow()`. This means you get proper Iceberg semantics: schema evolution, partition pruning, and snapshot isolation — not just raw Parquet file reads. + +## Quick Start + +### Install + +```bash +pip install "feast[iceberg]" +``` + +### Define a Source + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import IcebergSource + +driver_stats = IcebergSource( + warehouse="my_catalog", + namespace="ml_features", + table="driver_hourly_stats", + catalog_type="rest", + endpoint="https://my-iceberg-catalog.example.com", + token_env_var="CATALOG_TOKEN", + timestamp_field="event_timestamp", +) +``` + +### Use It + +```python +from datetime import timedelta +from feast import Entity, FeatureView, Field +from feast.types import Float64, Int64 + +driver = Entity(name="driver", join_keys=["driver_id"]) + +driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=365), + schema=[ + Field(name="driver_id", dtype=Int64), + Field(name="conv_rate", dtype=Float64), + Field(name="acc_rate", dtype=Float64), + Field(name="avg_daily_trips", dtype=Int64), + ], + source=driver_stats, + online=True, +) +``` + +```yaml +# feature_store.yaml +project: my_project +registry: data/registry.db +provider: local +online_store: + type: sqlite + path: data/online_store.db +offline_store: + type: duckdb +``` + +```bash +feast apply +``` + +Then use it like any other Feast source: + +```python +from feast import FeatureStore +import pandas as pd +from datetime import datetime, timezone + +store = FeatureStore(repo_path=".") + +# Training data +training_df = store.get_historical_features( + entity_df=pd.DataFrame({ + "driver_id": [1001, 1002, 1003], + "event_timestamp": [datetime(2026, 7, 1, tzinfo=timezone.utc)] * 3, + }), + features=["driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate"], +).to_df() + +# Materialize to online store +store.materialize_incremental(end_date=datetime.now(tz=timezone.utc)) + +# Online serving +online = store.get_online_features( + features=["driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate"], + entity_rows=[{"driver_id": 1001}], +).to_dict() +``` + +## Catalog Examples + +### AWS Glue + +```python +glue_source = IcebergSource( + warehouse="my_glue_database", + namespace="ml_features", + table="driver_stats", + catalog_type="glue", + catalog_properties={"region_name": "us-east-1"}, + timestamp_field="event_timestamp", +) +``` + +### Hive Metastore + +```python +hive_source = IcebergSource( + endpoint="thrift://hive-metastore:9083", + warehouse="warehouse", + namespace="features", + table="driver_stats", + catalog_type="hive", + timestamp_field="event_timestamp", +) +``` + +### Apache Polaris / Nessie + +```python +polaris_source = IcebergSource( + endpoint="https://polaris.example.com", + warehouse="my_catalog", + namespace="ml", + table="features", + catalog_type="rest", + token_env_var="POLARIS_TOKEN", + timestamp_field="event_timestamp", +) +``` + +### Local Development (SQLite-backed) + +For development and CI/CD, use a local PyIceberg SQL catalog — no external service required: + +```python +local_source = IcebergSource( + warehouse="dev_warehouse", + namespace="default", + table="driver_stats", + catalog_type="sql", + catalog_name="dev_catalog", + catalog_properties={ + "uri": "sqlite:////tmp/iceberg_catalog.db", + "warehouse": "file:///tmp/iceberg_warehouse", + }, + timestamp_field="event_timestamp", +) +``` + +## Use Case: Unity Catalog Integration + +Unity Catalog users get everything above with simpler configuration. `UnityCatalogSource` extends `IcebergSource` with UC-specific defaults: + +- Default connection via `DATABRICKS_HOST` and `DATABRICKS_TOKEN` environment variables — no manual endpoint or token setup +- Three-level naming (`warehouse.namespace.table`) maps directly to UC's catalog/schema/table hierarchy + +### Databricks Setup + +```bash +export DATABRICKS_HOST="https://your-workspace.cloud.databricks.com" +export DATABRICKS_TOKEN="dapi_your_token_here" +``` + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import UnityCatalogSource + +driver_stats_source = UnityCatalogSource( + warehouse="ml_catalog", + namespace="driver_features", + table="driver_hourly_stats", + timestamp_field="event_timestamp", + created_timestamp_column="created", + description="Hourly aggregated driver statistics", +) +``` + +That's it. No endpoint or token parameters needed — they come from the environment variables. + +### Optional: Governance Metadata Sync + +If you want `feast apply` to annotate your UC tables with `feast.*` properties (project name, feature view, primary keys, owner), you can use the `UnityCatalogProvider`: + +```yaml +# feature_store.yaml +provider: unity_catalog +``` + +This is optional. For most users, `provider: local` is sufficient. Reads, materialization, and online serving work the same regardless of which provider you use. + +### OSS Unity Catalog + +The integration also works with the open-source Unity Catalog, with some differences: + +| Capability | Databricks UC | OSS UC | +|---|---|---| +| Read via Iceberg REST | Built-in | Requires `uniform_iceberg_metadata_location` in H2 DB | +| Read via SQL catalog | Yes | Yes | +| Credential vending | Yes | Not available | + +For OSS UC, set `credential_vending=False` and `token_env_var=None`: + +```python +source = UnityCatalogSource( + warehouse="unity", + namespace="default", + table="driver_hourly_stats", + endpoint="http://localhost:8080/api/2.1/unity-catalog/iceberg", + token_env_var=None, + credential_vending=False, + catalog_type="sql", # recommended for OSS UC + catalog_name="my_catalog", + catalog_properties={ + "uri": "sqlite:////tmp/pyiceberg_catalog.db", + "warehouse": "file:///tmp/warehouse", + }, + timestamp_field="event_timestamp", + register_as_feature_table=False, +) +``` + +## How It Works + +### Read Path + +All data reads go through PyIceberg, regardless of catalog type: + +``` +IcebergSource.get_catalog_client() + → PyIceberg catalog (REST / SQL / Hive / Glue / DynamoDB) + → table.scan().to_arrow() + → Arrow table + → DuckDB ibis memtable or Spark DataFrame +``` + +If the catalog is misconfigured, you get a clear error — consistent with how every other Feast data source works. + +### Catalog Name Isolation + +Each source has a `catalog_name` parameter (default: `"feast_iceberg"`). This is the instance name PyIceberg uses when loading the catalog. If you have multiple sources pointing at different catalogs, use different names to avoid collisions: + +```python +production = IcebergSource(catalog_name="prod_catalog", ...) +staging = IcebergSource(catalog_name="staging_catalog", ...) +``` + +## What's Not Supported + +- **Write-back to Iceberg/UC tables.** Feast reads from existing tables; it doesn't write feature data back. The `write_to_offline_store` API only supports `FileSource` (DuckDB) and `SparkSource` (Spark). Your data engineering pipelines create and populate the tables. +- **Table creation.** Tables must exist before Feast can read from them. `feast apply` registers the feature view in Feast's registry, not the table in the catalog. + +## Configuration Reference + +### IcebergSource + +| Parameter | Default | Description | +|---|---|---| +| `warehouse` | *required* | Catalog or warehouse name | +| `namespace` | *required* | Schema or namespace | +| `table` | *required* | Table name | +| `catalog_type` | `"rest"` | Backend: `"rest"`, `"sql"`, `"hive"`, `"glue"`, `"dynamodb"` | +| `catalog_name` | `"feast_iceberg"` | PyIceberg instance name (unique per catalog to avoid collisions) | +| `endpoint` | `None` | Catalog endpoint URL | +| `catalog_properties` | `{}` | Additional catalog config (e.g., `{"uri": "sqlite:///..."}`) | +| `token_env_var` | `None` | Env var containing auth token | +| `credential_vending` | `True` | Request scoped storage credentials | +| `timestamp_field` | `None` | Event timestamp column | +| `created_timestamp_column` | `None` | Creation timestamp for deduplication | + +### UnityCatalogSource (extends IcebergSource) + +All `IcebergSource` parameters plus: + +| Parameter | Default | Description | +|---|---|---| +| `endpoint` | From `DATABRICKS_HOST` | Defaults to `{DATABRICKS_HOST}/api/2.1/unity-catalog/iceberg` | +| `token_env_var` | `"DATABRICKS_TOKEN"` | Defaults to Databricks token env var | +| `register_as_feature_table` | `True` | Sync `feast.*` properties to UC on `feast apply` | +| `sync_lineage` | `True` | Record lineage in UC (Databricks only) | + +--- + +*Native Iceberg support is available in Feast 0.64+. Install with `pip install "feast[iceberg]"` and check the [Iceberg data source documentation](/reference/data-sources/iceberg) for the full API reference.* diff --git a/infra/website/public/images/blog/feast-dqm-monitoring-hero.png b/infra/website/public/images/blog/feast-dqm-monitoring-hero.png new file mode 100644 index 00000000000..86d7125a3c3 Binary files /dev/null and b/infra/website/public/images/blog/feast-dqm-monitoring-hero.png differ diff --git a/infra/website/public/images/blog/feast-dqm-ui-all-features.png b/infra/website/public/images/blog/feast-dqm-ui-all-features.png new file mode 100644 index 00000000000..4e728a2e86e Binary files /dev/null and b/infra/website/public/images/blog/feast-dqm-ui-all-features.png differ diff --git a/infra/website/public/images/blog/feast-dqm-ui-categorical-feature.png b/infra/website/public/images/blog/feast-dqm-ui-categorical-feature.png new file mode 100644 index 00000000000..a3e6f22b74c Binary files /dev/null and b/infra/website/public/images/blog/feast-dqm-ui-categorical-feature.png differ diff --git a/infra/website/public/images/blog/feast-dqm-ui-numeric-feature.png b/infra/website/public/images/blog/feast-dqm-ui-numeric-feature.png new file mode 100644 index 00000000000..0b5e7f0d23d Binary files /dev/null and b/infra/website/public/images/blog/feast-dqm-ui-numeric-feature.png differ diff --git a/infra/website/src/components/Navigation.astro b/infra/website/src/components/Navigation.astro index a5987bf6348..143fcbc047f 100644 --- a/infra/website/src/components/Navigation.astro +++ b/infra/website/src/components/Navigation.astro @@ -14,11 +14,29 @@ COMMUNITY - +
@@ -38,7 +56,7 @@ top: 0; left: 0; right: 0; - background-color: white; + background-color: var(--color-nav-bg); z-index: 1000; } @@ -80,6 +98,35 @@ margin-left: 32px; } + .nav-right { + display: flex; + align-items: center; + gap: 4px; + padding-right: var(--content-padding); + } + + .theme-toggle { + background: none; + border: none; + padding: 8px; + cursor: pointer; + color: var(--color-text); + display: flex; + align-items: center; + justify-content: center; + opacity: 0.7; + transition: opacity 0.2s ease; + } + + .theme-toggle:hover { + opacity: 1; + } + + .icon-sun { display: none; } + .icon-moon { display: block; } + :global([data-theme="dark"]) .icon-sun { display: block; } + :global([data-theme="dark"]) .icon-moon { display: none; } + .mobile-menu-button { display: block; background: none; @@ -87,7 +134,6 @@ padding: 8px; cursor: pointer; color: var(--color-text); - margin-right: var(--content-padding); } @media (min-width: 1024px) { @@ -95,7 +141,7 @@ display: flex; align-items: center; } - + .mobile-menu-button { display: none; } @@ -104,9 +150,9 @@ .mobile-menu { display: none; width: 100%; - background: white; + background: var(--color-nav-bg); padding: 16px 0; - border-top: 1px solid #eee; + border-top: 1px solid var(--color-border); position: absolute; top: 52px; left: 0; @@ -122,17 +168,27 @@ } .mobile-menu a:hover { - background-color: #f5f5f5; + background-color: var(--color-nav-hover); } \ No newline at end of file diff --git a/infra/website/src/layouts/BaseLayout.astro b/infra/website/src/layouts/BaseLayout.astro index 05d940005ba..96e30a6aee2 100644 --- a/infra/website/src/layouts/BaseLayout.astro +++ b/infra/website/src/layouts/BaseLayout.astro @@ -52,7 +52,17 @@ const { - + +
diff --git a/infra/website/src/layouts/BlogLayout.astro b/infra/website/src/layouts/BlogLayout.astro index c5f8379fc80..9fe68a9daa7 100644 --- a/infra/website/src/layouts/BlogLayout.astro +++ b/infra/website/src/layouts/BlogLayout.astro @@ -14,7 +14,7 @@ const { frontmatter } = Astro.props;

{frontmatter.title}

{frontmatter.date && ( -