diff --git a/.github/fork_workflows/fork_pr_integration_tests_aws.yml b/.github/fork_workflows/fork_pr_integration_tests_aws.yml index aa89ece1776..6eb8b8feff0 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_aws.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_aws.yml @@ -3,64 +3,6 @@ name: fork-pr-integration-tests-aws on: [pull_request] jobs: - build-docker-image: - if: github.repository == 'your github repo' # swap here with your project id - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - # pull_request_target runs the workflow in the context of the base repo - # as such actions/checkout needs to be explicit configured to retrieve - # code from the PR. - ref: refs/pull/${{ github.event.pull_request.number }}/merge - submodules: recursive - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - install: true - - name: Set up AWS SDK - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - name: Set ECR image tag - id: image-tag - run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - - name: Cache Public ECR Image - id: lambda_python_3_11 - uses: actions/cache@v2 - with: - path: ~/cache - key: lambda_python_3_11 - - name: Handle Cache Miss (pull public ECR image & save it to tar file) - if: steps.cache-primes.outputs.cache-hit != 'true' - run: | - mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.11 - docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - - name: Handle Cache Hit (load docker image from tar file) - if: steps.cache-primes.outputs.cache-hit == 'true' - run: | - docker load -i ~/cache/lambda_python_3_11.tar - - name: Build and push - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: feast-python-server - run: | - docker build \ - --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ - --load \ - . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} - outputs: - DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: if: github.repository == 'your github repo' # swap here with your project id runs-on: ${{ matrix.os }} @@ -138,8 +80,6 @@ jobs: docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster - name: Test python if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak - env: - FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }} run: | pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread -k "aws and not Snowflake and not BigQuery and not minio_registry" pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread -k "File and not Snowflake and not BigQuery and not minio_registry" diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 596eef2b52c..f04015a9892 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -116,7 +116,7 @@ jobs: needs: get-version strategy: matrix: - component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server] + component: [feature-server, feature-server-java, feature-transformation-server] env: REGISTRY: feastdev steps: diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 1d6850e4d8e..1b401997a7b 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -6,61 +6,8 @@ on: - master jobs: - build-lambda-docker-image: - if: github.repository == 'feast-dev/feast' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - install: true - - name: Set up AWS SDK - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - name: Set ECR image tag - id: image-tag - run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - - name: Cache Public ECR Image - id: lambda_python_3_11 - uses: actions/cache@v2 - with: - path: ~/cache - key: lambda_python_3_11 - - name: Handle Cache Miss (pull public ECR image & save it to tar file) - if: steps.cache-primes.outputs.cache-hit != 'true' - run: | - mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.11 - docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - - name: Handle Cache Hit (load docker image from tar file) - if: steps.cache-primes.outputs.cache-hit == 'true' - run: | - docker load -i ~/cache/lambda_python_3_11.tar - - name: Build and push - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: feast-python-server - run: | - docker build \ - --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ - --load \ - . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} - outputs: - DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: if: github.repository == 'feast-dev/feast' - needs: build-lambda-docker-image runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -125,7 +72,6 @@ jobs: docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster - name: Test python and go env: - FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }} SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} @@ -134,7 +80,6 @@ jobs: run: make test-python-integration - name: Benchmark python env: - FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }} SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 8a6ed2d7a73..11c91af2d7b 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -61,65 +61,9 @@ jobs: run: gcloud info - name: Run DynamoDB / Bigtable cleanup script run: python infra/scripts/cleanup_ci.py - build-docker-image: - if: github.repository == 'feast-dev/feast' - needs: [check_date] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: master - submodules: recursive - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - install: true - - name: Set up AWS SDK - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - name: Set ECR image tag - id: image-tag - run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - - name: Cache Public ECR Image - id: lambda_python_3_11 - uses: actions/cache@v4 - with: - path: ~/cache - key: lambda_python_3_11 - - name: Handle Cache Miss (pull public ECR image & save it to tar file) - if: steps.lambda_python_3_11.outputs.cache-hit != 'true' - run: | - mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.11 - docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - - name: Handle Cache Hit (load docker image from tar file) - if: steps.lambda_python_3_11.outputs.cache-hit == 'true' - run: | - docker load -i ~/cache/lambda_python_3_11.tar - - name: Build and push - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: feast-python-server - run: | - docker build \ - --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ - --load \ - . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} - outputs: - DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: if: github.repository == 'feast-dev/feast' - needs: [check_date, build-docker-image, cleanup_dynamo_tables] + needs: [check_date, cleanup_dynamo_tables] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -205,7 +149,6 @@ jobs: - name: Test python if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak env: - FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }} SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 3081d418fcf..f4a9132d292 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -13,75 +13,12 @@ on: # cancel-in-progress: true jobs: - build-docker-image: - # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. - if: - ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || - (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && - github.repository == 'feast-dev/feast' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - # pull_request_target runs the workflow in the context of the base repo - # as such actions/checkout needs to be explicit configured to retrieve - # code from the PR. - ref: refs/pull/${{ github.event.pull_request.number }}/merge - submodules: recursive - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - install: true - - name: Set up AWS SDK - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - name: Set ECR image tag - id: image-tag - run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - - name: Cache Public ECR Image - id: lambda_python_3_11 - uses: actions/cache@v2 - with: - path: ~/cache - key: lambda_python_3_11 - - name: Handle Cache Miss (pull public ECR image & save it to tar file) - if: steps.cache-primes.outputs.cache-hit != 'true' - run: | - mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.11 - docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar - - name: Handle Cache Hit (load docker image from tar file) - if: steps.cache-primes.outputs.cache-hit == 'true' - run: | - docker load -i ~/cache/lambda_python_3_11.tar - - name: Build and push - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: feast-python-server - run: | - docker build \ - --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ - --load \ - . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} - outputs: - DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. if: ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && github.repository == 'feast-dev/feast' - needs: build-docker-image runs-on: ${{ matrix.os }} strategy: fail-fast: false diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 914e5a233c7..e56296ec4b4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -49,7 +49,7 @@ jobs: needs: [get-version, publish-python-sdk] strategy: matrix: - component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server, feast-operator] + component: [feature-server, feature-server-java, feature-transformation-server, feast-operator] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: feastdev diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile new file mode 100644 index 00000000000..51796294700 --- /dev/null +++ b/.gitpod.Dockerfile @@ -0,0 +1,8 @@ +FROM gitpod/workspace-base +RUN sudo apt-get update && sudo apt-get install -y python3-dev python3-setuptools python3-pip python-is-python3 && sudo rm -rf /var/lib/apt/lists/* +RUN curl -LsSf https://astral.sh/uv/install.sh | sh +RUN curl -fsSL https://pixi.sh/install.sh | bash +ENV PATH=$PATH:/home/gitpod/.cargo/bin +RUN curl -s "https://get.sdkman.io" | bash +SHELL ["/bin/bash", "-c"] +RUN source "/home/gitpod/.sdkman/bin/sdkman-init.sh" && sdk install java 14.0.2-zulu \ No newline at end of file diff --git a/.gitpod.yml b/.gitpod.yml index b28dfbe49f5..480baefede4 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,42 +1,20 @@ # https://www.gitpod.io/docs/config-gitpod-file +image: + file: .gitpod.Dockerfile + tasks: - init: | - python -m venv venv - source venv/bin/activate - - pip install pre-commit + uv venv + uv pip install pre-commit pre-commit install --hook-type pre-commit --hook-type pre-push - pip install '.[dev]' - make compile-protos-python - make compile-protos-go - make compile-go-lib - env: - PYTHONUSERBASE: "/workspace/.pip-modules" - command: | - source venv/bin/activate - - git config --global alias.ci 'commit -s' - git config --global alias.sw switch - git config --global alias.st status - git config --global alias.co checkout - git config --global alias.br branch - git config --global alias.df diff -github: - prebuilds: - # enable for the default branch (defaults to true) - master: true - # enable for all branches in this repo (defaults to false) - branches: false - # enable for pull requests coming from this repo (defaults to true) - pullRequests: true - # enable for pull requests coming from forks (defaults to false) - pullRequestsFromForks: false - # add a check to pull requests (defaults to true) - addCheck: true - # add a "Review in Gitpod" button as a comment to pull requests (defaults to false) - addComment: false - # add a "Review in Gitpod" button to the pull request's description (defaults to false) - addBadge: false + source .venv/bin/activate + export PYTHON=3.10 && make install-python-ci-dependencies-uv-venv + # git config --global alias.ci 'commit -s' + # git config --global alias.sw switch + # git config --global alias.st status + # git config --global alias.co checkout + # git config --global alias.br branch + # git config --global alias.df diff vscode: extensions: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f46f2af604f..7ecde0ec5d3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,13 +7,13 @@ repos: name: Format stages: [ push ] language: system - entry: make format + entry: make format-python pass_filenames: false - id: lint name: Lint stages: [ push ] language: system - entry: make lint + entry: make lint-python pass_filenames: false - id: template name: Build Templates diff --git a/CHANGELOG.md b/CHANGELOG.md index fc569e5fbba..798df5d0247 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Changelog +# [0.39.0](https://github.com/feast-dev/feast/compare/v0.38.0...v0.39.0) (2024-06-18) + + +### Bug Fixes + +* Feast UI importlib change ([#4248](https://github.com/feast-dev/feast/issues/4248)) ([5d486b8](https://github.com/feast-dev/feast/commit/5d486b8a53d799a49cc93e0f5a41aede3bc753ef)) +* Feature server no_feature_log argument error ([#4255](https://github.com/feast-dev/feast/issues/4255)) ([15524ce](https://github.com/feast-dev/feast/commit/15524cec6ba81ab6aae24b390ee63868c60c49e5)) +* Feature UI Server image won't start in an OpenShift cluster ([#4250](https://github.com/feast-dev/feast/issues/4250)) ([4891f76](https://github.com/feast-dev/feast/commit/4891f766f79a1863293412664ff8592a9e446785)) +* Handles null values in data during GO Feature retrieval ([#4274](https://github.com/feast-dev/feast/issues/4274)) ([c491e57](https://github.com/feast-dev/feast/commit/c491e5726d792f731f961b66fdf6c0b772165e86)) +* Make Java gRPC client use timeouts as expected ([#4237](https://github.com/feast-dev/feast/issues/4237)) ([f5a37c1](https://github.com/feast-dev/feast/commit/f5a37c1ce129620a4e3ee4fbe88425017f5a5ec2)) +* Remove self assignment code line. ([#4238](https://github.com/feast-dev/feast/issues/4238)) ([e514f66](https://github.com/feast-dev/feast/commit/e514f66a4c39f31bed969721bfe5c9c563786693)) +* Set default values for feature_store.serve() function ([#4225](https://github.com/feast-dev/feast/issues/4225)) ([fa74438](https://github.com/feast-dev/feast/commit/fa744380ad48ee394a05b2f600db5fb0a99c03aa)) + + +### Features + +* Add online_read_async for dynamodb ([#4244](https://github.com/feast-dev/feast/issues/4244)) ([b5ef384](https://github.com/feast-dev/feast/commit/b5ef3843499f575d4cacb9988b98b9778f67ee3b)) +* Add the ability to list objects by `tags` ([#4246](https://github.com/feast-dev/feast/issues/4246)) ([fbf92da](https://github.com/feast-dev/feast/commit/fbf92da6a4468759bfc9738f9ca581f047efb2b0)) +* Added deadline to gRPC Java client ([#4217](https://github.com/feast-dev/feast/issues/4217)) ([ff429c9](https://github.com/feast-dev/feast/commit/ff429c9f85c99478d9814e698522905d23e8d384)) +* Adding vector search for sqlite ([#4176](https://github.com/feast-dev/feast/issues/4176)) ([2478831](https://github.com/feast-dev/feast/commit/2478831e8204bc8b9204ba048a74179ac3193367)) +* Change get_online_features signature, move online retrieval functions to utils ([#4278](https://github.com/feast-dev/feast/issues/4278)) ([7287662](https://github.com/feast-dev/feast/commit/7287662f25117660160441bd61c9109b63a20d0d)) +* Feature/adding remote online store ([#4226](https://github.com/feast-dev/feast/issues/4226)) ([9454d7c](https://github.com/feast-dev/feast/commit/9454d7cb8901c59f5e7c95096cd0078cbbe953fd)) +* List all feature views ([#4256](https://github.com/feast-dev/feast/issues/4256)) ([36a574d](https://github.com/feast-dev/feast/commit/36a574d6788afca5fe2fb8776386c9462cb2ff24)) +* Make RegistryServer writable ([#4231](https://github.com/feast-dev/feast/issues/4231)) ([79e1143](https://github.com/feast-dev/feast/commit/79e11439688650bc5dc62a6fa9a9a6f54c214a50)) +* Remote offline Store ([#4262](https://github.com/feast-dev/feast/issues/4262)) ([28a3d24](https://github.com/feast-dev/feast/commit/28a3d24b12b35e4154df2bfd66dedb80bcfa3292)) +* Set optional full-scan for deletion ([#4189](https://github.com/feast-dev/feast/issues/4189)) ([b9cadd5](https://github.com/feast-dev/feast/commit/b9cadd53250f619f5ffd39232efef5461f156fde)) + # [0.38.0](https://github.com/feast-dev/feast/compare/v0.37.0...v0.38.0) (2024-05-24) diff --git a/Makefile b/Makefile index aed58ed465b..b44aaf0ee5a 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,11 @@ install-python-ci-dependencies-uv: uv pip install --system --no-deps -e . python setup.py build_python_protos --inplace +install-python-ci-dependencies-uv-venv: + uv pip sync sdk/python/requirements/py$(PYTHON)-ci-requirements.txt + uv pip install --no-deps -e . + python setup.py build_python_protos --inplace + lock-python-ci-dependencies: uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt @@ -80,19 +85,16 @@ test-python-unit: python -m pytest -n 8 --color=yes sdk/python/tests test-python-integration: - python -m pytest -n 8 --integration -k "(not snowflake or not test_historical_features_main) and not minio_registry" --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests + python -m pytest -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread \ + -k "(not snowflake or not test_historical_features_main)" \ + sdk/python/tests test-python-integration-local: - @(docker info > /dev/null 2>&1 && \ - FEAST_IS_LOCAL_TEST=True \ - FEAST_LOCAL_ONLINE_CONTAINER=True \ - python -m pytest -n 8 --color=yes --integration \ - -k "not gcs_registry and \ - not s3_registry and \ - not test_lambda_materialization and \ - not test_snowflake_materialization" \ - sdk/python/tests \ - ) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!"; + FEAST_IS_LOCAL_TEST=True \ + FEAST_LOCAL_ONLINE_CONTAINER=True \ + python -m pytest -n 8 --color=yes --integration --durations=5 --dist loadgroup \ + -k "not test_lambda_materialization and not test_snowflake_materialization" \ + sdk/python/tests test-python-integration-container: @(docker info > /dev/null 2>&1 && \ diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 2e205dee0a1..06c5edcc8b0 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -85,6 +85,7 @@ * [PostgreSQL (contrib)](reference/offline-stores/postgres.md) * [Trino (contrib)](reference/offline-stores/trino.md) * [Azure Synapse + Azure SQL (contrib)](reference/offline-stores/mssql.md) + * [Remote Offline](reference/offline-stores/remote-offline-store.md) * [Online stores](reference/online-stores/README.md) * [Overview](reference/online-stores/overview.md) * [SQLite](reference/online-stores/sqlite.md) @@ -95,6 +96,7 @@ * [Datastore](reference/online-stores/datastore.md) * [DynamoDB](reference/online-stores/dynamodb.md) * [Bigtable](reference/online-stores/bigtable.md) + * [Remote](reference/online-stores/remote.md) * [PostgreSQL (contrib)](reference/online-stores/postgres.md) * [Cassandra + Astra DB (contrib)](reference/online-stores/cassandra.md) * [MySQL (contrib)](reference/online-stores/mysql.md) @@ -117,6 +119,8 @@ * [Python feature server](reference/feature-servers/python-feature-server.md) * [\[Alpha\] Go feature server](reference/feature-servers/go-feature-server.md) * [\[Alpha\] AWS Lambda feature server](reference/feature-servers/alpha-aws-lambda-feature-server.md) + * [Offline Feature Server](reference/feature-servers/offline-feature-server) + * [\[Beta\] Web UI](reference/alpha-web-ui.md) * [\[Alpha\] On demand feature view](reference/alpha-on-demand-feature-view.md) * [\[Alpha\] Data quality monitoring](reference/dqm.md) diff --git a/docs/getting-started/architecture-and-components/README.md b/docs/getting-started/architecture-and-components/README.md index a67761b2fc6..3a2ebcf6ed5 100644 --- a/docs/getting-started/architecture-and-components/README.md +++ b/docs/getting-started/architecture-and-components/README.md @@ -1,5 +1,9 @@ # Architecture +{% content-ref url="language.md" %} +[langauge.md](language.md) +{% endcontent-ref %} + {% content-ref url="overview.md" %} [overview.md](overview.md) {% endcontent-ref %} diff --git a/docs/getting-started/architecture-and-components/language.md b/docs/getting-started/architecture-and-components/language.md new file mode 100644 index 00000000000..916dff28d74 --- /dev/null +++ b/docs/getting-started/architecture-and-components/language.md @@ -0,0 +1,46 @@ +# Python: The Language of Production Machine Learning + +Use Python to serve your features online. + + +## Why should you use Python to Serve features for Machine Learning? +Python has emerged as the primary language for machine learning, and this extends to feature serving and there are five main reasons Feast recommends using a microservice in Feast. + +## 1. Python is the language of Machine Learning + +You should meet your users where they are. Python’s popularity in the machine learning community is undeniable. Its simplicity and readability make it an ideal language for writing and understanding complex algorithms. Python boasts a rich ecosystem of libraries such as TensorFlow, PyTorch, XGBoost, and scikit-learn, which provide robust support for developing and deploying machine learning models and we want Feast in this ecosystem. + +## 2. Precomputation is The Way + +Precomputing features is the recommended optimal path to ensure low latency performance. Reducing feature serving to a lightweight database lookup is the ideal pattern, which means the marginal overhead of Python should be tolerable. Precomputation ensures product experiences for downstream services are also fast. Slow user experiences are bad user experiences. Precompute and persist data as much as you can. + +## 3. Serving features in another language can lead to skew +Ensuring that features used during model training (offline serving) and online serving are available in production to make real-time predictions is critical. When features are initially developed, they are typically written in Python. This is due to the convenience and efficiency provided by Python's data manipulation libraries. However, in a production environment, there is often interest or pressure to rewrite these features in a different language, like Java, Go, or C++, for performance reasons. This reimplementation introduces a significant risk: training and serving skew. Note that there will always be some minor exceptions (e.g., any *Time Since Last Event* types of features) but this should not be the rule. + +Training and serving skew occurs when there are discrepancies between the features used during model training and those used during prediction. This can lead to degraded model performance, unreliable predictions, and reduced velocity in releasing new features and new models. The process of rewriting features in another language is prone to errors and inconsistencies, which exacerbate this issue. + +## 4. Reimplementation is Excessive + +Rewriting features in another language is not only risky but also resource-intensive. It requires significant time and effort from engineers to ensure that the features are correctly translated. This process can introduce bugs and inconsistencies, further increasing the risk of training and serving skew. Additionally, maintaining two versions of the same feature codebase adds unnecessary complexity and overhead. More importantly, the opportunity cost of this work is high and requires twice the amount of resourcing. Reimplementing code should only be done when the performance gains are worth the investment. Features should largely be precomputed so the latency performance gains should not be the highest impact work that your team can accomplish. + +## 5. Use existing Python Optimizations + +Rather than switching languages, it is more efficient to optimize the performance of your feature store while keeping Python as the primary language. Optimization is a two step process. + +### Step 1: Quantify latency bottlenecks in your feature calculations + +Use tools like [CProfile](https://docs.python.org/3/library/profile.html) to understand latency bottlenecks in your code. This will help you prioritize the biggest inefficiencies first. When we initially launched Python native transformations in Python, [profiling the code](https://github.com/feast-dev/feast/issues/4207#issuecomment-2155754504) helped us identify that Pandas resulted in a 10x overhead due to type conversion. + +### Step 2: Optimize your feature calculations + +As mentioned, precomputation is the recommended path. In some cases, you may want fully synchronous writes from your data producer to your online feature store, in which case you will want your feature computations and writes to be very fast. In this case, we recommend optimizing the feature calculation code first. + +You should optimize your code using libraries, tools, and caching. For example, identify whether your feature calculations can be optimized through vectorized calculations in NumPy; explore tools like Numba for faster execution; and cache frequently accessed data using tools like an lru_cache. + +Lastly, Feast will continue to optimize serving in Python and making the overall infrastructure more performant. This will better serve the community. + +So we recommend focusing on optimizing your feature-specific code, reporting latency bottlenecks to the maintainers, and contributing to help the infrastructure be more performant. + +By keeping features in Python and optimizing performance, you can ensure consistency between training and serving, reduce the risk of errors, and focus on launching more product experiences for your customers. + +Embrace Python for feature serving, and leverage its strengths to build robust and reliable machine learning systems. diff --git a/docs/getting-started/architecture-and-components/overview.md b/docs/getting-started/architecture-and-components/overview.md index b6e1c48e89d..f4d543cd5a5 100644 --- a/docs/getting-started/architecture-and-components/overview.md +++ b/docs/getting-started/architecture-and-components/overview.md @@ -30,5 +30,9 @@ A complete Feast deployment contains the following components: * **Offline Store:** The offline store persists batch data that has been ingested into Feast. This data is used for producing training datasets. For feature retrieval and materialization, Feast does not manage the offline store directly, but runs queries against it. However, offline stores can be configured to support writes if Feast configures logging functionality of served features. {% hint style="info" %} -Java and Go Clients are also available for online feature retrieval. +Java and Go Clients are also available for online feature retrieval. + +In general, we recommend [using Python](language.md) for your Feature Store microservice. + +As mentioned in the document, precomputing features is the recommended optimal path to ensure low latency performance. Reducing feature serving to a lightweight database lookup is the ideal pattern, which means the marginal overhead of Python should be tolerable. Because of this we believe the pros of Python outweigh the costs, as reimplementing feature logic is undesirable. {% endhint %} diff --git a/docs/getting-started/faq.md b/docs/getting-started/faq.md index 9b7eb834bf2..8948eed5880 100644 --- a/docs/getting-started/faq.md +++ b/docs/getting-started/faq.md @@ -8,6 +8,9 @@ We encourage you to ask questions on [GitHub](https://github.com/feast-dev/feast ## Getting started +### Which programming language should I use to run Feast in a microservice architecture? +[We recommend Python](language.md). + ### Do you have any examples of how Feast should be used? The [quickstart](quickstart.md) is the easiest way to learn about Feast. For more detailed tutorials, please check out the [tutorials](../tutorials/tutorials-overview/) page. diff --git a/docs/project/release-process.md b/docs/project/release-process.md index d3ff34bbc38..e6f75ffd413 100644 --- a/docs/project/release-process.md +++ b/docs/project/release-process.md @@ -4,6 +4,12 @@ For Feast maintainers, these are the concrete steps for making a new release. +### 0. Cutting a minor release +You only need to hit the `release` workflow using [the GitHub action](https://github.com/feast-dev/feast/blob/master/.github/workflows/release.yml). +First test with a `dry-run` then run it live. This is all you need to do. All deployments to dockerhub, PyPI, and npm are handled by the workflows. + +Also note that as a part of the workflow, the [infra/scripts/release/bump_file_versions.py](https://github.com/feast-dev/feast/blob/master/infra/scripts/release/bump_file_versions.py) file will increment the Feast versions in the appropriate files. + ### 1. (for patch releases) Cherry-pick changes into the branch from master If you were cutting Feast 0.22.3, for example, you might do: 1. `git checkout v0.22-branch` (or `git pull upstream v0.22-branch --rebase` if you've cut a release before) @@ -16,6 +22,8 @@ If you were cutting Feast 0.22.3, for example, you might do: After this step, you will have all the changes you need in the branch. +Note, for patches you *do not need to run the `bump_file_versions.py` script.* + ### 2. Pre-release verification (currently broken) A lot of things can go wrong. One of the most common is getting the wheels to build correctly (and not accidentally building dev wheels from improper tagging or local code changes during the release process). diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index 37d9b9cdf87..b9ce7f408a0 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -13,7 +13,9 @@ Below are supported vector databases and implemented features: | Elasticsearch | [x] | [x] | | Milvus | [ ] | [ ] | | Faiss | [ ] | [ ] | +| SQLite | [x] | [ ] | +Note: SQLite is in limited access and only working on Python 3.10. It will be updated as [sqlite_vec](https://github.com/asg017/sqlite-vec/) progresses. ## Example @@ -108,4 +110,20 @@ def print_online_features(features): print(key, " : ", value) print_online_features(features) +``` + +### Configuration +We offer two Online Store options for Vector Databases. PGVector and SQLite. + +#### Installation with SQLite +If you are using `pyenv` to manage your Python versions, you can install the SQLite extension with the following command: +```bash +PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" \ + LDFLAGS="-L/opt/homebrew/opt/sqlite/lib" \ + CPPFLAGS="-I/opt/homebrew/opt/sqlite/include" \ + pyenv install 3.10.14 +``` +And you can the Feast install package via: +```bash +pip install feast[sqlite_vec] ``` \ No newline at end of file diff --git a/docs/reference/feast-cli-commands.md b/docs/reference/feast-cli-commands.md index 7bdea19e610..afcfcfef640 100644 --- a/docs/reference/feast-cli-commands.md +++ b/docs/reference/feast-cli-commands.md @@ -66,6 +66,10 @@ List all registered entities ```text feast entities list + +Options: + --tags TEXT Filter by tags (e.g. --tags 'key:value' --tags 'key:value, + key:value, ...'). Items return when ALL tags match. ``` ```text @@ -79,11 +83,15 @@ List all registered feature views ```text feast feature-views list + +Options: + --tags TEXT Filter by tags (e.g. --tags 'key:value' --tags 'key:value, + key:value, ...'). Items return when ALL tags match. ``` ```text -NAME ENTITIES -driver_hourly_stats ['driver_id'] +NAME ENTITIES TYPE +driver_hourly_stats {'driver'} FeatureView ``` ## Init diff --git a/docs/reference/feature-servers/README.md b/docs/reference/feature-servers/README.md index f9a40104c3a..d5a4312f73a 100644 --- a/docs/reference/feature-servers/README.md +++ b/docs/reference/feature-servers/README.md @@ -12,4 +12,8 @@ Feast users can choose to retrieve features from a feature server, as opposed to {% content-ref url="alpha-aws-lambda-feature-server.md" %} [alpha-aws-lambda-feature-server.md](alpha-aws-lambda-feature-server.md) +{% endcontent-ref %} + +{% content-ref url="offline-feature-server.md" %} +[offline-feature-server.md](offline-feature-server.md) {% endcontent-ref %} \ No newline at end of file diff --git a/docs/reference/feature-servers/offline-feature-server.md b/docs/reference/feature-servers/offline-feature-server.md new file mode 100644 index 00000000000..6c2fdf7a259 --- /dev/null +++ b/docs/reference/feature-servers/offline-feature-server.md @@ -0,0 +1,35 @@ +# Offline feature server + +## Description + +The Offline feature server is an Apache Arrow Flight Server that uses the gRPC communication protocol to exchange data. +This server wraps calls to existing offline store implementations and exposes interfaces as Arrow Flight endpoints. + +## How to configure the server + +## CLI + +There is a CLI command that starts the Offline feature server: `feast serve_offline`. By default, remote offline server uses port 8815, the port can be overridden with a `--port` flag. + +## Deploying as a service on Kubernetes + +The Offline feature server can be deployed using helm chart see this [helm chart](https://github.com/feast-dev/feast/blob/master/infra/charts/feast-feature-server). + +User need to set `feast_mode=offline`, when installing Offline feature server as shown in the helm command below: + +``` +helm install feast-offline-server feast-charts/feast-feature-server --set feast_mode=offline --set feature_store_yaml_base64=$(base64 > feature_store.yaml) +``` + +## Server Example + +The complete example can be find under [remote-offline-store-example](../../../examples/remote-offline-store) + +## How to configure the client + +Please see the detail how to configure offline store client [remote-offline-store.md](../offline-stores/remote-offline-store.md) + +## Functionality Matrix + +The set of functionalities supported by remote offline stores is the same as those supported by offline stores with the SDK, which are described in detail [here](../offline-stores/overview.md#functionality). + diff --git a/docs/reference/offline-stores/remote-offline-store.md b/docs/reference/offline-stores/remote-offline-store.md new file mode 100644 index 00000000000..0179e0f06f8 --- /dev/null +++ b/docs/reference/offline-stores/remote-offline-store.md @@ -0,0 +1,28 @@ +# Remote Offline Store + +## Description + +The Remote Offline Store is an Arrow Flight client for the offline store that implements the `RemoteOfflineStore` class using the existing `OfflineStore` interface. +The client implements various methods, including `get_historical_features`, `pull_latest_from_table_or_query`, `write_logged_features`, and `offline_write_batch`. + +## How to configure the client + +User needs to create client side `feature_store.yaml` file and set the `offline_store` type `remote` and provide the server connection configuration +including adding the host and specifying the port (default is 8815) required by the Arrow Flight client to connect with the Arrow Flight server. + +{% code title="feature_store.yaml" %} +```yaml +offline_store: + type: remote + host: localhost + port: 8815 +``` +{% endcode %} + +## Client Example + +The complete example can be find under [remote-offline-store-example](../../../examples/remote-offline-store) + +## How to configure the server + +Please see the detail how to configure offline feature server [offline-feature-server.md](../feature-servers/offline-feature-server.md) \ No newline at end of file diff --git a/docs/reference/offline-stores/snowflake.md b/docs/reference/offline-stores/snowflake.md index 9f2dafee671..39bbe3f8a03 100644 --- a/docs/reference/offline-stores/snowflake.md +++ b/docs/reference/offline-stores/snowflake.md @@ -34,6 +34,21 @@ offline_store: The full set of configuration options is available in [SnowflakeOfflineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.offline_stores.snowflake.SnowflakeOfflineStoreConfig). + +## Limitation +Please be aware that here is a restriction/limitation for using SQL query string in Feast with Snowflake. Try to avoid the usage of single quote in SQL query string. For example, the following query string will fail: +``` +SELECT + some_column +FROM + some_table +WHERE + other_column = 'value' +``` +That 'value' will fail in Snowflake. Instead, please use pairs of dollar signs like `$$value$$` as [mentioned in Snowflake document](https://docs.snowflake.com/en/sql-reference/data-types-text#label-dollar-quoted-string-constants). + + + ## Functionality Matrix The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index 686e820f4e7..b5f4eb8de89 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -61,3 +61,7 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli {% content-ref url="scylladb.md" %} [scylladb.md](scylladb.md) {% endcontent-ref %} + +{% content-ref url="remote.md" %} +[remote.md](remote.md) +{% endcontent-ref %} diff --git a/docs/reference/online-stores/remote.md b/docs/reference/online-stores/remote.md new file mode 100644 index 00000000000..c560fa6f223 --- /dev/null +++ b/docs/reference/online-stores/remote.md @@ -0,0 +1,21 @@ +# Remote online store + +## Description + +This remote online store will let you interact with remote feature server. At this moment this only supports the read operation. You can use this online store and able retrieve online features `store.get_online_features` from remote feature server. + +## Examples + +The registry is pointing to registry of remote feature store. If it is not accessible then should be configured to use remote registry. + +{% code title="feature_store.yaml" %} +```yaml +project: my-local-project + registry: /remote/data/registry.db + provider: local + online_store: + path: http://localhost:6566 + type: remote + entity_key_serialization_version: 2 +``` +{% endcode %} \ No newline at end of file diff --git a/examples/remote-offline-store/README.md b/examples/remote-offline-store/README.md new file mode 100644 index 00000000000..c07d7f30419 --- /dev/null +++ b/examples/remote-offline-store/README.md @@ -0,0 +1,98 @@ +# Feast Remote Offline Store Server + +This example demonstrates the steps using an [Arrow Flight](https://arrow.apache.org/blog/2019/10/13/introducing-arrow-flight/) server/client as the remote Feast offline store. + +## Launch the offline server locally + +1. **Create Feast Project**: Using the `feast init` command for example the [offline_server](./offline_server) folder contains a sample Feast repository. + +2. **Start Remote Offline Server**: Use the `feast server_offline` command to start remote offline requests. This command will: + - Spin up an `Arrow Flight` server at the default port 8815. + +3. **Initialize Offline Server**: The offline server can be initialized by providing the `feature_store.yml` file via an environment variable named `FEATURE_STORE_YAML_BASE64`. A temporary directory will be created with the provided YAML file named `feature_store.yml`. + +Example + +```console +cd offline_server +feast -c feature_repo apply +``` + +```console +feast -c feature_repo serve_offline +``` + +Sample output: +```console +Serving on grpc+tcp://127.0.0.1:8815 +``` + +## Launch a remote offline client + +The [offline_client](./offline_client) folder includes a test python function that uses an offline store of type `remote`, leveraging the remote server as the +actual data provider. + + +The test class is located under [offline_client](./offline_client/) and uses a remote configuration of the offline store to delegate the actual +implementation to the offline store server: +```yaml +offline_store: + type: remote + host: localhost + port: 8815 +``` + +The test code in [test.py](./offline_client/test.py) initializes the store from the local configuration and then fetches the historical features +from the store like any other Feast client, but the actual implementation is delegated to the offline server +```py +store = FeatureStore(repo_path=".") +training_df = store.get_historical_features(entity_df, features).to_df() +``` + + +Run client +`cd offline_client; + python test.py` + +Sample output: + +```console +config.offline_store is +----- Feature schema ----- + + +RangeIndex: 3 entries, 0 to 2 +Data columns (total 10 columns): + # Column Non-Null Count Dtype +--- ------ -------------- ----- + 0 driver_id 3 non-null int64 + 1 event_timestamp 3 non-null datetime64[ns, UTC] + 2 label_driver_reported_satisfaction 3 non-null int64 + 3 val_to_add 3 non-null int64 + 4 val_to_add_2 3 non-null int64 + 5 conv_rate 3 non-null float32 + 6 acc_rate 3 non-null float32 + 7 avg_daily_trips 3 non-null int32 + 8 conv_rate_plus_val1 3 non-null float64 + 9 conv_rate_plus_val2 3 non-null float64 +dtypes: datetime64[ns, UTC](1), float32(2), float64(2), int32(1), int64(4) +memory usage: 332.0 bytes +None + +----- Features ----- + + driver_id event_timestamp label_driver_reported_satisfaction ... avg_daily_trips conv_rate_plus_val1 conv_rate_plus_val2 +0 1001 2021-04-12 10:59:42+00:00 1 ... 590 1.022378 10.022378 +1 1002 2021-04-12 08:12:10+00:00 5 ... 974 2.762213 20.762213 +2 1003 2021-04-12 16:40:26+00:00 3 ... 127 3.419828 30.419828 + +[3 rows x 10 columns] +------training_df---- + driver_id event_timestamp label_driver_reported_satisfaction ... avg_daily_trips conv_rate_plus_val1 conv_rate_plus_val2 +0 1001 2021-04-12 10:59:42+00:00 1 ... 590 1.022378 10.022378 +1 1002 2021-04-12 08:12:10+00:00 5 ... 974 2.762213 20.762213 +2 1003 2021-04-12 16:40:26+00:00 3 ... 127 3.419828 30.419828 + +[3 rows x 10 columns] +``` + diff --git a/sdk/python/feast/infra/registry/contrib/postgres/__init__.py b/examples/remote-offline-store/offline_client/__init__.py similarity index 100% rename from sdk/python/feast/infra/registry/contrib/postgres/__init__.py rename to examples/remote-offline-store/offline_client/__init__.py diff --git a/examples/remote-offline-store/offline_client/feature_store.yaml b/examples/remote-offline-store/offline_client/feature_store.yaml new file mode 100644 index 00000000000..24ee5d70426 --- /dev/null +++ b/examples/remote-offline-store/offline_client/feature_store.yaml @@ -0,0 +1,10 @@ +project: offline_server +# By default, the registry is a file (but can be turned into a more scalable SQL-backed registry) +registry: ../offline_server/feature_repo/data/registry.db +# The provider primarily specifies default offline / online stores & storing the registry in a given cloud +provider: local +offline_store: + type: remote + host: localhost + port: 8815 +entity_key_serialization_version: 2 diff --git a/examples/remote-offline-store/offline_client/test.py b/examples/remote-offline-store/offline_client/test.py new file mode 100644 index 00000000000..172ee73bf09 --- /dev/null +++ b/examples/remote-offline-store/offline_client/test.py @@ -0,0 +1,40 @@ +from datetime import datetime +from feast import FeatureStore +import pandas as pd + +entity_df = pd.DataFrame.from_dict( + { + "driver_id": [1001, 1002, 1003], + "event_timestamp": [ + datetime(2021, 4, 12, 10, 59, 42), + datetime(2021, 4, 12, 8, 12, 10), + datetime(2021, 4, 12, 16, 40, 26), + ], + "label_driver_reported_satisfaction": [1, 5, 3], + "val_to_add": [1, 2, 3], + "val_to_add_2": [10, 20, 30], + } +) + +features = [ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + "transformed_conv_rate:conv_rate_plus_val1", + "transformed_conv_rate:conv_rate_plus_val2", +] + +store = FeatureStore(repo_path=".") + +training_df = store.get_historical_features(entity_df, features).to_df() + +print("----- Feature schema -----\n") +print(training_df.info()) + +print() +print("----- Features -----\n") +print(training_df.head()) + +print("------training_df----") + +print(training_df) diff --git a/examples/remote-offline-store/offline_server/__init__.py b/examples/remote-offline-store/offline_server/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/examples/remote-offline-store/offline_server/feature_repo/__init__.py b/examples/remote-offline-store/offline_server/feature_repo/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/examples/remote-offline-store/offline_server/feature_repo/data/driver_stats.parquet b/examples/remote-offline-store/offline_server/feature_repo/data/driver_stats.parquet new file mode 100644 index 00000000000..19279202d84 Binary files /dev/null and b/examples/remote-offline-store/offline_server/feature_repo/data/driver_stats.parquet differ diff --git a/examples/remote-offline-store/offline_server/feature_repo/data/online_store.db b/examples/remote-offline-store/offline_server/feature_repo/data/online_store.db new file mode 100644 index 00000000000..d230f45b934 Binary files /dev/null and b/examples/remote-offline-store/offline_server/feature_repo/data/online_store.db differ diff --git a/examples/remote-offline-store/offline_server/feature_repo/example_repo.py b/examples/remote-offline-store/offline_server/feature_repo/example_repo.py new file mode 100644 index 00000000000..c06ebc788b2 --- /dev/null +++ b/examples/remote-offline-store/offline_server/feature_repo/example_repo.py @@ -0,0 +1,140 @@ +# This is an example feature definition file + +from datetime import timedelta + +import pandas as pd +import os + +from feast import ( + Entity, + FeatureService, + FeatureView, + Field, + FileSource, + PushSource, + RequestSource, +) +from feast.on_demand_feature_view import on_demand_feature_view +from feast.types import Float32, Float64, Int64 + +# Define an entity for the driver. You can think of an entity as a primary key used to +# fetch features. +driver = Entity(name="driver", join_keys=["driver_id"]) + +# Read data from parquet files. Parquet is convenient for local development mode. For +# production, you can use your favorite DWH, such as BigQuery. See Feast documentation +# for more info. +driver_stats_source = FileSource( + name="driver_hourly_stats_source", + path=f"{os.path.dirname(os.path.abspath(__file__))}/data/driver_stats.parquet", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) + +# Our parquet files contain sample data that includes a driver_id column, timestamps and +# three feature column. Here we define a Feature View that will allow us to serve this +# data to our model online. +driver_stats_fv = FeatureView( + # The unique name of this feature view. Two feature views in a single + # project cannot have the same name + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=1), + # The list of features defined below act as a schema to both define features + # for both materialization of features into a store, and are used as references + # during retrieval for building a training dataset or serving features + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64, description="Average daily trips"), + ], + online=True, + source=driver_stats_source, + # Tags are user defined key/value pairs that are attached to each + # feature view + tags={"team": "driver_performance"}, +) + +# Define a request data source which encodes features / information only +# available at request time (e.g. part of the user initiated HTTP request) +input_request = RequestSource( + name="vals_to_add", + schema=[ + Field(name="val_to_add", dtype=Int64), + Field(name="val_to_add_2", dtype=Int64), + ], +) + + +# Define an on demand feature view which can generate new features based on +# existing feature views and RequestSource features +@on_demand_feature_view( + sources=[driver_stats_fv, input_request], + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], +) +def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] + df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] + return df + + +# This groups features into a model version +driver_activity_v1 = FeatureService( + name="driver_activity_v1", + features=[ + driver_stats_fv[["conv_rate"]], # Sub-selects a feature from a feature view + transformed_conv_rate, # Selects all features from the feature view + ], +) +driver_activity_v2 = FeatureService( + name="driver_activity_v2", features=[driver_stats_fv, transformed_conv_rate] +) + +# Defines a way to push data (to be available offline, online or both) into Feast. +driver_stats_push_source = PushSource( + name="driver_stats_push_source", + batch_source=driver_stats_source, +) + +# Defines a slightly modified version of the feature view from above, where the source +# has been changed to the push source. This allows fresh features to be directly pushed +# to the online store for this feature view. +driver_stats_fresh_fv = FeatureView( + name="driver_hourly_stats_fresh", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_push_source, # Changed from above + tags={"team": "driver_performance"}, +) + + +# Define an on demand feature view which can generate new features based on +# existing feature views and RequestSource features +@on_demand_feature_view( + sources=[driver_stats_fresh_fv, input_request], # relies on fresh version of FV + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], +) +def transformed_conv_rate_fresh(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] + df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] + return df + + +driver_activity_v3 = FeatureService( + name="driver_activity_v3", + features=[driver_stats_fresh_fv, transformed_conv_rate_fresh], +) diff --git a/examples/remote-offline-store/offline_server/feature_repo/feature_store.yaml b/examples/remote-offline-store/offline_server/feature_repo/feature_store.yaml new file mode 100644 index 00000000000..a751706d07a --- /dev/null +++ b/examples/remote-offline-store/offline_server/feature_repo/feature_store.yaml @@ -0,0 +1,9 @@ +project: offline_server +# By default, the registry is a file (but can be turned into a more scalable SQL-backed registry) +registry: data/registry.db +# The provider primarily specifies default offline / online stores & storing the registry in a given cloud +provider: local +online_store: + type: sqlite + path: data/online_store.db +entity_key_serialization_version: 2 diff --git a/go/types/typeconversion.go b/go/types/typeconversion.go index 45eeac52c6f..18b4769b4d7 100644 --- a/go/types/typeconversion.go +++ b/go/types/typeconversion.go @@ -11,6 +11,9 @@ import ( ) func ProtoTypeToArrowType(sample *types.Value) (arrow.DataType, error) { + if sample.Val == nil { + return nil, nil + } switch sample.Val.(type) { case *types.Value_BytesVal: return arrow.BinaryTypes.Binary, nil @@ -91,81 +94,71 @@ func ValueTypeEnumToArrowType(t types.ValueType_Enum) (arrow.DataType, error) { } func CopyProtoValuesToArrowArray(builder array.Builder, values []*types.Value) error { - switch fieldBuilder := builder.(type) { - case *array.BooleanBuilder: - for _, v := range values { - fieldBuilder.Append(v.GetBoolVal()) - } - case *array.BinaryBuilder: - for _, v := range values { - fieldBuilder.Append(v.GetBytesVal()) - } - case *array.StringBuilder: - for _, v := range values { - fieldBuilder.Append(v.GetStringVal()) - } - case *array.Int32Builder: - for _, v := range values { - fieldBuilder.Append(v.GetInt32Val()) - } - case *array.Int64Builder: - for _, v := range values { - fieldBuilder.Append(v.GetInt64Val()) - } - case *array.Float32Builder: - for _, v := range values { - fieldBuilder.Append(v.GetFloatVal()) + for _, value := range values { + if value == nil || value.Val == nil { + builder.AppendNull() + continue } - case *array.Float64Builder: - for _, v := range values { - fieldBuilder.Append(v.GetDoubleVal()) - } - case *array.TimestampBuilder: - for _, v := range values { - fieldBuilder.Append(arrow.Timestamp(v.GetUnixTimestampVal())) - } - case *array.ListBuilder: - for _, list := range values { + + switch fieldBuilder := builder.(type) { + + case *array.BooleanBuilder: + fieldBuilder.Append(value.GetBoolVal()) + case *array.BinaryBuilder: + fieldBuilder.Append(value.GetBytesVal()) + case *array.StringBuilder: + fieldBuilder.Append(value.GetStringVal()) + case *array.Int32Builder: + fieldBuilder.Append(value.GetInt32Val()) + case *array.Int64Builder: + fieldBuilder.Append(value.GetInt64Val()) + case *array.Float32Builder: + fieldBuilder.Append(value.GetFloatVal()) + case *array.Float64Builder: + fieldBuilder.Append(value.GetDoubleVal()) + case *array.TimestampBuilder: + fieldBuilder.Append(arrow.Timestamp(value.GetUnixTimestampVal())) + case *array.ListBuilder: fieldBuilder.Append(true) switch valueBuilder := fieldBuilder.ValueBuilder().(type) { case *array.BooleanBuilder: - for _, v := range list.GetBoolListVal().GetVal() { + for _, v := range value.GetBoolListVal().GetVal() { valueBuilder.Append(v) } case *array.BinaryBuilder: - for _, v := range list.GetBytesListVal().GetVal() { + for _, v := range value.GetBytesListVal().GetVal() { valueBuilder.Append(v) } case *array.StringBuilder: - for _, v := range list.GetStringListVal().GetVal() { + for _, v := range value.GetStringListVal().GetVal() { valueBuilder.Append(v) } case *array.Int32Builder: - for _, v := range list.GetInt32ListVal().GetVal() { + for _, v := range value.GetInt32ListVal().GetVal() { valueBuilder.Append(v) } case *array.Int64Builder: - for _, v := range list.GetInt64ListVal().GetVal() { + for _, v := range value.GetInt64ListVal().GetVal() { valueBuilder.Append(v) } case *array.Float32Builder: - for _, v := range list.GetFloatListVal().GetVal() { + for _, v := range value.GetFloatListVal().GetVal() { valueBuilder.Append(v) } case *array.Float64Builder: - for _, v := range list.GetDoubleListVal().GetVal() { + for _, v := range value.GetDoubleListVal().GetVal() { valueBuilder.Append(v) } case *array.TimestampBuilder: - for _, v := range list.GetUnixTimestampListVal().GetVal() { + for _, v := range value.GetUnixTimestampListVal().GetVal() { valueBuilder.Append(arrow.Timestamp(v)) } } + default: + return fmt.Errorf("unsupported array builder: %s", builder) } - default: - return fmt.Errorf("unsupported array builder: %s", builder) } return nil } @@ -249,41 +242,68 @@ func ArrowValuesToProtoValues(arr arrow.Array) ([]*types.Value, error) { switch arr.DataType() { case arrow.PrimitiveTypes.Int32: - for _, v := range arr.(*array.Int32).Int32Values() { - values = append(values, &types.Value{Val: &types.Value_Int32Val{Int32Val: v}}) + for idx := 0; idx < arr.Len(); idx++ { + if arr.IsNull(idx) { + values = append(values, &types.Value{}) + } else { + values = append(values, &types.Value{Val: &types.Value_Int32Val{Int32Val: arr.(*array.Int32).Value(idx)}}) + } } case arrow.PrimitiveTypes.Int64: - for _, v := range arr.(*array.Int64).Int64Values() { - values = append(values, &types.Value{Val: &types.Value_Int64Val{Int64Val: v}}) + for idx := 0; idx < arr.Len(); idx++ { + if arr.IsNull(idx) { + values = append(values, &types.Value{}) + } else { + values = append(values, &types.Value{Val: &types.Value_Int64Val{Int64Val: arr.(*array.Int64).Value(idx)}}) + } } case arrow.PrimitiveTypes.Float32: - for _, v := range arr.(*array.Float32).Float32Values() { - values = append(values, &types.Value{Val: &types.Value_FloatVal{FloatVal: v}}) + for idx := 0; idx < arr.Len(); idx++ { + if arr.IsNull(idx) { + values = append(values, &types.Value{}) + } else { + values = append(values, &types.Value{Val: &types.Value_FloatVal{FloatVal: arr.(*array.Float32).Value(idx)}}) + } } case arrow.PrimitiveTypes.Float64: - for _, v := range arr.(*array.Float64).Float64Values() { - values = append(values, &types.Value{Val: &types.Value_DoubleVal{DoubleVal: v}}) + for idx := 0; idx < arr.Len(); idx++ { + if arr.IsNull(idx) { + values = append(values, &types.Value{}) + } else { + values = append(values, &types.Value{Val: &types.Value_DoubleVal{DoubleVal: arr.(*array.Float64).Value(idx)}}) + } } case arrow.FixedWidthTypes.Boolean: for idx := 0; idx < arr.Len(); idx++ { - values = append(values, - &types.Value{Val: &types.Value_BoolVal{BoolVal: arr.(*array.Boolean).Value(idx)}}) + if arr.IsNull(idx) { + values = append(values, &types.Value{}) + } else { + values = append(values, &types.Value{Val: &types.Value_BoolVal{BoolVal: arr.(*array.Boolean).Value(idx)}}) + } } case arrow.BinaryTypes.Binary: for idx := 0; idx < arr.Len(); idx++ { - values = append(values, - &types.Value{Val: &types.Value_BytesVal{BytesVal: arr.(*array.Binary).Value(idx)}}) + if arr.IsNull(idx) { + values = append(values, &types.Value{}) + } else { + values = append(values, &types.Value{Val: &types.Value_BytesVal{BytesVal: arr.(*array.Binary).Value(idx)}}) + } } case arrow.BinaryTypes.String: for idx := 0; idx < arr.Len(); idx++ { - values = append(values, - &types.Value{Val: &types.Value_StringVal{StringVal: arr.(*array.String).Value(idx)}}) + if arr.IsNull(idx) { + values = append(values, &types.Value{}) + } else { + values = append(values, &types.Value{Val: &types.Value_StringVal{StringVal: arr.(*array.String).Value(idx)}}) + } } case arrow.FixedWidthTypes.Timestamp_s: for idx := 0; idx < arr.Len(); idx++ { - values = append(values, - &types.Value{Val: &types.Value_UnixTimestampVal{ - UnixTimestampVal: int64(arr.(*array.Timestamp).Value(idx))}}) + if arr.IsNull(idx) { + values = append(values, &types.Value{}) + } else { + values = append(values, &types.Value{Val: &types.Value_UnixTimestampVal{UnixTimestampVal: int64(arr.(*array.Timestamp).Value(idx))}}) + } } case arrow.Null: for idx := 0; idx < arr.Len(); idx++ { @@ -306,7 +326,9 @@ func ProtoValuesToArrowArray(protoValues []*types.Value, arrowAllocator memory.A if err != nil { return nil, err } - break + if fieldType != nil { + break + } } } diff --git a/go/types/typeconversion_test.go b/go/types/typeconversion_test.go index 1f89593ea01..4869369c186 100644 --- a/go/types/typeconversion_test.go +++ b/go/types/typeconversion_test.go @@ -1,27 +1,46 @@ package types import ( + "math" "testing" "time" "github.com/apache/arrow/go/v8/arrow/memory" - "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/proto" "github.com/feast-dev/feast/go/protos/feast/types" ) +var nil_or_null_val = &types.Value{} + var ( PROTO_VALUES = [][]*types.Value{ + {{Val: nil}}, + {{Val: nil}, {Val: nil}}, + {nil_or_null_val, nil_or_null_val}, + {nil_or_null_val, {Val: nil}}, + {{Val: &types.Value_Int32Val{10}}, {Val: nil}, nil_or_null_val, {Val: &types.Value_Int32Val{20}}}, + {{Val: &types.Value_Int32Val{10}}, nil_or_null_val}, + {nil_or_null_val, {Val: &types.Value_Int32Val{20}}}, {{Val: &types.Value_Int32Val{10}}, {Val: &types.Value_Int32Val{20}}}, + {{Val: &types.Value_Int64Val{10}}, nil_or_null_val}, {{Val: &types.Value_Int64Val{10}}, {Val: &types.Value_Int64Val{20}}}, + {nil_or_null_val, {Val: &types.Value_FloatVal{2.0}}}, {{Val: &types.Value_FloatVal{1.0}}, {Val: &types.Value_FloatVal{2.0}}}, + {{Val: &types.Value_FloatVal{1.0}}, {Val: &types.Value_FloatVal{2.0}}, {Val: &types.Value_FloatVal{float32(math.NaN())}}}, {{Val: &types.Value_DoubleVal{1.0}}, {Val: &types.Value_DoubleVal{2.0}}}, + {{Val: &types.Value_DoubleVal{1.0}}, {Val: &types.Value_DoubleVal{2.0}}, {Val: &types.Value_DoubleVal{math.NaN()}}}, + {{Val: &types.Value_DoubleVal{1.0}}, nil_or_null_val}, + {nil_or_null_val, {Val: &types.Value_StringVal{"bbb"}}}, {{Val: &types.Value_StringVal{"aaa"}}, {Val: &types.Value_StringVal{"bbb"}}}, + {{Val: &types.Value_BytesVal{[]byte{1, 2, 3}}}, nil_or_null_val}, {{Val: &types.Value_BytesVal{[]byte{1, 2, 3}}}, {Val: &types.Value_BytesVal{[]byte{4, 5, 6}}}}, + {nil_or_null_val, {Val: &types.Value_BoolVal{false}}}, {{Val: &types.Value_BoolVal{true}}, {Val: &types.Value_BoolVal{false}}}, - {{Val: &types.Value_UnixTimestampVal{time.Now().Unix()}}, - {Val: &types.Value_UnixTimestampVal{time.Now().Unix()}}}, + {{Val: &types.Value_UnixTimestampVal{time.Now().Unix()}}, nil_or_null_val}, + {{Val: &types.Value_UnixTimestampVal{time.Now().Unix()}}, {Val: &types.Value_UnixTimestampVal{time.Now().Unix()}}}, + {{Val: &types.Value_UnixTimestampVal{time.Now().Unix()}}, {Val: &types.Value_UnixTimestampVal{time.Now().Unix()}}, {Val: &types.Value_UnixTimestampVal{-9223372036854775808}}}, { {Val: &types.Value_Int32ListVal{&types.Int32List{Val: []int32{0, 1, 2}}}}, @@ -55,6 +74,11 @@ var ( {Val: &types.Value_UnixTimestampListVal{&types.Int64List{Val: []int64{time.Now().Unix()}}}}, {Val: &types.Value_UnixTimestampListVal{&types.Int64List{Val: []int64{time.Now().Unix()}}}}, }, + { + {Val: &types.Value_UnixTimestampListVal{&types.Int64List{Val: []int64{time.Now().Unix(), time.Now().Unix()}}}}, + {Val: &types.Value_UnixTimestampListVal{&types.Int64List{Val: []int64{time.Now().Unix(), time.Now().Unix()}}}}, + {Val: &types.Value_UnixTimestampListVal{&types.Int64List{Val: []int64{-9223372036854775808, time.Now().Unix()}}}}, + }, } ) diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index fca8f0c98c5..aa39b158dd7 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.38.0 +version: 0.39.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index 457aeff2452..121b0cc0cd9 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.38.0` +Current chart version is `0.39.0` ## Installation @@ -13,9 +13,18 @@ helm repo update Install Feast Feature Server on Kubernetes -A base64 encoded version of the `feature_store.yaml` file is needed. Helm install example: +- Feast Deployment Mode: The Feast Feature Server supports multiple deployment modes using the `feast_mode` property. Supported modes are `online` (default), `offline`, `ui`, and `registry`. +Users can set the `feast_mode` based on their deployment choice. The `online` mode is the default and maintains backward compatibility with previous Feast Feature Server implementations. + +- Feature Store File: A base64 encoded version of the `feature_store.yaml` file is needed. + +Helm install examples: ``` -helm install feast-feature-server feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 feature_store.yaml) +helm install feast-feature-server feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 > feature_store.yaml) +helm install feast-offline-server feast-charts/feast-feature-server --set feast_mode=offline --set feature_store_yaml_base64=$(base64 > feature_store.yaml) +helm install feast-ui-server feast-charts/feast-feature-server --set feast_mode=ui --set feature_store_yaml_base64=$(base64 > feature_store.yaml) +helm install feast-registry-server feast-charts/feast-feature-server --set feast_mode=registry --set feature_store_yaml_base64=$(base64 > feature_store.yaml) + ``` ## Tutorial @@ -26,11 +35,12 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | Key | Type | Default | Description | |-----|------|---------|-------------| | affinity | object | `{}` | | +| feast_mode | string | `"online"` | Feast supported deployment modes - online (default), offline, ui and registry | | feature_store_yaml_base64 | string | `""` | [required] a base64 encoded version of feature_store.yaml | | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.38.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.39.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/README.md.gotmpl b/infra/charts/feast-feature-server/README.md.gotmpl index fb877208e06..be2fdae2482 100644 --- a/infra/charts/feast-feature-server/README.md.gotmpl +++ b/infra/charts/feast-feature-server/README.md.gotmpl @@ -13,9 +13,18 @@ helm repo update Install Feast Feature Server on Kubernetes -A base64 encoded version of the `feature_store.yaml` file is needed. Helm install example: +- Feast Deployment Mode: The Feast Feature Server supports multiple deployment modes using the `feast_mode` property. Supported modes are `online` (default), `offline`, `ui`, and `registry`. +Users can set the `feast_mode` based on their deployment choice. The `online` mode is the default and maintains backward compatibility with previous Feast Feature Server implementations. + +- Feature Store File: A base64 encoded version of the `feature_store.yaml` file is needed. + +Helm install examples: ``` -helm install feast-feature-server feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 feature_store.yaml) +helm install feast-feature-server feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 > feature_store.yaml) +helm install feast-offline-server feast-charts/feast-feature-server --set feast_mode=offline --set feature_store_yaml_base64=$(base64 > feature_store.yaml) +helm install feast-ui-server feast-charts/feast-feature-server --set feast_mode=ui --set feature_store_yaml_base64=$(base64 > feature_store.yaml) +helm install feast-registry-server feast-charts/feast-feature-server --set feast_mode=registry --set feature_store_yaml_base64=$(base64 > feature_store.yaml) + ``` ## Tutorial diff --git a/infra/charts/feast-feature-server/templates/deployment.yaml b/infra/charts/feast-feature-server/templates/deployment.yaml index 94c56de9dda..85b323610d2 100644 --- a/infra/charts/feast-feature-server/templates/deployment.yaml +++ b/infra/charts/feast-feature-server/templates/deployment.yaml @@ -33,19 +33,46 @@ spec: env: - name: FEATURE_STORE_YAML_BASE64 value: {{ .Values.feature_store_yaml_base64 }} - command: ["feast", "serve", "-h", "0.0.0.0"] + command: + {{- if eq .Values.feast_mode "offline" }} + - "feast" + - "serve_offline" + - "-h" + - "0.0.0.0" + {{- else if eq .Values.feast_mode "ui" }} + - "feast" + - "ui" + - "-h" + - "0.0.0.0" + {{- else if eq .Values.feast_mode "registry" }} + - "feast" + - "serve_registry" + {{- else }} + - "feast" + - "serve" + - "-h" + - "0.0.0.0" + {{- end }} ports: - - name: http + - name: {{ .Values.feast_mode }} + {{- if eq .Values.feast_mode "offline" }} + containerPort: 8815 + {{- else if eq .Values.feast_mode "ui" }} + containerPort: 8888 + {{- else if eq .Values.feast_mode "registry" }} + containerPort: 6570 + {{- else }} containerPort: 6566 + {{- end }} protocol: TCP livenessProbe: tcpSocket: - port: http + port: {{ .Values.feast_mode }} initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.livenessProbe.periodSeconds }} readinessProbe: tcpSocket: - port: http + port: {{ .Values.feast_mode }} initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.readinessProbe.periodSeconds }} resources: diff --git a/infra/charts/feast-feature-server/templates/service.yaml b/infra/charts/feast-feature-server/templates/service.yaml index db0ac8b10b8..68f096264e0 100644 --- a/infra/charts/feast-feature-server/templates/service.yaml +++ b/infra/charts/feast-feature-server/templates/service.yaml @@ -8,7 +8,7 @@ spec: type: {{ .Values.service.type }} ports: - port: {{ .Values.service.port }} - targetPort: http + targetPort: {{ .Values.feast_mode }} protocol: TCP name: http selector: diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 168164ffe9d..33430749d8d 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.38.0 + tag: 0.39.0 imagePullSecrets: [] nameOverride: "" @@ -18,6 +18,9 @@ fullnameOverride: "" # feature_store_yaml_base64 -- [required] a base64 encoded version of feature_store.yaml feature_store_yaml_base64: "" +# feast_mode -- Feast supported deployment modes - online (default), offline, ui and registry +feast_mode: "online" + podAnnotations: {} podSecurityContext: {} diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 109b6713933..c724a748a6b 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.38.0 +version: 0.39.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 70296aa130c..d611b69d847 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.38.0` +Feature store for machine learning Current chart version is `0.39.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.38.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.38.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.39.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.39.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index 3df922d7994..5616d463015 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.38.0 -appVersion: v0.38.0 +version: 0.39.0 +appVersion: v0.39.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 8266efeda3d..f4a8ea8cda8 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.38.0](https://img.shields.io/badge/Version-0.38.0-informational?style=flat-square) ![AppVersion: v0.38.0](https://img.shields.io/badge/AppVersion-v0.38.0-informational?style=flat-square) +![Version: 0.39.0](https://img.shields.io/badge/Version-0.39.0-informational?style=flat-square) ![AppVersion: v0.39.0](https://img.shields.io/badge/AppVersion-v0.39.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.38.0"` | Image tag | +| image.tag | string | `"0.39.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index fac64c18c7b..cd60eaf93f8 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.38.0 + tag: 0.39.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 91f0781f523..2e3211697f3 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.38.0 -appVersion: v0.38.0 +version: 0.39.0 +appVersion: v0.39.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index 7b33e4b4a13..dec106617e5 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.38.0](https://img.shields.io/badge/Version-0.38.0-informational?style=flat-square) ![AppVersion: v0.38.0](https://img.shields.io/badge/AppVersion-v0.38.0-informational?style=flat-square) +![Version: 0.39.0](https://img.shields.io/badge/Version-0.39.0-informational?style=flat-square) ![AppVersion: v0.39.0](https://img.shields.io/badge/AppVersion-v0.39.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.38.0"` | Image tag | +| image.tag | string | `"0.39.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index 8c116cf7783..a6935a9993c 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.38.0 + tag: 0.39.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index d9c5f747b8a..1f579a5f3c3 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.38.0 + version: 0.39.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.38.0 + version: 0.39.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/infra/scripts/pixi/pixi.lock b/infra/scripts/pixi/pixi.lock index 19a32f32ae8..f1ce2d26585 100644 --- a/infra/scripts/pixi/pixi.lock +++ b/infra/scripts/pixi/pixi.lock @@ -1,4 +1,4 @@ -version: 4 +version: 5 environments: default: channels: @@ -11,6 +11,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda py310: channels: - url: https://conda.anaconda.org/conda-forge/ @@ -38,6 +41,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.14-h2469fbe_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 py311: channels: - url: https://conda.anaconda.org/conda-forge/ @@ -66,6 +84,22 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.6.2-hebf3989_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.9-h932a869_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 py39: channels: - url: https://conda.anaconda.org/conda-forge/ @@ -93,6 +127,21 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.9.19-hd7ebdb9_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 packages: - kind: conda name: _libgcc_mutex @@ -123,6 +172,19 @@ packages: license_family: BSD size: 23621 timestamp: 1650670423406 +- kind: conda + name: bzip2 + version: 1.0.8 + build: h93a5062_5 + build_number: 5 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda + sha256: bfa84296a638bea78a8bb29abc493ee95f2a0218775642474a840411b950fe5f + md5: 1bbc659ca658bfd49a481b5ef7a0f40f + license: bzip2-1.0.6 + license_family: BSD + size: 122325 + timestamp: 1699280294368 - kind: conda name: bzip2 version: 1.0.8 @@ -149,6 +211,17 @@ packages: license: ISC size: 155432 timestamp: 1706843687645 +- kind: conda + name: ca-certificates + version: 2024.2.2 + build: hf0a4a13_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda + sha256: 49bc3439816ac72d0c0e0f144b8cc870fdcc4adec2e861407ec818d8116b2204 + md5: fb416a1795f18dcc5a038bc2dc54edf9 + license: ISC + size: 155725 + timestamp: 1706844034242 - kind: conda name: ld_impl_linux-64 version: '2.40' @@ -177,6 +250,20 @@ packages: license_family: GPL size: 713322 timestamp: 1713651222435 +- kind: conda + name: libcxx + version: 17.0.6 + build: h5f092b4_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + sha256: 119d3d9306f537d4c89dc99ed99b94c396d262f0b06f7833243646f68884f2c2 + md5: a96fd5dda8ce56c86a971e0fa02751d0 + depends: + - __osx >=11.0 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + size: 1248885 + timestamp: 1715020154867 - kind: conda name: libexpat version: 2.6.2 @@ -193,6 +280,33 @@ packages: license_family: MIT size: 73730 timestamp: 1710362120304 +- kind: conda + name: libexpat + version: 2.6.2 + build: hebf3989_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.6.2-hebf3989_0.conda + sha256: ba7173ac30064ea901a4c9fb5a51846dcc25512ceb565759be7d18cbf3e5415e + md5: e3cde7cfa87f82f7cb13d482d5e0ad09 + constrains: + - expat 2.6.2.* + license: MIT + license_family: MIT + size: 63655 + timestamp: 1710362424980 +- kind: conda + name: libffi + version: 3.4.2 + build: h3422bc3_5 + build_number: 5 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 + sha256: 41b3d13efb775e340e4dba549ab5c029611ea6918703096b2eaa9c015c0750ca + md5: 086914b672be056eb70fd4285b6783b6 + license: MIT + license_family: MIT + size: 39020 + timestamp: 1636488587153 - kind: conda name: libffi version: 3.4.2 @@ -288,6 +402,19 @@ packages: license_family: GPL size: 33408 timestamp: 1697359010159 +- kind: conda + name: libsqlite + version: 3.45.3 + build: h091b4b1_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda + sha256: 4337f466eb55bbdc74e168b52ec8c38f598e3664244ec7a2536009036e2066cc + md5: c8c1186c7f3351f6ffddb97b1f54fc58 + depends: + - libzlib >=1.2.13,<2.0.0a0 + license: Unlicense + size: 824794 + timestamp: 1713367748819 - kind: conda name: libsqlite version: 3.45.3 @@ -360,6 +487,23 @@ packages: license_family: Other size: 61588 timestamp: 1686575217516 +- kind: conda + name: libzlib + version: 1.2.13 + build: hfb2fe0b_6 + build_number: 6 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda + sha256: 8b29a2386d99b8f58178951dcf19117b532cd9c4aa07623bf1667eae99755d32 + md5: 9c4e121cd926cab631bd1c4a61d18b17 + depends: + - __osx >=11.0 + constrains: + - zlib 1.2.13 *_6 + license: Zlib + license_family: Other + size: 46768 + timestamp: 1716874151980 - kind: conda name: ncurses version: 6.4.20240210 @@ -373,6 +517,17 @@ packages: license: X11 AND BSD-3-Clause size: 895669 timestamp: 1710866638986 +- kind: conda + name: ncurses + version: '6.5' + build: hb89a1cb_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda + sha256: 87d7cf716d9d930dab682cb57b3b8d3a61940b47d6703f3529a155c938a6990a + md5: b13ad5724ac9ae98b6b4fd87e4500ba4 + license: X11 AND BSD-3-Clause + size: 795131 + timestamp: 1715194898402 - kind: conda name: openssl version: 3.2.1 @@ -408,6 +563,24 @@ packages: license_family: Apache size: 2895187 timestamp: 1714466138265 +- kind: conda + name: openssl + version: 3.3.0 + build: hfb2fe0b_3 + build_number: 3 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda + sha256: 6f41c163ab57e7499dff092be4498614651f0f6432e12c2b9f06859a8bc39b75 + md5: 730f618b008b3c13c1e3f973408ddd67 + depends: + - __osx >=11.0 + - ca-certificates + constrains: + - pyopenssl >=22.1 + license: Apache-2.0 + license_family: Apache + size: 2893954 + timestamp: 1716468329572 - kind: conda name: python version: 3.9.19 @@ -437,6 +610,54 @@ packages: license: Python-2.0 size: 23800555 timestamp: 1710940120866 +- kind: conda + name: python + version: 3.9.19 + build: hd7ebdb9_0_cpython + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.9.19-hd7ebdb9_0_cpython.conda + sha256: 3b93f7a405f334043758dfa8aaca050429a954a37721a6462ebd20e94ef7c5a0 + md5: 45c4d173b12154f746be3b49b1190634 + depends: + - bzip2 >=1.0.8,<2.0a0 + - libffi >=3.4,<4.0a0 + - libsqlite >=3.45.2,<4.0a0 + - libzlib >=1.2.13,<2.0.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.9.* *_cp39 + license: Python-2.0 + size: 11847835 + timestamp: 1710939779164 +- kind: conda + name: python + version: 3.10.14 + build: h2469fbe_0_cpython + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.14-h2469fbe_0_cpython.conda + sha256: 454d609fe25daedce9e886efcbfcadad103ed0362e7cb6d2bcddec90b1ecd3ee + md5: 4ae999c8227c6d8c7623d32d51d25ea9 + depends: + - bzip2 >=1.0.8,<2.0a0 + - libffi >=3.4,<4.0a0 + - libsqlite >=3.45.2,<4.0a0 + - libzlib >=1.2.13,<2.0.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.10.* *_cp310 + license: Python-2.0 + size: 12336005 + timestamp: 1710939659384 - kind: conda name: python version: 3.10.14 @@ -466,6 +687,32 @@ packages: license: Python-2.0 size: 25517742 timestamp: 1710939725109 +- kind: conda + name: python + version: 3.11.9 + build: h932a869_0_cpython + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.9-h932a869_0_cpython.conda + sha256: a436ceabde1f056a0ac3e347dadc780ee2a135a421ddb6e9a469370769829e3c + md5: 293e0713ae804b5527a673e7605c04fc + depends: + - __osx >=11.0 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.6.2,<3.0a0 + - libffi >=3.4,<4.0a0 + - libsqlite >=3.45.3,<4.0a0 + - libzlib >=1.2.13,<2.0.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.11.* *_cp311 + license: Python-2.0 + size: 14644189 + timestamp: 1713552154779 - kind: conda name: python version: 3.11.9 @@ -512,6 +759,36 @@ packages: license_family: GPL size: 281456 timestamp: 1679532220005 +- kind: conda + name: readline + version: '8.2' + build: h92ec313_1 + build_number: 1 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda + sha256: a1dfa679ac3f6007362386576a704ad2d0d7a02e98f5d0b115f207a2da63e884 + md5: 8cbb776a2f641b943d413b3e19df71f4 + depends: + - ncurses >=6.3,<7.0a0 + license: GPL-3.0-only + license_family: GPL + size: 250351 + timestamp: 1679532511311 +- kind: conda + name: tk + version: 8.6.13 + build: h5083fa2_1 + build_number: 1 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda + sha256: 72457ad031b4c048e5891f3f6cb27a53cb479db68a52d965f796910e71a403a8 + md5: b50a57ba89c32b62428b71a875291c9b + depends: + - libzlib >=1.2.13,<1.3.0a0 + license: TCL + license_family: BSD + size: 3145523 + timestamp: 1699202432999 - kind: conda name: tk version: 8.6.13 @@ -554,6 +831,22 @@ packages: license: Apache-2.0 OR MIT size: 11891252 timestamp: 1714233659570 +- kind: conda + name: uv + version: 0.1.45 + build: hc069d6b_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + sha256: 80dfc19f2ef473e86e718361847d1d598e95ffd0c0f5de7d07cda35d25f6aef5 + md5: 9192238a60bc6da9c41092990c31eb41 + depends: + - __osx >=11.0 + - libcxx >=16 + constrains: + - __osx >=11.0 + license: Apache-2.0 OR MIT + size: 9231858 + timestamp: 1716265232676 - kind: conda name: xz version: 5.2.6 @@ -567,3 +860,14 @@ packages: license: LGPL-2.1 and GPL-2.0 size: 418368 timestamp: 1660346797927 +- kind: conda + name: xz + version: 5.2.6 + build: h57fd34a_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 + sha256: 59d78af0c3e071021cfe82dc40134c19dab8cdf804324b62940f5c8cd71803ec + md5: 39c6b54e94014701dd157f4f576ed211 + license: LGPL-2.1 and GPL-2.0 + size: 235693 + timestamp: 1660346961024 diff --git a/infra/scripts/pixi/pixi.toml b/infra/scripts/pixi/pixi.toml index f0d360fff3d..10179339f70 100644 --- a/infra/scripts/pixi/pixi.toml +++ b/infra/scripts/pixi/pixi.toml @@ -1,7 +1,7 @@ [project] name = "pixi-feast" channels = ["conda-forge"] -platforms = ["linux-64"] +platforms = ["linux-64", "osx-arm64"] [tasks] diff --git a/java/pom.xml b/java/pom.xml index 6aabb87d0cc..492e756ba57 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.38.0 + 0.39.0 https://github.com/feast-dev/feast UTF-8 diff --git a/java/serving-client/src/main/java/dev/feast/FeastClient.java b/java/serving-client/src/main/java/dev/feast/FeastClient.java index c10a76ecf81..c14d3be5867 100644 --- a/java/serving-client/src/main/java/dev/feast/FeastClient.java +++ b/java/serving-client/src/main/java/dev/feast/FeastClient.java @@ -49,6 +49,7 @@ public class FeastClient implements AutoCloseable { private final ManagedChannel channel; private final ServingServiceBlockingStub stub; + private final long requestTimeout; /** * Create a client to access Feast Serving. @@ -63,7 +64,22 @@ public static FeastClient create(String host, int port) { } /** - * Create a authenticated client that can access Feast serving with authentication enabled. + * Create a client to access Feast Serving. + * + * @param host hostname or ip address of Feast serving GRPC server + * @param port port number of Feast serving GRPC server + * @param requestTimeout maximum duration for online retrievals from the GRPC server in + * milliseconds, use 0 for no timeout + * @return {@link FeastClient} + */ + public static FeastClient create(String host, int port, long requestTimeout) { + // configure client with no security config. + return FeastClient.createSecure( + host, port, SecurityConfig.newBuilder().build(), requestTimeout); + } + + /** + * Create an authenticated client that can access Feast serving with authentication enabled. * * @param host hostname or ip address of Feast serving GRPC server * @param port port number of Feast serving GRPC server @@ -72,6 +88,27 @@ public static FeastClient create(String host, int port) { * @return {@link FeastClient} */ public static FeastClient createSecure(String host, int port, SecurityConfig securityConfig) { + return FeastClient.createSecure(host, port, securityConfig, 0); + } + + /** + * Create an authenticated client that can access Feast serving with authentication enabled. + * + * @param host hostname or ip address of Feast serving GRPC server + * @param port port number of Feast serving GRPC server + * @param securityConfig security options to configure the Feast client. See {@link + * SecurityConfig} for options. + * @param requestTimeout maximum duration for online retrievals from the GRPC server in + * milliseconds + * @return {@link FeastClient} + */ + public static FeastClient createSecure( + String host, int port, SecurityConfig securityConfig, long requestTimeout) { + + if (requestTimeout < 0) { + throw new IllegalArgumentException("Request timeout can't be negative"); + } + // Configure client TLS ManagedChannel channel = null; if (securityConfig.isTLSEnabled()) { @@ -98,7 +135,7 @@ public static FeastClient createSecure(String host, int port, SecurityConfig sec channel = ManagedChannelBuilder.forAddress(host, port).usePlaintext().build(); } - return new FeastClient(channel, securityConfig.getCredentials()); + return new FeastClient(channel, securityConfig.getCredentials(), requestTimeout); } /** @@ -129,7 +166,10 @@ public List getOnlineFeatures(List featureRefs, List entities) requestBuilder.putAllEntities(getEntityValuesMap(entities)); - GetOnlineFeaturesResponse response = stub.getOnlineFeatures(requestBuilder.build()); + ServingServiceGrpc.ServingServiceBlockingStub timedStub = + requestTimeout != 0 ? stub.withDeadlineAfter(requestTimeout, TimeUnit.MILLISECONDS) : stub; + + GetOnlineFeaturesResponse response = timedStub.getOnlineFeatures(requestBuilder.build()); List results = Lists.newArrayList(); if (response.getResultsCount() == 0) { @@ -202,7 +242,13 @@ public List getOnlineFeatures(List featureRefs, List rows, Str } protected FeastClient(ManagedChannel channel, Optional credentials) { + this(channel, credentials, 0); + } + + protected FeastClient( + ManagedChannel channel, Optional credentials, long requestTimeout) { this.channel = channel; + this.requestTimeout = requestTimeout; TracingClientInterceptor tracingInterceptor = TracingClientInterceptor.newBuilder().withTracer(GlobalTracer.get()).build(); diff --git a/java/serving-client/src/test/java/dev/feast/FeastClientTest.java b/java/serving-client/src/test/java/dev/feast/FeastClientTest.java index 1dfb9989c95..cbd4b0016e5 100644 --- a/java/serving-client/src/test/java/dev/feast/FeastClientTest.java +++ b/java/serving-client/src/test/java/dev/feast/FeastClientTest.java @@ -45,6 +45,7 @@ public class FeastClientTest { private final String AUTH_TOKEN = "test token"; + private final long TIMEOUT_MILLIS = 300; @Rule public GrpcCleanupRule grpcRule; private AtomicBoolean isAuthenticated; @@ -86,7 +87,7 @@ public void setup() throws Exception { ManagedChannel channel = this.grpcRule.register( InProcessChannelBuilder.forName(serverName).directExecutor().build()); - this.client = new FeastClient(channel, Optional.empty()); + this.client = new FeastClient(channel, Optional.empty(), TIMEOUT_MILLIS); } @Test diff --git a/java/serving/pom.xml b/java/serving/pom.xml index 6929d65d934..93e4f81efef 100644 --- a/java/serving/pom.xml +++ b/java/serving/pom.xml @@ -131,7 +131,7 @@ com.azure azure-identity - 1.11.3 + 1.12.2 diff --git a/protos/feast/core/Transformation.proto b/protos/feast/core/Transformation.proto index 5cb53e690fa..7033f553f16 100644 --- a/protos/feast/core/Transformation.proto +++ b/protos/feast/core/Transformation.proto @@ -5,8 +5,6 @@ option go_package = "github.com/feast-dev/feast/go/protos/feast/core"; option java_outer_classname = "FeatureTransformationProto"; option java_package = "feast.proto.core"; -import "google/protobuf/duration.proto"; - // Serialized representation of python function. message UserDefinedFunctionV2 { // The function name diff --git a/protos/feast/registry/RegistryServer.proto b/protos/feast/registry/RegistryServer.proto index e99987eb2da..44529f5409c 100644 --- a/protos/feast/registry/RegistryServer.proto +++ b/protos/feast/registry/RegistryServer.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package feast.registry; import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; import "feast/core/Registry.proto"; import "feast/core/Entity.proto"; import "feast/core/DataSource.proto"; @@ -16,16 +17,22 @@ import "feast/core/InfraObject.proto"; service RegistryServer{ // Entity RPCs + rpc ApplyEntity (ApplyEntityRequest) returns (google.protobuf.Empty) {} rpc GetEntity (GetEntityRequest) returns (feast.core.Entity) {} rpc ListEntities (ListEntitiesRequest) returns (ListEntitiesResponse) {} + rpc DeleteEntity (DeleteEntityRequest) returns (google.protobuf.Empty) {} // DataSource RPCs + rpc ApplyDataSource (ApplyDataSourceRequest) returns (google.protobuf.Empty) {} rpc GetDataSource (GetDataSourceRequest) returns (feast.core.DataSource) {} rpc ListDataSources (ListDataSourcesRequest) returns (ListDataSourcesResponse) {} + rpc DeleteDataSource (DeleteDataSourceRequest) returns (google.protobuf.Empty) {} // FeatureView RPCs + rpc ApplyFeatureView (ApplyFeatureViewRequest) returns (google.protobuf.Empty) {} rpc GetFeatureView (GetFeatureViewRequest) returns (feast.core.FeatureView) {} rpc ListFeatureViews (ListFeatureViewsRequest) returns (ListFeatureViewsResponse) {} + rpc DeleteFeatureView (DeleteFeatureViewRequest) returns (google.protobuf.Empty) {} // StreamFeatureView RPCs rpc GetStreamFeatureView (GetStreamFeatureViewRequest) returns (feast.core.StreamFeatureView) {} @@ -36,19 +43,28 @@ service RegistryServer{ rpc ListOnDemandFeatureViews (ListOnDemandFeatureViewsRequest) returns (ListOnDemandFeatureViewsResponse) {} // FeatureService RPCs + rpc ApplyFeatureService (ApplyFeatureServiceRequest) returns (google.protobuf.Empty) {} rpc GetFeatureService (GetFeatureServiceRequest) returns (feast.core.FeatureService) {} rpc ListFeatureServices (ListFeatureServicesRequest) returns (ListFeatureServicesResponse) {} + rpc DeleteFeatureService (DeleteFeatureServiceRequest) returns (google.protobuf.Empty) {} // SavedDataset RPCs + rpc ApplySavedDataset (ApplySavedDatasetRequest) returns (google.protobuf.Empty) {} rpc GetSavedDataset (GetSavedDatasetRequest) returns (feast.core.SavedDataset) {} rpc ListSavedDatasets (ListSavedDatasetsRequest) returns (ListSavedDatasetsResponse) {} + rpc DeleteSavedDataset (DeleteSavedDatasetRequest) returns (google.protobuf.Empty) {} // ValidationReference RPCs + rpc ApplyValidationReference (ApplyValidationReferenceRequest) returns (google.protobuf.Empty) {} rpc GetValidationReference (GetValidationReferenceRequest) returns (feast.core.ValidationReference) {} rpc ListValidationReferences (ListValidationReferencesRequest) returns (ListValidationReferencesResponse) {} - + rpc DeleteValidationReference (DeleteValidationReferenceRequest) returns (google.protobuf.Empty) {} + + rpc ApplyMaterialization (ApplyMaterializationRequest) returns (google.protobuf.Empty) {} rpc ListProjectMetadata (ListProjectMetadataRequest) returns (ListProjectMetadataResponse) {} + rpc UpdateInfra (UpdateInfraRequest) returns (google.protobuf.Empty) {} rpc GetInfra (GetInfraRequest) returns (feast.core.Infra) {} + rpc Commit (google.protobuf.Empty) returns (google.protobuf.Empty) {} rpc Refresh (RefreshRequest) returns (google.protobuf.Empty) {} rpc Proto (google.protobuf.Empty) returns (feast.core.Registry) {} @@ -58,6 +74,12 @@ message RefreshRequest { string project = 1; } +message UpdateInfraRequest { + feast.core.Infra infra = 1; + string project = 2; + bool commit = 3; +} + message GetInfraRequest { string project = 1; bool allow_cache = 2; @@ -72,6 +94,20 @@ message ListProjectMetadataResponse { repeated feast.core.ProjectMetadata project_metadata = 1; } +message ApplyMaterializationRequest { + feast.core.FeatureView feature_view = 1; + string project = 2; + google.protobuf.Timestamp start_date = 3; + google.protobuf.Timestamp end_date = 4; + bool commit = 5; +} + +message ApplyEntityRequest { + feast.core.Entity entity = 1; + string project = 2; + bool commit = 3; +} + message GetEntityRequest { string name = 1; string project = 2; @@ -81,14 +117,27 @@ message GetEntityRequest { message ListEntitiesRequest { string project = 1; bool allow_cache = 2; + map tags = 3; } message ListEntitiesResponse { repeated feast.core.Entity entities = 1; } +message DeleteEntityRequest { + string name = 1; + string project = 2; + bool commit = 3; +} + // DataSources +message ApplyDataSourceRequest { + feast.core.DataSource data_source = 1; + string project = 2; + bool commit = 3; +} + message GetDataSourceRequest { string name = 1; string project = 2; @@ -98,14 +147,31 @@ message GetDataSourceRequest { message ListDataSourcesRequest { string project = 1; bool allow_cache = 2; + map tags = 3; } message ListDataSourcesResponse { repeated feast.core.DataSource data_sources = 1; } +message DeleteDataSourceRequest { + string name = 1; + string project = 2; + bool commit = 3; +} + // FeatureViews +message ApplyFeatureViewRequest { + oneof base_feature_view { + feast.core.FeatureView feature_view = 1; + feast.core.OnDemandFeatureView on_demand_feature_view = 2; + feast.core.StreamFeatureView stream_feature_view = 3; + } + string project = 4; + bool commit = 5; +} + message GetFeatureViewRequest { string name = 1; string project = 2; @@ -115,12 +181,19 @@ message GetFeatureViewRequest { message ListFeatureViewsRequest { string project = 1; bool allow_cache = 2; + map tags = 3; } message ListFeatureViewsResponse { repeated feast.core.FeatureView feature_views = 1; } +message DeleteFeatureViewRequest { + string name = 1; + string project = 2; + bool commit = 3; +} + // StreamFeatureView message GetStreamFeatureViewRequest { @@ -132,6 +205,7 @@ message GetStreamFeatureViewRequest { message ListStreamFeatureViewsRequest { string project = 1; bool allow_cache = 2; + map tags = 3; } message ListStreamFeatureViewsResponse { @@ -149,6 +223,7 @@ message GetOnDemandFeatureViewRequest { message ListOnDemandFeatureViewsRequest { string project = 1; bool allow_cache = 2; + map tags = 3; } message ListOnDemandFeatureViewsResponse { @@ -157,6 +232,12 @@ message ListOnDemandFeatureViewsResponse { // FeatureServices +message ApplyFeatureServiceRequest { + feast.core.FeatureService feature_service = 1; + string project = 2; + bool commit = 3; +} + message GetFeatureServiceRequest { string name = 1; string project = 2; @@ -166,14 +247,27 @@ message GetFeatureServiceRequest { message ListFeatureServicesRequest { string project = 1; bool allow_cache = 2; + map tags = 3; } message ListFeatureServicesResponse { repeated feast.core.FeatureService feature_services = 1; } +message DeleteFeatureServiceRequest { + string name = 1; + string project = 2; + bool commit = 3; +} + // SavedDataset +message ApplySavedDatasetRequest { + feast.core.SavedDataset saved_dataset = 1; + string project = 2; + bool commit = 3; +} + message GetSavedDatasetRequest { string name = 1; string project = 2; @@ -189,8 +283,20 @@ message ListSavedDatasetsResponse { repeated feast.core.SavedDataset saved_datasets = 1; } +message DeleteSavedDatasetRequest { + string name = 1; + string project = 2; + bool commit = 3; +} + // ValidationReference +message ApplyValidationReferenceRequest { + feast.core.ValidationReference validation_reference = 1; + string project = 2; + bool commit = 3; +} + message GetValidationReferenceRequest { string name = 1; string project = 2; @@ -205,3 +311,9 @@ message ListValidationReferencesRequest { message ListValidationReferencesResponse { repeated feast.core.ValidationReference validation_references = 1; } + +message DeleteValidationReferenceRequest { + string name = 1; + string project = 2; + bool commit = 3; +} \ No newline at end of file diff --git a/sdk/python/docs/index.rst b/sdk/python/docs/index.rst index 4cedffb1fc0..1ef6bd16c80 100644 --- a/sdk/python/docs/index.rst +++ b/sdk/python/docs/index.rst @@ -182,12 +182,6 @@ S3 Registry Store .. autoclass:: feast.infra.registry.s3.S3RegistryStore :members: -PostgreSQL Registry Store ------------------------ - -.. autoclass:: feast.infra.registry.contrib.postgres.postgres_registry_store.PostgreSQLRegistryStore - :members: - Provider ================== diff --git a/sdk/python/docs/source/feast.embedded_go.rst b/sdk/python/docs/source/feast.embedded_go.rst new file mode 100644 index 00000000000..3b18d280ab0 --- /dev/null +++ b/sdk/python/docs/source/feast.embedded_go.rst @@ -0,0 +1,29 @@ +feast.embedded\_go package +========================== + +Submodules +---------- + +feast.embedded\_go.online\_features\_service module +--------------------------------------------------- + +.. automodule:: feast.embedded_go.online_features_service + :members: + :undoc-members: + :show-inheritance: + +feast.embedded\_go.type\_map module +----------------------------------- + +.. automodule:: feast.embedded_go.type_map + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.embedded_go + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.contrib.rst b/sdk/python/docs/source/feast.infra.contrib.rst new file mode 100644 index 00000000000..7b2fa3cc9c5 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.contrib.rst @@ -0,0 +1,45 @@ +feast.infra.contrib package +=========================== + +Submodules +---------- + +feast.infra.contrib.azure\_provider module +------------------------------------------ + +.. automodule:: feast.infra.contrib.azure_provider + :members: + :undoc-members: + :show-inheritance: + +feast.infra.contrib.grpc\_server module +--------------------------------------- + +.. automodule:: feast.infra.contrib.grpc_server + :members: + :undoc-members: + :show-inheritance: + +feast.infra.contrib.spark\_kafka\_processor module +-------------------------------------------------- + +.. automodule:: feast.infra.contrib.spark_kafka_processor + :members: + :undoc-members: + :show-inheritance: + +feast.infra.contrib.stream\_processor module +-------------------------------------------- + +.. automodule:: feast.infra.contrib.stream_processor + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.contrib + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.feature_servers.aws_lambda.rst b/sdk/python/docs/source/feast.infra.feature_servers.aws_lambda.rst new file mode 100644 index 00000000000..de90bfc0002 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.feature_servers.aws_lambda.rst @@ -0,0 +1,29 @@ +feast.infra.feature\_servers.aws\_lambda package +================================================ + +Submodules +---------- + +feast.infra.feature\_servers.aws\_lambda.app module +--------------------------------------------------- + +.. automodule:: feast.infra.feature_servers.aws_lambda.app + :members: + :undoc-members: + :show-inheritance: + +feast.infra.feature\_servers.aws\_lambda.config module +------------------------------------------------------ + +.. automodule:: feast.infra.feature_servers.aws_lambda.config + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.feature_servers.aws_lambda + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.feature_servers.gcp_cloudrun.rst b/sdk/python/docs/source/feast.infra.feature_servers.gcp_cloudrun.rst new file mode 100644 index 00000000000..f7fdaf5b361 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.feature_servers.gcp_cloudrun.rst @@ -0,0 +1,29 @@ +feast.infra.feature\_servers.gcp\_cloudrun package +================================================== + +Submodules +---------- + +feast.infra.feature\_servers.gcp\_cloudrun.app module +----------------------------------------------------- + +.. automodule:: feast.infra.feature_servers.gcp_cloudrun.app + :members: + :undoc-members: + :show-inheritance: + +feast.infra.feature\_servers.gcp\_cloudrun.config module +-------------------------------------------------------- + +.. automodule:: feast.infra.feature_servers.gcp_cloudrun.config + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.feature_servers.gcp_cloudrun + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.feature_servers.local_process.rst b/sdk/python/docs/source/feast.infra.feature_servers.local_process.rst new file mode 100644 index 00000000000..003b9dcb2ee --- /dev/null +++ b/sdk/python/docs/source/feast.infra.feature_servers.local_process.rst @@ -0,0 +1,21 @@ +feast.infra.feature\_servers.local\_process package +=================================================== + +Submodules +---------- + +feast.infra.feature\_servers.local\_process.config module +--------------------------------------------------------- + +.. automodule:: feast.infra.feature_servers.local_process.config + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.feature_servers.local_process + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.feature_servers.multicloud.rst b/sdk/python/docs/source/feast.infra.feature_servers.multicloud.rst new file mode 100644 index 00000000000..9d34623f562 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.feature_servers.multicloud.rst @@ -0,0 +1,10 @@ +feast.infra.feature\_servers.multicloud package +=============================================== + +Module contents +--------------- + +.. automodule:: feast.infra.feature_servers.multicloud + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.feature_servers.rst b/sdk/python/docs/source/feast.infra.feature_servers.rst new file mode 100644 index 00000000000..334b5859053 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.feature_servers.rst @@ -0,0 +1,32 @@ +feast.infra.feature\_servers package +==================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.feature_servers.aws_lambda + feast.infra.feature_servers.gcp_cloudrun + feast.infra.feature_servers.local_process + feast.infra.feature_servers.multicloud + +Submodules +---------- + +feast.infra.feature\_servers.base\_config module +------------------------------------------------ + +.. automodule:: feast.infra.feature_servers.base_config + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.feature_servers + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.materialization.contrib.rst b/sdk/python/docs/source/feast.infra.materialization.contrib.rst index f9d77006610..e1dcba67a89 100644 --- a/sdk/python/docs/source/feast.infra.materialization.contrib.rst +++ b/sdk/python/docs/source/feast.infra.materialization.contrib.rst @@ -1,10 +1,10 @@ feast.infra.materialization.contrib package -========================================== +=========================================== -Subpackages ------------ +Module contents +--------------- -.. toctree:: - :maxdepth: 4 - - feast.infra.materialization.contrib.bytewax +.. automodule:: feast.infra.materialization.contrib + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.materialization.kubernetes.rst b/sdk/python/docs/source/feast.infra.materialization.kubernetes.rst new file mode 100644 index 00000000000..abb0d61c0f4 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.materialization.kubernetes.rst @@ -0,0 +1,45 @@ +feast.infra.materialization.kubernetes package +============================================== + +Submodules +---------- + +feast.infra.materialization.kubernetes.k8s\_materialization\_engine module +-------------------------------------------------------------------------- + +.. automodule:: feast.infra.materialization.kubernetes.k8s_materialization_engine + :members: + :undoc-members: + :show-inheritance: + +feast.infra.materialization.kubernetes.k8s\_materialization\_job module +----------------------------------------------------------------------- + +.. automodule:: feast.infra.materialization.kubernetes.k8s_materialization_job + :members: + :undoc-members: + :show-inheritance: + +feast.infra.materialization.kubernetes.k8s\_materialization\_task module +------------------------------------------------------------------------ + +.. automodule:: feast.infra.materialization.kubernetes.k8s_materialization_task + :members: + :undoc-members: + :show-inheritance: + +feast.infra.materialization.kubernetes.main module +-------------------------------------------------- + +.. automodule:: feast.infra.materialization.kubernetes.main + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.materialization.kubernetes + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.materialization.rst b/sdk/python/docs/source/feast.infra.materialization.rst index 6e526c367cd..30054b6dd3a 100644 --- a/sdk/python/docs/source/feast.infra.materialization.rst +++ b/sdk/python/docs/source/feast.infra.materialization.rst @@ -7,7 +7,8 @@ Subpackages .. toctree:: :maxdepth: 4 - feast.infra.materialization.lambda + feast.infra.materialization.contrib + feast.infra.materialization.kubernetes Submodules ---------- diff --git a/sdk/python/docs/source/feast.infra.offline_stores.rst b/sdk/python/docs/source/feast.infra.offline_stores.rst index 7949c9efb32..052a114cfb3 100644 --- a/sdk/python/docs/source/feast.infra.offline_stores.rst +++ b/sdk/python/docs/source/feast.infra.offline_stores.rst @@ -28,6 +28,14 @@ feast.infra.offline\_stores.bigquery\_source module :undoc-members: :show-inheritance: +feast.infra.offline\_stores.duckdb module +----------------------------------------- + +.. automodule:: feast.infra.offline_stores.duckdb + :members: + :undoc-members: + :show-inheritance: + feast.infra.offline\_stores.file module --------------------------------------- @@ -44,6 +52,14 @@ feast.infra.offline\_stores.file\_source module :undoc-members: :show-inheritance: +feast.infra.offline\_stores.ibis module +--------------------------------------- + +.. automodule:: feast.infra.offline_stores.ibis + :members: + :undoc-members: + :show-inheritance: + feast.infra.offline\_stores.offline\_store module ------------------------------------------------- @@ -76,6 +92,14 @@ feast.infra.offline\_stores.redshift\_source module :undoc-members: :show-inheritance: +feast.infra.offline\_stores.remote module +----------------------------------------- + +.. automodule:: feast.infra.offline_stores.remote + :members: + :undoc-members: + :show-inheritance: + feast.infra.offline\_stores.snowflake module -------------------------------------------- diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.ikv_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.ikv_online_store.rst new file mode 100644 index 00000000000..e7f858d1cf4 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.ikv_online_store.rst @@ -0,0 +1,21 @@ +feast.infra.online\_stores.contrib.ikv\_online\_store package +============================================================= + +Submodules +---------- + +feast.infra.online\_stores.contrib.ikv\_online\_store.ikv module +---------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.ikv_online_store.ikv + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.contrib.ikv_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst index b6c8a404ee4..d614438e3d5 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst @@ -10,6 +10,7 @@ Subpackages feast.infra.online_stores.contrib.cassandra_online_store feast.infra.online_stores.contrib.hazelcast_online_store feast.infra.online_stores.contrib.hbase_online_store + feast.infra.online_stores.contrib.ikv_online_store feast.infra.online_stores.contrib.mysql_online_store feast.infra.online_stores.contrib.rockset_online_store @@ -24,6 +25,22 @@ feast.infra.online\_stores.contrib.cassandra\_repo\_configuration module :undoc-members: :show-inheritance: +feast.infra.online\_stores.contrib.elasticsearch module +------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.elasticsearch + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.contrib.elasticsearch\_repo\_configuration module +---------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.elasticsearch_repo_configuration + :members: + :undoc-members: + :show-inheritance: + feast.infra.online\_stores.contrib.hazelcast\_repo\_configuration module ------------------------------------------------------------------------ @@ -48,6 +65,14 @@ feast.infra.online\_stores.contrib.mysql\_repo\_configuration module :undoc-members: :show-inheritance: +feast.infra.online\_stores.contrib.pgvector\_repo\_configuration module +----------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.pgvector_repo_configuration + :members: + :undoc-members: + :show-inheritance: + feast.infra.online\_stores.contrib.postgres module -------------------------------------------------- @@ -64,14 +89,6 @@ feast.infra.online\_stores.contrib.postgres\_repo\_configuration module :undoc-members: :show-inheritance: -feast.infra.online\_stores.contrib.rockset\_repo\_configuration module ----------------------------------------------------------------------- - -.. automodule:: feast.infra.online_stores.contrib.rockset_repo_configuration - :members: - :undoc-members: - :show-inheritance: - Module contents --------------- diff --git a/sdk/python/docs/source/feast.infra.online_stores.rst b/sdk/python/docs/source/feast.infra.online_stores.rst index 59ac1868f58..801d187a7c8 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.rst @@ -60,6 +60,14 @@ feast.infra.online\_stores.redis module :undoc-members: :show-inheritance: +feast.infra.online\_stores.remote module +---------------------------------------- + +.. automodule:: feast.infra.online_stores.remote + :members: + :undoc-members: + :show-inheritance: + feast.infra.online\_stores.snowflake module ------------------------------------------- diff --git a/sdk/python/docs/source/feast.infra.registry.contrib.azure.rst b/sdk/python/docs/source/feast.infra.registry.contrib.azure.rst new file mode 100644 index 00000000000..f9280925ad3 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.registry.contrib.azure.rst @@ -0,0 +1,21 @@ +feast.infra.registry.contrib.azure package +========================================== + +Submodules +---------- + +feast.infra.registry.contrib.azure.azure\_registry\_store module +---------------------------------------------------------------- + +.. automodule:: feast.infra.registry.contrib.azure.azure_registry_store + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.registry.contrib.azure + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.registry.contrib.rst b/sdk/python/docs/source/feast.infra.registry.contrib.rst new file mode 100644 index 00000000000..83417109b86 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.registry.contrib.rst @@ -0,0 +1,18 @@ +feast.infra.registry.contrib package +==================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.registry.contrib.azure + +Module contents +--------------- + +.. automodule:: feast.infra.registry.contrib + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.registry.rst b/sdk/python/docs/source/feast.infra.registry.rst index 7a2d9689975..26f459fe71b 100644 --- a/sdk/python/docs/source/feast.infra.registry.rst +++ b/sdk/python/docs/source/feast.infra.registry.rst @@ -1,6 +1,14 @@ feast.infra.registry package ============================ +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.registry.contrib + Submodules ---------- @@ -12,6 +20,14 @@ feast.infra.registry.base\_registry module :undoc-members: :show-inheritance: +feast.infra.registry.caching\_registry module +--------------------------------------------- + +.. automodule:: feast.infra.registry.caching_registry + :members: + :undoc-members: + :show-inheritance: + feast.infra.registry.file module -------------------------------- @@ -28,6 +44,14 @@ feast.infra.registry.gcs module :undoc-members: :show-inheritance: +feast.infra.registry.proto\_registry\_utils module +-------------------------------------------------- + +.. automodule:: feast.infra.registry.proto_registry_utils + :members: + :undoc-members: + :show-inheritance: + feast.infra.registry.registry module ------------------------------------ @@ -44,6 +68,14 @@ feast.infra.registry.registry\_store module :undoc-members: :show-inheritance: +feast.infra.registry.remote module +---------------------------------- + +.. automodule:: feast.infra.registry.remote + :members: + :undoc-members: + :show-inheritance: + feast.infra.registry.s3 module ------------------------------ @@ -52,6 +84,14 @@ feast.infra.registry.s3 module :undoc-members: :show-inheritance: +feast.infra.registry.snowflake module +------------------------------------- + +.. automodule:: feast.infra.registry.snowflake + :members: + :undoc-members: + :show-inheritance: + feast.infra.registry.sql module ------------------------------- diff --git a/sdk/python/docs/source/feast.infra.rst b/sdk/python/docs/source/feast.infra.rst index 50e1f37f1c6..a1dfc864926 100644 --- a/sdk/python/docs/source/feast.infra.rst +++ b/sdk/python/docs/source/feast.infra.rst @@ -7,6 +7,9 @@ Subpackages .. toctree:: :maxdepth: 4 + feast.infra.contrib + feast.infra.feature_servers + feast.infra.materialization feast.infra.offline_stores feast.infra.online_stores feast.infra.registry diff --git a/sdk/python/docs/source/feast.infra.utils.rst b/sdk/python/docs/source/feast.infra.utils.rst index e4116e7a172..083259bfaae 100644 --- a/sdk/python/docs/source/feast.infra.utils.rst +++ b/sdk/python/docs/source/feast.infra.utils.rst @@ -8,6 +8,7 @@ Subpackages :maxdepth: 4 feast.infra.utils.postgres + feast.infra.utils.snowflake Submodules ---------- diff --git a/sdk/python/docs/source/feast.infra.utils.snowflake.registry.rst b/sdk/python/docs/source/feast.infra.utils.snowflake.registry.rst new file mode 100644 index 00000000000..17605e61c32 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.utils.snowflake.registry.rst @@ -0,0 +1,10 @@ +feast.infra.utils.snowflake.registry package +============================================ + +Module contents +--------------- + +.. automodule:: feast.infra.utils.snowflake.registry + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.utils.snowflake.rst b/sdk/python/docs/source/feast.infra.utils.snowflake.rst new file mode 100644 index 00000000000..4dca045ab75 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.utils.snowflake.rst @@ -0,0 +1,30 @@ +feast.infra.utils.snowflake package +=================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.utils.snowflake.registry + feast.infra.utils.snowflake.snowpark + +Submodules +---------- + +feast.infra.utils.snowflake.snowflake\_utils module +--------------------------------------------------- + +.. automodule:: feast.infra.utils.snowflake.snowflake_utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.utils.snowflake + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.utils.snowflake.snowpark.rst b/sdk/python/docs/source/feast.infra.utils.snowflake.snowpark.rst new file mode 100644 index 00000000000..81ffbfebf75 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.utils.snowflake.snowpark.rst @@ -0,0 +1,21 @@ +feast.infra.utils.snowflake.snowpark package +============================================ + +Submodules +---------- + +feast.infra.utils.snowflake.snowpark.snowflake\_udfs module +----------------------------------------------------------- + +.. automodule:: feast.infra.utils.snowflake.snowpark.snowflake_udfs + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.utils.snowflake.snowpark + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.feast.core.rst b/sdk/python/docs/source/feast.protos.feast.core.rst index 5da16d2a267..a8691c20feb 100644 --- a/sdk/python/docs/source/feast.protos.feast.core.rst +++ b/sdk/python/docs/source/feast.protos.feast.core.rst @@ -292,6 +292,22 @@ feast.protos.feast.core.StreamFeatureView\_pb2\_grpc module :undoc-members: :show-inheritance: +feast.protos.feast.core.Transformation\_pb2 module +-------------------------------------------------- + +.. automodule:: feast.protos.feast.core.Transformation_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.Transformation\_pb2\_grpc module +-------------------------------------------------------- + +.. automodule:: feast.protos.feast.core.Transformation_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + feast.protos.feast.core.ValidationProfile\_pb2 module ----------------------------------------------------- diff --git a/sdk/python/docs/source/feast.protos.feast.registry.rst b/sdk/python/docs/source/feast.protos.feast.registry.rst new file mode 100644 index 00000000000..07d0d1420f3 --- /dev/null +++ b/sdk/python/docs/source/feast.protos.feast.registry.rst @@ -0,0 +1,29 @@ +feast.protos.feast.registry package +=================================== + +Submodules +---------- + +feast.protos.feast.registry.RegistryServer\_pb2 module +------------------------------------------------------ + +.. automodule:: feast.protos.feast.registry.RegistryServer_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.registry.RegistryServer\_pb2\_grpc module +------------------------------------------------------------ + +.. automodule:: feast.protos.feast.registry.RegistryServer_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.protos.feast.registry + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.feast.rst b/sdk/python/docs/source/feast.protos.feast.rst index f519165db8e..6a50967670f 100644 --- a/sdk/python/docs/source/feast.protos.feast.rst +++ b/sdk/python/docs/source/feast.protos.feast.rst @@ -8,6 +8,7 @@ Subpackages :maxdepth: 4 feast.protos.feast.core + feast.protos.feast.registry feast.protos.feast.serving feast.protos.feast.storage feast.protos.feast.types diff --git a/sdk/python/docs/source/feast.protos.feast.serving.rst b/sdk/python/docs/source/feast.protos.feast.serving.rst index 792335b189d..fccea892be0 100644 --- a/sdk/python/docs/source/feast.protos.feast.serving.rst +++ b/sdk/python/docs/source/feast.protos.feast.serving.rst @@ -20,6 +20,22 @@ feast.protos.feast.serving.Connector\_pb2\_grpc module :undoc-members: :show-inheritance: +feast.protos.feast.serving.GrpcServer\_pb2 module +------------------------------------------------- + +.. automodule:: feast.protos.feast.serving.GrpcServer_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.serving.GrpcServer\_pb2\_grpc module +------------------------------------------------------- + +.. automodule:: feast.protos.feast.serving.GrpcServer_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + feast.protos.feast.serving.ServingService\_pb2 module ----------------------------------------------------- diff --git a/sdk/python/docs/source/feast.rst b/sdk/python/docs/source/feast.rst index 4730fdf725d..95fbea8d7a3 100644 --- a/sdk/python/docs/source/feast.rst +++ b/sdk/python/docs/source/feast.rst @@ -9,9 +9,11 @@ Subpackages feast.diff feast.dqm + feast.embedded_go feast.infra feast.loaders feast.protos + feast.transformation feast.ui Submodules @@ -209,6 +211,14 @@ feast.names module :undoc-members: :show-inheritance: +feast.offline\_server module +---------------------------- + +.. automodule:: feast.offline_server + :members: + :undoc-members: + :show-inheritance: + feast.on\_demand\_feature\_view module -------------------------------------- @@ -241,6 +251,14 @@ feast.proto\_json module :undoc-members: :show-inheritance: +feast.registry\_server module +----------------------------- + +.. automodule:: feast.registry_server + :members: + :undoc-members: + :show-inheritance: + feast.repo\_config module ------------------------- @@ -265,14 +283,6 @@ feast.repo\_operations module :undoc-members: :show-inheritance: -feast.repo\_upgrade module --------------------------- - -.. automodule:: feast.repo_upgrade - :members: - :undoc-members: - :show-inheritance: - feast.saved\_dataset module --------------------------- diff --git a/sdk/python/docs/source/feast.transformation.rst b/sdk/python/docs/source/feast.transformation.rst new file mode 100644 index 00000000000..ef2278fa92b --- /dev/null +++ b/sdk/python/docs/source/feast.transformation.rst @@ -0,0 +1,37 @@ +feast.transformation package +============================ + +Submodules +---------- + +feast.transformation.pandas\_transformation module +-------------------------------------------------- + +.. automodule:: feast.transformation.pandas_transformation + :members: + :undoc-members: + :show-inheritance: + +feast.transformation.python\_transformation module +-------------------------------------------------- + +.. automodule:: feast.transformation.python_transformation + :members: + :undoc-members: + :show-inheritance: + +feast.transformation.substrait\_transformation module +----------------------------------------------------- + +.. automodule:: feast.transformation.substrait_transformation + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.transformation + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/index.rst b/sdk/python/docs/source/index.rst index 4cedffb1fc0..1ef6bd16c80 100644 --- a/sdk/python/docs/source/index.rst +++ b/sdk/python/docs/source/index.rst @@ -182,12 +182,6 @@ S3 Registry Store .. autoclass:: feast.infra.registry.s3.S3RegistryStore :members: -PostgreSQL Registry Store ------------------------ - -.. autoclass:: feast.infra.registry.contrib.postgres.postgres_registry_store.PostgreSQLRegistryStore - :members: - Provider ================== diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index f239c2dfad5..f0655c40f24 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -27,6 +27,7 @@ from feast import utils from feast.constants import ( DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT, + DEFAULT_OFFLINE_SERVER_PORT, DEFAULT_REGISTRY_SERVER_PORT, ) from feast.errors import FeastObjectNotFoundException, FeastProviderLoginError @@ -46,6 +47,12 @@ from feast.utils import maybe_local_tz _logger = logging.getLogger(__name__) +tagsOption = click.option( + "--tags", + help="Filter by tags (e.g. --tags 'key:value' --tags 'key:value, key:value, ...'). Items return when ALL tags match.", + default=[""], + multiple=True, +) class NoOptionDefaultFormat(click.Command): @@ -225,14 +232,16 @@ def data_source_describe(ctx: click.Context, name: str): @data_sources_cmd.command(name="list") +@tagsOption @click.pass_context -def data_source_list(ctx: click.Context): +def data_source_list(ctx: click.Context, tags: list[str]): """ List all data sources """ store = create_feature_store(ctx) table = [] - for datasource in store.list_data_sources(): + tags_filter = utils.tags_list_to_dict(tags) + for datasource in store.list_data_sources(tags=tags_filter): table.append([datasource.name, datasource.__class__]) from tabulate import tabulate @@ -271,14 +280,16 @@ def entity_describe(ctx: click.Context, name: str): @entities_cmd.command(name="list") +@tagsOption @click.pass_context -def entity_list(ctx: click.Context): +def entity_list(ctx: click.Context, tags: list[str]): """ List all entities """ store = create_feature_store(ctx) table = [] - for entity in store.list_entities(): + tags_filter = utils.tags_list_to_dict(tags) + for entity in store.list_entities(tags=tags_filter): table.append([entity.name, entity.description, entity.value_type]) from tabulate import tabulate @@ -319,14 +330,16 @@ def feature_service_describe(ctx: click.Context, name: str): @feature_services_cmd.command(name="list") +@tagsOption @click.pass_context -def feature_service_list(ctx: click.Context): +def feature_service_list(ctx: click.Context, tags: list[str]): """ List all feature services """ store = create_feature_store(ctx) feature_services = [] - for feature_service in store.list_feature_services(): + tags_filter = utils.tags_list_to_dict(tags) + for feature_service in store.list_feature_services(tags=tags_filter): feature_names = [] for projection in feature_service.feature_view_projections: feature_names.extend( @@ -370,16 +383,18 @@ def feature_view_describe(ctx: click.Context, name: str): @feature_views_cmd.command(name="list") +@tagsOption @click.pass_context -def feature_view_list(ctx: click.Context): +def feature_view_list(ctx: click.Context, tags: list[str]): """ List all feature views """ store = create_feature_store(ctx) table = [] + tags_filter = utils.tags_list_to_dict(tags) for feature_view in [ - *store.list_feature_views(), - *store.list_on_demand_feature_views(), + *store.list_batch_feature_views(tags=tags_filter), + *store.list_on_demand_feature_views(tags=tags_filter), ]: entities = set() if isinstance(feature_view, FeatureView): @@ -433,14 +448,16 @@ def on_demand_feature_view_describe(ctx: click.Context, name: str): @on_demand_feature_views_cmd.command(name="list") +@tagsOption @click.pass_context -def on_demand_feature_view_list(ctx: click.Context): +def on_demand_feature_view_list(ctx: click.Context, tags: list[str]): """ [Experimental] List all on demand feature views """ store = create_feature_store(ctx) table = [] - for on_demand_feature_view in store.list_on_demand_feature_views(): + tags_filter = utils.tags_list_to_dict(tags) + for on_demand_feature_view in store.list_on_demand_feature_views(tags=tags_filter): table.append([on_demand_feature_view.name]) from tabulate import tabulate @@ -644,12 +661,6 @@ def init_command(project_directory, minimal: bool, template: str): show_default=True, help="Disable the Uvicorn access log", ) -@click.option( - "--no-feature-log", - is_flag=True, - show_default=True, - help="Disable logging served features", -) @click.option( "--workers", "-w", @@ -680,7 +691,6 @@ def serve_command( port: int, type_: str, no_access_log: bool, - no_feature_log: bool, workers: int, keep_alive_timeout: int, registry_ttl_sec: int = 5, @@ -693,7 +703,6 @@ def serve_command( port=port, type_=type_, no_access_log=no_access_log, - no_feature_log=no_feature_log, workers=workers, keep_alive_timeout=keep_alive_timeout, registry_ttl_sec=registry_ttl_sec, @@ -773,6 +782,34 @@ def serve_registry_command(ctx: click.Context, port: int): store.serve_registry(port) +@cli.command("serve_offline") +@click.option( + "--host", + "-h", + type=click.STRING, + default="127.0.0.1", + show_default=True, + help="Specify a host for the server", +) +@click.option( + "--port", + "-p", + type=click.INT, + default=DEFAULT_OFFLINE_SERVER_PORT, + help="Specify a port for the server", +) +@click.pass_context +def serve_offline_command( + ctx: click.Context, + host: str, + port: int, +): + """Start a remote server locally on a given host, port.""" + store = create_feature_store(ctx) + + store.serve_offline(host, port) + + @cli.command("validate") @click.option( "--feature-service", diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 6aad3e60bbf..fa8674d91d2 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -41,6 +41,9 @@ # Default registry server port DEFAULT_REGISTRY_SERVER_PORT = 6570 +# Default offline server port +DEFAULT_OFFLINE_SERVER_PORT = 8815 + # Environment variable for feature server docker image tag DOCKER_IMAGE_TAG_ENV_NAME: str = "FEAST_SERVER_DOCKER_IMAGE_TAG" diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index 98a8c0caf49..bf20e51df98 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -2,7 +2,6 @@ import sys import threading import traceback -import warnings from contextlib import asynccontextmanager from typing import List, Optional @@ -97,9 +96,9 @@ def get_online_features(body=Depends(get_body)): full_feature_names = body.get("full_feature_names", False) - response_proto = store._get_online_features( + response_proto = store.get_online_features( features=features, - entity_values=body["entities"], + entity_rows=body["entities"], full_feature_names=full_feature_names, ).proto @@ -147,10 +146,6 @@ def push(body=Depends(get_body)): @app.post("/write-to-online-store") def write_to_online_store(body=Depends(get_body)): - warnings.warn( - "write_to_online_store is deprecated. Please consider using /push instead", - RuntimeWarning, - ) try: request = WriteToFeatureStoreRequest(**json.loads(body)) df = pd.DataFrame(request.df) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 343aa04d604..b7e4ef619f0 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -11,11 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import copy import itertools +import logging import os import warnings -from collections import Counter, defaultdict from datetime import datetime, timedelta from pathlib import Path from typing import ( @@ -27,7 +26,6 @@ Mapping, Optional, Sequence, - Set, Tuple, Union, cast, @@ -56,20 +54,15 @@ from feast.errors import ( DataFrameSerializationError, DataSourceRepeatNamesException, - EntityNotFoundException, - FeatureNameCollisionError, FeatureViewNotFoundException, PushSourceNotFoundException, RequestDataNotFoundInEntityDfException, - RequestDataNotFoundInEntityRowsException, ) from feast.feast_object import FeastObject from feast.feature_service import FeatureService from feast.feature_view import ( DUMMY_ENTITY, - DUMMY_ENTITY_ID, DUMMY_ENTITY_NAME, - DUMMY_ENTITY_VAL, FeatureView, ) from feast.inference import ( @@ -88,14 +81,11 @@ FieldStatus, GetOnlineFeaturesResponse, ) -from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import RepeatedValue, Value from feast.repo_config import RepoConfig, load_repo_config from feast.repo_contents import RepoContents from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference from feast.stream_feature_view import StreamFeatureView -from feast.type_map import python_values_to_proto_values -from feast.value_type import ValueType from feast.version import get_version warnings.simplefilter("once", DeprecationWarning) @@ -214,23 +204,29 @@ def refresh_registry(self): self._registry = registry - def list_entities(self, allow_cache: bool = False) -> List[Entity]: + def list_entities( + self, allow_cache: bool = False, tags: Optional[dict[str, str]] = None + ) -> List[Entity]: """ Retrieves the list of entities from the registry. Args: allow_cache: Whether to allow returning entities from a cached registry. + tags: Filter by tags. Returns: A list of entities. """ - return self._list_entities(allow_cache) + return self._list_entities(allow_cache, tags=tags) def _list_entities( - self, allow_cache: bool = False, hide_dummy_entity: bool = True + self, + allow_cache: bool = False, + hide_dummy_entity: bool = True, + tags: Optional[dict[str, str]] = None, ) -> List[Entity]: all_entities = self._registry.list_entities( - self.project, allow_cache=allow_cache + self.project, allow_cache=allow_cache, tags=tags ) return [ entity @@ -238,35 +234,117 @@ def _list_entities( if entity.name != DUMMY_ENTITY_NAME or not hide_dummy_entity ] - def list_feature_services(self) -> List[FeatureService]: + def list_feature_services( + self, tags: Optional[dict[str, str]] = None + ) -> List[FeatureService]: """ Retrieves the list of feature services from the registry. + Args: + tags: Filter by tags. + Returns: A list of feature services. """ - return self._registry.list_feature_services(self.project) + return self._registry.list_feature_services(self.project, tags=tags) + + def list_all_feature_views( + self, allow_cache: bool = False, tags: Optional[dict[str, str]] = None + ) -> List[Union[FeatureView, StreamFeatureView, OnDemandFeatureView]]: + """ + Retrieves the list of feature views from the registry. + + Args: + allow_cache: Whether to allow returning entities from a cached registry. + + Returns: + A list of feature views. + """ + return self._list_all_feature_views(allow_cache, tags=tags) - def list_feature_views(self, allow_cache: bool = False) -> List[FeatureView]: + def list_feature_views( + self, allow_cache: bool = False, tags: Optional[dict[str, str]] = None + ) -> List[FeatureView]: """ Retrieves the list of feature views from the registry. Args: allow_cache: Whether to allow returning entities from a cached registry. + tags: Filter by tags. Returns: A list of feature views. """ - return self._list_feature_views(allow_cache) + logging.warning( + "list_feature_views will make breaking changes. Please use list_batch_feature_views instead. " + "list_feature_views will behave like list_all_feature_views in the future." + ) + return utils._list_feature_views( + self._registry, self.project, allow_cache, tags=tags + ) + + def list_batch_feature_views( + self, allow_cache: bool = False, tags: Optional[dict[str, str]] = None + ) -> List[FeatureView]: + """ + Retrieves the list of feature views from the registry. + + Args: + allow_cache: Whether to allow returning entities from a cached registry. + tags: Filter by tags. + + Returns: + A list of feature views. + """ + return self._list_batch_feature_views(allow_cache=allow_cache, tags=tags) + + def _list_all_feature_views( + self, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, + ) -> List[Union[FeatureView, StreamFeatureView, OnDemandFeatureView]]: + all_feature_views = ( + utils._list_feature_views( + self._registry, self.project, allow_cache, tags=tags + ) + + self._list_stream_feature_views(allow_cache, tags=tags) + + self.list_on_demand_feature_views(allow_cache, tags=tags) + ) + return all_feature_views def _list_feature_views( self, allow_cache: bool = False, hide_dummy_entity: bool = True, + tags: Optional[dict[str, str]] = None, + ) -> List[FeatureView]: + logging.warning( + "_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. " + "_list_feature_views will behave like _list_all_feature_views in the future." + ) + feature_views = [] + for fv in self._registry.list_feature_views( + self.project, allow_cache=allow_cache, tags=tags + ): + if ( + hide_dummy_entity + and fv.entities + and fv.entities[0] == DUMMY_ENTITY_NAME + ): + fv.entities = [] + fv.entity_columns = [] + feature_views.append(fv) + return feature_views + + def _list_batch_feature_views( + self, + allow_cache: bool = False, + hide_dummy_entity: bool = True, + tags: Optional[dict[str, str]] = None, ) -> List[FeatureView]: feature_views = [] for fv in self._registry.list_feature_views( - self.project, allow_cache=allow_cache + self.project, allow_cache=allow_cache, tags=tags ): if ( hide_dummy_entity @@ -282,10 +360,11 @@ def _list_stream_feature_views( self, allow_cache: bool = False, hide_dummy_entity: bool = True, + tags: Optional[dict[str, str]] = None, ) -> List[StreamFeatureView]: stream_feature_views = [] for sfv in self._registry.list_stream_feature_views( - self.project, allow_cache=allow_cache + self.project, allow_cache=allow_cache, tags=tags ): if hide_dummy_entity and sfv.entities[0] == DUMMY_ENTITY_NAME: sfv.entities = [] @@ -294,20 +373,24 @@ def _list_stream_feature_views( return stream_feature_views def list_on_demand_feature_views( - self, allow_cache: bool = False + self, allow_cache: bool = False, tags: Optional[dict[str, str]] = None ) -> List[OnDemandFeatureView]: """ Retrieves the list of on demand feature views from the registry. + Args: + allow_cache: Whether to allow returning entities from a cached registry. + tags: Filter by tags. + Returns: A list of on demand feature views. """ return self._registry.list_on_demand_feature_views( - self.project, allow_cache=allow_cache + self.project, allow_cache=allow_cache, tags=tags ) def list_stream_feature_views( - self, allow_cache: bool = False + self, allow_cache: bool = False, tags: Optional[dict[str, str]] = None ) -> List[StreamFeatureView]: """ Retrieves the list of stream feature views from the registry. @@ -315,19 +398,24 @@ def list_stream_feature_views( Returns: A list of stream feature views. """ - return self._list_stream_feature_views(allow_cache) + return self._list_stream_feature_views(allow_cache, tags=tags) - def list_data_sources(self, allow_cache: bool = False) -> List[DataSource]: + def list_data_sources( + self, allow_cache: bool = False, tags: Optional[dict[str, str]] = None + ) -> List[DataSource]: """ Retrieves the list of data sources from the registry. Args: allow_cache: Whether to allow returning data sources from a cached registry. + tags: Filter by tags. Returns: A list of data sources. """ - return self._registry.list_data_sources(self.project, allow_cache=allow_cache) + return self._registry.list_data_sources( + self.project, allow_cache=allow_cache, tags=tags + ) def get_entity(self, name: str, allow_registry_cache: bool = False) -> Entity: """ @@ -483,39 +571,6 @@ def delete_feature_service(self, name: str): """ return self._registry.delete_feature_service(name, self.project) - def _get_features( - self, - features: Union[List[str], FeatureService], - allow_cache: bool = False, - ) -> List[str]: - _features = features - - if not _features: - raise ValueError("No features specified for retrieval") - - _feature_refs = [] - if isinstance(_features, FeatureService): - feature_service_from_registry = self.get_feature_service( - _features.name, allow_cache - ) - if feature_service_from_registry != _features: - warnings.warn( - "The FeatureService object that has been passed in as an argument is " - "inconsistent with the version from the registry. Potentially a newer version " - "of the FeatureService has been applied to the registry." - ) - for projection in feature_service_from_registry.feature_view_projections: - _feature_refs.extend( - [ - f"{projection.name_to_use()}:{f.name}" - for f in projection.features - ] - ) - else: - assert isinstance(_features, list) - _feature_refs = _features - return _feature_refs - def _should_use_plan(self): """Returns True if plan and _apply_diffs should be used, False otherwise.""" # Currently only the local provider with sqlite online store supports plan and _apply_diffs. @@ -609,8 +664,8 @@ def _get_feature_views_to_materialize( feature_views_to_materialize: List[FeatureView] = [] if feature_views is None: - feature_views_to_materialize = self._list_feature_views( - hide_dummy_entity=False + feature_views_to_materialize = utils._list_feature_views( + self._registry, self.project, hide_dummy_entity=False ) feature_views_to_materialize = [ fv for fv in feature_views_to_materialize if fv.online @@ -1020,16 +1075,16 @@ def get_historical_features( ... ) >>> feature_data = retrieval_job.to_df() """ - _feature_refs = self._get_features(features) + _feature_refs = utils._get_features(self._registry, self.project, features) ( all_feature_views, all_on_demand_feature_views, - ) = self._get_feature_views_to_use(features) + ) = utils._get_feature_views_to_use(self._registry, self.project, features) # TODO(achal): _group_feature_refs returns the on demand feature views, but it's not passed into the provider. # This is a weird interface quirk - we should revisit the `get_historical_features` to # pass in the on demand feature views as well. - fvs, odfvs = _group_feature_refs( + fvs, odfvs = utils._group_feature_refs( _feature_refs, all_feature_views, all_on_demand_feature_views, @@ -1050,7 +1105,7 @@ def get_historical_features( feature_view_name=odfv.name, ) - _validate_feature_refs(_feature_refs, full_feature_names) + utils._validate_feature_refs(_feature_refs, full_feature_names) provider = self._get_provider() job = provider.get_historical_features( @@ -1457,7 +1512,10 @@ def write_to_offline_store( def get_online_features( self, features: Union[List[str], FeatureService], - entity_rows: List[Dict[str, Any]], + entity_rows: Union[ + List[Dict[str, Any]], + Mapping[str, Union[Sequence[Any], Sequence[Value], RepeatedValue]], + ], full_feature_names: bool = False, ) -> OnlineResponse: """ @@ -1501,230 +1559,19 @@ def get_online_features( ... ) >>> online_response_dict = online_response.to_dict() """ - columnar: Dict[str, List[Any]] = {k: [] for k in entity_rows[0].keys()} - for entity_row in entity_rows: - for key, value in entity_row.items(): - try: - columnar[key].append(value) - except KeyError as e: - raise ValueError("All entity_rows must have the same keys.") from e - - return self._get_online_features( - features=features, - entity_values=columnar, - full_feature_names=full_feature_names, - native_entity_values=True, - ) - - async def get_online_features_async( - self, - features: Union[List[str], FeatureService], - entity_rows: List[Dict[str, Any]], - full_feature_names: bool = False, - ) -> OnlineResponse: - """ - [Alpha] Retrieves the latest online feature data asynchronously. - - Note: This method will download the full feature registry the first time it is run. If you are using a - remote registry like GCS or S3 then that may take a few seconds. The registry remains cached up to a TTL - duration (which can be set to infinity). If the cached registry is stale (more time than the TTL has - passed), then a new registry will be downloaded synchronously by this method. This download may - introduce latency to online feature retrieval. In order to avoid synchronous downloads, please call - refresh_registry() prior to the TTL being reached. Remember it is possible to set the cache TTL to - infinity (cache forever). - - Args: - features: The list of features that should be retrieved from the online store. These features can be - specified either as a list of string feature references or as a feature service. String feature - references must have format "feature_view:feature", e.g. "customer_fv:daily_transactions". - entity_rows: A list of dictionaries where each key-value is an entity-name, entity-value pair. - full_feature_names: If True, feature names will be prefixed with the corresponding feature view name, - changing them from the format "feature" to "feature_view__feature" (e.g. "daily_transactions" - changes to "customer_fv__daily_transactions"). - - Returns: - OnlineResponse containing the feature data in records. - - Raises: - Exception: No entity with the specified name exists. - """ - columnar: Dict[str, List[Any]] = {k: [] for k in entity_rows[0].keys()} - for entity_row in entity_rows: - for key, value in entity_row.items(): - try: - columnar[key].append(value) - except KeyError as e: - raise ValueError("All entity_rows must have the same keys.") from e - - return await self._get_online_features_async( - features=features, - entity_values=columnar, - full_feature_names=full_feature_names, - native_entity_values=True, - ) - - def _get_online_request_context( - self, features: Union[List[str], FeatureService], full_feature_names: bool - ): - _feature_refs = self._get_features(features, allow_cache=True) - - ( - requested_feature_views, - requested_on_demand_feature_views, - ) = self._get_feature_views_to_use( - features=features, allow_cache=True, hide_dummy_entity=False - ) - - ( - entity_name_to_join_key_map, - entity_type_map, - join_keys_set, - ) = self._get_entity_maps(requested_feature_views) - - _validate_feature_refs(_feature_refs, full_feature_names) - ( - grouped_refs, - grouped_odfv_refs, - ) = _group_feature_refs( - _feature_refs, - requested_feature_views, - requested_on_demand_feature_views, - ) - - requested_result_row_names = { - feat_ref.replace(":", "__") for feat_ref in _feature_refs - } - if not full_feature_names: - requested_result_row_names = { - name.rpartition("__")[-1] for name in requested_result_row_names - } - - feature_views = list(view for view, _ in grouped_refs) - - needed_request_data = self.get_needed_request_data(grouped_odfv_refs) - - entityless_case = DUMMY_ENTITY_NAME in [ - entity_name - for feature_view in feature_views - for entity_name in feature_view.entities - ] - - return ( - _feature_refs, - requested_on_demand_feature_views, - entity_name_to_join_key_map, - entity_type_map, - join_keys_set, - grouped_refs, - requested_result_row_names, - needed_request_data, - entityless_case, - ) - - def _prepare_entities_to_read_from_online_store( - self, - features: Union[List[str], FeatureService], - entity_values: Mapping[ - str, Union[Sequence[Any], Sequence[Value], RepeatedValue] - ], - full_feature_names: bool = False, - native_entity_values: bool = True, - ): - ( - feature_refs, - requested_on_demand_feature_views, - entity_name_to_join_key_map, - entity_type_map, - join_keys_set, - grouped_refs, - requested_result_row_names, - needed_request_data, - entityless_case, - ) = self._get_online_request_context(features, full_feature_names) - - # Extract Sequence from RepeatedValue Protobuf. - entity_value_lists: Dict[str, Union[List[Any], List[Value]]] = { - k: list(v) if isinstance(v, Sequence) else list(v.val) - for k, v in entity_values.items() - } - - entity_proto_values: Dict[str, List[Value]] - if native_entity_values: - # Convert values to Protobuf once. - entity_proto_values = { - k: python_values_to_proto_values( - v, entity_type_map.get(k, ValueType.UNKNOWN) - ) - for k, v in entity_value_lists.items() - } - else: - entity_proto_values = entity_value_lists - - num_rows = _validate_entity_values(entity_proto_values) - - join_key_values: Dict[str, List[Value]] = {} - request_data_features: Dict[str, List[Value]] = {} - # Entity rows may be either entities or request data. - for join_key_or_entity_name, values in entity_proto_values.items(): - # Found request data - if join_key_or_entity_name in needed_request_data: - request_data_features[join_key_or_entity_name] = values - else: - if join_key_or_entity_name in join_keys_set: - join_key = join_key_or_entity_name - else: + if isinstance(entity_rows, list): + columnar: Dict[str, List[Any]] = {k: [] for k in entity_rows[0].keys()} + for entity_row in entity_rows: + for key, value in entity_row.items(): try: - join_key = entity_name_to_join_key_map[join_key_or_entity_name] - except KeyError: - raise EntityNotFoundException( - join_key_or_entity_name, self.project - ) - else: - warnings.warn( - "Using entity name is deprecated. Use join_key instead." - ) + columnar[key].append(value) + except KeyError as e: + raise ValueError( + "All entity_rows must have the same keys." + ) from e - # All join keys should be returned in the result. - requested_result_row_names.add(join_key) - join_key_values[join_key] = values + entity_rows = columnar - self.ensure_request_data_values_exist( - needed_request_data, request_data_features - ) - - # Populate online features response proto with join keys and request data features - online_features_response = GetOnlineFeaturesResponse(results=[]) - self._populate_result_rows_from_columnar( - online_features_response=online_features_response, - data=dict(**join_key_values, **request_data_features), - ) - - # Add the Entityless case after populating result rows to avoid having to remove - # it later. - if entityless_case: - join_key_values[DUMMY_ENTITY_ID] = python_values_to_proto_values( - [DUMMY_ENTITY_VAL] * num_rows, DUMMY_ENTITY.value_type - ) - - return ( - join_key_values, - grouped_refs, - entity_name_to_join_key_map, - requested_on_demand_feature_views, - feature_refs, - requested_result_row_names, - online_features_response, - ) - - def _get_online_features( - self, - features: Union[List[str], FeatureService], - entity_values: Mapping[ - str, Union[Sequence[Any], Sequence[Value], RepeatedValue] - ], - full_feature_names: bool = False, - native_entity_values: bool = True, - ): ( join_key_values, grouped_refs, @@ -1733,17 +1580,19 @@ def _get_online_features( feature_refs, requested_result_row_names, online_features_response, - ) = self._prepare_entities_to_read_from_online_store( + ) = utils._prepare_entities_to_read_from_online_store( + registry=self._registry, + project=self.project, features=features, - entity_values=entity_values, + entity_values=entity_rows, full_feature_names=full_feature_names, - native_entity_values=native_entity_values, + native_entity_values=True, ) provider = self._get_provider() for table, requested_features in grouped_refs: # Get the correct set of entity values with the correct join keys. - table_entity_values, idxs = self._get_unique_entities( + table_entity_values, idxs = utils._get_unique_entities( table, join_key_values, entity_name_to_join_key_map, @@ -1758,7 +1607,7 @@ def _get_online_features( ) # Populate the result_rows with the Features from the OnlineStore inplace. - self._populate_response_from_feature_data( + utils._populate_response_from_feature_data( feature_data, idxs, online_features_response, @@ -1768,27 +1617,66 @@ def _get_online_features( ) if requested_on_demand_feature_views: - self._augment_response_with_on_demand_transforms( + utils._augment_response_with_on_demand_transforms( online_features_response, feature_refs, requested_on_demand_feature_views, full_feature_names, ) - self._drop_unneeded_columns( + utils._drop_unneeded_columns( online_features_response, requested_result_row_names ) return OnlineResponse(online_features_response) - async def _get_online_features_async( + async def get_online_features_async( self, features: Union[List[str], FeatureService], - entity_values: Mapping[ - str, Union[Sequence[Any], Sequence[Value], RepeatedValue] + entity_rows: Union[ + List[Dict[str, Any]], + Mapping[str, Union[Sequence[Any], Sequence[Value], RepeatedValue]], ], full_feature_names: bool = False, - native_entity_values: bool = True, - ): + ) -> OnlineResponse: + """ + [Alpha] Retrieves the latest online feature data asynchronously. + + Note: This method will download the full feature registry the first time it is run. If you are using a + remote registry like GCS or S3 then that may take a few seconds. The registry remains cached up to a TTL + duration (which can be set to infinity). If the cached registry is stale (more time than the TTL has + passed), then a new registry will be downloaded synchronously by this method. This download may + introduce latency to online feature retrieval. In order to avoid synchronous downloads, please call + refresh_registry() prior to the TTL being reached. Remember it is possible to set the cache TTL to + infinity (cache forever). + + Args: + features: The list of features that should be retrieved from the online store. These features can be + specified either as a list of string feature references or as a feature service. String feature + references must have format "feature_view:feature", e.g. "customer_fv:daily_transactions". + entity_rows: A list of dictionaries where each key-value is an entity-name, entity-value pair. + full_feature_names: If True, feature names will be prefixed with the corresponding feature view name, + changing them from the format "feature" to "feature_view__feature" (e.g. "daily_transactions" + changes to "customer_fv__daily_transactions"). + + Returns: + OnlineResponse containing the feature data in records. + + Raises: + Exception: No entity with the specified name exists. + """ + if isinstance(entity_rows, list): + columnar: Dict[str, List[Any]] = {k: [] for k in entity_rows[0].keys()} + for entity_row in entity_rows: + for key, value in entity_row.items(): + try: + columnar[key].append(value) + except KeyError as e: + raise ValueError( + "All entity_rows must have the same keys." + ) from e + + entity_rows = columnar + ( join_key_values, grouped_refs, @@ -1797,17 +1685,19 @@ async def _get_online_features_async( feature_refs, requested_result_row_names, online_features_response, - ) = self._prepare_entities_to_read_from_online_store( + ) = utils._prepare_entities_to_read_from_online_store( + registry=self._registry, + project=self.project, features=features, - entity_values=entity_values, + entity_values=entity_rows, full_feature_names=full_feature_names, - native_entity_values=native_entity_values, + native_entity_values=True, ) provider = self._get_provider() for table, requested_features in grouped_refs: # Get the correct set of entity values with the correct join keys. - table_entity_values, idxs = self._get_unique_entities( + table_entity_values, idxs = utils._get_unique_entities( table, join_key_values, entity_name_to_join_key_map, @@ -1822,7 +1712,7 @@ async def _get_online_features_async( ) # Populate the result_rows with the Features from the OnlineStore inplace. - self._populate_response_from_feature_data( + utils._populate_response_from_feature_data( feature_data, idxs, online_features_response, @@ -1832,14 +1722,14 @@ async def _get_online_features_async( ) if requested_on_demand_feature_views: - self._augment_response_with_on_demand_transforms( + utils._augment_response_with_on_demand_transforms( online_features_response, feature_refs, requested_on_demand_feature_views, full_feature_names, ) - self._drop_unneeded_columns( + utils._drop_unneeded_columns( online_features_response, requested_result_row_names ) return OnlineResponse(online_features_response) @@ -1862,37 +1752,37 @@ def retrieve_online_documents( top_k: The number of closest document features to retrieve. distance_metric: The distance metric to use for retrieval. """ - return self._retrieve_online_documents( - feature=feature, - query=query, - top_k=top_k, - distance_metric=distance_metric, - ) - - def _retrieve_online_documents( - self, - feature: str, - query: Union[str, List[float]], - top_k: int, - distance_metric: Optional[str] = None, - ): if isinstance(query, str): raise ValueError( "Using embedding functionality is not supported for document retrieval. Please embed the query before calling retrieve_online_documents." ) ( - requested_feature_views, + available_feature_views, _, - ) = self._get_feature_views_to_use( - features=[feature], allow_cache=True, hide_dummy_entity=False + ) = utils._get_feature_views_to_use( + registry=self._registry, + project=self.project, + features=[feature], + allow_cache=True, + hide_dummy_entity=False, ) + requested_feature_view_name = ( + feature.split(":")[0] if isinstance(feature, str) else feature + ) + for feature_view in available_feature_views: + if feature_view.name == requested_feature_view_name: + requested_feature_view = feature_view + if not requested_feature_view: + raise ValueError( + f"Feature view {requested_feature_view} not found in the registry." + ) requested_feature = ( feature.split(":")[1] if isinstance(feature, str) else feature ) provider = self._get_provider() document_features = self._retrieve_from_online_store( provider, - requested_feature_views[0], + requested_feature_view, requested_feature, query, top_k, @@ -1906,218 +1796,16 @@ def _retrieve_online_documents( document_feature_vals = [feature[2] for feature in document_features] document_feature_distance_vals = [feature[4] for feature in document_features] online_features_response = GetOnlineFeaturesResponse(results=[]) - self._populate_result_rows_from_columnar( + utils._populate_result_rows_from_columnar( online_features_response=online_features_response, data={requested_feature: document_feature_vals}, ) - self._populate_result_rows_from_columnar( + utils._populate_result_rows_from_columnar( online_features_response=online_features_response, data={"distance": document_feature_distance_vals}, ) return OnlineResponse(online_features_response) - @staticmethod - def _get_columnar_entity_values( - rowise: Optional[List[Dict[str, Any]]], columnar: Optional[Dict[str, List[Any]]] - ) -> Dict[str, List[Any]]: - if (rowise is None and columnar is None) or ( - rowise is not None and columnar is not None - ): - raise ValueError( - "Exactly one of `columnar_entity_values` and `rowise_entity_values` must be set." - ) - - if rowise is not None: - # Convert entity_rows from rowise to columnar. - res = defaultdict(list) - for entity_row in rowise: - for key, value in entity_row.items(): - res[key].append(value) - return res - return cast(Dict[str, List[Any]], columnar) - - def _get_entity_maps( - self, feature_views - ) -> Tuple[Dict[str, str], Dict[str, ValueType], Set[str]]: - # TODO(felixwang9817): Support entities that have different types for different feature views. - entities = self._list_entities(allow_cache=True, hide_dummy_entity=False) - entity_name_to_join_key_map: Dict[str, str] = {} - entity_type_map: Dict[str, ValueType] = {} - for entity in entities: - entity_name_to_join_key_map[entity.name] = entity.join_key - for feature_view in feature_views: - for entity_name in feature_view.entities: - entity = self._registry.get_entity( - entity_name, self.project, allow_cache=True - ) - # User directly uses join_key as the entity reference in the entity_rows for the - # entity mapping case. - entity_name = feature_view.projection.join_key_map.get( - entity.join_key, entity.name - ) - join_key = feature_view.projection.join_key_map.get( - entity.join_key, entity.join_key - ) - entity_name_to_join_key_map[entity_name] = join_key - for entity_column in feature_view.entity_columns: - entity_type_map[entity_column.name] = ( - entity_column.dtype.to_value_type() - ) - - return ( - entity_name_to_join_key_map, - entity_type_map, - set(entity_name_to_join_key_map.values()), - ) - - @staticmethod - def _get_table_entity_values( - table: FeatureView, - entity_name_to_join_key_map: Dict[str, str], - join_key_proto_values: Dict[str, List[Value]], - ) -> Dict[str, List[Value]]: - # The correct join_keys expected by the OnlineStore for this Feature View. - table_join_keys = [ - entity_name_to_join_key_map[entity_name] for entity_name in table.entities - ] - - # If the FeatureView has a Projection then the join keys may be aliased. - alias_to_join_key_map = {v: k for k, v in table.projection.join_key_map.items()} - - # Subset to columns which are relevant to this FeatureView and - # give them the correct names. - entity_values = { - alias_to_join_key_map.get(k, k): v - for k, v in join_key_proto_values.items() - if alias_to_join_key_map.get(k, k) in table_join_keys - } - return entity_values - - @staticmethod - def _populate_result_rows_from_columnar( - online_features_response: GetOnlineFeaturesResponse, - data: Dict[str, List[Value]], - ): - timestamp = Timestamp() # Only initialize this timestamp once. - # Add more values to the existing result rows - for feature_name, feature_values in data.items(): - online_features_response.metadata.feature_names.val.append(feature_name) - online_features_response.results.append( - GetOnlineFeaturesResponse.FeatureVector( - values=feature_values, - statuses=[FieldStatus.PRESENT] * len(feature_values), - event_timestamps=[timestamp] * len(feature_values), - ) - ) - - @staticmethod - def get_needed_request_data( - grouped_odfv_refs: List[Tuple[OnDemandFeatureView, List[str]]], - ) -> Set[str]: - needed_request_data: Set[str] = set() - for odfv, _ in grouped_odfv_refs: - odfv_request_data_schema = odfv.get_request_data_schema() - needed_request_data.update(odfv_request_data_schema.keys()) - return needed_request_data - - @staticmethod - def ensure_request_data_values_exist( - needed_request_data: Set[str], - request_data_features: Dict[str, List[Any]], - ): - if len(needed_request_data) != len(request_data_features.keys()): - missing_features = [ - x for x in needed_request_data if x not in request_data_features - ] - raise RequestDataNotFoundInEntityRowsException( - feature_names=missing_features - ) - - def _get_unique_entities( - self, - table: FeatureView, - join_key_values: Dict[str, List[Value]], - entity_name_to_join_key_map: Dict[str, str], - ) -> Tuple[Tuple[Dict[str, Value], ...], Tuple[List[int], ...]]: - """Return the set of unique composite Entities for a Feature View and the indexes at which they appear. - - This method allows us to query the OnlineStore for data we need only once - rather than requesting and processing data for the same combination of - Entities multiple times. - """ - # Get the correct set of entity values with the correct join keys. - table_entity_values = self._get_table_entity_values( - table, - entity_name_to_join_key_map, - join_key_values, - ) - - # Convert back to rowise. - keys = table_entity_values.keys() - # Sort the rowise data to allow for grouping but keep original index. This lambda is - # sufficient as Entity types cannot be complex (ie. lists). - rowise = list(enumerate(zip(*table_entity_values.values()))) - rowise.sort( - key=lambda row: tuple(getattr(x, x.WhichOneof("val")) for x in row[1]) - ) - - # Identify unique entities and the indexes at which they occur. - unique_entities: Tuple[Dict[str, Value], ...] - indexes: Tuple[List[int], ...] - unique_entities, indexes = tuple( - zip( - *[ - (dict(zip(keys, k)), [_[0] for _ in g]) - for k, g in itertools.groupby(rowise, key=lambda x: x[1]) - ] - ) - ) - return unique_entities, indexes - - def _get_entity_key_protos( - self, - entity_rows: Iterable[Mapping[str, Value]], - ) -> List[EntityKeyProto]: - # Instantiate one EntityKeyProto per Entity. - entity_key_protos = [ - EntityKeyProto(join_keys=row.keys(), entity_values=row.values()) - for row in entity_rows - ] - return entity_key_protos - - def _convert_rows_to_protobuf( - self, - requested_features: List[str], - read_rows: List[Tuple[Optional[datetime], Optional[Dict[str, Value]]]], - ) -> List[Tuple[List[Timestamp], List["FieldStatus.ValueType"], List[Value]]]: - # Each row is a set of features for a given entity key. - # We only need to convert the data to Protobuf once. - null_value = Value() - read_row_protos = [] - for read_row in read_rows: - row_ts_proto = Timestamp() - row_ts, feature_data = read_row - # TODO (Ly): reuse whatever timestamp if row_ts is None? - if row_ts is not None: - row_ts_proto.FromDatetime(row_ts) - event_timestamps = [row_ts_proto] * len(requested_features) - if feature_data is None: - statuses = [FieldStatus.NOT_FOUND] * len(requested_features) - values = [null_value] * len(requested_features) - else: - statuses = [] - values = [] - for feature_name in requested_features: - # Make sure order of data is the same as requested_features. - if feature_name not in feature_data: - statuses.append(FieldStatus.NOT_FOUND) - values.append(null_value) - else: - statuses.append(FieldStatus.PRESENT) - values.append(feature_data[feature_name]) - read_row_protos.append((event_timestamps, statuses, values)) - return read_row_protos - def _read_from_online_store( self, entity_rows: Iterable[Mapping[str, Value]], @@ -2134,7 +1822,7 @@ def _read_from_online_store( combination of Entities in `entity_rows` in the same order as they are provided. """ - entity_key_protos = self._get_entity_key_protos(entity_rows) + entity_key_protos = utils._get_entity_key_protos(entity_rows) # Fetch data for Entities. read_rows = provider.online_read( @@ -2144,7 +1832,7 @@ def _read_from_online_store( requested_features=requested_features, ) - return self._convert_rows_to_protobuf(requested_features, read_rows) + return utils._convert_rows_to_protobuf(requested_features, read_rows) async def _read_from_online_store_async( self, @@ -2153,7 +1841,7 @@ async def _read_from_online_store_async( requested_features: List[str], table: FeatureView, ) -> List[Tuple[List[Timestamp], List["FieldStatus.ValueType"], List[Value]]]: - entity_key_protos = self._get_entity_key_protos(entity_rows) + entity_key_protos = utils._get_entity_key_protos(entity_rows) # Fetch data for Entities. read_rows = await provider.online_read_async( @@ -2163,7 +1851,7 @@ async def _read_from_online_store_async( requested_features=requested_features, ) - return self._convert_rows_to_protobuf(requested_features, read_rows) + return utils._convert_rows_to_protobuf(requested_features, read_rows) def _retrieve_from_online_store( self, @@ -2207,253 +1895,15 @@ def _retrieve_from_online_store( ) return read_row_protos - @staticmethod - def _populate_response_from_feature_data( - feature_data: Iterable[ - Tuple[ - Iterable[Timestamp], Iterable["FieldStatus.ValueType"], Iterable[Value] - ] - ], - indexes: Iterable[List[int]], - online_features_response: GetOnlineFeaturesResponse, - full_feature_names: bool, - requested_features: Iterable[str], - table: FeatureView, - ): - """Populate the GetOnlineFeaturesResponse with feature data. - - This method assumes that `_read_from_online_store` returns data for each - combination of Entities in `entity_rows` in the same order as they - are provided. - - Args: - feature_data: A list of data in Protobuf form which was retrieved from the OnlineStore. - indexes: A list of indexes which should be the same length as `feature_data`. Each list - of indexes corresponds to a set of result rows in `online_features_response`. - online_features_response: The object to populate. - full_feature_names: A boolean that provides the option to add the feature view prefixes to the feature names, - changing them from the format "feature" to "feature_view__feature" (e.g., "daily_transactions" changes to - "customer_fv__daily_transactions"). - requested_features: The names of the features in `feature_data`. This should be ordered in the same way as the - data in `feature_data`. - table: The FeatureView that `feature_data` was retrieved from. - """ - # Add the feature names to the response. - requested_feature_refs = [ - f"{table.projection.name_to_use()}__{feature_name}" - if full_feature_names - else feature_name - for feature_name in requested_features - ] - online_features_response.metadata.feature_names.val.extend( - requested_feature_refs - ) - - timestamps, statuses, values = zip(*feature_data) - - # Populate the result with data fetched from the OnlineStore - # which is guaranteed to be aligned with `requested_features`. - for ( - feature_idx, - (timestamp_vector, statuses_vector, values_vector), - ) in enumerate(zip(zip(*timestamps), zip(*statuses), zip(*values))): - online_features_response.results.append( - GetOnlineFeaturesResponse.FeatureVector( - values=apply_list_mapping(values_vector, indexes), - statuses=apply_list_mapping(statuses_vector, indexes), - event_timestamps=apply_list_mapping(timestamp_vector, indexes), - ) - ) - - @staticmethod - def _augment_response_with_on_demand_transforms( - online_features_response: GetOnlineFeaturesResponse, - feature_refs: List[str], - requested_on_demand_feature_views: List[OnDemandFeatureView], - full_feature_names: bool, - ): - """Computes on demand feature values and adds them to the result rows. - - Assumes that 'online_features_response' already contains the necessary request data and input feature - views for the on demand feature views. Unneeded feature values such as request data and - unrequested input feature views will be removed from 'online_features_response'. - - Args: - online_features_response: Protobuf object to populate - feature_refs: List of all feature references to be returned. - requested_on_demand_feature_views: List of all odfvs that have been requested. - full_feature_names: A boolean that provides the option to add the feature view prefixes to the feature names, - changing them from the format "feature" to "feature_view__feature" (e.g., "daily_transactions" changes to - "customer_fv__daily_transactions"). - """ - requested_odfv_map = { - odfv.name: odfv for odfv in requested_on_demand_feature_views - } - requested_odfv_feature_names = requested_odfv_map.keys() - - odfv_feature_refs = defaultdict(list) - for feature_ref in feature_refs: - view_name, feature_name = feature_ref.split(":") - if view_name in requested_odfv_feature_names: - odfv_feature_refs[view_name].append( - f"{requested_odfv_map[view_name].projection.name_to_use()}__{feature_name}" - if full_feature_names - else feature_name - ) - - initial_response = OnlineResponse(online_features_response) - initial_response_arrow: Optional[pa.Table] = None - initial_response_dict: Optional[Dict[str, List[Any]]] = None - - # Apply on demand transformations and augment the result rows - odfv_result_names = set() - for odfv_name, _feature_refs in odfv_feature_refs.items(): - odfv = requested_odfv_map[odfv_name] - if odfv.mode == "python": - if initial_response_dict is None: - initial_response_dict = initial_response.to_dict() - transformed_features_dict: Dict[str, List[Any]] = odfv.transform_dict( - initial_response_dict - ) - elif odfv.mode in {"pandas", "substrait"}: - if initial_response_arrow is None: - initial_response_arrow = initial_response.to_arrow() - transformed_features_arrow = odfv.transform_arrow( - initial_response_arrow, full_feature_names - ) - else: - raise Exception( - f"Invalid OnDemandFeatureMode: {odfv.mode}. Expected one of 'pandas', 'python', or 'substrait'." - ) - - transformed_features = ( - transformed_features_dict - if odfv.mode == "python" - else transformed_features_arrow - ) - transformed_columns = ( - transformed_features.column_names - if isinstance(transformed_features, pa.Table) - else transformed_features - ) - selected_subset = [f for f in transformed_columns if f in _feature_refs] - - proto_values = [] - for selected_feature in selected_subset: - feature_vector = transformed_features[selected_feature] - proto_values.append( - python_values_to_proto_values(feature_vector, ValueType.UNKNOWN) - if odfv.mode == "python" - else python_values_to_proto_values( - feature_vector.to_numpy(), ValueType.UNKNOWN - ) - ) - - odfv_result_names |= set(selected_subset) - - online_features_response.metadata.feature_names.val.extend(selected_subset) - for feature_idx in range(len(selected_subset)): - online_features_response.results.append( - GetOnlineFeaturesResponse.FeatureVector( - values=proto_values[feature_idx], - statuses=[FieldStatus.PRESENT] * len(proto_values[feature_idx]), - event_timestamps=[Timestamp()] * len(proto_values[feature_idx]), - ) - ) - - @staticmethod - def _drop_unneeded_columns( - online_features_response: GetOnlineFeaturesResponse, - requested_result_row_names: Set[str], - ): - """ - Unneeded feature values such as request data and unrequested input feature views will - be removed from 'online_features_response'. - - Args: - online_features_response: Protobuf object to populate - requested_result_row_names: Fields from 'result_rows' that have been requested, and - therefore should not be dropped. - """ - # Drop values that aren't needed - unneeded_feature_indices = [ - idx - for idx, val in enumerate( - online_features_response.metadata.feature_names.val - ) - if val not in requested_result_row_names - ] - - for idx in reversed(unneeded_feature_indices): - del online_features_response.metadata.feature_names.val[idx] - del online_features_response.results[idx] - - def _get_feature_views_to_use( - self, - features: Optional[Union[List[str], FeatureService]], - allow_cache=False, - hide_dummy_entity: bool = True, - ) -> Tuple[List[FeatureView], List[OnDemandFeatureView]]: - fvs = { - fv.name: fv - for fv in [ - *self._list_feature_views(allow_cache, hide_dummy_entity), - *self._registry.list_stream_feature_views( - project=self.project, allow_cache=allow_cache - ), - ] - } - - od_fvs = { - fv.name: fv - for fv in self._registry.list_on_demand_feature_views( - project=self.project, allow_cache=allow_cache - ) - } - - if isinstance(features, FeatureService): - fvs_to_use, od_fvs_to_use = [], [] - for fv_name, projection in [ - (projection.name, projection) - for projection in features.feature_view_projections - ]: - if fv_name in fvs: - fvs_to_use.append( - fvs[fv_name].with_projection(copy.copy(projection)) - ) - elif fv_name in od_fvs: - odfv = od_fvs[fv_name].with_projection(copy.copy(projection)) - od_fvs_to_use.append(odfv) - # Let's make sure to include an FVs which the ODFV requires Features from. - for projection in odfv.source_feature_view_projections.values(): - fv = fvs[projection.name].with_projection(copy.copy(projection)) - if fv not in fvs_to_use: - fvs_to_use.append(fv) - else: - raise ValueError( - f"The provided feature service {features.name} contains a reference to a feature view" - f"{fv_name} which doesn't exist. Please make sure that you have created the feature view" - f'{fv_name} and that you have registered it by running "apply".' - ) - views_to_use = (fvs_to_use, od_fvs_to_use) - else: - views_to_use = ( - [*fvs.values()], - [*od_fvs.values()], - ) - - return views_to_use - def serve( self, host: str, port: int, - type_: str, - no_access_log: bool, - no_feature_log: bool, - workers: int, - keep_alive_timeout: int, - registry_ttl_sec: int, + type_: str = "http", + no_access_log: bool = True, + workers: int = 1, + keep_alive_timeout: int = 30, + registry_ttl_sec: int = 2, ) -> None: """Start the feature consumption server locally on a given port.""" type_ = type_.lower() @@ -2507,6 +1957,12 @@ def serve_registry(self, port: int) -> None: registry_server.start_server(self, port) + def serve_offline(self, host: str, port: int) -> None: + """Start offline server locally on a given port.""" + from feast import offline_server + + offline_server.start_server(self, host, port) + def serve_transformations(self, port: int) -> None: """Start the feature transformation server locally on a given port.""" warnings.warn( @@ -2622,101 +2078,6 @@ def get_validation_reference( return ref -def _validate_entity_values(join_key_values: Dict[str, List[Value]]): - set_of_row_lengths = {len(v) for v in join_key_values.values()} - if len(set_of_row_lengths) > 1: - raise ValueError("All entity rows must have the same columns.") - return set_of_row_lengths.pop() - - -def _validate_feature_refs(feature_refs: List[str], full_feature_names: bool = False): - """ - Validates that there are no collisions among the feature references. - - Args: - feature_refs: List of feature references to validate. Feature references must have format - "feature_view:feature", e.g. "customer_fv:daily_transactions". - full_feature_names: If True, the full feature references are compared for collisions; if False, - only the feature names are compared. - - Raises: - FeatureNameCollisionError: There is a collision among the feature references. - """ - collided_feature_refs = [] - - if full_feature_names: - collided_feature_refs = [ - ref for ref, occurrences in Counter(feature_refs).items() if occurrences > 1 - ] - else: - feature_names = [ref.split(":")[1] for ref in feature_refs] - collided_feature_names = [ - ref - for ref, occurrences in Counter(feature_names).items() - if occurrences > 1 - ] - - for feature_name in collided_feature_names: - collided_feature_refs.extend( - [ref for ref in feature_refs if ref.endswith(":" + feature_name)] - ) - - if len(collided_feature_refs) > 0: - raise FeatureNameCollisionError(collided_feature_refs, full_feature_names) - - -def _group_feature_refs( - features: List[str], - all_feature_views: List[FeatureView], - all_on_demand_feature_views: List[OnDemandFeatureView], -) -> Tuple[ - List[Tuple[FeatureView, List[str]]], List[Tuple[OnDemandFeatureView, List[str]]] -]: - """Get list of feature views and corresponding feature names based on feature references""" - - # view name to view proto - view_index = {view.projection.name_to_use(): view for view in all_feature_views} - - # on demand view to on demand view proto - on_demand_view_index = { - view.projection.name_to_use(): view for view in all_on_demand_feature_views - } - - # view name to feature names - views_features = defaultdict(set) - - # on demand view name to feature names - on_demand_view_features = defaultdict(set) - - for ref in features: - view_name, feat_name = ref.split(":") - if view_name in view_index: - view_index[view_name].projection.get_feature(feat_name) # For validation - views_features[view_name].add(feat_name) - elif view_name in on_demand_view_index: - on_demand_view_index[view_name].projection.get_feature( - feat_name - ) # For validation - on_demand_view_features[view_name].add(feat_name) - # Let's also add in any FV Feature dependencies here. - for input_fv_projection in on_demand_view_index[ - view_name - ].source_feature_view_projections.values(): - for input_feat in input_fv_projection.features: - views_features[input_fv_projection.name].add(input_feat.name) - else: - raise FeatureViewNotFoundException(view_name) - - fvs_result: List[Tuple[FeatureView, List[str]]] = [] - odfvs_result: List[Tuple[OnDemandFeatureView, List[str]]] = [] - - for view_name, feature_names in views_features.items(): - fvs_result.append((view_index[view_name], list(feature_names))) - for view_name, feature_names in on_demand_view_features.items(): - odfvs_result.append((on_demand_view_index[view_name], list(feature_names))) - return fvs_result, odfvs_result - - def _print_materialization_log( start_date, end_date, num_feature_views: int, online_store: str ): @@ -2759,15 +2120,3 @@ def _validate_data_sources(data_sources: List[DataSource]): raise DataSourceRepeatNamesException(case_insensitive_ds_name) else: ds_names.add(case_insensitive_ds_name) - - -def apply_list_mapping( - lst: Iterable[Any], mapping_indexes: Iterable[List[int]] -) -> Iterable[Any]: - output_len = sum(len(item) for item in mapping_indexes) - output = [None] * output_len - for elem, destinations in zip(lst, mapping_indexes): - for idx in destinations: - output[idx] = elem - - return output diff --git a/sdk/python/feast/infra/contrib/grpc_server.py b/sdk/python/feast/infra/contrib/grpc_server.py index 2bd1b27755b..b6ed6cb25d4 100644 --- a/sdk/python/feast/infra/contrib/grpc_server.py +++ b/sdk/python/feast/infra/contrib/grpc_server.py @@ -114,7 +114,7 @@ def GetOnlineFeatures(self, request: GetOnlineFeaturesRequest, context): else: features = list(request.features.val) - result = self.fs._get_online_features( + result = self.fs.get_online_features( features, request.entities, request.full_feature_names, diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index 8a441479184..c272f4ed66d 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -16,4 +16,6 @@ RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | RUN apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb RUN apt update RUN apt -y install libarrow-dev -RUN mkdir -m 775 /.cache \ No newline at end of file +# modify permissions to support running with a random uid +RUN mkdir -m 775 /.cache +RUN chmod g+w $(python -c "import feast.ui as _; print(_.__path__)" | tr -d "[']")/build/projects-list.json diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev index 948e3569a64..858a5ae7d1a 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -16,4 +16,7 @@ RUN apt install -y -V ca-certificates lsb-release wget RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb RUN apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb RUN apt update -RUN apt -y install libarrow-dev \ No newline at end of file +RUN apt -y install libarrow-dev +# modify permissions to support running with a random uid +RUN mkdir -m 775 /.cache +RUN chmod g+w $(python -c "import feast.ui as _; print(_.__path__)" | tr -d "[']")/build/projects-list.json diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py index f95a750fd14..b43c874ddc3 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py @@ -47,7 +47,6 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = "ts", diff --git a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py index bf892e9d969..ccf826c068f 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py @@ -64,7 +64,6 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = "ts", diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py index a23d90e1868..c94b04329e0 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py @@ -91,7 +91,6 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = "ts", @@ -115,7 +114,7 @@ def create_offline_store_config(self) -> PostgreSQLOfflineStoreConfig: def get_prefixed_table_name(self, suffix: str) -> str: return f"{self.project_name}_{suffix}" - def create_online_store(self) -> PostgreSQLOnlineStoreConfig: + def create_online_store(self) -> PostgreSQLOnlineStoreConfig: # type: ignore assert self.container return PostgreSQLOnlineStoreConfig( type="postgres", diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py index b9785218857..7093e40b99e 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py @@ -69,7 +69,6 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = "ts", diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py index 0dee517eb37..c8fc15a6350 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py @@ -81,7 +81,6 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = "ts", diff --git a/sdk/python/feast/infra/offline_stores/remote.py b/sdk/python/feast/infra/offline_stores/remote.py new file mode 100644 index 00000000000..dc657017d9b --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/remote.py @@ -0,0 +1,407 @@ +import json +import logging +import uuid +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.flight as fl +import pyarrow.parquet +from pydantic import StrictInt, StrictStr + +from feast import OnDemandFeatureView +from feast.data_source import DataSource +from feast.feature_logging import ( + FeatureServiceLoggingSource, + LoggingConfig, + LoggingSource, +) +from feast.feature_view import FeatureView +from feast.infra.offline_stores import offline_utils +from feast.infra.offline_stores.offline_store import ( + OfflineStore, + RetrievalJob, + RetrievalMetadata, +) +from feast.infra.registry.base_registry import BaseRegistry +from feast.repo_config import FeastConfigBaseModel, RepoConfig +from feast.saved_dataset import SavedDatasetStorage + +logger = logging.getLogger(__name__) + + +class RemoteOfflineStoreConfig(FeastConfigBaseModel): + type: Literal["remote"] = "remote" + host: StrictStr + """ str: remote offline store server port, e.g. the host URL for offline store of arrow flight server. """ + + port: Optional[StrictInt] = None + """ str: remote offline store server port.""" + + +class RemoteRetrievalJob(RetrievalJob): + def __init__( + self, + client: fl.FlightClient, + api: str, + api_parameters: Dict[str, Any], + entity_df: Union[pd.DataFrame, str] = None, + table: pa.Table = None, + metadata: Optional[RetrievalMetadata] = None, + ): + # Initialize the client connection + self.client = client + self.api = api + self.api_parameters = api_parameters + self.entity_df = entity_df + self.table = table + self._metadata = metadata + + # Invoked to realize the Pandas DataFrame + def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: + # We use arrow format because it gives better control of the table schema + return self._to_arrow_internal().to_pandas() + + # Invoked to synchronously execute the underlying query and return the result as an arrow table + # This is where do_get service is invoked + def _to_arrow_internal(self, timeout: Optional[int] = None) -> pa.Table: + return _send_retrieve_remote( + self.api, self.api_parameters, self.entity_df, self.table, self.client + ) + + @property + def on_demand_feature_views(self) -> List[OnDemandFeatureView]: + return [] + + @property + def metadata(self) -> Optional[RetrievalMetadata]: + return self._metadata + + @property + def full_feature_names(self) -> bool: + return self.api_parameters["full_feature_names"] + + def persist( + self, + storage: SavedDatasetStorage, + allow_overwrite: bool = False, + timeout: Optional[int] = None, + ): + """ + Arrow flight action is being used to perform the persist action remotely + """ + + api_parameters = { + "data_source_name": storage.to_data_source().name, + "allow_overwrite": allow_overwrite, + "timeout": timeout, + } + + # Add api parameters to command + for key, value in self.api_parameters.items(): + api_parameters[key] = value + + api_parameters["retrieve_func"] = self.api + + _call_put( + api=RemoteRetrievalJob.persist.__name__, + api_parameters=api_parameters, + client=self.client, + table=self.table, + entity_df=self.entity_df, + ) + + +class RemoteOfflineStore(OfflineStore): + @staticmethod + def get_historical_features( + config: RepoConfig, + feature_views: List[FeatureView], + feature_refs: List[str], + entity_df: Union[pd.DataFrame, str], + registry: BaseRegistry, + project: str, + full_feature_names: bool = False, + ) -> RemoteRetrievalJob: + assert isinstance(config.offline_store, RemoteOfflineStoreConfig) + + # Initialize the client connection + client = RemoteOfflineStore.init_client(config) + + feature_view_names = [fv.name for fv in feature_views] + name_aliases = [fv.projection.name_alias for fv in feature_views] + + api_parameters = { + "feature_view_names": feature_view_names, + "feature_refs": feature_refs, + "project": project, + "full_feature_names": full_feature_names, + "name_aliases": name_aliases, + } + + return RemoteRetrievalJob( + client=client, + api=OfflineStore.get_historical_features.__name__, + api_parameters=api_parameters, + entity_df=entity_df, + metadata=_create_retrieval_metadata(feature_refs, entity_df), + ) + + @staticmethod + def pull_all_from_table_or_query( + config: RepoConfig, + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + timestamp_field: str, + start_date: datetime, + end_date: datetime, + ) -> RetrievalJob: + assert isinstance(config.offline_store, RemoteOfflineStoreConfig) + + # Initialize the client connection + client = RemoteOfflineStore.init_client(config) + + api_parameters = { + "data_source_name": data_source.name, + "join_key_columns": join_key_columns, + "feature_name_columns": feature_name_columns, + "timestamp_field": timestamp_field, + "start_date": start_date.isoformat(), + "end_date": end_date.isoformat(), + } + + return RemoteRetrievalJob( + client=client, + api=OfflineStore.pull_all_from_table_or_query.__name__, + api_parameters=api_parameters, + ) + + @staticmethod + def pull_latest_from_table_or_query( + config: RepoConfig, + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + timestamp_field: str, + created_timestamp_column: Optional[str], + start_date: datetime, + end_date: datetime, + ) -> RetrievalJob: + assert isinstance(config.offline_store, RemoteOfflineStoreConfig) + + # Initialize the client connection + client = RemoteOfflineStore.init_client(config) + + api_parameters = { + "data_source_name": data_source.name, + "join_key_columns": join_key_columns, + "feature_name_columns": feature_name_columns, + "timestamp_field": timestamp_field, + "created_timestamp_column": created_timestamp_column, + "start_date": start_date.isoformat(), + "end_date": end_date.isoformat(), + } + + return RemoteRetrievalJob( + client=client, + api=OfflineStore.pull_latest_from_table_or_query.__name__, + api_parameters=api_parameters, + ) + + @staticmethod + def write_logged_features( + config: RepoConfig, + data: Union[pyarrow.Table, Path], + source: LoggingSource, + logging_config: LoggingConfig, + registry: BaseRegistry, + ): + assert isinstance(config.offline_store, RemoteOfflineStoreConfig) + assert isinstance(source, FeatureServiceLoggingSource) + + if isinstance(data, Path): + data = pyarrow.parquet.read_table(data, use_threads=False, pre_buffer=False) + + # Initialize the client connection + client = RemoteOfflineStore.init_client(config) + + api_parameters = { + "feature_service_name": source._feature_service.name, + } + + _call_put( + api=OfflineStore.write_logged_features.__name__, + api_parameters=api_parameters, + client=client, + table=data, + entity_df=None, + ) + + @staticmethod + def offline_write_batch( + config: RepoConfig, + feature_view: FeatureView, + table: pyarrow.Table, + progress: Optional[Callable[[int], Any]], + ): + assert isinstance(config.offline_store, RemoteOfflineStoreConfig) + + # Initialize the client connection + client = RemoteOfflineStore.init_client(config) + + feature_view_names = [feature_view.name] + name_aliases = [feature_view.projection.name_alias] + + api_parameters = { + "feature_view_names": feature_view_names, + "progress": progress, + "name_aliases": name_aliases, + } + + _call_put( + api=OfflineStore.offline_write_batch.__name__, + api_parameters=api_parameters, + client=client, + table=table, + entity_df=None, + ) + + @staticmethod + def init_client(config): + location = f"grpc://{config.offline_store.host}:{config.offline_store.port}" + client = fl.connect(location=location) + logger.info(f"Connecting FlightClient at {location}") + return client + + +def _create_retrieval_metadata(feature_refs: List[str], entity_df: pd.DataFrame): + entity_schema = _get_entity_schema( + entity_df=entity_df, + ) + + event_timestamp_col = offline_utils.infer_event_timestamp_from_entity_df( + entity_schema=entity_schema, + ) + + timestamp_range = _get_entity_df_event_timestamp_range( + entity_df, event_timestamp_col + ) + + return RetrievalMetadata( + features=feature_refs, + keys=list(set(entity_df.columns) - {event_timestamp_col}), + min_event_timestamp=timestamp_range[0], + max_event_timestamp=timestamp_range[1], + ) + + +def _get_entity_schema(entity_df: pd.DataFrame) -> Dict[str, np.dtype]: + return dict(zip(entity_df.columns, entity_df.dtypes)) + + +def _get_entity_df_event_timestamp_range( + entity_df: Union[pd.DataFrame, str], + entity_df_event_timestamp_col: str, +) -> Tuple[datetime, datetime]: + if not isinstance(entity_df, pd.DataFrame): + raise ValueError( + f"Please provide an entity_df of type {type(pd.DataFrame)} instead of type {type(entity_df)}" + ) + + entity_df_event_timestamp = entity_df.loc[ + :, entity_df_event_timestamp_col + ].infer_objects() + if pd.api.types.is_string_dtype(entity_df_event_timestamp): + entity_df_event_timestamp = pd.to_datetime(entity_df_event_timestamp, utc=True) + + return ( + entity_df_event_timestamp.min().to_pydatetime(), + entity_df_event_timestamp.max().to_pydatetime(), + ) + + +def _send_retrieve_remote( + api: str, + api_parameters: Dict[str, Any], + entity_df: Union[pd.DataFrame, str], + table: pa.Table, + client: fl.FlightClient, +): + command_descriptor = _call_put(api, api_parameters, client, entity_df, table) + return _call_get(client, command_descriptor) + + +def _call_get(client: fl.FlightClient, command_descriptor: fl.FlightDescriptor): + flight = client.get_flight_info(command_descriptor) + ticket = flight.endpoints[0].ticket + reader = client.do_get(ticket) + return reader.read_all() + + +def _call_put( + api: str, + api_parameters: Dict[str, Any], + client: fl.FlightClient, + entity_df: Union[pd.DataFrame, str], + table: pa.Table, +): + # Generate unique command identifier + command_id = str(uuid.uuid4()) + command = { + "command_id": command_id, + "api": api, + } + # Add api parameters to command + for key, value in api_parameters.items(): + command[key] = value + + command_descriptor = fl.FlightDescriptor.for_command( + json.dumps( + command, + ) + ) + + _put_parameters(command_descriptor, entity_df, table, client) + return command_descriptor + + +def _put_parameters( + command_descriptor: fl.FlightDescriptor, + entity_df: Union[pd.DataFrame, str], + table: pa.Table, + client: fl.FlightClient, +): + updatedTable: pa.Table + + if entity_df is not None: + updatedTable = pa.Table.from_pandas(entity_df) + elif table is not None: + updatedTable = table + else: + updatedTable = _create_empty_table() + + writer, _ = client.do_put( + command_descriptor, + updatedTable.schema, + ) + + writer.write_table(updatedTable) + writer.close() + + +def _create_empty_table(): + schema = pa.schema( + { + "key": pa.string(), + } + ) + + keys = ["mock_key"] + + table = pa.Table.from_pydict(dict(zip(schema.names, keys)), schema=schema) + + return table diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index 0ee9af185d3..b2488543b02 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -33,6 +33,8 @@ try: import boto3 + from aiobotocore import session + from boto3.dynamodb.types import TypeDeserializer from botocore.config import Config from botocore.exceptions import ClientError except ImportError as e: @@ -80,6 +82,7 @@ class DynamoDBOnlineStore(OnlineStore): _dynamodb_client = None _dynamodb_resource = None + _aioboto_session = None def update( self, @@ -223,6 +226,7 @@ def online_read( """ online_config = config.online_store assert isinstance(online_config, DynamoDBOnlineStoreConfig) + dynamodb_resource = self._get_dynamodb_resource( online_config.region, online_config.endpoint_url ) @@ -230,62 +234,95 @@ def online_read( _get_table_name(online_config, config, table) ) - result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] - entity_ids = [ - compute_entity_id( - entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, - ) - for entity_key in entity_keys - ] batch_size = online_config.batch_size + entity_ids = self._to_entity_ids(config, entity_keys) entity_ids_iter = iter(entity_ids) + result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] + while True: batch = list(itertools.islice(entity_ids_iter, batch_size)) - batch_result: List[ - Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]] - ] = [] + # No more items to insert if len(batch) == 0: break - batch_entity_ids = { - table_instance.name: { - "Keys": [{"entity_id": entity_id} for entity_id in batch], - "ConsistentRead": online_config.consistent_reads, - } - } + batch_entity_ids = self._to_resource_batch_get_payload( + online_config, table_instance.name, batch + ) response = dynamodb_resource.batch_get_item( RequestItems=batch_entity_ids, ) - response = response.get("Responses") - table_responses = response.get(table_instance.name) - if table_responses: - table_responses = self._sort_dynamodb_response( - table_responses, entity_ids - ) - entity_idx = 0 - for tbl_res in table_responses: - entity_id = tbl_res["entity_id"] - while entity_id != batch[entity_idx]: - batch_result.append((None, None)) - entity_idx += 1 - res = {} - for feature_name, value_bin in tbl_res["values"].items(): - val = ValueProto() - val.ParseFromString(value_bin.value) - res[feature_name] = val - batch_result.append( - (datetime.fromisoformat(tbl_res["event_ts"]), res) - ) - entity_idx += 1 - - # Not all entities in a batch may have responses - # Pad with remaining values in batch that were not found - batch_size_nones = ((None, None),) * (len(batch) - len(batch_result)) - batch_result.extend(batch_size_nones) + batch_result = self._process_batch_get_response( + table_instance.name, response, entity_ids, batch + ) result.extend(batch_result) return result + async def online_read_async( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + """ + Reads features values for the given entity keys asynchronously. + + Args: + config: The config for the current feature store. + table: The feature view whose feature values should be read. + entity_keys: The list of entity keys for which feature values should be read. + requested_features: The list of features that should be read. + + Returns: + A list of the same length as entity_keys. Each item in the list is a tuple where the first + item is the event timestamp for the row, and the second item is a dict mapping feature names + to values, which are returned in proto format. + """ + online_config = config.online_store + assert isinstance(online_config, DynamoDBOnlineStoreConfig) + + batch_size = online_config.batch_size + entity_ids = self._to_entity_ids(config, entity_keys) + entity_ids_iter = iter(entity_ids) + result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] + table_name = _get_table_name(online_config, config, table) + + deserialize = TypeDeserializer().deserialize + + def to_tbl_resp(raw_client_response): + return { + "entity_id": deserialize(raw_client_response["entity_id"]), + "event_ts": deserialize(raw_client_response["event_ts"]), + "values": deserialize(raw_client_response["values"]), + } + + async with self._get_aiodynamodb_client(online_config.region) as client: + while True: + batch = list(itertools.islice(entity_ids_iter, batch_size)) + + # No more items to insert + if len(batch) == 0: + break + batch_entity_ids = self._to_client_batch_get_payload( + online_config, table_name, batch + ) + response = await client.batch_get_item( + RequestItems=batch_entity_ids, + ) + batch_result = self._process_batch_get_response( + table_name, response, entity_ids, batch, to_tbl_response=to_tbl_resp + ) + result.extend(batch_result) + return result + + def _get_aioboto_session(self): + if self._aioboto_session is None: + self._aioboto_session = session.get_session() + return self._aioboto_session + + def _get_aiodynamodb_client(self, region: str): + return self._get_aioboto_session().create_client("dynamodb", region_name=region) + def _get_dynamodb_client(self, region: str, endpoint_url: Optional[str] = None): if self._dynamodb_client is None: self._dynamodb_client = _initialize_dynamodb_client(region, endpoint_url) @@ -298,13 +335,19 @@ def _get_dynamodb_resource(self, region: str, endpoint_url: Optional[str] = None ) return self._dynamodb_resource - def _sort_dynamodb_response(self, responses: list, order: list) -> Any: + def _sort_dynamodb_response( + self, + responses: list, + order: list, + to_tbl_response: Callable = lambda raw_dict: raw_dict, + ) -> Any: """DynamoDB Batch Get Item doesn't return items in a particular order.""" # Assign an index to order order_with_index = {value: idx for idx, value in enumerate(order)} # Sort table responses by index table_responses_ordered: Any = [ - (order_with_index[tbl_res["entity_id"]], tbl_res) for tbl_res in responses + (order_with_index[tbl_res["entity_id"]], tbl_res) + for tbl_res in map(to_tbl_response, responses) ] table_responses_ordered = sorted( table_responses_ordered, key=lambda tup: tup[0] @@ -341,6 +384,64 @@ def _write_batch_non_duplicates( if progress: progress(1) + def _process_batch_get_response( + self, table_name, response, entity_ids, batch, **sort_kwargs + ): + response = response.get("Responses") + table_responses = response.get(table_name) + + batch_result = [] + if table_responses: + table_responses = self._sort_dynamodb_response( + table_responses, entity_ids, **sort_kwargs + ) + entity_idx = 0 + for tbl_res in table_responses: + entity_id = tbl_res["entity_id"] + while entity_id != batch[entity_idx]: + batch_result.append((None, None)) + entity_idx += 1 + res = {} + for feature_name, value_bin in tbl_res["values"].items(): + val = ValueProto() + val.ParseFromString(value_bin.value) + res[feature_name] = val + batch_result.append((datetime.fromisoformat(tbl_res["event_ts"]), res)) + entity_idx += 1 + # Not all entities in a batch may have responses + # Pad with remaining values in batch that were not found + batch_size_nones = ((None, None),) * (len(batch) - len(batch_result)) + batch_result.extend(batch_size_nones) + return batch_result + + @staticmethod + def _to_entity_ids(config: RepoConfig, entity_keys: List[EntityKeyProto]): + return [ + compute_entity_id( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + for entity_key in entity_keys + ] + + @staticmethod + def _to_resource_batch_get_payload(online_config, table_name, batch): + return { + table_name: { + "Keys": [{"entity_id": entity_id} for entity_id in batch], + "ConsistentRead": online_config.consistent_reads, + } + } + + @staticmethod + def _to_client_batch_get_payload(online_config, table_name, batch): + return { + table_name: { + "Keys": [{"entity_id": {"S": entity_id}} for entity_id in batch], + "ConsistentRead": online_config.consistent_reads, + } + } + def _initialize_dynamodb_client(region: str, endpoint_url: Optional[str] = None): return boto3.client( diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index 7428eb8bea4..5f0156f6204 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -77,6 +77,9 @@ class RedisOnlineStoreConfig(FeastConfigBaseModel): key_ttl_seconds: Optional[int] = None """(Optional) redis key bin ttl (in seconds) for expiring entities""" + full_scan_for_deletion: Optional[bool] = True + """(Optional) whether to scan for deletion of features""" + class RedisOnlineStore(OnlineStore): """ @@ -162,9 +165,13 @@ def update( entities_to_keep: Entities to keep partial: Whether to do a partial update """ + online_store_config = config.online_store + + assert isinstance(online_store_config, RedisOnlineStoreConfig) - for table in tables_to_delete: - self.delete_table(config, table) + if online_store_config.full_scan_for_deletion: + for table in tables_to_delete: + self.delete_table(config, table) def teardown( self, diff --git a/sdk/python/feast/infra/online_stores/remote.py b/sdk/python/feast/infra/online_stores/remote.py new file mode 100644 index 00000000000..19e1b7d5159 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/remote.py @@ -0,0 +1,167 @@ +# Copyright 2021 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import logging +from datetime import datetime +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple + +import requests +from pydantic import StrictStr + +from feast import Entity, FeatureView, RepoConfig +from feast.infra.online_stores.online_store import OnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel +from feast.type_map import python_values_to_proto_values +from feast.value_type import ValueType + +logger = logging.getLogger(__name__) + + +class RemoteOnlineStoreConfig(FeastConfigBaseModel): + """Remote Online store config for remote online store""" + + type: Literal["remote"] = "remote" + """Online store type selector""" + + path: StrictStr = "http://localhost:6566" + """ str: Path to metadata store. + If type is 'remote', then this is a URL for registry server """ + + +class RemoteOnlineStore(OnlineStore): + """ + remote online store implementation wrapper to communicate with feast online server. + """ + + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + raise NotImplementedError + + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + assert isinstance(config.online_store, RemoteOnlineStoreConfig) + config.online_store.__class__ = RemoteOnlineStoreConfig + + req_body = self._construct_online_read_api_json_request( + entity_keys, table, requested_features + ) + response = requests.post( + f"{config.online_store.path}/get-online-features", data=req_body + ) + if response.status_code == 200: + logger.debug("Able to retrieve the online features from feature server.") + response_json = json.loads(response.text) + event_ts = self._get_event_ts(response_json) + # Iterating over results and converting the API results in column format to row format. + result_tuples: List[ + Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]] + ] = [] + for feature_value_index in range(len(entity_keys)): + feature_values_dict: Dict[str, ValueProto] = dict() + for index, feature_name in enumerate( + response_json["metadata"]["feature_names"] + ): + if ( + requested_features is not None + and feature_name in requested_features + ): + if ( + response_json["results"][index]["statuses"][ + feature_value_index + ] + == "PRESENT" + ): + message = python_values_to_proto_values( + [ + response_json["results"][index]["values"][ + feature_value_index + ] + ], + ValueType.UNKNOWN, + ) + feature_values_dict[feature_name] = message[0] + else: + feature_values_dict[feature_name] = ValueProto() + result_tuples.append((event_ts, feature_values_dict)) + return result_tuples + else: + error_msg = f"Unable to retrieve the online store data using feature server API. Error_code={response.status_code}, error_message={response.reason}" + logger.error(error_msg) + raise RuntimeError(error_msg) + + def _construct_online_read_api_json_request( + self, + entity_keys: List[EntityKeyProto], + table: FeatureView, + requested_features: Optional[List[str]] = None, + ) -> str: + api_requested_features = [] + if requested_features is not None: + for requested_feature in requested_features: + api_requested_features.append(f"{table.name}:{requested_feature}") + + entity_values = [] + entity_key = "" + for row in entity_keys: + entity_key = row.join_keys[0] + entity_values.append( + getattr(row.entity_values[0], row.entity_values[0].WhichOneof("val")) + ) + + req_body = json.dumps( + { + "features": api_requested_features, + "entities": {entity_key: entity_values}, + } + ) + return req_body + + def _get_event_ts(self, response_json) -> datetime: + event_ts = "" + if len(response_json["results"]) > 1: + event_ts = response_json["results"][1]["event_timestamps"][0] + return datetime.fromisoformat(event_ts.replace("Z", "+00:00")) + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + pass + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ): + pass diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 63d3ef03f51..41af14aaf16 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -14,10 +14,14 @@ import itertools import os import sqlite3 +import struct +import sys from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union +import sqlite_vec +from google.protobuf.internal.containers import RepeatedScalarFieldContainer from pydantic import StrictStr from feast import Entity @@ -29,6 +33,7 @@ from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.core.SqliteTable_pb2 import SqliteTable as SqliteTableProto from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import FloatList as FloatListProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.utils import to_naive_utc @@ -45,6 +50,12 @@ class SqliteOnlineStoreConfig(FeastConfigBaseModel): path: StrictStr = "data/online.db" """ (optional) Path to sqlite db """ + vec_enabled: Optional[bool] = False + """ (optional) Enable or disable sqlite-vss for vector search""" + + vector_len: Optional[int] = 512 + """ (optional) Length of the vector to be stored in the database""" + class SqliteOnlineStore(OnlineStore): """ @@ -73,6 +84,10 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) + if sys.version_info[0:2] == (3, 10): + self._conn.enable_load_extension(True) # type: ignore + sqlite_vec.load(self._conn) + return self._conn def online_write_batch( @@ -80,7 +95,12 @@ def online_write_batch( config: RepoConfig, table: FeatureView, data: List[ - Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + Tuple[ + EntityKeyProto, + Dict[str, ValueProto], + datetime, + Optional[datetime], + ] ], progress: Optional[Callable[[int], Any]], ) -> None: @@ -98,36 +118,74 @@ def online_write_batch( if created_ts is not None: created_ts = to_naive_utc(created_ts) + table_name = _table_id(project, table) for feature_name, val in values.items(): - conn.execute( - f""" - UPDATE {_table_id(project, table)} - SET value = ?, event_ts = ?, created_ts = ? - WHERE (entity_key = ? AND feature_name = ?) - """, - ( - # SET - val.SerializeToString(), - timestamp, - created_ts, - # WHERE - entity_key_bin, - feature_name, - ), - ) - - conn.execute( - f"""INSERT OR IGNORE INTO {_table_id(project, table)} - (entity_key, feature_name, value, event_ts, created_ts) - VALUES (?, ?, ?, ?, ?)""", - ( - entity_key_bin, - feature_name, - val.SerializeToString(), - timestamp, - created_ts, - ), - ) + if config.online_store.vec_enabled: + vector_bin = serialize_f32( + val.float_list_val.val, config.online_store.vector_len + ) # type: ignore + conn.execute( + f""" + UPDATE {table_name} + SET value = ?, vector_value = ?, event_ts = ?, created_ts = ? + WHERE (entity_key = ? AND feature_name = ?) + """, + ( + # SET + val.SerializeToString(), + vector_bin, + timestamp, + created_ts, + # WHERE + entity_key_bin, + feature_name, + ), + ) + + conn.execute( + f"""INSERT OR IGNORE INTO {table_name} + (entity_key, feature_name, value, vector_value, event_ts, created_ts) + VALUES (?, ?, ?, ?, ?, ?)""", + ( + entity_key_bin, + feature_name, + val.SerializeToString(), + vector_bin, + timestamp, + created_ts, + ), + ) + + else: + conn.execute( + f""" + UPDATE {table_name} + SET value = ?, event_ts = ?, created_ts = ? + WHERE (entity_key = ? AND feature_name = ?) + """, + ( + # SET + val.SerializeToString(), + timestamp, + created_ts, + # WHERE + entity_key_bin, + feature_name, + ), + ) + + conn.execute( + f"""INSERT OR IGNORE INTO {table_name} + (entity_key, feature_name, value, event_ts, created_ts) + VALUES (?, ?, ?, ?, ?)""", + ( + entity_key_bin, + feature_name, + val.SerializeToString(), + timestamp, + created_ts, + ), + ) if progress: progress(1) @@ -195,7 +253,7 @@ def update( for table in tables_to_keep: conn.execute( - f"CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key BLOB, feature_name TEXT, value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" + f"CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" ) conn.execute( f"CREATE INDEX IF NOT EXISTS {_table_id(project, table)}_ek ON {_table_id(project, table)} (entity_key);" @@ -232,6 +290,124 @@ def teardown( except FileNotFoundError: pass + def retrieve_online_documents( + self, + config: RepoConfig, + table: FeatureView, + requested_feature: str, + embedding: List[float], + top_k: int, + distance_metric: Optional[str] = None, + ) -> List[ + Tuple[ + Optional[datetime], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ]: + """ + + Args: + config: Feast configuration object + table: FeatureView object as the table to search + requested_feature: The requested feature as the column to search + embedding: The query embedding to search for + top_k: The number of items to return + Returns: + List of tuples containing the event timestamp, the document feature, the vector value, and the distance + """ + project = config.project + + if not config.online_store.vec_enabled: + raise ValueError("sqlite-vss is not enabled in the online store config") + + conn = self._get_conn(config) + cur = conn.cursor() + + # Convert the embedding to a binary format instead of using SerializeToString() + query_embedding_bin = serialize_f32(embedding, config.online_store.vector_len) + table_name = _table_id(project, table) + + cur.execute( + f""" + CREATE VIRTUAL TABLE vec_example using vec0( + vector_value float[{config.online_store.vector_len}] + ); + """ + ) + + # Currently I can only insert the embedding value without crashing SQLite, will report a bug + cur.execute( + f""" + INSERT INTO vec_example(rowid, vector_value) + select rowid, vector_value from {table_name} + """ + ) + cur.execute( + """ + INSERT INTO vec_example(rowid, vector_value) + VALUES (?, ?) + """, + (0, query_embedding_bin), + ) + + # Have to join this with the {table_name} to get the feature name and entity_key + # Also the `top_k` doesn't appear to be working for some reason + cur.execute( + f""" + select + fv.entity_key, + f.vector_value, + fv.value, + f.distance, + fv.event_ts + from ( + select + rowid, + vector_value, + distance + from vec_example + where vector_value match ? + order by distance + limit ? + ) f + left join {table_name} fv + on f.rowid = fv.rowid + """, + (query_embedding_bin, top_k), + ) + + rows = cur.fetchall() + + result: List[ + Tuple[ + Optional[datetime], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ] = [] + + for entity_key, _, string_value, distance, event_ts in rows: + feature_value_proto = ValueProto() + feature_value_proto.ParseFromString(string_value if string_value else b"") + vector_value_proto = ValueProto( + float_list_val=FloatListProto(val=embedding) + ) + distance_value_proto = ValueProto(float_val=distance) + + result.append( + ( + event_ts, + feature_value_proto, + vector_value_proto, + distance_value_proto, + ) + ) + + return result + def _initialize_conn(db_path: str): Path(db_path).parent.mkdir(exist_ok=True) @@ -246,6 +422,19 @@ def _table_id(project: str, table: FeatureView) -> str: return f"{project}_{table.name}" +def serialize_f32( + vector: Union[RepeatedScalarFieldContainer[float], List[float]], vector_length: int +) -> bytes: + """serializes a list of floats into a compact "raw bytes" format""" + return struct.pack(f"{vector_length}f", *vector) + + +def deserialize_f32(byte_vector: bytes, vector_length: int) -> List[float]: + """deserializes a list of floats from a compact "raw bytes" format""" + num_floats = vector_length // 4 # 4 bytes per float + return list(struct.unpack(f"{num_floats}f", byte_vector)) + + class SqliteTable(InfraObject): """ A Sqlite table managed by Feast. @@ -292,8 +481,11 @@ def from_proto(sqlite_table_proto: SqliteTableProto) -> Any: ) def update(self): + if sys.version_info[0:2] == (3, 10): + self.conn.enable_load_extension(True) + sqlite_vec.load(self.conn) self.conn.execute( - f"CREATE TABLE IF NOT EXISTS {self.name} (entity_key BLOB, feature_name TEXT, value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" + f"CREATE TABLE IF NOT EXISTS {self.name} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" ) self.conn.execute( f"CREATE INDEX IF NOT EXISTS {self.name}_ek ON {self.name} (entity_key);" diff --git a/sdk/python/feast/infra/registry/base_registry.py b/sdk/python/feast/infra/registry/base_registry.py index ed1fc3ab879..bc08796e39d 100644 --- a/sdk/python/feast/infra/registry/base_registry.py +++ b/sdk/python/feast/infra/registry/base_registry.py @@ -84,13 +84,19 @@ def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Enti raise NotImplementedError @abstractmethod - def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity]: + def list_entities( + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, + ) -> List[Entity]: """ Retrieve a list of entities from the registry Args: allow_cache: Whether to allow returning entities from a cached registry project: Filter entities based on project name + tags: Filter by tags Returns: List of entities @@ -143,7 +149,10 @@ def get_data_source( @abstractmethod def list_data_sources( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[DataSource]: """ Retrieve a list of data sources from the registry @@ -151,6 +160,7 @@ def list_data_sources( Args: project: Filter data source based on project name allow_cache: Whether to allow returning data sources from a cached registry + tags: Filter by tags Returns: List of data sources @@ -203,7 +213,10 @@ def get_feature_service( @abstractmethod def list_feature_services( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[FeatureService]: """ Retrieve a list of feature services from the registry @@ -211,6 +224,7 @@ def list_feature_services( Args: allow_cache: Whether to allow returning entities from a cached registry project: Filter entities based on project name + tags: Filter by tags Returns: List of feature services @@ -265,7 +279,10 @@ def get_stream_feature_view( @abstractmethod def list_stream_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[StreamFeatureView]: """ Retrieve a list of stream feature views from the registry @@ -273,6 +290,7 @@ def list_stream_feature_views( Args: project: Filter stream feature views based on project name allow_cache: Whether to allow returning stream feature views from a cached registry + tags: Filter by tags Returns: List of stream feature views @@ -300,7 +318,10 @@ def get_on_demand_feature_view( @abstractmethod def list_on_demand_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[OnDemandFeatureView]: """ Retrieve a list of on demand feature views from the registry @@ -308,6 +329,7 @@ def list_on_demand_feature_views( Args: project: Filter on demand feature views based on project name allow_cache: Whether to allow returning on demand feature views from a cached registry + tags: Filter by tags Returns: List of on demand feature views @@ -335,7 +357,10 @@ def get_feature_view( @abstractmethod def list_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[FeatureView]: """ Retrieve a list of feature views from the registry @@ -343,6 +368,7 @@ def list_feature_views( Args: allow_cache: Allow returning feature views from the cached registry project: Filter feature views based on project name + tags: Filter by tags Returns: List of feature views @@ -406,18 +432,14 @@ def get_saved_dataset( """ raise NotImplementedError - def delete_saved_dataset(self, name: str, project: str, allow_cache: bool = False): + def delete_saved_dataset(self, name: str, project: str, commit: bool = True): """ Delete a saved dataset. Args: name: Name of dataset project: Feast project that this dataset belongs to - allow_cache: Whether to allow returning this dataset from a cached registry - - Returns: - Returns either the specified SavedDataset, or raises an exception if - none is found + commit: Whether the change should be persisted immediately """ raise NotImplementedError @@ -602,7 +624,8 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]: self._message_to_sorted_dict(data_source.to_proto()) ) for entity in sorted( - self.list_entities(project=project), key=lambda entity: entity.name + self.list_entities(project=project), + key=lambda entity: entity.name, ): registry_dict["entities"].append( self._message_to_sorted_dict(entity.to_proto()) diff --git a/sdk/python/feast/infra/registry/caching_registry.py b/sdk/python/feast/infra/registry/caching_registry.py index 0f660128086..6336dd7fee5 100644 --- a/sdk/python/feast/infra/registry/caching_registry.py +++ b/sdk/python/feast/infra/registry/caching_registry.py @@ -48,18 +48,23 @@ def get_data_source( return self._get_data_source(name, project) @abstractmethod - def _list_data_sources(self, project: str) -> List[DataSource]: + def _list_data_sources( + self, project: str, tags: Optional[dict[str, str]] + ) -> List[DataSource]: pass def list_data_sources( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[DataSource]: if allow_cache: self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_data_sources( - self.cached_registry_proto, project + self.cached_registry_proto, project, tags ) - return self._list_data_sources(project) + return self._list_data_sources(project, tags) @abstractmethod def _get_entity(self, name: str, project: str) -> Entity: @@ -74,16 +79,23 @@ def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Enti return self._get_entity(name, project) @abstractmethod - def _list_entities(self, project: str) -> List[Entity]: + def _list_entities( + self, project: str, tags: Optional[dict[str, str]] + ) -> List[Entity]: pass - def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity]: + def list_entities( + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, + ) -> List[Entity]: if allow_cache: self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_entities( - self.cached_registry_proto, project + self.cached_registry_proto, project, tags ) - return self._list_entities(project) + return self._list_entities(project, tags) @abstractmethod def _get_feature_view(self, name: str, project: str) -> FeatureView: @@ -100,18 +112,23 @@ def get_feature_view( return self._get_feature_view(name, project) @abstractmethod - def _list_feature_views(self, project: str) -> List[FeatureView]: + def _list_feature_views( + self, project: str, tags: Optional[dict[str, str]] + ) -> List[FeatureView]: pass def list_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[FeatureView]: if allow_cache: self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_feature_views( - self.cached_registry_proto, project + self.cached_registry_proto, project, tags ) - return self._list_feature_views(project) + return self._list_feature_views(project, tags) @abstractmethod def _get_on_demand_feature_view( @@ -130,18 +147,23 @@ def get_on_demand_feature_view( return self._get_on_demand_feature_view(name, project) @abstractmethod - def _list_on_demand_feature_views(self, project: str) -> List[OnDemandFeatureView]: + def _list_on_demand_feature_views( + self, project: str, tags: Optional[dict[str, str]] + ) -> List[OnDemandFeatureView]: pass def list_on_demand_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[OnDemandFeatureView]: if allow_cache: self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_on_demand_feature_views( - self.cached_registry_proto, project + self.cached_registry_proto, project, tags ) - return self._list_on_demand_feature_views(project) + return self._list_on_demand_feature_views(project, tags) @abstractmethod def _get_stream_feature_view(self, name: str, project: str) -> StreamFeatureView: @@ -158,18 +180,23 @@ def get_stream_feature_view( return self._get_stream_feature_view(name, project) @abstractmethod - def _list_stream_feature_views(self, project: str) -> List[StreamFeatureView]: + def _list_stream_feature_views( + self, project: str, tags: Optional[dict[str, str]] + ) -> List[StreamFeatureView]: pass def list_stream_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[StreamFeatureView]: if allow_cache: self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_stream_feature_views( - self.cached_registry_proto, project + self.cached_registry_proto, project, tags ) - return self._list_stream_feature_views(project) + return self._list_stream_feature_views(project, tags) @abstractmethod def _get_feature_service(self, name: str, project: str) -> FeatureService: @@ -186,18 +213,23 @@ def get_feature_service( return self._get_feature_service(name, project) @abstractmethod - def _list_feature_services(self, project: str) -> List[FeatureService]: + def _list_feature_services( + self, project: str, tags: Optional[dict[str, str]] + ) -> List[FeatureService]: pass def list_feature_services( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[FeatureService]: if allow_cache: self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_feature_services( - self.cached_registry_proto, project + self.cached_registry_proto, project, tags ) - return self._list_feature_services(project) + return self._list_feature_services(project, tags) @abstractmethod def _get_saved_dataset(self, name: str, project: str) -> SavedDataset: diff --git a/sdk/python/feast/infra/registry/contrib/postgres/postgres_registry_store.py b/sdk/python/feast/infra/registry/contrib/postgres/postgres_registry_store.py deleted file mode 100644 index 877e0a018a8..00000000000 --- a/sdk/python/feast/infra/registry/contrib/postgres/postgres_registry_store.py +++ /dev/null @@ -1,128 +0,0 @@ -import warnings -from typing import Optional - -import psycopg2 -from psycopg2 import sql - -from feast.infra.registry.registry_store import RegistryStore -from feast.infra.utils.postgres.connection_utils import _get_conn -from feast.infra.utils.postgres.postgres_config import PostgreSQLConfig -from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto -from feast.repo_config import RegistryConfig - - -class PostgresRegistryConfig(RegistryConfig): - host: str - port: int - database: str - db_schema: str - user: str - password: str - sslmode: Optional[str] - sslkey_path: Optional[str] - sslcert_path: Optional[str] - sslrootcert_path: Optional[str] - - -class PostgreSQLRegistryStore(RegistryStore): - def __init__(self, config: PostgresRegistryConfig, registry_path: str): - self.db_config = PostgreSQLConfig( - host=config.host, - port=config.port, - database=config.database, - db_schema=config.db_schema, - user=config.user, - password=config.password, - sslmode=getattr(config, "sslmode", None), - sslkey_path=getattr(config, "sslkey_path", None), - sslcert_path=getattr(config, "sslcert_path", None), - sslrootcert_path=getattr(config, "sslrootcert_path", None), - ) - warnings.warn( - "PostgreSQLRegistryStore is deprecated and will be removed in the future releases. Please use SqlRegistry instead.", - DeprecationWarning, - ) - - self.table_name = config.path - self.cache_ttl_seconds = config.cache_ttl_seconds - - def get_registry_proto(self) -> RegistryProto: - registry_proto = RegistryProto() - try: - with _get_conn(self.db_config) as conn, conn.cursor() as cur: - cur.execute( - sql.SQL( - """ - SELECT registry - FROM {} - WHERE version = (SELECT max(version) FROM {}) - """ - ).format( - sql.Identifier(self.table_name), - sql.Identifier(self.table_name), - ) - ) - row = cur.fetchone() - if row: - registry_proto = registry_proto.FromString(row[0]) - except psycopg2.errors.UndefinedTable: - pass - return registry_proto - - def update_registry_proto(self, registry_proto: RegistryProto): - """ - Overwrites the current registry proto with the proto passed in. This method - writes to the registry path. - - Args: - registry_proto: the new RegistryProto - """ - schema_name = self.db_config.db_schema or self.db_config.user - with _get_conn(self.db_config) as conn, conn.cursor() as cur: - cur.execute( - """ - SELECT schema_name - FROM information_schema.schemata - WHERE schema_name = %s - """, - (schema_name,), - ) - schema_exists = cur.fetchone() - if not schema_exists: - cur.execute( - sql.SQL("CREATE SCHEMA IF NOT EXISTS {} AUTHORIZATION {}").format( - sql.Identifier(schema_name), - sql.Identifier(self.db_config.user), - ), - ) - - cur.execute( - sql.SQL( - """ - CREATE TABLE IF NOT EXISTS {} ( - version BIGSERIAL PRIMARY KEY, - registry BYTEA NOT NULL - ); - """ - ).format(sql.Identifier(self.table_name)), - ) - # Do we want to keep track of the history or just keep the latest? - cur.execute( - sql.SQL( - """ - INSERT INTO {} (registry) - VALUES (%s); - """ - ).format(sql.Identifier(self.table_name)), - [registry_proto.SerializeToString()], - ) - - def teardown(self): - with _get_conn(self.db_config) as conn, conn.cursor() as cur: - cur.execute( - sql.SQL( - """ - DROP TABLE IF EXISTS {}; - """ - ).format(sql.Identifier(self.table_name)) - ) diff --git a/sdk/python/feast/infra/registry/proto_registry_utils.py b/sdk/python/feast/infra/registry/proto_registry_utils.py index 60e9cfa3abc..0e85f5b0a9f 100644 --- a/sdk/python/feast/infra/registry/proto_registry_utils.py +++ b/sdk/python/feast/infra/registry/proto_registry_utils.py @@ -2,6 +2,7 @@ from functools import wraps from typing import List, Optional +from feast import utils from feast.data_source import DataSource from feast.entity import Entity from feast.errors import ( @@ -42,6 +43,30 @@ def wrapper(registry_proto: RegistryProto, project: str): return wrapper +def registry_proto_cache_with_tags(func): + cache_key = None + cache_value = None + + @wraps(func) + def wrapper( + registry_proto: RegistryProto, + project: str, + tags: Optional[dict[str, str]], + ): + nonlocal cache_key, cache_value + + key = tuple([id(registry_proto), registry_proto.version_id, project, tags]) + + if key == cache_key: + return cache_value + else: + cache_value = func(registry_proto, project, tags) + cache_key = key + return cache_value + + return wrapper + + def init_project_metadata(cached_registry_proto: RegistryProto, project: str): new_project_uuid = f"{uuid.uuid4()}" cached_registry_proto.project_metadata.append( @@ -145,68 +170,84 @@ def get_validation_reference( raise ValidationReferenceNotFound(name, project=project) -@registry_proto_cache +@registry_proto_cache_with_tags def list_feature_services( - registry_proto: RegistryProto, project: str + registry_proto: RegistryProto, project: str, tags: Optional[dict[str, str]] ) -> List[FeatureService]: feature_services = [] for feature_service_proto in registry_proto.feature_services: - if feature_service_proto.spec.project == project: + if feature_service_proto.spec.project == project and utils.has_all_tags( + feature_service_proto.spec.tags, tags + ): feature_services.append(FeatureService.from_proto(feature_service_proto)) return feature_services -@registry_proto_cache +@registry_proto_cache_with_tags def list_feature_views( - registry_proto: RegistryProto, project: str + registry_proto: RegistryProto, project: str, tags: Optional[dict[str, str]] ) -> List[FeatureView]: feature_views: List[FeatureView] = [] for feature_view_proto in registry_proto.feature_views: - if feature_view_proto.spec.project == project: + if feature_view_proto.spec.project == project and utils.has_all_tags( + feature_view_proto.spec.tags, tags + ): feature_views.append(FeatureView.from_proto(feature_view_proto)) return feature_views -@registry_proto_cache +@registry_proto_cache_with_tags def list_stream_feature_views( - registry_proto: RegistryProto, project: str + registry_proto: RegistryProto, project: str, tags: Optional[dict[str, str]] ) -> List[StreamFeatureView]: stream_feature_views = [] for stream_feature_view in registry_proto.stream_feature_views: - if stream_feature_view.spec.project == project: + if stream_feature_view.spec.project == project and utils.has_all_tags( + stream_feature_view.spec.tags, tags + ): stream_feature_views.append( StreamFeatureView.from_proto(stream_feature_view) ) return stream_feature_views -@registry_proto_cache +@registry_proto_cache_with_tags def list_on_demand_feature_views( - registry_proto: RegistryProto, project: str + registry_proto: RegistryProto, project: str, tags: Optional[dict[str, str]] ) -> List[OnDemandFeatureView]: on_demand_feature_views = [] for on_demand_feature_view in registry_proto.on_demand_feature_views: - if on_demand_feature_view.spec.project == project: + if on_demand_feature_view.spec.project == project and utils.has_all_tags( + on_demand_feature_view.spec.tags, tags + ): on_demand_feature_views.append( OnDemandFeatureView.from_proto(on_demand_feature_view) ) return on_demand_feature_views -@registry_proto_cache -def list_entities(registry_proto: RegistryProto, project: str) -> List[Entity]: +@registry_proto_cache_with_tags +def list_entities( + registry_proto: RegistryProto, project: str, tags: Optional[dict[str, str]] +) -> List[Entity]: entities = [] for entity_proto in registry_proto.entities: - if entity_proto.spec.project == project: + if entity_proto.spec.project == project and utils.has_all_tags( + entity_proto.spec.tags, tags + ): entities.append(Entity.from_proto(entity_proto)) return entities -@registry_proto_cache -def list_data_sources(registry_proto: RegistryProto, project: str) -> List[DataSource]: +@registry_proto_cache_with_tags +def list_data_sources( + registry_proto: RegistryProto, project: str, tags: Optional[dict[str, str]] +) -> List[DataSource]: data_sources = [] for data_source_proto in registry_proto.data_sources: - if data_source_proto.project == project: + if data_source_proto.project == project and utils.has_all_tags( + data_source_proto.tags, tags + ): data_sources.append(DataSource.from_proto(data_source_proto)) return data_sources diff --git a/sdk/python/feast/infra/registry/registry.py b/sdk/python/feast/infra/registry/registry.py index b1efbb2c7c3..39cdedb4906 100644 --- a/sdk/python/feast/infra/registry/registry.py +++ b/sdk/python/feast/infra/registry/registry.py @@ -54,7 +54,6 @@ "GCSRegistryStore": "feast.infra.registry.gcs.GCSRegistryStore", "S3RegistryStore": "feast.infra.registry.s3.S3RegistryStore", "FileRegistryStore": "feast.infra.registry.file.FileRegistryStore", - "PostgreSQLRegistryStore": "feast.infra.registry.contrib.postgres.postgres_registry_store.PostgreSQLRegistryStore", "AzureRegistryStore": "feast.infra.registry.contrib.azure.azure_registry_store.AzBlobRegistryStore", } @@ -273,19 +272,27 @@ def apply_entity(self, entity: Entity, project: str, commit: bool = True): if commit: self.commit() - def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity]: + def list_entities( + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, + ) -> List[Entity]: registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - return proto_registry_utils.list_entities(registry_proto, project) + return proto_registry_utils.list_entities(registry_proto, project, tags) def list_data_sources( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[DataSource]: registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - return proto_registry_utils.list_data_sources(registry_proto, project) + return proto_registry_utils.list_data_sources(registry_proto, project, tags) def apply_data_source( self, data_source: DataSource, project: str, commit: bool = True @@ -345,12 +352,15 @@ def apply_feature_service( self.commit() def list_feature_services( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[FeatureService]: registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - return proto_registry_utils.list_feature_services(registry_proto, project) + return proto_registry_utils.list_feature_services(registry_proto, project, tags) def get_feature_service( self, name: str, project: str, allow_cache: bool = False @@ -419,21 +429,29 @@ def apply_feature_view( self.commit() def list_stream_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[StreamFeatureView]: registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - return proto_registry_utils.list_stream_feature_views(registry_proto, project) + return proto_registry_utils.list_stream_feature_views( + registry_proto, project, tags + ) def list_on_demand_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[OnDemandFeatureView]: registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) return proto_registry_utils.list_on_demand_feature_views( - registry_proto, project + registry_proto, project, tags ) def get_on_demand_feature_view( @@ -514,12 +532,15 @@ def apply_materialization( raise FeatureViewNotFoundException(feature_view.name, project) def list_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[FeatureView]: registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - return proto_registry_utils.list_feature_views(registry_proto, project) + return proto_registry_utils.list_feature_views(registry_proto, project, tags) def get_feature_view( self, name: str, project: str, allow_cache: bool = False diff --git a/sdk/python/feast/infra/registry/remote.py b/sdk/python/feast/infra/registry/remote.py index f93e1ab1c03..0eddf03cf64 100644 --- a/sdk/python/feast/infra/registry/remote.py +++ b/sdk/python/feast/infra/registry/remote.py @@ -4,12 +4,12 @@ import grpc from google.protobuf.empty_pb2 import Empty +from google.protobuf.timestamp_pb2 import Timestamp from pydantic import StrictStr from feast.base_feature_view import BaseFeatureView from feast.data_source import DataSource from feast.entity import Entity -from feast.errors import ReadOnlyRegistryException from feast.feature_service import FeatureService from feast.feature_view import FeatureView from feast.infra.infra_object import Infra @@ -43,10 +43,18 @@ def __init__( self.stub = RegistryServer_pb2_grpc.RegistryServerStub(self.channel) def apply_entity(self, entity: Entity, project: str, commit: bool = True): - raise ReadOnlyRegistryException() + request = RegistryServer_pb2.ApplyEntityRequest( + entity=entity.to_proto(), project=project, commit=commit + ) + + self.stub.ApplyEntity(request) def delete_entity(self, name: str, project: str, commit: bool = True): - raise ReadOnlyRegistryException() + request = RegistryServer_pb2.DeleteEntityRequest( + name=name, project=project, commit=commit + ) + + self.stub.DeleteEntity(request) def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Entity: request = RegistryServer_pb2.GetEntityRequest( @@ -57,9 +65,14 @@ def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Enti return Entity.from_proto(response) - def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity]: + def list_entities( + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, + ) -> List[Entity]: request = RegistryServer_pb2.ListEntitiesRequest( - project=project, allow_cache=allow_cache + project=project, allow_cache=allow_cache, tags=tags ) response = self.stub.ListEntities(request) @@ -69,10 +82,18 @@ def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity] def apply_data_source( self, data_source: DataSource, project: str, commit: bool = True ): - raise ReadOnlyRegistryException() + request = RegistryServer_pb2.ApplyDataSourceRequest( + data_source=data_source.to_proto(), project=project, commit=commit + ) + + self.stub.ApplyDataSource(request) def delete_data_source(self, name: str, project: str, commit: bool = True): - raise ReadOnlyRegistryException() + request = RegistryServer_pb2.DeleteDataSourceRequest( + name=name, project=project, commit=commit + ) + + self.stub.DeleteDataSource(request) def get_data_source( self, name: str, project: str, allow_cache: bool = False @@ -86,10 +107,13 @@ def get_data_source( return DataSource.from_proto(response) def list_data_sources( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[DataSource]: request = RegistryServer_pb2.ListDataSourcesRequest( - project=project, allow_cache=allow_cache + project=project, allow_cache=allow_cache, tags=tags ) response = self.stub.ListDataSources(request) @@ -101,10 +125,18 @@ def list_data_sources( def apply_feature_service( self, feature_service: FeatureService, project: str, commit: bool = True ): - raise ReadOnlyRegistryException() + request = RegistryServer_pb2.ApplyFeatureServiceRequest( + feature_service=feature_service.to_proto(), project=project, commit=commit + ) + + self.stub.ApplyFeatureService(request) def delete_feature_service(self, name: str, project: str, commit: bool = True): - raise ReadOnlyRegistryException() + request = RegistryServer_pb2.DeleteFeatureServiceRequest( + name=name, project=project, commit=commit + ) + + self.stub.DeleteFeatureService(request) def get_feature_service( self, name: str, project: str, allow_cache: bool = False @@ -118,10 +150,13 @@ def get_feature_service( return FeatureService.from_proto(response) def list_feature_services( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[FeatureService]: request = RegistryServer_pb2.ListFeatureServicesRequest( - project=project, allow_cache=allow_cache + project=project, allow_cache=allow_cache, tags=tags ) response = self.stub.ListFeatureServices(request) @@ -134,10 +169,35 @@ def list_feature_services( def apply_feature_view( self, feature_view: BaseFeatureView, project: str, commit: bool = True ): - raise ReadOnlyRegistryException() + if isinstance(feature_view, StreamFeatureView): + arg_name = "stream_feature_view" + elif isinstance(feature_view, FeatureView): + arg_name = "feature_view" + elif isinstance(feature_view, OnDemandFeatureView): + arg_name = "on_demand_feature_view" + + request = RegistryServer_pb2.ApplyFeatureViewRequest( + feature_view=feature_view.to_proto() + if arg_name == "feature_view" + else None, + stream_feature_view=feature_view.to_proto() + if arg_name == "stream_feature_view" + else None, + on_demand_feature_view=feature_view.to_proto() + if arg_name == "on_demand_feature_view" + else None, + project=project, + commit=commit, + ) + + self.stub.ApplyFeatureView(request) def delete_feature_view(self, name: str, project: str, commit: bool = True): - raise ReadOnlyRegistryException() + request = RegistryServer_pb2.DeleteFeatureViewRequest( + name=name, project=project, commit=commit + ) + + self.stub.DeleteFeatureView(request) def get_stream_feature_view( self, name: str, project: str, allow_cache: bool = False @@ -151,10 +211,13 @@ def get_stream_feature_view( return StreamFeatureView.from_proto(response) def list_stream_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[StreamFeatureView]: request = RegistryServer_pb2.ListStreamFeatureViewsRequest( - project=project, allow_cache=allow_cache + project=project, allow_cache=allow_cache, tags=tags ) response = self.stub.ListStreamFeatureViews(request) @@ -176,10 +239,13 @@ def get_on_demand_feature_view( return OnDemandFeatureView.from_proto(response) def list_on_demand_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[OnDemandFeatureView]: request = RegistryServer_pb2.ListOnDemandFeatureViewsRequest( - project=project, allow_cache=allow_cache + project=project, allow_cache=allow_cache, tags=tags ) response = self.stub.ListOnDemandFeatureViews(request) @@ -201,10 +267,13 @@ def get_feature_view( return FeatureView.from_proto(response) def list_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[FeatureView]: request = RegistryServer_pb2.ListFeatureViewsRequest( - project=project, allow_cache=allow_cache + project=project, allow_cache=allow_cache, tags=tags ) response = self.stub.ListFeatureViews(request) @@ -222,7 +291,20 @@ def apply_materialization( end_date: datetime, commit: bool = True, ): - raise ReadOnlyRegistryException() + start_date_timestamp = Timestamp() + end_date_timestamp = Timestamp() + start_date_timestamp.FromDatetime(start_date) + end_date_timestamp.FromDatetime(end_date) + + request = RegistryServer_pb2.ApplyMaterializationRequest( + feature_view=feature_view.to_proto(), + project=project, + start_date=start_date_timestamp, + end_date=end_date_timestamp, + commit=commit, + ) + + self.stub.ApplyMaterialization(request) def apply_saved_dataset( self, @@ -230,10 +312,18 @@ def apply_saved_dataset( project: str, commit: bool = True, ): - raise ReadOnlyRegistryException() + request = RegistryServer_pb2.ApplySavedDatasetRequest( + saved_dataset=saved_dataset.to_proto(), project=project, commit=commit + ) - def delete_saved_dataset(self, name: str, project: str, allow_cache: bool = False): - raise ReadOnlyRegistryException() + self.stub.ApplyFeatureService(request) + + def delete_saved_dataset(self, name: str, project: str, commit: bool = True): + request = RegistryServer_pb2.DeleteSavedDatasetRequest( + name=name, project=project, commit=commit + ) + + self.stub.DeleteSavedDataset(request) def get_saved_dataset( self, name: str, project: str, allow_cache: bool = False @@ -266,10 +356,20 @@ def apply_validation_reference( project: str, commit: bool = True, ): - raise ReadOnlyRegistryException() + request = RegistryServer_pb2.ApplyValidationReferenceRequest( + validation_reference=validation_reference.to_proto(), + project=project, + commit=commit, + ) + + self.stub.ApplyValidationReference(request) def delete_validation_reference(self, name: str, project: str, commit: bool = True): - raise ReadOnlyRegistryException() + request = RegistryServer_pb2.DeleteValidationReferenceRequest( + name=name, project=project, commit=commit + ) + + self.stub.DeleteValidationReference(request) def get_validation_reference( self, name: str, project: str, allow_cache: bool = False @@ -308,7 +408,11 @@ def list_project_metadata( return [ProjectMetadata.from_proto(pm) for pm in response.project_metadata] def update_infra(self, infra: Infra, project: str, commit: bool = True): - raise ReadOnlyRegistryException() + request = RegistryServer_pb2.UpdateInfraRequest( + infra=infra.to_proto(), project=project, commit=commit + ) + + self.stub.UpdateInfra(request) def get_infra(self, project: str, allow_cache: bool = False) -> Infra: request = RegistryServer_pb2.GetInfraRequest( @@ -336,9 +440,12 @@ def proto(self) -> RegistryProto: return self.stub.Proto(Empty()) def commit(self): - raise ReadOnlyRegistryException() + self.stub.Commit(Empty()) def refresh(self, project: Optional[str] = None): request = RegistryServer_pb2.RefreshRequest(project=str(project)) self.stub.Refresh(request) + + def teardown(self): + pass diff --git a/sdk/python/feast/infra/registry/snowflake.py b/sdk/python/feast/infra/registry/snowflake.py index 87d89af9c87..aaf6c4c48dc 100644 --- a/sdk/python/feast/infra/registry/snowflake.py +++ b/sdk/python/feast/infra/registry/snowflake.py @@ -10,6 +10,7 @@ from pydantic import ConfigDict, Field, StrictStr import feast +from feast import utils from feast.base_feature_view import BaseFeatureView from feast.data_source import DataSource from feast.entity import Entity @@ -619,34 +620,50 @@ def _get_object( # list operations def list_data_sources( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[DataSource]: if allow_cache: self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_data_sources( - self.cached_registry_proto, project + self.cached_registry_proto, project, tags ) return self._list_objects( - "DATA_SOURCES", project, DataSourceProto, DataSource, "DATA_SOURCE_PROTO" + "DATA_SOURCES", + project, + DataSourceProto, + DataSource, + "DATA_SOURCE_PROTO", + tags=tags, ) - def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity]: + def list_entities( + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, + ) -> List[Entity]: if allow_cache: self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_entities( - self.cached_registry_proto, project + self.cached_registry_proto, project, tags ) return self._list_objects( - "ENTITIES", project, EntityProto, Entity, "ENTITY_PROTO" + "ENTITIES", project, EntityProto, Entity, "ENTITY_PROTO", tags=tags ) def list_feature_services( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[FeatureService]: if allow_cache: self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_feature_services( - self.cached_registry_proto, project + self.cached_registry_proto, project, tags ) return self._list_objects( "FEATURE_SERVICES", @@ -654,15 +671,19 @@ def list_feature_services( FeatureServiceProto, FeatureService, "FEATURE_SERVICE_PROTO", + tags=tags, ) def list_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[FeatureView]: if allow_cache: self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_feature_views( - self.cached_registry_proto, project + self.cached_registry_proto, project, tags ) return self._list_objects( "FEATURE_VIEWS", @@ -670,15 +691,19 @@ def list_feature_views( FeatureViewProto, FeatureView, "FEATURE_VIEW_PROTO", + tags=tags, ) def list_on_demand_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[OnDemandFeatureView]: if allow_cache: self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_on_demand_feature_views( - self.cached_registry_proto, project + self.cached_registry_proto, project, tags ) return self._list_objects( "ON_DEMAND_FEATURE_VIEWS", @@ -686,6 +711,7 @@ def list_on_demand_feature_views( OnDemandFeatureViewProto, OnDemandFeatureView, "ON_DEMAND_FEATURE_VIEW_PROTO", + tags=tags, ) def list_saved_datasets( @@ -705,12 +731,15 @@ def list_saved_datasets( ) def list_stream_feature_views( - self, project: str, allow_cache: bool = False + self, + project: str, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, ) -> List[StreamFeatureView]: if allow_cache: self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_stream_feature_views( - self.cached_registry_proto, project + self.cached_registry_proto, project, tags ) return self._list_objects( "STREAM_FEATURE_VIEWS", @@ -718,6 +747,7 @@ def list_stream_feature_views( StreamFeatureViewProto, StreamFeatureView, "STREAM_FEATURE_VIEW_PROTO", + tags=tags, ) def list_validation_references( @@ -738,6 +768,7 @@ def _list_objects( proto_class: Any, python_class: Any, proto_field_name: str, + tags: Optional[dict[str, str]] = None, ): self._maybe_init_project_metadata(project) with GetSnowflakeConnection(self.registry_config) as conn: @@ -750,14 +781,15 @@ def _list_objects( project_id = '{project}' """ df = execute_snowflake_statement(conn, query).fetch_pandas_all() - if not df.empty: - return [ - python_class.from_proto( + objects = [] + for row in df.iterrows(): + obj = python_class.from_proto( proto_class.FromString(row[1][proto_field_name]) ) - for row in df.iterrows() - ] + if utils.has_all_tags(obj.tags, tags): + objects.append(obj) + return objects return [] def apply_materialization( diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index 26f9da19e18..d0af6872c1c 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -21,6 +21,7 @@ ) from sqlalchemy.engine import Engine +from feast import utils from feast.base_feature_view import BaseFeatureView from feast.data_source import DataSource from feast.entity import Entity @@ -220,13 +221,16 @@ def _get_stream_feature_view(self, name: str, project: str): not_found_exception=FeatureViewNotFoundException, ) - def _list_stream_feature_views(self, project: str) -> List[StreamFeatureView]: + def _list_stream_feature_views( + self, project: str, tags: Optional[dict[str, str]] + ) -> List[StreamFeatureView]: return self._list_objects( stream_feature_views, project, StreamFeatureViewProto, StreamFeatureView, "feature_view_proto", + tags=tags, ) def apply_entity(self, entity: Entity, project: str, commit: bool = True): @@ -321,9 +325,11 @@ def _list_validation_references(self, project: str) -> List[ValidationReference] proto_field_name="validation_reference_proto", ) - def _list_entities(self, project: str) -> List[Entity]: + def _list_entities( + self, project: str, tags: Optional[dict[str, str]] + ) -> List[Entity]: return self._list_objects( - entities, project, EntityProto, Entity, "entity_proto" + entities, project, EntityProto, Entity, "entity_proto", tags=tags ) def delete_entity(self, name: str, project: str, commit: bool = True): @@ -365,9 +371,16 @@ def _get_data_source(self, name: str, project: str) -> DataSource: not_found_exception=DataSourceObjectNotFoundException, ) - def _list_data_sources(self, project: str) -> List[DataSource]: + def _list_data_sources( + self, project: str, tags: Optional[dict[str, str]] + ) -> List[DataSource]: return self._list_objects( - data_sources, project, DataSourceProto, DataSource, "data_source_proto" + data_sources, + project, + DataSourceProto, + DataSource, + "data_source_proto", + tags=tags, ) def apply_data_source( @@ -407,18 +420,28 @@ def delete_data_source(self, name: str, project: str, commit: bool = True): if rows.rowcount < 1: raise DataSourceObjectNotFoundException(name, project) - def _list_feature_services(self, project: str) -> List[FeatureService]: + def _list_feature_services( + self, project: str, tags: Optional[dict[str, str]] + ) -> List[FeatureService]: return self._list_objects( feature_services, project, FeatureServiceProto, FeatureService, "feature_service_proto", + tags=tags, ) - def _list_feature_views(self, project: str) -> List[FeatureView]: + def _list_feature_views( + self, project: str, tags: Optional[dict[str, str]] + ) -> List[FeatureView]: return self._list_objects( - feature_views, project, FeatureViewProto, FeatureView, "feature_view_proto" + feature_views, + project, + FeatureViewProto, + FeatureView, + "feature_view_proto", + tags=tags, ) def _list_saved_datasets(self, project: str) -> List[SavedDataset]: @@ -430,13 +453,16 @@ def _list_saved_datasets(self, project: str) -> List[SavedDataset]: "saved_dataset_proto", ) - def _list_on_demand_feature_views(self, project: str) -> List[OnDemandFeatureView]: + def _list_on_demand_feature_views( + self, project: str, tags: Optional[dict[str, str]] + ) -> List[OnDemandFeatureView]: return self._list_objects( on_demand_feature_views, project, OnDemandFeatureViewProto, OnDemandFeatureView, "feature_view_proto", + tags=tags, ) def _list_project_metadata(self, project: str) -> List[ProjectMetadata]: @@ -796,18 +822,21 @@ def _list_objects( proto_class: Any, python_class: Any, proto_field_name: str, + tags: Optional[dict[str, str]] = None, ): self._maybe_init_project_metadata(project) with self.engine.begin() as conn: stmt = select(table).where(table.c.project_id == project) rows = conn.execute(stmt).all() if rows: - return [ - python_class.from_proto( + objects = [] + for row in rows: + obj = python_class.from_proto( proto_class.FromString(row._mapping[proto_field_name]) ) - for row in rows - ] + if utils.has_all_tags(obj.tags, tags): + objects.append(obj) + return objects return [] def _set_last_updated_metadata(self, last_updated: datetime, project: str): diff --git a/sdk/python/feast/offline_server.py b/sdk/python/feast/offline_server.py new file mode 100644 index 00000000000..718da1b109f --- /dev/null +++ b/sdk/python/feast/offline_server.py @@ -0,0 +1,332 @@ +import ast +import json +import logging +import traceback +from datetime import datetime +from typing import Any, Dict, List + +import pyarrow as pa +import pyarrow.flight as fl + +from feast import FeatureStore, FeatureView, utils +from feast.feature_logging import FeatureServiceLoggingSource +from feast.feature_view import DUMMY_ENTITY_NAME +from feast.infra.offline_stores.offline_utils import get_offline_store_from_config +from feast.saved_dataset import SavedDatasetStorage + +logger = logging.getLogger(__name__) + + +class OfflineServer(fl.FlightServerBase): + def __init__(self, store: FeatureStore, location: str, **kwargs): + super(OfflineServer, self).__init__(location, **kwargs) + self._location = location + # A dictionary of configured flights, e.g. API calls received and not yet served + self.flights: Dict[str, Any] = {} + self.store = store + self.offline_store = get_offline_store_from_config(store.config.offline_store) + + @classmethod + def descriptor_to_key(self, descriptor: fl.FlightDescriptor): + return ( + descriptor.descriptor_type.value, + descriptor.command, + tuple(descriptor.path or tuple()), + ) + + def _make_flight_info(self, key: Any, descriptor: fl.FlightDescriptor): + endpoints = [fl.FlightEndpoint(repr(key), [self._location])] + # TODO calculate actual schema from the given features + schema = pa.schema([]) + + return fl.FlightInfo(schema, descriptor, endpoints, -1, -1) + + def get_flight_info( + self, context: fl.ServerCallContext, descriptor: fl.FlightDescriptor + ): + key = OfflineServer.descriptor_to_key(descriptor) + if key in self.flights: + return self._make_flight_info(key, descriptor) + raise KeyError("Flight not found.") + + def list_flights(self, context: fl.ServerCallContext, criteria: bytes): + for key, table in self.flights.items(): + if key[1] is not None: + descriptor = fl.FlightDescriptor.for_command(key[1]) + else: + descriptor = fl.FlightDescriptor.for_path(*key[2]) + + yield self._make_flight_info(key, descriptor) + + # Expects to receive request parameters and stores them in the flights dictionary + # Indexed by the unique command + def do_put( + self, + context: fl.ServerCallContext, + descriptor: fl.FlightDescriptor, + reader: fl.MetadataRecordBatchReader, + writer: fl.FlightMetadataWriter, + ): + key = OfflineServer.descriptor_to_key(descriptor) + command = json.loads(key[1]) + if "api" in command: + data = reader.read_all() + logger.debug(f"do_put: command is{command}, data is {data}") + self.flights[key] = data + + self._call_api(command, key) + else: + logger.warning(f"No 'api' field in command: {command}") + + def _call_api(self, command: dict, key: str): + remove_data = False + try: + api = command["api"] + if api == OfflineServer.offline_write_batch.__name__: + self.offline_write_batch(command, key) + remove_data = True + elif api == OfflineServer.write_logged_features.__name__: + self.write_logged_features(command, key) + remove_data = True + elif api == OfflineServer.persist.__name__: + self.persist(command["retrieve_func"], command, key) + remove_data = True + except Exception as e: + remove_data = True + logger.exception(e) + traceback.print_exc() + raise e + finally: + if remove_data: + # Get service is consumed, so we clear the corresponding flight and data + del self.flights[key] + + def get_feature_view_by_name( + self, fv_name: str, name_alias: str, project: str + ) -> FeatureView: + """ + Retrieves a feature view by name, including all subclasses of FeatureView. + + Args: + fv_name: Name of feature view + name_alias: Alias to be applied to the projection of the registered view + project: Feast project that this feature view belongs to + + Returns: + Returns either the specified feature view, or raises an exception if + none is found + """ + try: + fv = self.store.registry.get_feature_view(name=fv_name, project=project) + if name_alias is not None: + for fs in self.store.registry.list_feature_services(project=project): + for p in fs.feature_view_projections: + if p.name_alias == name_alias: + logger.debug( + f"Found matching FeatureService {fs.name} with projection {p}" + ) + fv = fv.with_projection(p) + return fv + except Exception: + try: + return self.store.registry.get_stream_feature_view( + name=fv_name, project=project + ) + except Exception as e: + logger.error( + f"Cannot find any FeatureView by name {fv_name} in project {project}" + ) + raise e + + def list_feature_views_by_name( + self, feature_view_names: List[str], name_aliases: List[str], project: str + ) -> List[FeatureView]: + return [ + remove_dummies( + self.get_feature_view_by_name( + fv_name=fv_name, name_alias=name_aliases[index], project=project + ) + ) + for index, fv_name in enumerate(feature_view_names) + ] + + # Extracts the API parameters from the flights dictionary, delegates the execution to the FeatureStore instance + # and returns the stream of data + def do_get(self, context: fl.ServerCallContext, ticket: fl.Ticket): + key = ast.literal_eval(ticket.ticket.decode()) + if key not in self.flights: + logger.error(f"Unknown key {key}") + return None + + command = json.loads(key[1]) + api = command["api"] + logger.debug(f"get command is {command}") + logger.debug(f"requested api is {api}") + try: + if api == OfflineServer.get_historical_features.__name__: + table = self.get_historical_features(command, key).to_arrow() + elif api == OfflineServer.pull_all_from_table_or_query.__name__: + table = self.pull_all_from_table_or_query(command).to_arrow() + elif api == OfflineServer.pull_latest_from_table_or_query.__name__: + table = self.pull_latest_from_table_or_query(command).to_arrow() + else: + raise NotImplementedError + except Exception as e: + logger.exception(e) + traceback.print_exc() + raise e + + # Get service is consumed, so we clear the corresponding flight and data + del self.flights[key] + return fl.RecordBatchStream(table) + + def offline_write_batch(self, command: dict, key: str): + feature_view_names = command["feature_view_names"] + assert ( + len(feature_view_names) == 1 + ), "feature_view_names list should only have one item" + name_aliases = command["name_aliases"] + assert len(name_aliases) == 1, "name_aliases list should only have one item" + project = self.store.config.project + feature_views = self.list_feature_views_by_name( + feature_view_names=feature_view_names, + name_aliases=name_aliases, + project=project, + ) + + assert len(feature_views) == 1 + table = self.flights[key] + self.offline_store.offline_write_batch( + self.store.config, feature_views[0], table, command["progress"] + ) + + def write_logged_features(self, command: dict, key: str): + table = self.flights[key] + feature_service = self.store.get_feature_service( + command["feature_service_name"] + ) + + assert feature_service.logging_config is not None + + self.offline_store.write_logged_features( + config=self.store.config, + data=table, + source=FeatureServiceLoggingSource( + feature_service, self.store.config.project + ), + logging_config=feature_service.logging_config, + registry=self.store.registry, + ) + + def pull_all_from_table_or_query(self, command: dict): + return self.offline_store.pull_all_from_table_or_query( + self.store.config, + self.store.get_data_source(command["data_source_name"]), + command["join_key_columns"], + command["feature_name_columns"], + command["timestamp_field"], + utils.make_tzaware(datetime.fromisoformat(command["start_date"])), + utils.make_tzaware(datetime.fromisoformat(command["end_date"])), + ) + + def pull_latest_from_table_or_query(self, command: dict): + return self.offline_store.pull_latest_from_table_or_query( + self.store.config, + self.store.get_data_source(command["data_source_name"]), + command["join_key_columns"], + command["feature_name_columns"], + command["timestamp_field"], + command["created_timestamp_column"], + utils.make_tzaware(datetime.fromisoformat(command["start_date"])), + utils.make_tzaware(datetime.fromisoformat(command["end_date"])), + ) + + def list_actions(self, context): + return [ + ( + OfflineServer.offline_write_batch.__name__, + "Writes the specified arrow table to the data source underlying the specified feature view.", + ), + ( + OfflineServer.write_logged_features.__name__, + "Writes logged features to a specified destination in the offline store.", + ), + ( + OfflineServer.persist.__name__, + "Synchronously executes the underlying query and persists the result in the same offline store at the " + "specified destination.", + ), + ] + + def get_historical_features(self, command: dict, key: str): + # Extract parameters from the internal flights dictionary + entity_df_value = self.flights[key] + entity_df = pa.Table.to_pandas(entity_df_value) + feature_view_names = command["feature_view_names"] + name_aliases = command["name_aliases"] + feature_refs = command["feature_refs"] + project = command["project"] + full_feature_names = command["full_feature_names"] + feature_views = self.list_feature_views_by_name( + feature_view_names=feature_view_names, + name_aliases=name_aliases, + project=project, + ) + retJob = self.offline_store.get_historical_features( + config=self.store.config, + feature_views=feature_views, + feature_refs=feature_refs, + entity_df=entity_df, + registry=self.store.registry, + project=project, + full_feature_names=full_feature_names, + ) + return retJob + + def persist(self, retrieve_func: str, command: dict, key: str): + try: + if retrieve_func == OfflineServer.get_historical_features.__name__: + ret_job = self.get_historical_features(command, key) + elif ( + retrieve_func == OfflineServer.pull_latest_from_table_or_query.__name__ + ): + ret_job = self.pull_latest_from_table_or_query(command) + elif retrieve_func == OfflineServer.pull_all_from_table_or_query.__name__: + ret_job = self.pull_all_from_table_or_query(command) + else: + raise NotImplementedError + + data_source = self.store.get_data_source(command["data_source_name"]) + storage = SavedDatasetStorage.from_data_source(data_source) + ret_job.persist(storage, command["allow_overwrite"], command["timeout"]) + except Exception as e: + logger.exception(e) + traceback.print_exc() + raise e + + def do_action(self, context: fl.ServerCallContext, action: fl.Action): + pass + + def do_drop_dataset(self, dataset): + pass + + +def remove_dummies(fv: FeatureView) -> FeatureView: + """ + Removes dummmy IDs from FeatureView instances created with FeatureView.from_proto + """ + if DUMMY_ENTITY_NAME in fv.entities: + fv.entities = [] + fv.entity_columns = [] + return fv + + +def start_server( + store: FeatureStore, + host: str, + port: int, +): + location = "grpc+tcp://{}:{}".format(host, port) + server = OfflineServer(store, location) + logger.info(f"Offline store server serving on {location}") + server.serve() diff --git a/sdk/python/feast/registry_server.py b/sdk/python/feast/registry_server.py index 7de0cc43e14..1b6798b022c 100644 --- a/sdk/python/feast/registry_server.py +++ b/sdk/python/feast/registry_server.py @@ -1,47 +1,95 @@ from concurrent import futures +from datetime import datetime import grpc from google.protobuf.empty_pb2 import Empty from feast import FeatureStore +from feast.data_source import DataSource +from feast.entity import Entity +from feast.feature_service import FeatureService +from feast.feature_view import FeatureView +from feast.infra.infra_object import Infra +from feast.infra.registry.base_registry import BaseRegistry +from feast.on_demand_feature_view import OnDemandFeatureView from feast.protos.feast.registry import RegistryServer_pb2, RegistryServer_pb2_grpc +from feast.saved_dataset import SavedDataset, ValidationReference +from feast.stream_feature_view import StreamFeatureView class RegistryServer(RegistryServer_pb2_grpc.RegistryServerServicer): - def __init__(self, store: FeatureStore) -> None: + def __init__(self, registry: BaseRegistry) -> None: super().__init__() - self.proxied_registry = store.registry + self.proxied_registry = registry + + def ApplyEntity(self, request: RegistryServer_pb2.ApplyEntityRequest, context): + self.proxied_registry.apply_entity( + entity=Entity.from_proto(request.entity), + project=request.project, + commit=request.commit, + ) + return Empty() def GetEntity(self, request: RegistryServer_pb2.GetEntityRequest, context): return self.proxied_registry.get_entity( name=request.name, project=request.project, allow_cache=request.allow_cache ).to_proto() - def ListEntities(self, request, context): + def ListEntities(self, request: RegistryServer_pb2.ListEntitiesRequest, context): return RegistryServer_pb2.ListEntitiesResponse( entities=[ entity.to_proto() for entity in self.proxied_registry.list_entities( - project=request.project, allow_cache=request.allow_cache + project=request.project, + allow_cache=request.allow_cache, + tags=dict(request.tags), ) ] ) + def DeleteEntity(self, request: RegistryServer_pb2.DeleteEntityRequest, context): + self.proxied_registry.delete_entity( + name=request.name, project=request.project, commit=request.commit + ) + return Empty() + + def ApplyDataSource( + self, request: RegistryServer_pb2.ApplyDataSourceRequest, context + ): + self.proxied_registry.apply_data_source( + data_source=DataSource.from_proto(request.data_source), + project=request.project, + commit=request.commit, + ) + return Empty() + def GetDataSource(self, request: RegistryServer_pb2.GetDataSourceRequest, context): return self.proxied_registry.get_data_source( name=request.name, project=request.project, allow_cache=request.allow_cache ).to_proto() - def ListDataSources(self, request, context): + def ListDataSources( + self, request: RegistryServer_pb2.ListDataSourcesRequest, context + ): return RegistryServer_pb2.ListDataSourcesResponse( data_sources=[ data_source.to_proto() for data_source in self.proxied_registry.list_data_sources( - project=request.project, allow_cache=request.allow_cache + project=request.project, + allow_cache=request.allow_cache, + tags=dict(request.tags), ) ] ) + def DeleteDataSource( + self, request: RegistryServer_pb2.DeleteDataSourceRequest, context + ): + self.proxied_registry.delete_data_source( + name=request.name, project=request.project, commit=request.commit + ) + return Empty() + def GetFeatureView( self, request: RegistryServer_pb2.GetFeatureViewRequest, context ): @@ -49,16 +97,46 @@ def GetFeatureView( name=request.name, project=request.project, allow_cache=request.allow_cache ).to_proto() - def ListFeatureViews(self, request, context): + def ApplyFeatureView( + self, request: RegistryServer_pb2.ApplyFeatureViewRequest, context + ): + feature_view_type = request.WhichOneof("base_feature_view") + if feature_view_type == "feature_view": + feature_view = FeatureView.from_proto(request.feature_view) + elif feature_view_type == "on_demand_feature_view": + feature_view = OnDemandFeatureView.from_proto( + request.on_demand_feature_view + ) + elif feature_view_type == "stream_feature_view": + feature_view = StreamFeatureView.from_proto(request.stream_feature_view) + + self.proxied_registry.apply_feature_view( + feature_view=feature_view, project=request.project, commit=request.commit + ) + return Empty() + + def ListFeatureViews( + self, request: RegistryServer_pb2.ListFeatureViewsRequest, context + ): return RegistryServer_pb2.ListFeatureViewsResponse( feature_views=[ feature_view.to_proto() for feature_view in self.proxied_registry.list_feature_views( - project=request.project, allow_cache=request.allow_cache + project=request.project, + allow_cache=request.allow_cache, + tags=dict(request.tags), ) ] ) + def DeleteFeatureView( + self, request: RegistryServer_pb2.DeleteFeatureViewRequest, context + ): + self.proxied_registry.delete_feature_view( + name=request.name, project=request.project, commit=request.commit + ) + return Empty() + def GetStreamFeatureView( self, request: RegistryServer_pb2.GetStreamFeatureViewRequest, context ): @@ -66,12 +144,16 @@ def GetStreamFeatureView( name=request.name, project=request.project, allow_cache=request.allow_cache ).to_proto() - def ListStreamFeatureViews(self, request, context): + def ListStreamFeatureViews( + self, request: RegistryServer_pb2.ListStreamFeatureViewsRequest, context + ): return RegistryServer_pb2.ListStreamFeatureViewsResponse( stream_feature_views=[ stream_feature_view.to_proto() for stream_feature_view in self.proxied_registry.list_stream_feature_views( - project=request.project, allow_cache=request.allow_cache + project=request.project, + allow_cache=request.allow_cache, + tags=dict(request.tags), ) ] ) @@ -83,16 +165,30 @@ def GetOnDemandFeatureView( name=request.name, project=request.project, allow_cache=request.allow_cache ).to_proto() - def ListOnDemandFeatureViews(self, request, context): + def ListOnDemandFeatureViews( + self, request: RegistryServer_pb2.ListOnDemandFeatureViewsRequest, context + ): return RegistryServer_pb2.ListOnDemandFeatureViewsResponse( on_demand_feature_views=[ on_demand_feature_view.to_proto() for on_demand_feature_view in self.proxied_registry.list_on_demand_feature_views( - project=request.project, allow_cache=request.allow_cache + project=request.project, + allow_cache=request.allow_cache, + tags=dict(request.tags), ) ] ) + def ApplyFeatureService( + self, request: RegistryServer_pb2.ApplyFeatureServiceRequest, context + ): + self.proxied_registry.apply_feature_service( + feature_service=FeatureService.from_proto(request.feature_service), + project=request.project, + commit=request.commit, + ) + return Empty() + def GetFeatureService( self, request: RegistryServer_pb2.GetFeatureServiceRequest, context ): @@ -107,11 +203,31 @@ def ListFeatureServices( feature_services=[ feature_service.to_proto() for feature_service in self.proxied_registry.list_feature_services( - project=request.project, allow_cache=request.allow_cache + project=request.project, + allow_cache=request.allow_cache, + tags=dict(request.tags), ) ] ) + def DeleteFeatureService( + self, request: RegistryServer_pb2.DeleteFeatureServiceRequest, context + ): + self.proxied_registry.delete_feature_service( + name=request.name, project=request.project, commit=request.commit + ) + return Empty() + + def ApplySavedDataset( + self, request: RegistryServer_pb2.ApplySavedDatasetRequest, context + ): + self.proxied_registry.apply_saved_dataset( + saved_dataset=SavedDataset.from_proto(request.saved_dataset), + project=request.project, + commit=request.commit, + ) + return Empty() + def GetSavedDataset( self, request: RegistryServer_pb2.GetSavedDatasetRequest, context ): @@ -131,6 +247,26 @@ def ListSavedDatasets( ] ) + def DeleteSavedDataset( + self, request: RegistryServer_pb2.DeleteSavedDatasetRequest, context + ): + self.proxied_registry.delete_saved_dataset( + name=request.name, project=request.project, commit=request.commit + ) + return Empty() + + def ApplyValidationReference( + self, request: RegistryServer_pb2.ApplyValidationReferenceRequest, context + ): + self.proxied_registry.apply_validation_reference( + validation_reference=ValidationReference.from_proto( + request.validation_reference + ), + project=request.project, + commit=request.commit, + ) + return Empty() + def GetValidationReference( self, request: RegistryServer_pb2.GetValidationReferenceRequest, context ): @@ -150,6 +286,14 @@ def ListValidationReferences( ] ) + def DeleteValidationReference( + self, request: RegistryServer_pb2.DeleteValidationReferenceRequest, context + ): + self.proxied_registry.delete_validation_reference( + name=request.name, project=request.project, commit=request.commit + ) + return Empty() + def ListProjectMetadata( self, request: RegistryServer_pb2.ListProjectMetadataRequest, context ): @@ -162,11 +306,39 @@ def ListProjectMetadata( ] ) + def ApplyMaterialization( + self, request: RegistryServer_pb2.ApplyMaterializationRequest, context + ): + self.proxied_registry.apply_materialization( + feature_view=FeatureView.from_proto(request.feature_view), + project=request.project, + start_date=datetime.fromtimestamp( + request.start_date.seconds + request.start_date.nanos / 1e9 + ), + end_date=datetime.fromtimestamp( + request.end_date.seconds + request.end_date.nanos / 1e9 + ), + commit=request.commit, + ) + return Empty() + + def UpdateInfra(self, request: RegistryServer_pb2.UpdateInfraRequest, context): + self.proxied_registry.update_infra( + infra=Infra.from_proto(request.infra), + project=request.project, + commit=request.commit, + ) + return Empty() + def GetInfra(self, request: RegistryServer_pb2.GetInfraRequest, context): return self.proxied_registry.get_infra( project=request.project, allow_cache=request.allow_cache ).to_proto() + def Commit(self, request, context): + self.proxied_registry.commit() + return Empty() + def Refresh(self, request, context): self.proxied_registry.refresh(request.project) return Empty() @@ -178,7 +350,7 @@ def Proto(self, request, context): def start_server(store: FeatureStore, port: int): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) RegistryServer_pb2_grpc.add_RegistryServerServicer_to_server( - RegistryServer(store), server + RegistryServer(store.registry), server ) server.add_insecure_port(f"[::]:{port}") server.start() diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 6ef81794bf8..d5b3160b566 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -64,6 +64,7 @@ "hazelcast": "feast.infra.online_stores.contrib.hazelcast_online_store.hazelcast_online_store.HazelcastOnlineStore", "ikv": "feast.infra.online_stores.contrib.ikv_online_store.ikv.IKVOnlineStore", "elasticsearch": "feast.infra.online_stores.contrib.elasticsearch.ElasticSearchOnlineStore", + "remote": "feast.infra.online_stores.remote.RemoteOnlineStore", } OFFLINE_STORE_CLASS_FOR_TYPE = { @@ -77,6 +78,7 @@ "athena": "feast.infra.offline_stores.contrib.athena_offline_store.athena.AthenaOfflineStore", "mssql": "feast.infra.offline_stores.contrib.mssql_offline_store.mssql.MsSqlServerOfflineStore", "duckdb": "feast.infra.offline_stores.duckdb.DuckDBOfflineStore", + "remote": "feast.infra.offline_stores.remote.RemoteOfflineStore", } FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE = { diff --git a/sdk/python/feast/templates/local/bootstrap.py b/sdk/python/feast/templates/local/bootstrap.py index 125eb7c2e72..ee2847c19c8 100644 --- a/sdk/python/feast/templates/local/bootstrap.py +++ b/sdk/python/feast/templates/local/bootstrap.py @@ -24,6 +24,7 @@ def bootstrap(): example_py_file = repo_path / "example_repo.py" replace_str_in_file(example_py_file, "%PARQUET_PATH%", str(driver_stats_path)) + replace_str_in_file(example_py_file, "%LOGGING_PATH%", str(data_path)) if __name__ == "__main__": diff --git a/sdk/python/feast/templates/local/feature_repo/example_repo.py b/sdk/python/feast/templates/local/feature_repo/example_repo.py index 5aed3371b14..debe9d45e92 100644 --- a/sdk/python/feast/templates/local/feature_repo/example_repo.py +++ b/sdk/python/feast/templates/local/feature_repo/example_repo.py @@ -13,6 +13,8 @@ PushSource, RequestSource, ) +from feast.feature_logging import LoggingConfig +from feast.infra.offline_stores.file_source import FileLoggingDestination from feast.on_demand_feature_view import on_demand_feature_view from feast.types import Float32, Float64, Int64 @@ -88,6 +90,9 @@ def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: driver_stats_fv[["conv_rate"]], # Sub-selects a feature from a feature view transformed_conv_rate, # Selects all features from the feature view ], + logging_config=LoggingConfig( + destination=FileLoggingDestination(path="%LOGGING_PATH%") + ), ) driver_activity_v2 = FeatureService( name="driver_activity_v2", features=[driver_stats_fv, transformed_conv_rate] diff --git a/sdk/python/feast/templates/postgres/feature_repo/feature_store.yaml b/sdk/python/feast/templates/postgres/feature_repo/feature_store.yaml index 0ccd4a6d499..f14510f820e 100644 --- a/sdk/python/feast/templates/postgres/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/postgres/feature_repo/feature_store.yaml @@ -1,14 +1,12 @@ project: my_project provider: local registry: - registry_store_type: PostgreSQLRegistryStore - path: feast_registry - host: DB_HOST - port: DB_PORT - database: DB_NAME - db_schema: DB_SCHEMA - user: DB_USERNAME - password: DB_PASSWORD + registry_type: sql + path: postgresql://postgres:mysecretpassword@127.0.0.1:55001/feast + cache_ttl_seconds: 60 + sqlalchemy_config_kwargs: + echo: false + pool_pre_ping: true online_store: type: postgres host: DB_HOST diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index e7fdf971209..a0859f2f7ad 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -13,6 +13,7 @@ # limitations under the License. import json +import logging from collections import defaultdict from datetime import datetime, timezone from typing import ( @@ -53,6 +54,8 @@ # null timestamps get converted to -9223372036854775808 NULL_TIMESTAMP_INT_VALUE: int = np.datetime64("NaT").astype(int) +logger = logging.getLogger(__name__) + def feast_value_type_to_python_type(field_value_proto: ProtoValue) -> Any: """ @@ -77,9 +80,11 @@ def feast_value_type_to_python_type(field_value_proto: ProtoValue) -> Any: # Convert UNIX_TIMESTAMP values to `datetime` if val_attr == "unix_timestamp_list_val": val = [ - datetime.fromtimestamp(v, tz=timezone.utc) - if v != NULL_TIMESTAMP_INT_VALUE - else None + ( + datetime.fromtimestamp(v, tz=timezone.utc) + if v != NULL_TIMESTAMP_INT_VALUE + else None + ) for v in val ] elif val_attr == "unix_timestamp_val": @@ -295,9 +300,11 @@ def _type_err(item, dtype): ValueType.INT32: ("int32_val", lambda x: int(x), None), ValueType.INT64: ( "int64_val", - lambda x: int(x.timestamp()) - if isinstance(x, pd._libs.tslibs.timestamps.Timestamp) - else int(x), + lambda x: ( + int(x.timestamp()) + if isinstance(x, pd._libs.tslibs.timestamps.Timestamp) + else int(x) + ), None, ), ValueType.FLOAT: ("float_val", lambda x: float(x), None), @@ -373,10 +380,18 @@ def _python_value_to_proto_value( if sample is not None and not all( type(item) in valid_types for item in sample ): - first_invalid = next( - item for item in sample if type(item) not in valid_types - ) - raise _type_err(first_invalid, valid_types[0]) + # to_numpy() in utils._convert_arrow_to_proto() upcasts values of type Array of INT32 or INT64 with NULL values to Float64 automatically. + for item in sample: + if type(item) not in valid_types: + if feast_value_type in [ + ValueType.INT32_LIST, + ValueType.INT64_LIST, + ]: + if not any(np.isnan(item) for item in sample): + logger.error( + "Array of Int32 or Int64 type has NULL values. to_numpy() upcasts to Float64 automatically." + ) + raise _type_err(item, valid_types[0]) if feast_value_type == ValueType.UNIX_TIMESTAMP_LIST: int_timestamps_lists = ( @@ -390,15 +405,21 @@ def _python_value_to_proto_value( if feast_value_type == ValueType.BOOL_LIST: # ProtoValue does not support conversion of np.bool_ so we need to convert it to support np.bool_. return [ - ProtoValue(**{field_name: proto_type(val=[bool(e) for e in value])}) # type: ignore - if value is not None - else ProtoValue() + ( + ProtoValue( + **{field_name: proto_type(val=[bool(e) for e in value])} # type: ignore + ) + if value is not None + else ProtoValue() + ) for value in values ] return [ - ProtoValue(**{field_name: proto_type(val=value)}) # type: ignore - if value is not None - else ProtoValue() + ( + ProtoValue(**{field_name: proto_type(val=value)}) # type: ignore + if value is not None + else ProtoValue() + ) for value in values ] @@ -433,15 +454,17 @@ def _python_value_to_proto_value( if feast_value_type == ValueType.BOOL: # ProtoValue does not support conversion of np.bool_ so we need to convert it to support np.bool_. return [ - ProtoValue( - **{ - field_name: func( - bool(value) if type(value) is np.bool_ else value # type: ignore - ) - } + ( + ProtoValue( + **{ + field_name: func( + bool(value) if type(value) is np.bool_ else value # type: ignore + ) + } + ) + if not pd.isnull(value) + else ProtoValue() ) - if not pd.isnull(value) - else ProtoValue() for value in values ] if feast_value_type in PYTHON_SCALAR_VALUE_TYPE_TO_PROTO_VALUE: diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index d4b5decaac1..66daf7b993e 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.38.0", + "@feast-dev/feast-ui": "0.39.0", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index cb1e3154049..005035db2d1 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1451,10 +1451,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.38.0": - version "0.38.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.38.0.tgz#3a2b8325b15a1e789741523bd5113b54a80b4325" - integrity sha512-i2F4yMwbaWOOPE+FOyDxrqAsb1GETDUsZ/AYJQJiQYyWgXtVFBZpShrJcOQkOwBvV5eX/2jtj9o7SaFQpUcM8A== +"@feast-dev/feast-ui@0.39.0": + version "0.39.0" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.39.0.tgz#9ab9bdcfd866399383b489f192e3d907590ac841" + integrity sha512-ggTyiv+D/i6sF5WZRxEFmVKMVgWmrdP3bnUzbDYnMpJ6A1UKFOdj29Ukh4F8DXDvrAskV1LjF+DZVkaD5lF4TQ== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^55.0.1" diff --git a/sdk/python/feast/ui_server.py b/sdk/python/feast/ui_server.py index 1e0d87a64e3..35b51a8021a 100644 --- a/sdk/python/feast/ui_server.py +++ b/sdk/python/feast/ui_server.py @@ -51,7 +51,7 @@ def shutdown_event(): async_refresh() - ui_dir_ref = importlib_resources.files(__name__) / "ui/build/" + ui_dir_ref = importlib_resources.files(__spec__.parent) / "ui/build/" # type: ignore[name-defined] with importlib_resources.as_file(ui_dir_ref) as ui_dir: # Initialize with the projects-list.json file with ui_dir.joinpath("projects-list.json").open(mode="w") as f: diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 47faa7d8c48..a6c893c954c 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -1,24 +1,53 @@ +import copy +import itertools +import logging import os import typing -from collections import defaultdict +import warnings +from collections import Counter, defaultdict from datetime import datetime from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union +from typing import ( + Any, + Dict, + Iterable, + List, + Mapping, + Optional, + Sequence, + Set, + Tuple, + Union, + cast, +) import pandas as pd import pyarrow from dateutil.tz import tzlocal +from google.protobuf.timestamp_pb2 import Timestamp from pytz import utc from feast.constants import FEAST_FS_YAML_FILE_PATH_ENV_NAME from feast.entity import Entity +from feast.errors import ( + EntityNotFoundException, + FeatureNameCollisionError, + FeatureViewNotFoundException, + RequestDataNotFoundInEntityRowsException, +) +from feast.protos.feast.serving.ServingService_pb2 import ( + FieldStatus, + GetOnlineFeaturesResponse, +) from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import RepeatedValue as RepeatedValueProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.type_map import python_values_to_proto_values from feast.value_type import ValueType from feast.version import get_version if typing.TYPE_CHECKING: + from feast.feature_service import FeatureService from feast.feature_view import FeatureView from feast.on_demand_feature_view import OnDemandFeatureView @@ -256,3 +285,770 @@ def _convert_arrow_to_proto( created_timestamps = [None] * table.num_rows return list(zip(entity_keys, features, event_timestamps, created_timestamps)) + + +def _validate_entity_values(join_key_values: Dict[str, List[ValueProto]]): + set_of_row_lengths = {len(v) for v in join_key_values.values()} + if len(set_of_row_lengths) > 1: + raise ValueError("All entity rows must have the same columns.") + return set_of_row_lengths.pop() + + +def _validate_feature_refs(feature_refs: List[str], full_feature_names: bool = False): + """ + Validates that there are no collisions among the feature references. + + Args: + feature_refs: List of feature references to validate. Feature references must have format + "feature_view:feature", e.g. "customer_fv:daily_transactions". + full_feature_names: If True, the full feature references are compared for collisions; if False, + only the feature names are compared. + + Raises: + FeatureNameCollisionError: There is a collision among the feature references. + """ + collided_feature_refs = [] + + if full_feature_names: + collided_feature_refs = [ + ref for ref, occurrences in Counter(feature_refs).items() if occurrences > 1 + ] + else: + feature_names = [ref.split(":")[1] for ref in feature_refs] + collided_feature_names = [ + ref + for ref, occurrences in Counter(feature_names).items() + if occurrences > 1 + ] + + for feature_name in collided_feature_names: + collided_feature_refs.extend( + [ref for ref in feature_refs if ref.endswith(":" + feature_name)] + ) + + if len(collided_feature_refs) > 0: + raise FeatureNameCollisionError(collided_feature_refs, full_feature_names) + + +def _group_feature_refs( + features: List[str], + all_feature_views: List["FeatureView"], + all_on_demand_feature_views: List["OnDemandFeatureView"], +) -> Tuple[ + List[Tuple["FeatureView", List[str]]], List[Tuple["OnDemandFeatureView", List[str]]] +]: + """Get list of feature views and corresponding feature names based on feature references""" + + # view name to view proto + view_index = {view.projection.name_to_use(): view for view in all_feature_views} + + # on demand view to on demand view proto + on_demand_view_index = { + view.projection.name_to_use(): view for view in all_on_demand_feature_views + } + + # view name to feature names + views_features = defaultdict(set) + + # on demand view name to feature names + on_demand_view_features = defaultdict(set) + + for ref in features: + view_name, feat_name = ref.split(":") + if view_name in view_index: + view_index[view_name].projection.get_feature(feat_name) # For validation + views_features[view_name].add(feat_name) + elif view_name in on_demand_view_index: + on_demand_view_index[view_name].projection.get_feature( + feat_name + ) # For validation + on_demand_view_features[view_name].add(feat_name) + # Let's also add in any FV Feature dependencies here. + for input_fv_projection in on_demand_view_index[ + view_name + ].source_feature_view_projections.values(): + for input_feat in input_fv_projection.features: + views_features[input_fv_projection.name].add(input_feat.name) + else: + raise FeatureViewNotFoundException(view_name) + + fvs_result: List[Tuple["FeatureView", List[str]]] = [] + odfvs_result: List[Tuple["OnDemandFeatureView", List[str]]] = [] + + for view_name, feature_names in views_features.items(): + fvs_result.append((view_index[view_name], list(feature_names))) + for view_name, feature_names in on_demand_view_features.items(): + odfvs_result.append((on_demand_view_index[view_name], list(feature_names))) + return fvs_result, odfvs_result + + +def apply_list_mapping( + lst: Iterable[Any], mapping_indexes: Iterable[List[int]] +) -> Iterable[Any]: + output_len = sum(len(item) for item in mapping_indexes) + output = [None] * output_len + for elem, destinations in zip(lst, mapping_indexes): + for idx in destinations: + output[idx] = elem + + return output + + +def _augment_response_with_on_demand_transforms( + online_features_response: GetOnlineFeaturesResponse, + feature_refs: List[str], + requested_on_demand_feature_views: List["OnDemandFeatureView"], + full_feature_names: bool, +): + """Computes on demand feature values and adds them to the result rows. + + Assumes that 'online_features_response' already contains the necessary request data and input feature + views for the on demand feature views. Unneeded feature values such as request data and + unrequested input feature views will be removed from 'online_features_response'. + + Args: + online_features_response: Protobuf object to populate + feature_refs: List of all feature references to be returned. + requested_on_demand_feature_views: List of all odfvs that have been requested. + full_feature_names: A boolean that provides the option to add the feature view prefixes to the feature names, + changing them from the format "feature" to "feature_view__feature" (e.g., "daily_transactions" changes to + "customer_fv__daily_transactions"). + """ + from feast.online_response import OnlineResponse + + requested_odfv_map = {odfv.name: odfv for odfv in requested_on_demand_feature_views} + requested_odfv_feature_names = requested_odfv_map.keys() + + odfv_feature_refs = defaultdict(list) + for feature_ref in feature_refs: + view_name, feature_name = feature_ref.split(":") + if view_name in requested_odfv_feature_names: + odfv_feature_refs[view_name].append( + f"{requested_odfv_map[view_name].projection.name_to_use()}__{feature_name}" + if full_feature_names + else feature_name + ) + + initial_response = OnlineResponse(online_features_response) + initial_response_arrow: Optional[pyarrow.Table] = None + initial_response_dict: Optional[Dict[str, List[Any]]] = None + + # Apply on demand transformations and augment the result rows + odfv_result_names = set() + for odfv_name, _feature_refs in odfv_feature_refs.items(): + odfv = requested_odfv_map[odfv_name] + if odfv.mode == "python": + if initial_response_dict is None: + initial_response_dict = initial_response.to_dict() + transformed_features_dict: Dict[str, List[Any]] = odfv.transform_dict( + initial_response_dict + ) + elif odfv.mode in {"pandas", "substrait"}: + if initial_response_arrow is None: + initial_response_arrow = initial_response.to_arrow() + transformed_features_arrow = odfv.transform_arrow( + initial_response_arrow, full_feature_names + ) + else: + raise Exception( + f"Invalid OnDemandFeatureMode: {odfv.mode}. Expected one of 'pandas', 'python', or 'substrait'." + ) + + transformed_features = ( + transformed_features_dict + if odfv.mode == "python" + else transformed_features_arrow + ) + transformed_columns = ( + transformed_features.column_names + if isinstance(transformed_features, pyarrow.Table) + else transformed_features + ) + selected_subset = [f for f in transformed_columns if f in _feature_refs] + + proto_values = [] + for selected_feature in selected_subset: + feature_vector = transformed_features[selected_feature] + proto_values.append( + python_values_to_proto_values(feature_vector, ValueType.UNKNOWN) + if odfv.mode == "python" + else python_values_to_proto_values( + feature_vector.to_numpy(), ValueType.UNKNOWN + ) + ) + + odfv_result_names |= set(selected_subset) + + online_features_response.metadata.feature_names.val.extend(selected_subset) + for feature_idx in range(len(selected_subset)): + online_features_response.results.append( + GetOnlineFeaturesResponse.FeatureVector( + values=proto_values[feature_idx], + statuses=[FieldStatus.PRESENT] * len(proto_values[feature_idx]), + event_timestamps=[Timestamp()] * len(proto_values[feature_idx]), + ) + ) + + +def _get_entity_maps( + registry, + project, + feature_views, +) -> Tuple[Dict[str, str], Dict[str, ValueType], Set[str]]: + # TODO(felixwang9817): Support entities that have different types for different feature views. + entities = registry.list_entities(project, allow_cache=True) + entity_name_to_join_key_map: Dict[str, str] = {} + entity_type_map: Dict[str, ValueType] = {} + for entity in entities: + entity_name_to_join_key_map[entity.name] = entity.join_key + for feature_view in feature_views: + for entity_name in feature_view.entities: + entity = registry.get_entity(entity_name, project, allow_cache=True) + # User directly uses join_key as the entity reference in the entity_rows for the + # entity mapping case. + entity_name = feature_view.projection.join_key_map.get( + entity.join_key, entity.name + ) + join_key = feature_view.projection.join_key_map.get( + entity.join_key, entity.join_key + ) + entity_name_to_join_key_map[entity_name] = join_key + for entity_column in feature_view.entity_columns: + entity_type_map[entity_column.name] = entity_column.dtype.to_value_type() + + return ( + entity_name_to_join_key_map, + entity_type_map, + set(entity_name_to_join_key_map.values()), + ) + + +def _get_table_entity_values( + table: "FeatureView", + entity_name_to_join_key_map: Dict[str, str], + join_key_proto_values: Dict[str, List[ValueProto]], +) -> Dict[str, List[ValueProto]]: + # The correct join_keys expected by the OnlineStore for this Feature View. + table_join_keys = [ + entity_name_to_join_key_map[entity_name] for entity_name in table.entities + ] + + # If the FeatureView has a Projection then the join keys may be aliased. + alias_to_join_key_map = {v: k for k, v in table.projection.join_key_map.items()} + + # Subset to columns which are relevant to this FeatureView and + # give them the correct names. + entity_values = { + alias_to_join_key_map.get(k, k): v + for k, v in join_key_proto_values.items() + if alias_to_join_key_map.get(k, k) in table_join_keys + } + return entity_values + + +def _get_unique_entities( + table: "FeatureView", + join_key_values: Dict[str, List[ValueProto]], + entity_name_to_join_key_map: Dict[str, str], +) -> Tuple[Tuple[Dict[str, ValueProto], ...], Tuple[List[int], ...]]: + """Return the set of unique composite Entities for a Feature View and the indexes at which they appear. + + This method allows us to query the OnlineStore for data we need only once + rather than requesting and processing data for the same combination of + Entities multiple times. + """ + # Get the correct set of entity values with the correct join keys. + table_entity_values = _get_table_entity_values( + table, + entity_name_to_join_key_map, + join_key_values, + ) + + # Convert back to rowise. + keys = table_entity_values.keys() + # Sort the rowise data to allow for grouping but keep original index. This lambda is + # sufficient as Entity types cannot be complex (ie. lists). + rowise = list(enumerate(zip(*table_entity_values.values()))) + rowise.sort(key=lambda row: tuple(getattr(x, x.WhichOneof("val")) for x in row[1])) + + # Identify unique entities and the indexes at which they occur. + unique_entities: Tuple[Dict[str, ValueProto], ...] + indexes: Tuple[List[int], ...] + unique_entities, indexes = tuple( + zip( + *[ + (dict(zip(keys, k)), [_[0] for _ in g]) + for k, g in itertools.groupby(rowise, key=lambda x: x[1]) + ] + ) + ) + return unique_entities, indexes + + +def _drop_unneeded_columns( + online_features_response: GetOnlineFeaturesResponse, + requested_result_row_names: Set[str], +): + """ + Unneeded feature values such as request data and unrequested input feature views will + be removed from 'online_features_response'. + + Args: + online_features_response: Protobuf object to populate + requested_result_row_names: Fields from 'result_rows' that have been requested, and + therefore should not be dropped. + """ + # Drop values that aren't needed + unneeded_feature_indices = [ + idx + for idx, val in enumerate(online_features_response.metadata.feature_names.val) + if val not in requested_result_row_names + ] + + for idx in reversed(unneeded_feature_indices): + del online_features_response.metadata.feature_names.val[idx] + del online_features_response.results[idx] + + +def _populate_result_rows_from_columnar( + online_features_response: GetOnlineFeaturesResponse, + data: Dict[str, List[ValueProto]], +): + timestamp = Timestamp() # Only initialize this timestamp once. + # Add more values to the existing result rows + for feature_name, feature_values in data.items(): + online_features_response.metadata.feature_names.val.append(feature_name) + online_features_response.results.append( + GetOnlineFeaturesResponse.FeatureVector( + values=feature_values, + statuses=[FieldStatus.PRESENT] * len(feature_values), + event_timestamps=[timestamp] * len(feature_values), + ) + ) + + +def get_needed_request_data( + grouped_odfv_refs: List[Tuple["OnDemandFeatureView", List[str]]], +) -> Set[str]: + needed_request_data: Set[str] = set() + for odfv, _ in grouped_odfv_refs: + odfv_request_data_schema = odfv.get_request_data_schema() + needed_request_data.update(odfv_request_data_schema.keys()) + return needed_request_data + + +def ensure_request_data_values_exist( + needed_request_data: Set[str], + request_data_features: Dict[str, List[Any]], +): + if len(needed_request_data) != len(request_data_features.keys()): + missing_features = [ + x for x in needed_request_data if x not in request_data_features + ] + raise RequestDataNotFoundInEntityRowsException(feature_names=missing_features) + + +def _populate_response_from_feature_data( + feature_data: Iterable[ + Tuple[ + Iterable[Timestamp], Iterable["FieldStatus.ValueType"], Iterable[ValueProto] + ] + ], + indexes: Iterable[List[int]], + online_features_response: GetOnlineFeaturesResponse, + full_feature_names: bool, + requested_features: Iterable[str], + table: "FeatureView", +): + """Populate the GetOnlineFeaturesResponse with feature data. + + This method assumes that `_read_from_online_store` returns data for each + combination of Entities in `entity_rows` in the same order as they + are provided. + + Args: + feature_data: A list of data in Protobuf form which was retrieved from the OnlineStore. + indexes: A list of indexes which should be the same length as `feature_data`. Each list + of indexes corresponds to a set of result rows in `online_features_response`. + online_features_response: The object to populate. + full_feature_names: A boolean that provides the option to add the feature view prefixes to the feature names, + changing them from the format "feature" to "feature_view__feature" (e.g., "daily_transactions" changes to + "customer_fv__daily_transactions"). + requested_features: The names of the features in `feature_data`. This should be ordered in the same way as the + data in `feature_data`. + table: The FeatureView that `feature_data` was retrieved from. + """ + # Add the feature names to the response. + requested_feature_refs = [ + f"{table.projection.name_to_use()}__{feature_name}" + if full_feature_names + else feature_name + for feature_name in requested_features + ] + online_features_response.metadata.feature_names.val.extend(requested_feature_refs) + + timestamps, statuses, values = zip(*feature_data) + + # Populate the result with data fetched from the OnlineStore + # which is guaranteed to be aligned with `requested_features`. + for ( + feature_idx, + (timestamp_vector, statuses_vector, values_vector), + ) in enumerate(zip(zip(*timestamps), zip(*statuses), zip(*values))): + online_features_response.results.append( + GetOnlineFeaturesResponse.FeatureVector( + values=apply_list_mapping(values_vector, indexes), + statuses=apply_list_mapping(statuses_vector, indexes), + event_timestamps=apply_list_mapping(timestamp_vector, indexes), + ) + ) + + +def _get_features( + registry, + project, + features: Union[List[str], "FeatureService"], + allow_cache: bool = False, +) -> List[str]: + from feast.feature_service import FeatureService + + _features = features + + if not _features: + raise ValueError("No features specified for retrieval") + + _feature_refs = [] + if isinstance(_features, FeatureService): + feature_service_from_registry = registry.get_feature_service( + _features.name, project, allow_cache + ) + if feature_service_from_registry != _features: + warnings.warn( + "The FeatureService object that has been passed in as an argument is " + "inconsistent with the version from the registry. Potentially a newer version " + "of the FeatureService has been applied to the registry." + ) + for projection in feature_service_from_registry.feature_view_projections: + _feature_refs.extend( + [f"{projection.name_to_use()}:{f.name}" for f in projection.features] + ) + else: + assert isinstance(_features, list) + _feature_refs = _features + return _feature_refs + + +def _list_feature_views( + registry, + project, + allow_cache: bool = False, + hide_dummy_entity: bool = True, + tags: Optional[dict[str, str]] = None, +) -> List["FeatureView"]: + from feast.feature_view import DUMMY_ENTITY_NAME + + logging.warning( + "_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. " + "_list_feature_views will behave like _list_all_feature_views in the future." + ) + feature_views = [] + for fv in registry.list_feature_views(project, allow_cache=allow_cache, tags=tags): + if hide_dummy_entity and fv.entities and fv.entities[0] == DUMMY_ENTITY_NAME: + fv.entities = [] + fv.entity_columns = [] + feature_views.append(fv) + return feature_views + + +def _get_feature_views_to_use( + registry, + project, + features: Optional[Union[List[str], "FeatureService"]], + allow_cache=False, + hide_dummy_entity: bool = True, +) -> Tuple[List["FeatureView"], List["OnDemandFeatureView"]]: + from feast.feature_service import FeatureService + + fvs = { + fv.name: fv + for fv in [ + *_list_feature_views(registry, project, allow_cache, hide_dummy_entity), + *registry.list_stream_feature_views( + project=project, allow_cache=allow_cache + ), + ] + } + + od_fvs = { + fv.name: fv + for fv in registry.list_on_demand_feature_views( + project=project, allow_cache=allow_cache + ) + } + + if isinstance(features, FeatureService): + fvs_to_use, od_fvs_to_use = [], [] + for fv_name, projection in [ + (projection.name, projection) + for projection in features.feature_view_projections + ]: + if fv_name in fvs: + fvs_to_use.append(fvs[fv_name].with_projection(copy.copy(projection))) + elif fv_name in od_fvs: + odfv = od_fvs[fv_name].with_projection(copy.copy(projection)) + od_fvs_to_use.append(odfv) + # Let's make sure to include an FVs which the ODFV requires Features from. + for projection in odfv.source_feature_view_projections.values(): + fv = fvs[projection.name].with_projection(copy.copy(projection)) + if fv not in fvs_to_use: + fvs_to_use.append(fv) + else: + raise ValueError( + f"The provided feature service {features.name} contains a reference to a feature view" + f"{fv_name} which doesn't exist. Please make sure that you have created the feature view" + f'{fv_name} and that you have registered it by running "apply".' + ) + views_to_use = (fvs_to_use, od_fvs_to_use) + else: + views_to_use = ( + [*fvs.values()], + [*od_fvs.values()], + ) + + return views_to_use + + +def _get_online_request_context( + registry, + project, + features: Union[List[str], "FeatureService"], + full_feature_names: bool, +): + from feast.feature_view import DUMMY_ENTITY_NAME + + _feature_refs = _get_features(registry, project, features, allow_cache=True) + + ( + requested_feature_views, + requested_on_demand_feature_views, + ) = _get_feature_views_to_use( + registry=registry, + project=project, + features=features, + allow_cache=True, + hide_dummy_entity=False, + ) + + ( + entity_name_to_join_key_map, + entity_type_map, + join_keys_set, + ) = _get_entity_maps(registry, project, requested_feature_views) + + _validate_feature_refs(_feature_refs, full_feature_names) + ( + grouped_refs, + grouped_odfv_refs, + ) = _group_feature_refs( + _feature_refs, + requested_feature_views, + requested_on_demand_feature_views, + ) + + requested_result_row_names = { + feat_ref.replace(":", "__") for feat_ref in _feature_refs + } + if not full_feature_names: + requested_result_row_names = { + name.rpartition("__")[-1] for name in requested_result_row_names + } + + feature_views = list(view for view, _ in grouped_refs) + + needed_request_data = get_needed_request_data(grouped_odfv_refs) + + entityless_case = DUMMY_ENTITY_NAME in [ + entity_name + for feature_view in feature_views + for entity_name in feature_view.entities + ] + + return ( + _feature_refs, + requested_on_demand_feature_views, + entity_name_to_join_key_map, + entity_type_map, + join_keys_set, + grouped_refs, + requested_result_row_names, + needed_request_data, + entityless_case, + ) + + +def _prepare_entities_to_read_from_online_store( + registry, + project, + features: Union[List[str], "FeatureService"], + entity_values: Mapping[ + str, Union[Sequence[Any], Sequence[ValueProto], RepeatedValueProto] + ], + full_feature_names: bool = False, + native_entity_values: bool = True, +): + from feast.feature_view import DUMMY_ENTITY, DUMMY_ENTITY_ID, DUMMY_ENTITY_VAL + + ( + feature_refs, + requested_on_demand_feature_views, + entity_name_to_join_key_map, + entity_type_map, + join_keys_set, + grouped_refs, + requested_result_row_names, + needed_request_data, + entityless_case, + ) = _get_online_request_context(registry, project, features, full_feature_names) + + # Extract Sequence from RepeatedValue Protobuf. + entity_value_lists: Dict[str, Union[List[Any], List[ValueProto]]] = { + k: list(v) if isinstance(v, Sequence) else list(v.val) + for k, v in entity_values.items() + } + + entity_proto_values: Dict[str, List[ValueProto]] + if native_entity_values: + # Convert values to Protobuf once. + entity_proto_values = { + k: python_values_to_proto_values( + v, entity_type_map.get(k, ValueType.UNKNOWN) + ) + for k, v in entity_value_lists.items() + } + else: + entity_proto_values = entity_value_lists + + num_rows = _validate_entity_values(entity_proto_values) + + join_key_values: Dict[str, List[ValueProto]] = {} + request_data_features: Dict[str, List[ValueProto]] = {} + # Entity rows may be either entities or request data. + for join_key_or_entity_name, values in entity_proto_values.items(): + # Found request data + if join_key_or_entity_name in needed_request_data: + request_data_features[join_key_or_entity_name] = values + else: + if join_key_or_entity_name in join_keys_set: + join_key = join_key_or_entity_name + else: + try: + join_key = entity_name_to_join_key_map[join_key_or_entity_name] + except KeyError: + raise EntityNotFoundException(join_key_or_entity_name, project) + else: + warnings.warn( + "Using entity name is deprecated. Use join_key instead." + ) + + # All join keys should be returned in the result. + requested_result_row_names.add(join_key) + join_key_values[join_key] = values + + ensure_request_data_values_exist(needed_request_data, request_data_features) + + # Populate online features response proto with join keys and request data features + online_features_response = GetOnlineFeaturesResponse(results=[]) + _populate_result_rows_from_columnar( + online_features_response=online_features_response, + data=dict(**join_key_values, **request_data_features), + ) + + # Add the Entityless case after populating result rows to avoid having to remove + # it later. + if entityless_case: + join_key_values[DUMMY_ENTITY_ID] = python_values_to_proto_values( + [DUMMY_ENTITY_VAL] * num_rows, DUMMY_ENTITY.value_type + ) + + return ( + join_key_values, + grouped_refs, + entity_name_to_join_key_map, + requested_on_demand_feature_views, + feature_refs, + requested_result_row_names, + online_features_response, + ) + + +def _get_entity_key_protos( + entity_rows: Iterable[Mapping[str, ValueProto]], +) -> List[EntityKeyProto]: + # Instantiate one EntityKeyProto per Entity. + entity_key_protos = [ + EntityKeyProto(join_keys=row.keys(), entity_values=row.values()) + for row in entity_rows + ] + return entity_key_protos + + +def _convert_rows_to_protobuf( + requested_features: List[str], + read_rows: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]], +) -> List[Tuple[List[Timestamp], List["FieldStatus.ValueType"], List[ValueProto]]]: + # Each row is a set of features for a given entity key. + # We only need to convert the data to Protobuf once. + null_value = ValueProto() + read_row_protos = [] + for read_row in read_rows: + row_ts_proto = Timestamp() + row_ts, feature_data = read_row + # TODO (Ly): reuse whatever timestamp if row_ts is None? + if row_ts is not None: + row_ts_proto.FromDatetime(row_ts) + event_timestamps = [row_ts_proto] * len(requested_features) + if feature_data is None: + statuses = [FieldStatus.NOT_FOUND] * len(requested_features) + values = [null_value] * len(requested_features) + else: + statuses = [] + values = [] + for feature_name in requested_features: + # Make sure order of data is the same as requested_features. + if feature_name not in feature_data: + statuses.append(FieldStatus.NOT_FOUND) + values.append(null_value) + else: + statuses.append(FieldStatus.PRESENT) + values.append(feature_data[feature_name]) + read_row_protos.append((event_timestamps, statuses, values)) + return read_row_protos + + +def has_all_tags( + object_tags: dict[str, str], requested_tags: Optional[dict[str, str]] = None +) -> bool: + if requested_tags is None: + return True + return all(object_tags.get(key, None) == val for key, val in requested_tags.items()) + + +def tags_list_to_dict( + tags_list: Optional[list[str]] = None, +) -> Optional[dict[str, str]]: + if not tags_list: + return None + tags_dict: dict[str, str] = {} + for tags_str in tags_list: + tags_dict.update(tags_str_to_dict(tags_str)) + return tags_dict + + +def tags_str_to_dict(tags: str = "") -> dict[str, str]: + tags_list = tags.strip().strip("()").replace('"', "").replace("'", "").split(",") + return { + key.strip(): value.strip() + for key, value in dict( + cast(tuple[str, str], tag.split(":", 1)) for tag in tags_list if ":" in tag + ).items() + } diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index e7ca9ca35b6..97bdfc159ba 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,5 +1,12 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt +aiobotocore==2.13.0 +aiohttp==3.9.5 + # via aiobotocore +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp alabaster==0.7.16 # via sphinx altair==4.2.2 @@ -26,11 +33,14 @@ asttokens==2.4.1 async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 - # via redis + # via + # aiohttp + # redis atpublic==4.1.0 # via ibis-framework attrs==23.2.0 # via + # aiohttp # jsonschema # referencing azure-core==1.30.1 @@ -38,7 +48,7 @@ azure-core==1.30.1 # azure-identity # azure-storage-blob azure-identity==1.16.0 -azure-storage-blob==12.19.1 +azure-storage-blob==12.20.0 babel==2.15.0 # via # jupyterlab-server @@ -53,6 +63,7 @@ boto3==1.34.99 # via moto botocore==1.34.99 # via + # aiobotocore # boto3 # moto # s3transfer @@ -65,6 +76,7 @@ cachetools==5.3.3 cassandra-driver==3.29.1 certifi==2024.2.2 # via + # elastic-transport # httpcore # httpx # kubernetes @@ -98,7 +110,7 @@ comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.5.1 +coverage[toml]==7.5.3 # via pytest-cov cryptography==42.0.7 # via @@ -130,16 +142,19 @@ distlib==0.3.8 # via virtualenv dnspython==2.6.1 # via email-validator -docker==7.0.0 +docker==7.1.0 # via testcontainers docutils==0.19 # via sphinx -duckdb==0.10.2 +duckdb==0.10.3 # via # duckdb-engine # ibis-framework -duckdb-engine==0.12.0 +duckdb-engine==0.12.1 # via ibis-framework +elastic-transport==8.13.1 + # via elasticsearch +elasticsearch==8.13.2 email-validator==2.1.1 # via fastapi entrypoints==0.4 @@ -166,6 +181,10 @@ filelock==3.14.0 firebase-admin==5.4.0 fqdn==1.5.1 # via jsonschema +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal fsspec==2023.12.2 # via dask geojson==2.5.0 @@ -183,7 +202,7 @@ google-api-core[grpc]==2.19.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.128.0 +google-api-python-client==2.131.0 # via firebase-admin google-auth==2.29.0 # via @@ -225,12 +244,12 @@ googleapis-common-protos[grpc]==1.63.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.13 +great-expectations==0.18.15 greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.63.0 +grpcio==1.64.0 # via # google-api-core # google-cloud-bigquery @@ -253,7 +272,7 @@ h11==0.14.0 # httpcore # uvicorn happybase==1.2.0 -hazelcast-python-client==5.3.0 +hazelcast-python-client==5.4.0 hiredis==2.3.2 httpcore==1.0.5 # via httpx @@ -280,6 +299,7 @@ idna==3.7 # jsonschema # requests # snowflake-connector-python + # yarl imagesize==1.4.1 # via sphinx importlib-metadata==7.1.0 @@ -288,12 +308,12 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.4 # via jupyterlab -ipython==8.24.0 +ipython==8.25.0 # via # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.2 +ipywidgets==8.1.3 # via great-expectations isodate==0.6.1 # via azure-storage-blob @@ -333,7 +353,7 @@ jsonschema[format-nongpl]==4.22.0 # nbformat jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.1 +jupyter-client==8.6.2 # via # ipykernel # jupyter-server @@ -351,7 +371,7 @@ jupyter-events==0.10.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.14.0 +jupyter-server==2.14.1 # via # jupyter-lsp # jupyterlab @@ -360,15 +380,15 @@ jupyter-server==2.14.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.8 +jupyterlab==4.2.1 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.27.1 +jupyterlab-server==2.27.2 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.10 +jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 locket==1.0.0 @@ -406,6 +426,10 @@ msal-extensions==1.1.0 # via azure-identity msgpack==1.0.8 # via cachecontrol +multidict==6.0.5 + # via + # aiohttp + # yarl multipledispatch==1.0.0 # via ibis-framework mypy==1.10.0 @@ -425,9 +449,9 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel -nodeenv==1.8.0 +nodeenv==1.9.0 # via pre-commit -notebook==7.1.3 +notebook==7.2.0 # via great-expectations notebook-shim==0.2.4 # via @@ -454,7 +478,6 @@ packaging==24.0 # build # dask # db-dtypes - # docker # duckdb-engine # google-cloud-bigquery # great-expectations @@ -509,7 +532,7 @@ portalocker==2.8.2 pre-commit==3.3.1 prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.45 # via ipython proto-plus==1.23.0 # via @@ -589,7 +612,7 @@ pyjwt[crypto]==2.8.0 # msal # snowflake-connector-python pymssql==2.3.0 -pymysql==1.1.0 +pymysql==1.1.1 pyodbc==5.1.0 pyopenssl==24.1.0 # via snowflake-connector-python @@ -710,18 +733,17 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruff==0.4.3 +ruff==0.4.6 s3transfer==0.10.1 # via boto3 -scipy==1.13.0 +scipy==1.13.1 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==69.5.1 +setuptools==70.0.0 # via # grpcio-tools # kubernetes - # nodeenv # pip-tools shellingham==1.5.4 # via typer @@ -744,7 +766,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.10.0 +snowflake-connector-python[pandas]==3.10.1 sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 @@ -771,11 +793,12 @@ sqlalchemy-views==0.3.2 # via ibis-framework sqlglot==20.11.0 # via ibis-framework +sqlite-vec==0.0.1a10 stack-data==0.6.3 # via ipython starlette==0.37.2 # via fastapi -substrait==0.17.0 +substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 tenacity==8.3.0 @@ -798,7 +821,7 @@ tomli==2.0.1 # pip-tools # pytest # pytest-env -tomlkit==0.12.4 +tomlkit==0.12.5 # via snowflake-connector-python toolz==0.12.1 # via @@ -806,7 +829,7 @@ toolz==0.12.1 # dask # ibis-framework # partd -tornado==6.4 +tornado==6.4.1 # via # ipykernel # jupyter-client @@ -848,7 +871,7 @@ types-pytz==2024.1.0.20240417 types-pyyaml==6.0.12.20240311 types-redis==4.6.0.20240425 types-requests==2.30.0.0 -types-setuptools==69.5.0.20240423 +types-setuptools==70.0.0.20240524 # via types-cffi types-tabulate==0.9.0.20240106 types-urllib3==1.26.25.14 @@ -888,6 +911,7 @@ urllib3==1.26.18 # via # botocore # docker + # elastic-transport # great-expectations # kubernetes # minio @@ -923,11 +947,15 @@ werkzeug==3.0.3 # via moto wheel==0.43.0 # via pip-tools -widgetsnbextension==4.0.10 +widgetsnbextension==4.0.11 # via ipywidgets wrapt==1.16.0 - # via testcontainers + # via + # aiobotocore + # testcontainers xmltodict==0.13.0 # via moto +yarl==1.9.4 + # via aiohttp zipp==3.18.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 56a8259ab43..99c9bfc3fee 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -187,4 +187,4 @@ watchfiles==0.21.0 websockets==12.0 # via uvicorn zipp==3.18.1 - # via importlib-metadata \ No newline at end of file + # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 3b76237f599..f6db0af6bc0 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,5 +1,12 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt +aiobotocore==2.13.0 +aiohttp==3.9.5 + # via aiobotocore +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp alabaster==0.7.16 # via sphinx altair==4.2.2 @@ -29,6 +36,7 @@ atpublic==4.1.0 # via ibis-framework attrs==23.2.0 # via + # aiohttp # jsonschema # referencing azure-core==1.30.1 @@ -36,7 +44,7 @@ azure-core==1.30.1 # azure-identity # azure-storage-blob azure-identity==1.16.0 -azure-storage-blob==12.19.1 +azure-storage-blob==12.20.0 babel==2.15.0 # via # jupyterlab-server @@ -51,6 +59,7 @@ boto3==1.34.99 # via moto botocore==1.34.99 # via + # aiobotocore # boto3 # moto # s3transfer @@ -63,6 +72,7 @@ cachetools==5.3.3 cassandra-driver==3.29.1 certifi==2024.2.2 # via + # elastic-transport # httpcore # httpx # kubernetes @@ -96,7 +106,7 @@ comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.5.1 +coverage[toml]==7.5.3 # via pytest-cov cryptography==42.0.7 # via @@ -122,22 +132,25 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.17.3 +deltalake==0.17.4 dill==0.3.8 distlib==0.3.8 # via virtualenv dnspython==2.6.1 # via email-validator -docker==7.0.0 +docker==7.1.0 # via testcontainers docutils==0.19 # via sphinx -duckdb==0.10.2 +duckdb==0.10.3 # via # duckdb-engine # ibis-framework -duckdb-engine==0.12.0 +duckdb-engine==0.12.1 # via ibis-framework +elastic-transport==8.13.1 + # via elasticsearch +elasticsearch==8.13.2 email-validator==2.1.1 # via fastapi entrypoints==0.4 @@ -159,6 +172,10 @@ filelock==3.14.0 firebase-admin==5.4.0 fqdn==1.5.1 # via jsonschema +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal fsspec==2023.12.2 # via dask geojson==2.5.0 @@ -176,7 +193,7 @@ google-api-core[grpc]==2.19.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.128.0 +google-api-python-client==2.131.0 # via firebase-admin google-auth==2.29.0 # via @@ -218,12 +235,12 @@ googleapis-common-protos[grpc]==1.63.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.13 +great-expectations==0.18.15 greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.63.0 +grpcio==1.64.0 # via # google-api-core # google-cloud-bigquery @@ -246,7 +263,7 @@ h11==0.14.0 # httpcore # uvicorn happybase==1.2.0 -hazelcast-python-client==5.3.0 +hazelcast-python-client==5.4.0 hiredis==2.3.2 httpcore==1.0.5 # via httpx @@ -273,6 +290,7 @@ idna==3.7 # jsonschema # requests # snowflake-connector-python + # yarl imagesize==1.4.1 # via sphinx importlib-metadata==7.1.0 @@ -281,12 +299,12 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.4 # via jupyterlab -ipython==8.24.0 +ipython==8.25.0 # via # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.2 +ipywidgets==8.1.3 # via great-expectations isodate==0.6.1 # via azure-storage-blob @@ -326,7 +344,7 @@ jsonschema[format-nongpl]==4.22.0 # nbformat jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.1 +jupyter-client==8.6.2 # via # ipykernel # jupyter-server @@ -344,7 +362,7 @@ jupyter-events==0.10.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.14.0 +jupyter-server==2.14.1 # via # jupyter-lsp # jupyterlab @@ -353,15 +371,15 @@ jupyter-server==2.14.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.8 +jupyterlab==4.2.1 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.27.1 +jupyterlab-server==2.27.2 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.10 +jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 locket==1.0.0 @@ -399,6 +417,10 @@ msal-extensions==1.1.0 # via azure-identity msgpack==1.0.8 # via cachecontrol +multidict==6.0.5 + # via + # aiohttp + # yarl multipledispatch==1.0.0 # via ibis-framework mypy==1.10.0 @@ -418,9 +440,9 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel -nodeenv==1.8.0 +nodeenv==1.9.0 # via pre-commit -notebook==7.1.3 +notebook==7.2.0 # via great-expectations notebook-shim==0.2.4 # via @@ -447,7 +469,6 @@ packaging==24.0 # build # dask # db-dtypes - # docker # duckdb-engine # google-cloud-bigquery # great-expectations @@ -502,7 +523,7 @@ portalocker==2.8.2 pre-commit==3.3.1 prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.45 # via ipython proto-plus==1.23.0 # via @@ -582,7 +603,7 @@ pyjwt[crypto]==2.8.0 # msal # snowflake-connector-python pymssql==2.3.0 -pymysql==1.1.0 +pymysql==1.1.1 pyodbc==5.1.0 pyopenssl==24.1.0 # via snowflake-connector-python @@ -659,7 +680,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications # jupyter-events -regex==2024.4.28 +regex==2024.5.15 requests==2.31.0 # via # azure-core @@ -703,18 +724,17 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruff==0.4.3 +ruff==0.4.6 s3transfer==0.10.1 # via boto3 -scipy==1.13.0 +scipy==1.13.1 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==69.5.1 +setuptools==70.0.0 # via # grpcio-tools # kubernetes - # nodeenv # pip-tools shellingham==1.5.4 # via typer @@ -737,7 +757,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.10.0 +snowflake-connector-python[pandas]==3.10.1 sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 @@ -764,11 +784,12 @@ sqlalchemy-views==0.3.2 # via ibis-framework sqlglot==20.11.0 # via ibis-framework +sqlite-vec==0.0.1a10 stack-data==0.6.3 # via ipython starlette==0.37.2 # via fastapi -substrait==0.17.0 +substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 tenacity==8.3.0 @@ -782,7 +803,7 @@ thriftpy2==0.5.0 tinycss2==1.3.0 # via nbconvert toml==0.10.2 -tomlkit==0.12.4 +tomlkit==0.12.5 # via snowflake-connector-python toolz==0.12.1 # via @@ -790,7 +811,7 @@ toolz==0.12.1 # dask # ibis-framework # partd -tornado==6.4 +tornado==6.4.1 # via # ipykernel # jupyter-client @@ -832,7 +853,7 @@ types-pytz==2024.1.0.20240417 types-pyyaml==6.0.12.20240311 types-redis==4.6.0.20240425 types-requests==2.30.0.0 -types-setuptools==69.5.0.20240423 +types-setuptools==70.0.0.20240524 # via types-cffi types-tabulate==0.9.0.20240106 types-urllib3==1.26.25.14 @@ -869,6 +890,7 @@ urllib3==1.26.18 # via # botocore # docker + # elastic-transport # great-expectations # kubernetes # minio @@ -904,11 +926,15 @@ werkzeug==3.0.3 # via moto wheel==0.43.0 # via pip-tools -widgetsnbextension==4.0.10 +widgetsnbextension==4.0.11 # via ipywidgets wrapt==1.16.0 - # via testcontainers + # via + # aiobotocore + # testcontainers xmltodict==0.13.0 # via moto +yarl==1.9.4 + # via aiohttp zipp==3.18.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index a628f0823db..135b65a0ccc 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,5 +1,12 @@ # This file was autogenerated by uv via the following command: # uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt +aiobotocore==2.13.0 +aiohttp==3.9.5 + # via aiobotocore +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp alabaster==0.7.16 # via sphinx altair==4.2.2 @@ -26,11 +33,14 @@ asttokens==2.4.1 async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 - # via redis + # via + # aiohttp + # redis atpublic==4.1.0 # via ibis-framework attrs==23.2.0 # via + # aiohttp # jsonschema # referencing azure-core==1.30.1 @@ -38,7 +48,7 @@ azure-core==1.30.1 # azure-identity # azure-storage-blob azure-identity==1.16.0 -azure-storage-blob==12.19.1 +azure-storage-blob==12.20.0 babel==2.15.0 # via # jupyterlab-server @@ -53,6 +63,7 @@ boto3==1.34.99 # via moto botocore==1.34.99 # via + # aiobotocore # boto3 # moto # s3transfer @@ -65,6 +76,7 @@ cachetools==5.3.3 cassandra-driver==3.29.1 certifi==2024.2.2 # via + # elastic-transport # httpcore # httpx # kubernetes @@ -98,7 +110,7 @@ comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.5.1 +coverage[toml]==7.5.3 # via pytest-cov cryptography==42.0.7 # via @@ -124,22 +136,25 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.17.3 +deltalake==0.17.4 dill==0.3.8 distlib==0.3.8 # via virtualenv dnspython==2.6.1 # via email-validator -docker==7.0.0 +docker==7.1.0 # via testcontainers docutils==0.19 # via sphinx -duckdb==0.10.2 +duckdb==0.10.3 # via # duckdb-engine # ibis-framework -duckdb-engine==0.12.0 +duckdb-engine==0.12.1 # via ibis-framework +elastic-transport==8.13.1 + # via elasticsearch +elasticsearch==8.13.2 email-validator==2.1.1 # via fastapi entrypoints==0.4 @@ -166,6 +181,10 @@ filelock==3.14.0 firebase-admin==5.4.0 fqdn==1.5.1 # via jsonschema +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal fsspec==2023.12.2 # via dask geojson==2.5.0 @@ -183,7 +202,7 @@ google-api-core[grpc]==2.19.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.128.0 +google-api-python-client==2.131.0 # via firebase-admin google-auth==2.29.0 # via @@ -225,12 +244,12 @@ googleapis-common-protos[grpc]==1.63.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.13 +great-expectations==0.18.15 greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.63.0 +grpcio==1.64.0 # via # google-api-core # google-cloud-bigquery @@ -253,7 +272,7 @@ h11==0.14.0 # httpcore # uvicorn happybase==1.2.0 -hazelcast-python-client==5.3.0 +hazelcast-python-client==5.4.0 hiredis==2.3.2 httpcore==1.0.5 # via httpx @@ -280,6 +299,7 @@ idna==3.7 # jsonschema # requests # snowflake-connector-python + # yarl imagesize==1.4.1 # via sphinx importlib-metadata==7.1.0 @@ -302,7 +322,7 @@ ipython==8.18.1 # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.2 +ipywidgets==8.1.3 # via great-expectations isodate==0.6.1 # via azure-storage-blob @@ -342,7 +362,7 @@ jsonschema[format-nongpl]==4.22.0 # nbformat jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.1 +jupyter-client==8.6.2 # via # ipykernel # jupyter-server @@ -360,7 +380,7 @@ jupyter-events==0.10.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.14.0 +jupyter-server==2.14.1 # via # jupyter-lsp # jupyterlab @@ -369,15 +389,15 @@ jupyter-server==2.14.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.8 +jupyterlab==4.2.1 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.27.1 +jupyterlab-server==2.27.2 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.10 +jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 locket==1.0.0 @@ -415,6 +435,10 @@ msal-extensions==1.1.0 # via azure-identity msgpack==1.0.8 # via cachecontrol +multidict==6.0.5 + # via + # aiohttp + # yarl multipledispatch==1.0.0 # via ibis-framework mypy==1.10.0 @@ -434,9 +458,9 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel -nodeenv==1.8.0 +nodeenv==1.9.0 # via pre-commit -notebook==7.1.3 +notebook==7.2.0 # via great-expectations notebook-shim==0.2.4 # via @@ -463,7 +487,6 @@ packaging==24.0 # build # dask # db-dtypes - # docker # duckdb-engine # google-cloud-bigquery # great-expectations @@ -518,7 +541,7 @@ portalocker==2.8.2 pre-commit==3.3.1 prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.45 # via ipython proto-plus==1.23.0 # via @@ -598,7 +621,7 @@ pyjwt[crypto]==2.8.0 # msal # snowflake-connector-python pymssql==2.3.0 -pymysql==1.1.0 +pymysql==1.1.1 pyodbc==5.1.0 pyopenssl==24.1.0 # via snowflake-connector-python @@ -675,7 +698,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications # jupyter-events -regex==2024.4.28 +regex==2024.5.15 requests==2.31.0 # via # azure-core @@ -721,18 +744,17 @@ ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.8 # via ruamel-yaml -ruff==0.4.3 +ruff==0.4.6 s3transfer==0.10.1 # via boto3 -scipy==1.13.0 +scipy==1.13.1 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==69.5.1 +setuptools==70.0.0 # via # grpcio-tools # kubernetes - # nodeenv # pip-tools shellingham==1.5.4 # via typer @@ -755,7 +777,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.10.0 +snowflake-connector-python[pandas]==3.10.1 sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 @@ -782,11 +804,12 @@ sqlalchemy-views==0.3.2 # via ibis-framework sqlglot==20.11.0 # via ibis-framework +sqlite-vec==0.0.1a10 stack-data==0.6.3 # via ipython starlette==0.37.2 # via fastapi -substrait==0.17.0 +substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 tenacity==8.3.0 @@ -809,7 +832,7 @@ tomli==2.0.1 # pip-tools # pytest # pytest-env -tomlkit==0.12.4 +tomlkit==0.12.5 # via snowflake-connector-python toolz==0.12.1 # via @@ -817,7 +840,7 @@ toolz==0.12.1 # dask # ibis-framework # partd -tornado==6.4 +tornado==6.4.1 # via # ipykernel # jupyter-client @@ -850,7 +873,7 @@ types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 # via mypy-protobuf -types-pymysql==1.1.0.20240425 +types-pymysql==1.1.0.20240524 types-pyopenssl==24.1.0.20240425 # via types-redis types-python-dateutil==2.9.0.20240316 @@ -859,13 +882,14 @@ types-pytz==2024.1.0.20240417 types-pyyaml==6.0.12.20240311 types-redis==4.6.0.20240425 types-requests==2.30.0.0 -types-setuptools==69.5.0.20240423 +types-setuptools==70.0.0.20240524 # via types-cffi types-tabulate==0.9.0.20240106 types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via + # aioitertools # anyio # async-lru # azure-core @@ -900,6 +924,7 @@ urllib3==1.26.18 # via # botocore # docker + # elastic-transport # great-expectations # kubernetes # minio @@ -936,11 +961,15 @@ werkzeug==3.0.3 # via moto wheel==0.43.0 # via pip-tools -widgetsnbextension==4.0.10 +widgetsnbextension==4.0.11 # via ipywidgets wrapt==1.16.0 - # via testcontainers + # via + # aiobotocore + # testcontainers xmltodict==0.13.0 # via moto +yarl==1.9.4 + # via aiohttp zipp==3.18.1 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 1092aac9d09..149a96626ef 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -190,4 +190,4 @@ watchfiles==0.21.0 websockets==12.0 # via uvicorn zipp==3.18.1 - # via importlib-metadata \ No newline at end of file + # via importlib-metadata diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 7c875fc9bde..48f482f5428 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -32,8 +32,8 @@ create_basic_driver_dataset, create_document_dataset, ) -from tests.integration.feature_repos.integration_test_repo_config import ( # noqa: E402 - IntegrationTestRepoConfig, +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, # noqa: E402 ) from tests.integration.feature_repos.repo_configuration import ( # noqa: E402 AVAILABLE_OFFLINE_STORES, @@ -45,8 +45,8 @@ construct_universal_feature_views, construct_universal_test_data, ) -from tests.integration.feature_repos.universal.data_sources.file import ( # noqa: E402 - FileDataSourceCreator, +from tests.integration.feature_repos.universal.data_sources.file import ( + FileDataSourceCreator, # noqa: E402 ) from tests.integration.feature_repos.universal.entities import ( # noqa: E402 customer, @@ -173,7 +173,7 @@ def simple_dataset_2() -> pd.DataFrame: def start_test_local_server(repo_path: str, port: int): fs = FeatureStore(repo_path) - fs.serve("localhost", port, no_access_log=True) + fs.serve(host="localhost", port=port) @pytest.fixture @@ -305,10 +305,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): @pytest.fixture def feature_server_endpoint(environment): - if ( - not environment.python_feature_server - or environment.test_repo_config.provider != "local" - ): + if not environment.python_feature_server or environment.provider != "local": yield environment.feature_store.get_feature_server_endpoint() return diff --git a/sdk/python/tests/example_repos/example_feature_repo_1.py b/sdk/python/tests/example_repos/example_feature_repo_1.py index fbf1fbb9b07..daf7b7e7e6f 100644 --- a/sdk/python/tests/example_repos/example_feature_repo_1.py +++ b/sdk/python/tests/example_repos/example_feature_repo_1.py @@ -4,7 +4,8 @@ from feast import Entity, FeatureService, FeatureView, Field, FileSource, PushSource from feast.on_demand_feature_view import on_demand_feature_view -from feast.types import Float32, Int64, String +from feast.types import Array, Float32, Int64, String +from tests.integration.feature_repos.universal.feature_views import TAGS # Note that file source paths are not validated, so there doesn't actually need to be any data # at the paths for these file sources. Since these paths are effectively fake, this example @@ -32,17 +33,29 @@ batch_source=driver_locations_source, ) +rag_documents_source = FileSource( + name="rag_documents_source", + path="data/rag_documents.parquet", + timestamp_field="event_timestamp", +) + driver = Entity( name="driver", # The name is derived from this argument, not object name. join_keys=["driver_id"], description="driver id", + tags=TAGS, ) customer = Entity( name="customer", # The name is derived from this argument, not object name. join_keys=["customer_id"], + tags=TAGS, ) +item = Entity( + name="item_id", # The name is derived from this argument, not object name. + join_keys=["item_id"], +) driver_locations = FeatureView( name="driver_locations", @@ -101,6 +114,17 @@ tags={}, ) +document_embeddings = FeatureView( + name="document_embeddings", + entities=[item], + schema=[ + Field(name="Embeddings", dtype=Array(Float32)), + Field(name="item_id", dtype=String), + ], + source=rag_documents_source, + ttl=timedelta(hours=24), +) + @on_demand_feature_view( sources=[customer_profile], @@ -116,5 +140,5 @@ def customer_profile_pandas_odfv(inputs: pd.DataFrame) -> pd.DataFrame: all_drivers_feature_service = FeatureService( name="driver_locations_service", features=[driver_locations], - tags={"release": "production"}, + tags=TAGS, ) diff --git a/sdk/python/tests/example_repos/example_feature_repo_with_feature_service_2.py b/sdk/python/tests/example_repos/example_feature_repo_with_feature_service_2.py index 3547c3de86a..49f5bbaf054 100644 --- a/sdk/python/tests/example_repos/example_feature_repo_with_feature_service_2.py +++ b/sdk/python/tests/example_repos/example_feature_repo_with_feature_service_2.py @@ -59,5 +59,5 @@ driver_hourly_stats_view[["conv_rate"]], global_stats_feature_view[["num_rides"]], ], - tags={"release": "production"}, + tags={"release": "qa"}, ) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 2f260e87a60..7123bd0fc15 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -34,8 +34,8 @@ from tests.integration.feature_repos.universal.data_sources.file import ( DuckDBDataSourceCreator, DuckDBDeltaDataSourceCreator, - DuckDBDeltaS3DataSourceCreator, FileDataSourceCreator, + RemoteOfflineStoreDataSourceCreator, ) from tests.integration.feature_repos.universal.data_sources.redshift import ( RedshiftDataSourceCreator, @@ -122,21 +122,21 @@ ("local", FileDataSourceCreator), ("local", DuckDBDataSourceCreator), ("local", DuckDBDeltaDataSourceCreator), + ("local", RemoteOfflineStoreDataSourceCreator), ] if os.getenv("FEAST_IS_LOCAL_TEST", "False") == "True": AVAILABLE_OFFLINE_STORES.extend( [ - ("local", DuckDBDeltaS3DataSourceCreator), + # todo: @tokoko to reenable + # ("local", DuckDBDeltaS3DataSourceCreator), ] ) AVAILABLE_ONLINE_STORES: Dict[ str, Tuple[Union[str, Dict[Any, Any]], Optional[Type[OnlineStoreCreator]]] -] = { - "sqlite": ({"type": "sqlite"}, None), -} +] = {"sqlite": ({"type": "sqlite"}, None)} # Only configure Cloud DWH if running full integration tests if os.getenv("FEAST_IS_LOCAL_TEST", "False") != "True": @@ -153,7 +153,6 @@ AVAILABLE_ONLINE_STORES["datastore"] = ("datastore", None) AVAILABLE_ONLINE_STORES["snowflake"] = (SNOWFLAKE_CONFIG, None) AVAILABLE_ONLINE_STORES["bigtable"] = (BIGTABLE_CONFIG, None) - # Uncomment to test using private Rockset account. Currently not enabled as # there is no dedicated Rockset instance for CI testing and there is no # containerized version of Rockset. @@ -487,7 +486,6 @@ def construct_test_environment( "arn:aws:iam::402087665549:role/lambda_execution_role", ), ) - else: feature_server = LocalFeatureServerConfig( feature_logging=FeatureLoggingConfig(enabled=True) @@ -500,9 +498,7 @@ def construct_test_environment( aws_registry_path = os.getenv( "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries" ) - registry: Union[str, RegistryConfig] = ( - f"{aws_registry_path}/{project}/registry.db" - ) + registry = RegistryConfig(path=f"{aws_registry_path}/{project}/registry.db") else: registry = RegistryConfig( path=str(Path(repo_dir_name) / "registry.db"), diff --git a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py index 62d458d6f4a..f1cab214299 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py @@ -18,7 +18,6 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = None, @@ -32,7 +31,6 @@ def create_data_source( df: The dataframe to be used to create the data source. destination_name: This str is used by the implementing classes to isolate the multiple dataframes from each other. - event_timestamp_column: (Deprecated) Pass through for the underlying data source. created_timestamp_column: Pass through for the underlying data source. field_mapping: Pass through for the underlying data source. timestamp_field: Pass through for the underlying data source. diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index 6f0ac02a003..f7ab55d868a 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -1,18 +1,22 @@ +import logging import os.path import shutil +import subprocess import tempfile import uuid +from pathlib import Path from typing import Any, Dict, List, Optional import pandas as pd import pyarrow as pa import pyarrow.parquet as pq +import yaml from minio import Minio from testcontainers.core.generic import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs from testcontainers.minio import MinioContainer -from feast import FileSource +from feast import FileSource, RepoConfig from feast.data_format import DeltaFormat, ParquetFormat from feast.data_source import DataSource from feast.feature_logging import LoggingDestination @@ -22,10 +26,15 @@ FileLoggingDestination, SavedDatasetFileStorage, ) -from feast.repo_config import FeastConfigBaseModel +from feast.infra.offline_stores.remote import RemoteOfflineStoreConfig +from feast.repo_config import FeastConfigBaseModel, RegistryConfig +from feast.wait import wait_retry_backoff # noqa: E402 from tests.integration.feature_repos.universal.data_source_creator import ( DataSourceCreator, ) +from tests.utils.http_server import check_port_open, free_port # noqa: E402 + +logger = logging.getLogger(__name__) class FileDataSourceCreator(DataSourceCreator): @@ -141,7 +150,8 @@ def __init__(self, project_name: str, *args, **kwargs): self.minio = MinioContainer() self.minio.start() client = self.minio.get_client() - client.make_bucket("test") + if not client.bucket_exists("test"): + client.make_bucket("test") host_ip = self.minio.get_container_host_ip() exposed_port = self.minio.get_exposed_port(self.minio.port) self.endpoint_url = f"http://{host_ip}:{exposed_port}" @@ -351,3 +361,69 @@ def create_offline_store_config(self): staging_location_endpoint_override=self.endpoint_url, ) return self.duckdb_offline_store_config + + +class RemoteOfflineStoreDataSourceCreator(FileDataSourceCreator): + def __init__(self, project_name: str, *args, **kwargs): + super().__init__(project_name) + self.server_port: int = 0 + self.proc = None + + def setup(self, registry: RegistryConfig): + parent_offline_config = super().create_offline_store_config() + config = RepoConfig( + project=self.project_name, + provider="local", + offline_store=parent_offline_config, + registry=registry.path, + entity_key_serialization_version=2, + ) + + repo_path = Path(tempfile.mkdtemp()) + with open(repo_path / "feature_store.yaml", "w") as outfile: + yaml.dump(config.dict(by_alias=True), outfile) + repo_path = str(repo_path.resolve()) + + self.server_port = free_port() + host = "0.0.0.0" + cmd = [ + "feast", + "-c" + repo_path, + "serve_offline", + "--host", + host, + "--port", + str(self.server_port), + ] + self.proc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL + ) + + _time_out_sec: int = 60 + # Wait for server to start + wait_retry_backoff( + lambda: (None, check_port_open(host, self.server_port)), + timeout_secs=_time_out_sec, + timeout_msg=f"Unable to start the feast remote offline server in {_time_out_sec} seconds at port={self.server_port}", + ) + return "grpc+tcp://{}:{}".format(host, self.server_port) + + def create_offline_store_config(self) -> FeastConfigBaseModel: + self.remote_offline_store_config = RemoteOfflineStoreConfig( + type="remote", host="0.0.0.0", port=self.server_port + ) + return self.remote_offline_store_config + + def teardown(self): + super().teardown() + if self.proc is not None: + self.proc.kill() + + # wait server to free the port + wait_retry_backoff( + lambda: ( + None, + not check_port_open("localhost", self.server_port), + ), + timeout_secs=30, + ) diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py index 8fe933fbba7..91d1a74f071 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py @@ -49,7 +49,6 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = "ts", diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py index 237be2ac016..e9c4ad21a31 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py @@ -47,7 +47,6 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = "ts", diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 2a0a9d1bd01..11ddcb0ecc6 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -25,6 +25,8 @@ location, ) +TAGS = {"release": "production"} + def driver_feature_view( data_source: DataSource, @@ -85,7 +87,8 @@ def conv_rate_plus_100_feature_view( schema=[] if infer_features else _features, sources=sources, feature_transformation=PandasTransformation( - udf=conv_rate_plus_100, udf_string="raw udf source" + udf=conv_rate_plus_100, + udf_string="raw udf source", # type: ignore ) if not use_substrait_odfv else SubstraitTransformation.from_ibis(conv_rate_plus_100_ibis, sources), @@ -124,10 +127,11 @@ def similarity_feature_view( return OnDemandFeatureView( name=similarity.__name__, - sources=sources, + sources=sources, # type: ignore schema=[] if infer_features else _fields, feature_transformation=PandasTransformation( - udf=similarity, udf_string="similarity raw udf" + udf=similarity, + udf_string="similarity raw udf", # type: ignore ), ) @@ -200,6 +204,7 @@ def create_driver_hourly_stats_feature_view(source, infer_features: bool = False ], source=source, ttl=timedelta(hours=2), + tags=TAGS, ) return driver_stats_feature_view @@ -219,6 +224,7 @@ def create_driver_hourly_stats_batch_feature_view( ], source=source, ttl=timedelta(hours=2), + tags=TAGS, ) return driver_stats_feature_view @@ -236,6 +242,7 @@ def create_customer_daily_profile_feature_view(source, infer_features: bool = Fa ], source=source, ttl=timedelta(days=2), + tags=TAGS, ) return customer_profile_feature_view @@ -252,6 +259,7 @@ def create_global_stats_feature_view(source, infer_features: bool = False): ], source=source, ttl=timedelta(days=2), + tags=TAGS, ) return global_stats_feature_view diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/elasticsearch.py b/sdk/python/tests/integration/feature_repos/universal/online_store/elasticsearch.py index c62a9009caf..cfbc7611a1f 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/elasticsearch.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/elasticsearch.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Any, Dict from testcontainers.elasticsearch import ElasticSearchContainer @@ -14,7 +14,7 @@ def __init__(self, project_name: str, **kwargs): "elasticsearch:8.3.3", ).with_exposed_ports(9200) - def create_online_store(self) -> Dict[str, str]: + def create_online_store(self) -> Dict[str, Any]: self.container.start() return { "host": "localhost", diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/postgres.py b/sdk/python/tests/integration/feature_repos/universal/online_store/postgres.py index 7b4156fffe0..e4098626411 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/postgres.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/postgres.py @@ -1,5 +1,5 @@ import os -from typing import Dict +from typing import Any, Dict from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs @@ -51,7 +51,7 @@ def __init__(self, project_name: str, **kwargs): ) ) - def create_online_store(self) -> Dict[str, str]: + def create_online_store(self) -> Dict[str, Any]: self.container.start() log_string_to_wait_for = "database system is ready to accept connections" wait_for_logs( diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py b/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py index 4932001e76f..0963a1cd1e3 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py @@ -1,13 +1,14 @@ from abc import ABC, abstractmethod +from typing import Any -from feast.repo_config import FeastConfigBaseModel +# from feast.repo_config import FeastConfigBaseModel class OnlineStoreCreator(ABC): def __init__(self, project_name: str, **kwargs): self.project_name = project_name - def create_online_store(self) -> FeastConfigBaseModel: + def create_online_store(self) -> dict[str, Any]: raise NotImplementedError @abstractmethod diff --git a/sdk/python/tests/integration/offline_store/test_feature_logging.py b/sdk/python/tests/integration/offline_store/test_feature_logging.py index eba994544da..32f506f90b2 100644 --- a/sdk/python/tests/integration/offline_store/test_feature_logging.py +++ b/sdk/python/tests/integration/offline_store/test_feature_logging.py @@ -34,8 +34,6 @@ def test_feature_service_logging(environment, universal_data_sources, pass_as_pa (_, datasets, data_sources) = universal_data_sources feature_views = construct_universal_feature_views(data_sources) - store.apply([customer(), driver(), location(), *feature_views.values()]) - feature_service = FeatureService( name="test_service", features=[ @@ -49,6 +47,17 @@ def test_feature_service_logging(environment, universal_data_sources, pass_as_pa ), ) + store.apply( + [customer(), driver(), location(), *feature_views.values()], feature_service + ) + + # Added to handle the case that the offline store is remote + store.registry.apply_feature_service(feature_service, store.config.project) + store.registry.apply_data_source( + feature_service.logging_config.destination.to_data_source(), + store.config.project, + ) + driver_df = datasets.driver_df driver_df["val_to_add"] = 50 driver_df = driver_df.join(conv_rate_plus_100(driver_df)) diff --git a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py index a6db7f2535c..bfb8a56200a 100644 --- a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py +++ b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py @@ -19,6 +19,9 @@ construct_universal_feature_views, table_name_from_data_source, ) +from tests.integration.feature_repos.universal.data_sources.file import ( + RemoteOfflineStoreDataSourceCreator, +) from tests.integration.feature_repos.universal.data_sources.snowflake import ( SnowflakeDataSourceCreator, ) @@ -157,22 +160,25 @@ def test_historical_features_main( timestamp_precision=timedelta(milliseconds=1), ) - assert_feature_service_correctness( - store, - feature_service, - full_feature_names, - entity_df_with_request_data, - expected_df, - event_timestamp, - ) - assert_feature_service_entity_mapping_correctness( - store, - feature_service_entity_mapping, - full_feature_names, - entity_df_with_request_data, - full_expected_df, - event_timestamp, - ) + if not isinstance( + environment.data_source_creator, RemoteOfflineStoreDataSourceCreator + ): + assert_feature_service_correctness( + store, + feature_service, + full_feature_names, + entity_df_with_request_data, + expected_df, + event_timestamp, + ) + assert_feature_service_entity_mapping_correctness( + store, + feature_service_entity_mapping, + full_feature_names, + entity_df_with_request_data, + full_expected_df, + event_timestamp, + ) table_from_df_entities: pd.DataFrame = job_from_df.to_arrow().to_pandas() validate_dataframes( @@ -375,8 +381,13 @@ def test_historical_features_persisting( (entities, datasets, data_sources) = universal_data_sources feature_views = construct_universal_feature_views(data_sources) + storage = environment.data_source_creator.create_saved_dataset_destination() + store.apply([driver(), customer(), location(), *feature_views.values()]) + # Added to handle the case that the offline store is remote + store.registry.apply_data_source(storage.to_data_source(), store.config.project) + entity_df = datasets.entity_df.drop( columns=["order_id", "origin_id", "destination_id"] ) @@ -398,7 +409,7 @@ def test_historical_features_persisting( saved_dataset = store.create_saved_dataset( from_=job, name="saved_dataset", - storage=environment.data_source_creator.create_saved_dataset_destination(), + storage=storage, tags={"env": "test"}, allow_overwrite=True, ) diff --git a/sdk/python/tests/integration/offline_store/test_validation.py b/sdk/python/tests/integration/offline_store/test_validation.py index fdf182be573..1731f823c89 100644 --- a/sdk/python/tests/integration/offline_store/test_validation.py +++ b/sdk/python/tests/integration/offline_store/test_validation.py @@ -45,8 +45,13 @@ def test_historical_retrieval_with_validation(environment, universal_data_source store = environment.feature_store (entities, datasets, data_sources) = universal_data_sources feature_views = construct_universal_feature_views(data_sources) + storage = environment.data_source_creator.create_saved_dataset_destination() + store.apply([driver(), customer(), location(), *feature_views.values()]) + # Added to handle the case that the offline store is remote + store.registry.apply_data_source(storage.to_data_source(), store.config.project) + # Create two identical retrieval jobs entity_df = datasets.entity_df.drop( columns=["order_id", "origin_id", "destination_id"] @@ -64,7 +69,7 @@ def test_historical_retrieval_with_validation(environment, universal_data_source store.create_saved_dataset( from_=reference_job, name="my_training_dataset", - storage=environment.data_source_creator.create_saved_dataset_destination(), + storage=storage, allow_overwrite=True, ) saved_dataset = store.get_saved_dataset("my_training_dataset") @@ -80,9 +85,13 @@ def test_historical_retrieval_fails_on_validation(environment, universal_data_so (entities, datasets, data_sources) = universal_data_sources feature_views = construct_universal_feature_views(data_sources) + storage = environment.data_source_creator.create_saved_dataset_destination() store.apply([driver(), customer(), location(), *feature_views.values()]) + # Added to handle the case that the offline store is remote + store.registry.apply_data_source(storage.to_data_source(), store.config.project) + entity_df = datasets.entity_df.drop( columns=["order_id", "origin_id", "destination_id"] ) @@ -95,7 +104,7 @@ def test_historical_retrieval_fails_on_validation(environment, universal_data_so store.create_saved_dataset( from_=reference_job, name="my_other_dataset", - storage=environment.data_source_creator.create_saved_dataset_destination(), + storage=storage, allow_overwrite=True, ) @@ -149,10 +158,19 @@ def test_logged_features_validation(environment, universal_data_sources): ), ) + storage = environment.data_source_creator.create_saved_dataset_destination() + store.apply( [driver(), customer(), location(), feature_service, *feature_views.values()] ) + # Added to handle the case that the offline store is remote + store.registry.apply_data_source( + feature_service.logging_config.destination.to_data_source(), + store.config.project, + ) + store.registry.apply_data_source(storage.to_data_source(), store.config.project) + entity_df = datasets.entity_df.drop( columns=["order_id", "origin_id", "destination_id"] ) @@ -180,7 +198,7 @@ def test_logged_features_validation(environment, universal_data_sources): entity_df=entity_df, features=store_fs, full_feature_names=True ), name="reference_for_validating_logged_features", - storage=environment.data_source_creator.create_saved_dataset_destination(), + storage=storage, allow_overwrite=True, ) diff --git a/sdk/python/tests/integration/online_store/test_remote_online_store.py b/sdk/python/tests/integration/online_store/test_remote_online_store.py new file mode 100644 index 00000000000..759a9c7a87b --- /dev/null +++ b/sdk/python/tests/integration/online_store/test_remote_online_store.py @@ -0,0 +1,233 @@ +import os +import subprocess +import tempfile +from datetime import datetime +from textwrap import dedent + +import pytest + +from feast.feature_store import FeatureStore +from feast.wait import wait_retry_backoff +from tests.utils.cli_repo_creator import CliRunner +from tests.utils.http_server import check_port_open, free_port + + +@pytest.mark.integration +def test_remote_online_store_read(): + with tempfile.TemporaryDirectory() as remote_server_tmp_dir, tempfile.TemporaryDirectory() as remote_client_tmp_dir: + server_store, server_url, registry_path = ( + _create_server_store_spin_feature_server(temp_dir=remote_server_tmp_dir) + ) + assert None not in (server_store, server_url, registry_path) + client_store = _create_remote_client_feature_store( + temp_dir=remote_client_tmp_dir, + server_registry_path=str(registry_path), + feature_server_url=server_url, + ) + assert client_store is not None + _assert_non_existing_entity_feature_views_entity( + client_store=client_store, server_store=server_store + ) + _assert_existing_feature_views_entity( + client_store=client_store, server_store=server_store + ) + _assert_non_existing_feature_views( + client_store=client_store, server_store=server_store + ) + + +def _assert_non_existing_entity_feature_views_entity( + client_store: FeatureStore, server_store: FeatureStore +): + features = [ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + ] + + entity_rows = [{"driver_id": 1234}] + _assert_client_server_online_stores_are_matching( + client_store=client_store, + server_store=server_store, + features=features, + entity_rows=entity_rows, + ) + + +def _assert_non_existing_feature_views( + client_store: FeatureStore, server_store: FeatureStore +): + features = [ + "driver_hourly_stats1:conv_rate", + "driver_hourly_stats1:acc_rate", + "driver_hourly_stats:avg_daily_trips", + ] + + entity_rows = [{"driver_id": 1001}, {"driver_id": 1002}] + + with pytest.raises( + Exception, match="Feature view driver_hourly_stats1 does not exist" + ): + client_store.get_online_features( + features=features, entity_rows=entity_rows + ).to_dict() + + with pytest.raises( + Exception, match="Feature view driver_hourly_stats1 does not exist" + ): + server_store.get_online_features( + features=features, entity_rows=entity_rows + ).to_dict() + + +def _assert_existing_feature_views_entity( + client_store: FeatureStore, server_store: FeatureStore +): + features = [ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + ] + + entity_rows = [{"driver_id": 1001}, {"driver_id": 1002}] + _assert_client_server_online_stores_are_matching( + client_store=client_store, + server_store=server_store, + features=features, + entity_rows=entity_rows, + ) + + features = ["driver_hourly_stats:conv_rate"] + _assert_client_server_online_stores_are_matching( + client_store=client_store, + server_store=server_store, + features=features, + entity_rows=entity_rows, + ) + + +def _assert_client_server_online_stores_are_matching( + client_store: FeatureStore, + server_store: FeatureStore, + features: list[str], + entity_rows: list, +): + online_features_from_client = client_store.get_online_features( + features=features, entity_rows=entity_rows + ).to_dict() + + assert online_features_from_client is not None + + online_features_from_server = server_store.get_online_features( + features=features, entity_rows=entity_rows + ).to_dict() + + assert online_features_from_server is not None + assert online_features_from_client is not None + assert online_features_from_client == online_features_from_server + + +def _create_server_store_spin_feature_server(temp_dir): + feast_server_port = free_port() + store = _default_store(str(temp_dir), "REMOTE_ONLINE_SERVER_PROJECT") + server_url = next( + _start_feature_server( + repo_path=str(store.repo_path), server_port=feast_server_port + ) + ) + print(f"Server started successfully, {server_url}") + return store, server_url, os.path.join(store.repo_path, "data", "registry.db") + + +def _default_store(temp_dir, project_name) -> FeatureStore: + runner = CliRunner() + result = runner.run(["init", project_name], cwd=temp_dir) + repo_path = os.path.join(temp_dir, project_name, "feature_repo") + assert result.returncode == 0 + + result = runner.run(["--chdir", repo_path, "apply"], cwd=temp_dir) + assert result.returncode == 0 + + fs = FeatureStore(repo_path=repo_path) + fs.materialize_incremental( + end_date=datetime.utcnow(), feature_views=["driver_hourly_stats"] + ) + return fs + + +def _create_remote_client_feature_store( + temp_dir, server_registry_path: str, feature_server_url: str +) -> FeatureStore: + project_name = "REMOTE_ONLINE_CLIENT_PROJECT" + runner = CliRunner() + result = runner.run(["init", project_name], cwd=temp_dir) + assert result.returncode == 0 + repo_path = os.path.join(temp_dir, project_name, "feature_repo") + _overwrite_remote_client_feature_store_yaml( + repo_path=str(repo_path), + registry_path=server_registry_path, + feature_server_url=feature_server_url, + ) + + result = runner.run(["--chdir", repo_path, "apply"], cwd=temp_dir) + assert result.returncode == 0 + + return FeatureStore(repo_path=repo_path) + + +def _overwrite_remote_client_feature_store_yaml( + repo_path: str, registry_path: str, feature_server_url: str +): + repo_config = os.path.join(repo_path, "feature_store.yaml") + with open(repo_config, "w") as repo_config: + repo_config.write( + dedent( + f""" + project: REMOTE_ONLINE_CLIENT_PROJECT + registry: {registry_path} + provider: local + online_store: + path: {feature_server_url} + type: remote + entity_key_serialization_version: 2 + """ + ) + ) + + +def _start_feature_server(repo_path: str, server_port: int): + host = "0.0.0.0" + cmd = [ + "feast", + "-c" + repo_path, + "serve", + "--host", + host, + "--port", + str(server_port), + ] + feast_server_process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL + ) + _time_out_sec: int = 60 + # Wait for server to start + wait_retry_backoff( + lambda: (None, check_port_open(host, server_port)), + timeout_secs=_time_out_sec, + timeout_msg=f"Unable to start the feast server in {_time_out_sec} seconds for remote online store type, port={server_port}", + ) + + yield f"http://localhost:{server_port}" + + if feast_server_process is not None: + feast_server_process.kill() + + # wait server to free the port + wait_retry_backoff( + lambda: ( + None, + not check_port_open("localhost", server_port), + ), + timeout_msg=f"Unable to stop the feast server in {_time_out_sec} seconds for remote online store type, port={server_port}", + timeout_secs=_time_out_sec, + ) diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 4822a8d4f71..e78c1053bf8 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -29,6 +29,7 @@ ) from tests.integration.feature_repos.universal.entities import driver, item from tests.integration.feature_repos.universal.feature_views import ( + TAGS, create_driver_hourly_stats_feature_view, create_item_embeddings_feature_view, driver_feature_view, @@ -150,9 +151,13 @@ def test_write_to_online_store_event_check(environment): entities=[e], source=file_source, ttl=timedelta(minutes=5), + tags=TAGS, ) # Register Feature View and Entity fs.apply([fv1, e]) + assert len(fs.list_all_feature_views(tags=TAGS)) == 1 + assert len(fs.list_feature_views(tags=TAGS)) == 1 + assert len(fs.list_batch_feature_views(tags=TAGS)) == 1 # data to ingest into Online Store (recent) data = { @@ -410,6 +415,7 @@ def setup_feature_store_universal_feature_views( feature_views = construct_universal_feature_views(data_sources) fs.apply([driver(), feature_views.driver, feature_views.global_fv]) + assert len(fs.list_batch_feature_views(TAGS)) == 2 data = { "driver_id": [1, 2], @@ -476,7 +482,7 @@ def test_online_retrieval_with_event_timestamps(environment, universal_data_sour @pytest.mark.integration -@pytest.mark.universal_online_stores(only=["redis"]) +@pytest.mark.universal_online_stores(only=["redis", "dynamodb"]) def test_async_online_retrieval_with_event_timestamps( environment, universal_data_sources ): @@ -499,6 +505,16 @@ def test_async_online_retrieval_with_event_timestamps( assert_feature_store_universal_feature_views_response(df) +@pytest.mark.integration +@pytest.mark.universal_online_stores +def test_online_list_retrieval(environment, universal_data_sources): + fs = setup_feature_store_universal_feature_views( + environment, universal_data_sources + ) + + assert len(fs.list_batch_feature_views(tags=TAGS)) == 2 + + @pytest.mark.integration @pytest.mark.universal_online_stores(only=["redis"]) def test_online_store_cleanup(environment, universal_data_sources): diff --git a/sdk/python/tests/integration/registration/test_feature_store.py b/sdk/python/tests/integration/registration/test_feature_store.py index bf0c2fb61fd..d7ffb83059b 100644 --- a/sdk/python/tests/integration/registration/test_feature_store.py +++ b/sdk/python/tests/integration/registration/test_feature_store.py @@ -11,68 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os -import time from datetime import timedelta from tempfile import mkstemp import pytest from pytest_lazyfixture import lazy_fixture -from feast import FileSource -from feast.data_format import ParquetFormat from feast.entity import Entity from feast.feature_store import FeatureStore from feast.feature_view import FeatureView -from feast.field import Field -from feast.infra.offline_stores.file import FileOfflineStoreConfig -from feast.infra.online_stores.dynamodb import DynamoDBOnlineStoreConfig from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig from feast.repo_config import RepoConfig -from feast.types import Array, Bytes, Float64, Int64, String +from feast.types import Float64, Int64, String from tests.utils.data_source_test_creator import prep_file_source -@pytest.mark.integration -@pytest.mark.parametrize( - "test_feature_store", - [ - lazy_fixture("feature_store_with_gcs_registry"), - lazy_fixture("feature_store_with_s3_registry"), - ], -) -def test_apply_entity_integration(test_feature_store): - entity = Entity( - name="driver_car_id", - description="Car driver id", - tags={"team": "matchmaking"}, - ) - - # Register Entity - test_feature_store.apply([entity]) - - entities = test_feature_store.list_entities() - - entity = entities[0] - assert ( - len(entities) == 1 - and entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" - ) - - entity = test_feature_store.get_entity("driver_car_id") - assert ( - entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" - ) - - test_feature_store.teardown() - - @pytest.mark.integration @pytest.mark.parametrize( "test_feature_store", @@ -109,81 +62,6 @@ def test_feature_view_inference_success(test_feature_store, dataframe_source): test_feature_store.teardown() -@pytest.mark.integration -@pytest.mark.parametrize( - "test_feature_store", - [ - lazy_fixture("feature_store_with_gcs_registry"), - lazy_fixture("feature_store_with_s3_registry"), - ], -) -def test_apply_feature_view_integration(test_feature_store): - # Create Feature Views - batch_source = FileSource( - file_format=ParquetFormat(), - path="file://feast/*", - timestamp_field="ts_col", - created_timestamp_column="timestamp", - ) - - entity = Entity(name="fs1_my_entity_1", join_keys=["test"]) - - fv1 = FeatureView( - name="my_feature_view_1", - schema=[ - Field(name="fs1_my_feature_1", dtype=Int64), - Field(name="fs1_my_feature_2", dtype=String), - Field(name="fs1_my_feature_3", dtype=Array(String)), - Field(name="fs1_my_feature_4", dtype=Array(Bytes)), - Field(name="test", dtype=Int64), - ], - entities=[entity], - tags={"team": "matchmaking"}, - source=batch_source, - ttl=timedelta(minutes=5), - ) - - # Register Feature View - test_feature_store.apply([fv1, entity]) - - feature_views = test_feature_store.list_feature_views() - - # List Feature Views - assert ( - len(feature_views) == 1 - and feature_views[0].name == "my_feature_view_1" - and feature_views[0].features[0].name == "fs1_my_feature_1" - and feature_views[0].features[0].dtype == Int64 - and feature_views[0].features[1].name == "fs1_my_feature_2" - and feature_views[0].features[1].dtype == String - and feature_views[0].features[2].name == "fs1_my_feature_3" - and feature_views[0].features[2].dtype == Array(String) - and feature_views[0].features[3].name == "fs1_my_feature_4" - and feature_views[0].features[3].dtype == Array(Bytes) - and feature_views[0].entities[0] == "fs1_my_entity_1" - ) - - feature_view = test_feature_store.get_feature_view("my_feature_view_1") - assert ( - feature_view.name == "my_feature_view_1" - and feature_view.features[0].name == "fs1_my_feature_1" - and feature_view.features[0].dtype == Int64 - and feature_view.features[1].name == "fs1_my_feature_2" - and feature_view.features[1].dtype == String - and feature_view.features[2].name == "fs1_my_feature_3" - and feature_view.features[2].dtype == Array(String) - and feature_view.features[3].name == "fs1_my_feature_4" - and feature_view.features[3].dtype == Array(Bytes) - and feature_view.entities[0] == "fs1_my_entity_1" - ) - - test_feature_store.delete_feature_view("my_feature_view_1") - feature_views = test_feature_store.list_feature_views() - assert len(feature_views) == 0 - - test_feature_store.teardown() - - @pytest.fixture def feature_store_with_local_registry(): fd, registry_path = mkstemp() @@ -197,46 +75,3 @@ def feature_store_with_local_registry(): entity_key_serialization_version=2, ) ) - - -@pytest.fixture -def feature_store_with_gcs_registry(): - from google.cloud import storage - - storage_client = storage.Client() - bucket_name = f"feast-registry-test-{int(time.time() * 1000)}" - bucket = storage_client.bucket(bucket_name) - bucket = storage_client.create_bucket(bucket) - bucket.add_lifecycle_delete_rule( - age=14 - ) # delete buckets automatically after 14 days - bucket.patch() - bucket.blob("registry.db") - - return FeatureStore( - config=RepoConfig( - registry=f"gs://{bucket_name}/registry.db", - project="default", - provider="gcp", - entity_key_serialization_version=2, - ) - ) - - -@pytest.fixture -def feature_store_with_s3_registry(): - aws_registry_path = os.getenv( - "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries" - ) - return FeatureStore( - config=RepoConfig( - registry=f"{aws_registry_path}/{int(time.time() * 1000)}/registry.db", - project="default", - provider="aws", - online_store=DynamoDBOnlineStoreConfig( - region=os.getenv("AWS_REGION", "us-west-2") - ), - offline_store=FileOfflineStoreConfig(), - entity_key_serialization_version=2, - ) - ) diff --git a/sdk/python/tests/integration/registration/test_universal_cli.py b/sdk/python/tests/integration/registration/test_universal_cli.py index e7331a07894..fc90108d787 100644 --- a/sdk/python/tests/integration/registration/test_universal_cli.py +++ b/sdk/python/tests/integration/registration/test_universal_cli.py @@ -7,7 +7,9 @@ from assertpy import assertpy from feast.feature_store import FeatureStore -from tests.integration.feature_repos.repo_configuration import Environment +from tests.integration.feature_repos.universal.data_sources.file import ( + FileDataSourceCreator, +) from tests.utils.basic_read_write_test import basic_rw_test from tests.utils.cli_repo_creator import CliRunner, get_example_repo from tests.utils.e2e_test_validation import ( @@ -17,8 +19,7 @@ @pytest.mark.integration -@pytest.mark.universal_offline_stores -def test_universal_cli(environment: Environment): +def test_universal_cli(): project = f"test_universal_cli_{str(uuid.uuid4()).replace('-', '')[:8]}" runner = CliRunner() @@ -28,9 +29,9 @@ def test_universal_cli(environment: Environment): feature_store_yaml = make_feature_store_yaml( project, repo_path, - environment.data_source_creator, - environment.provider, - environment.online_store, + FileDataSourceCreator("project"), + "local", + {"type": "sqlite"}, ) repo_config = repo_path / "feature_store.yaml" @@ -73,13 +74,13 @@ def test_universal_cli(environment: Environment): cwd=repo_path, ) assertpy.assert_that(result.returncode).is_equal_to(0) - assertpy.assert_that(fs.list_feature_views()).is_length(4) + assertpy.assert_that(fs.list_feature_views()).is_length(5) result = runner.run( ["data-sources", "describe", "customer_profile_source"], cwd=repo_path, ) assertpy.assert_that(result.returncode).is_equal_to(0) - assertpy.assert_that(fs.list_data_sources()).is_length(4) + assertpy.assert_that(fs.list_data_sources()).is_length(5) # entity & feature view describe commands should fail when objects don't exist result = runner.run(["entities", "describe", "foo"], cwd=repo_path) @@ -115,8 +116,7 @@ def test_universal_cli(environment: Environment): @pytest.mark.integration -@pytest.mark.universal_offline_stores -def test_odfv_apply(environment) -> None: +def test_odfv_apply() -> None: project = f"test_odfv_apply{str(uuid.uuid4()).replace('-', '')[:8]}" runner = CliRunner() @@ -126,9 +126,9 @@ def test_odfv_apply(environment) -> None: feature_store_yaml = make_feature_store_yaml( project, repo_path, - environment.data_source_creator, - environment.provider, - environment.online_store, + FileDataSourceCreator("project"), + "local", + {"type": "sqlite"}, ) repo_config = repo_path / "feature_store.yaml" diff --git a/sdk/python/tests/integration/registration/test_universal_registry.py b/sdk/python/tests/integration/registration/test_universal_registry.py index 1f0ccb4f6b5..cd741853cc5 100644 --- a/sdk/python/tests/integration/registration/test_universal_registry.py +++ b/sdk/python/tests/integration/registration/test_universal_registry.py @@ -18,6 +18,7 @@ from tempfile import mkstemp from unittest import mock +import grpc_testing import pandas as pd import pytest from pytest_lazyfixture import lazy_fixture @@ -36,8 +37,11 @@ from feast.infra.infra_object import Infra from feast.infra.online_stores.sqlite import SqliteTable from feast.infra.registry.registry import Registry +from feast.infra.registry.remote import RemoteRegistry, RemoteRegistryConfig from feast.infra.registry.sql import SqlRegistry from feast.on_demand_feature_view import on_demand_feature_view +from feast.protos.feast.registry import RegistryServer_pb2, RegistryServer_pb2_grpc +from feast.registry_server import RegistryServer from feast.repo_config import RegistryConfig from feast.stream_feature_view import Aggregation, StreamFeatureView from feast.types import Array, Bytes, Float32, Int32, Int64, String @@ -187,19 +191,83 @@ def sqlite_registry(): yield SqlRegistry(registry_config, "project", None) -@pytest.mark.integration -@pytest.mark.parametrize( - "test_registry", - [ +class GrpcMockChannel: + def __init__(self, service, servicer): + self.service = service + self.test_server = grpc_testing.server_from_dictionary( + {service: servicer}, + grpc_testing.strict_real_time(), + ) + + def unary_unary( + self, method: str, request_serializer=None, response_deserializer=None + ): + method_name = method.split("/")[-1] + method_descriptor = self.service.methods_by_name[method_name] + + def handler(request): + rpc = self.test_server.invoke_unary_unary( + method_descriptor, (), request, None + ) + + response, trailing_metadata, code, details = rpc.termination() + return response + + return handler + + +@pytest.fixture +def mock_remote_registry(): + fd, registry_path = mkstemp() + registry_config = RegistryConfig(path=registry_path, cache_ttl_seconds=600) + proxied_registry = Registry("project", registry_config, None) + + registry = RemoteRegistry( + registry_config=RemoteRegistryConfig(path=""), project=None, repo_path=None + ) + mock_channel = GrpcMockChannel( + RegistryServer_pb2.DESCRIPTOR.services_by_name["RegistryServer"], + RegistryServer(registry=proxied_registry), + ) + registry.stub = RegistryServer_pb2_grpc.RegistryServerStub(mock_channel) + yield registry + + +if os.getenv("FEAST_IS_LOCAL_TEST", "False") == "False": + all_fixtures = [lazy_fixture("s3_registry"), lazy_fixture("gcs_registry")] +else: + all_fixtures = [ lazy_fixture("local_registry"), - lazy_fixture("gcs_registry"), - lazy_fixture("s3_registry"), - lazy_fixture("minio_registry"), - lazy_fixture("pg_registry"), - lazy_fixture("mysql_registry"), + pytest.param( + lazy_fixture("minio_registry"), + marks=pytest.mark.xdist_group(name="minio_registry"), + ), + pytest.param( + lazy_fixture("pg_registry"), + marks=pytest.mark.xdist_group(name="pg_registry"), + ), + pytest.param( + lazy_fixture("mysql_registry"), + marks=pytest.mark.xdist_group(name="mysql_registry"), + ), lazy_fixture("sqlite_registry"), - ], -) + lazy_fixture("mock_remote_registry"), + ] + +sql_fixtures = [ + pytest.param( + lazy_fixture("pg_registry"), marks=pytest.mark.xdist_group(name="pg_registry") + ), + pytest.param( + lazy_fixture("mysql_registry"), + marks=pytest.mark.xdist_group(name="mysql_registry"), + ), + lazy_fixture("sqlite_registry"), +] + + +@pytest.mark.integration +@pytest.mark.parametrize("test_registry", all_fixtures) def test_apply_entity_success(test_registry): entity = Entity( name="driver_car_id", @@ -217,7 +285,7 @@ def test_apply_entity_success(test_registry): assert len(project_metadata[0].project_uuid) == 36 assert_project_uuid(project, project_uuid, test_registry) - entities = test_registry.list_entities(project) + entities = test_registry.list_entities(project, tags=entity.tags) assert_project_uuid(project, project_uuid, test_registry) entity = entities[0] @@ -258,15 +326,7 @@ def assert_project_uuid(project, project_uuid, test_registry): @pytest.mark.integration @pytest.mark.parametrize( "test_registry", - [ - lazy_fixture("local_registry"), - lazy_fixture("gcs_registry"), - lazy_fixture("s3_registry"), - lazy_fixture("minio_registry"), - lazy_fixture("pg_registry"), - lazy_fixture("mysql_registry"), - lazy_fixture("sqlite_registry"), - ], + all_fixtures, ) def test_apply_feature_view_success(test_registry): # Create Feature Views @@ -299,7 +359,7 @@ def test_apply_feature_view_success(test_registry): # Register Feature View test_registry.apply_feature_view(fv1, project) - feature_views = test_registry.list_feature_views(project) + feature_views = test_registry.list_feature_views(project, tags=fv1.tags) # List Feature Views assert ( @@ -353,15 +413,7 @@ def test_apply_feature_view_success(test_registry): @pytest.mark.integration @pytest.mark.parametrize( "test_registry", - [ - # lazy_fixture("local_registry"), - # lazy_fixture("gcs_registry"), - # lazy_fixture("s3_registry"), - # lazy_fixture("minio_registry"), - lazy_fixture("pg_registry"), - lazy_fixture("mysql_registry"), - lazy_fixture("sqlite_registry"), - ], + sql_fixtures, ) def test_apply_on_demand_feature_view_success(test_registry): # Create Feature Views @@ -443,15 +495,7 @@ def location_features_from_push(inputs: pd.DataFrame) -> pd.DataFrame: @pytest.mark.integration @pytest.mark.parametrize( "test_registry", - [ - lazy_fixture("local_registry"), - lazy_fixture("gcs_registry"), - lazy_fixture("s3_registry"), - lazy_fixture("minio_registry"), - lazy_fixture("pg_registry"), - lazy_fixture("mysql_registry"), - lazy_fixture("sqlite_registry"), - ], + all_fixtures, ) def test_apply_data_source(test_registry): # Create Feature Views @@ -486,7 +530,7 @@ def test_apply_data_source(test_registry): test_registry.apply_data_source(batch_source, project, commit=False) test_registry.apply_feature_view(fv1, project, commit=True) - registry_feature_views = test_registry.list_feature_views(project) + registry_feature_views = test_registry.list_feature_views(project, tags=fv1.tags) registry_data_sources = test_registry.list_data_sources(project) assert len(registry_feature_views) == 1 assert len(registry_data_sources) == 1 @@ -499,7 +543,7 @@ def test_apply_data_source(test_registry): batch_source.timestamp_field = "new_ts_col" test_registry.apply_data_source(batch_source, project, commit=False) test_registry.apply_feature_view(fv1, project, commit=True) - registry_feature_views = test_registry.list_feature_views(project) + registry_feature_views = test_registry.list_feature_views(project, tags=fv1.tags) registry_data_sources = test_registry.list_data_sources(project) assert len(registry_feature_views) == 1 assert len(registry_data_sources) == 1 @@ -514,15 +558,7 @@ def test_apply_data_source(test_registry): @pytest.mark.integration @pytest.mark.parametrize( "test_registry", - [ - lazy_fixture("local_registry"), - lazy_fixture("gcs_registry"), - lazy_fixture("s3_registry"), - lazy_fixture("minio_registry"), - lazy_fixture("pg_registry"), - lazy_fixture("mysql_registry"), - lazy_fixture("sqlite_registry"), - ], + all_fixtures, ) def test_modify_feature_views_success(test_registry): # Create Feature Views @@ -620,7 +656,7 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: ) # Make sure fv1 is untouched - feature_views = test_registry.list_feature_views(project) + feature_views = test_registry.list_feature_views(project, tags=fv1.tags) # List Feature Views assert ( @@ -645,15 +681,7 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame: @pytest.mark.integration @pytest.mark.parametrize( "test_registry", - [ - # lazy_fixture("local_registry"), - # lazy_fixture("gcs_registry"), - # lazy_fixture("s3_registry"), - # lazy_fixture("minio_registry"), - lazy_fixture("pg_registry"), - lazy_fixture("mysql_registry"), - lazy_fixture("sqlite_registry"), - ], + sql_fixtures, ) def test_update_infra(test_registry): # Create infra object @@ -684,15 +712,7 @@ def test_update_infra(test_registry): @pytest.mark.integration @pytest.mark.parametrize( "test_registry", - [ - # lazy_fixture("local_registry"), - # lazy_fixture("gcs_registry"), - # lazy_fixture("s3_registry"), - # lazy_fixture("minio_registry"), - lazy_fixture("pg_registry"), - lazy_fixture("mysql_registry"), - lazy_fixture("sqlite_registry"), - ], + sql_fixtures, ) def test_registry_cache(test_registry): # Create Feature Views @@ -702,6 +722,7 @@ def test_registry_cache(test_registry): path="file://feast/*", timestamp_field="ts_col", created_timestamp_column="timestamp", + tags={"team": "matchmaking"}, ) entity = Entity(name="fs1_my_entity_1", join_keys=["test"]) @@ -738,10 +759,10 @@ def test_registry_cache(test_registry): test_registry.refresh(project) # Now objects exist registry_feature_views_cached = test_registry.list_feature_views( - project, allow_cache=True + project, allow_cache=True, tags=fv1.tags ) registry_data_sources_cached = test_registry.list_data_sources( - project, allow_cache=True + project, allow_cache=True, tags=batch_source.tags ) assert len(registry_feature_views_cached) == 1 assert len(registry_data_sources_cached) == 1 @@ -756,15 +777,7 @@ def test_registry_cache(test_registry): @pytest.mark.integration @pytest.mark.parametrize( "test_registry", - [ - lazy_fixture("local_registry"), - lazy_fixture("gcs_registry"), - lazy_fixture("s3_registry"), - lazy_fixture("minio_registry"), - lazy_fixture("pg_registry"), - lazy_fixture("mysql_registry"), - lazy_fixture("sqlite_registry"), - ], + all_fixtures, ) def test_apply_stream_feature_view_success(test_registry): # Create Feature Views @@ -807,7 +820,7 @@ def simple_udf(x: int): mode="spark", source=stream_source, udf=simple_udf, - tags={}, + tags={"team": "matchmaking"}, ) project = "project" @@ -815,7 +828,9 @@ def simple_udf(x: int): # Register Feature View test_registry.apply_feature_view(sfv, project) - stream_feature_views = test_registry.list_stream_feature_views(project) + stream_feature_views = test_registry.list_stream_feature_views( + project, tags=sfv.tags + ) # List Feature Views assert len(stream_feature_views) == 1 @@ -826,3 +841,94 @@ def simple_udf(x: int): assert len(stream_feature_views) == 0 test_registry.teardown() + + +@pytest.mark.integration +def test_commit(): + fd, registry_path = mkstemp() + registry_config = RegistryConfig(path=registry_path, cache_ttl_seconds=600) + test_registry = Registry("project", registry_config, None) + + entity = Entity( + name="driver_car_id", + description="Car driver id", + tags={"team": "matchmaking"}, + ) + + project = "project" + + # Register Entity without commiting + test_registry.apply_entity(entity, project, commit=False) + assert test_registry.cached_registry_proto + assert len(test_registry.cached_registry_proto.project_metadata) == 1 + project_metadata = test_registry.cached_registry_proto.project_metadata[0] + project_uuid = project_metadata.project_uuid + assert len(project_uuid) == 36 + validate_project_uuid(project_uuid, test_registry) + + # Retrieving the entity should still succeed + entities = test_registry.list_entities(project, allow_cache=True, tags=entity.tags) + entity = entities[0] + assert ( + len(entities) == 1 + and entity.name == "driver_car_id" + and entity.description == "Car driver id" + and "team" in entity.tags + and entity.tags["team"] == "matchmaking" + ) + validate_project_uuid(project_uuid, test_registry) + + entity = test_registry.get_entity("driver_car_id", project, allow_cache=True) + assert ( + entity.name == "driver_car_id" + and entity.description == "Car driver id" + and "team" in entity.tags + and entity.tags["team"] == "matchmaking" + ) + validate_project_uuid(project_uuid, test_registry) + + # Create new registry that points to the same store + registry_with_same_store = Registry("project", registry_config, None) + + # Retrieving the entity should fail since the store is empty + entities = registry_with_same_store.list_entities(project) + assert len(entities) == 0 + validate_project_uuid(project_uuid, registry_with_same_store) + + # commit from the original registry + test_registry.commit() + + # Reconstruct the new registry in order to read the newly written store + registry_with_same_store = Registry("project", registry_config, None) + + # Retrieving the entity should now succeed + entities = registry_with_same_store.list_entities(project, tags=entity.tags) + entity = entities[0] + assert ( + len(entities) == 1 + and entity.name == "driver_car_id" + and entity.description == "Car driver id" + and "team" in entity.tags + and entity.tags["team"] == "matchmaking" + ) + validate_project_uuid(project_uuid, registry_with_same_store) + + entity = test_registry.get_entity("driver_car_id", project) + assert ( + entity.name == "driver_car_id" + and entity.description == "Car driver id" + and "team" in entity.tags + and entity.tags["team"] == "matchmaking" + ) + + test_registry.teardown() + + # Will try to reload registry, which will fail because the file has been deleted + with pytest.raises(FileNotFoundError): + test_registry._get_registry_proto(project=project) + + +def validate_project_uuid(project_uuid, test_registry): + assert len(test_registry.cached_registry_proto.project_metadata) == 1 + project_metadata = test_registry.cached_registry_proto.project_metadata[0] + assert project_metadata.project_uuid == project_uuid diff --git a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py index 79a3a27b67a..fd50d376322 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py @@ -29,6 +29,10 @@ RedshiftOfflineStoreConfig, RedshiftRetrievalJob, ) +from feast.infra.offline_stores.remote import ( + RemoteOfflineStoreConfig, + RemoteRetrievalJob, +) from feast.infra.offline_stores.snowflake import ( SnowflakeOfflineStoreConfig, SnowflakeRetrievalJob, @@ -104,6 +108,7 @@ def metadata(self) -> Optional[RetrievalMetadata]: PostgreSQLRetrievalJob, SparkRetrievalJob, TrinoRetrievalJob, + RemoteRetrievalJob, ] ) def retrieval_job(request, environment): @@ -203,6 +208,35 @@ def retrieval_job(request, environment): config=environment.config, full_feature_names=False, ) + elif request.param is RemoteRetrievalJob: + offline_store_config = RemoteOfflineStoreConfig( + type="remote", + host="localhost", + port=0, + ) + environment.config._offline_store = offline_store_config + + entity_df = pd.DataFrame.from_dict( + { + "id": [1], + "event_timestamp": ["datetime"], + "val_to_add": [1], + } + ) + + return RemoteRetrievalJob( + client=MagicMock(), + api_parameters={ + "str": "str", + }, + api="api", + table=pyarrow.Table.from_pandas(entity_df), + entity_df=entity_df, + metadata=RetrievalMetadata( + features=["1", "2", "3", "4"], + keys=["1", "2", "3", "4"], + ), + ) else: return request.param() diff --git a/sdk/python/tests/unit/infra/registry/test_remote.py b/sdk/python/tests/unit/infra/registry/test_remote.py deleted file mode 100644 index 16c6f0abfb0..00000000000 --- a/sdk/python/tests/unit/infra/registry/test_remote.py +++ /dev/null @@ -1,69 +0,0 @@ -import assertpy -import grpc_testing -import pytest - -from feast import Entity, FeatureStore -from feast.infra.registry.remote import RemoteRegistry, RemoteRegistryConfig -from feast.protos.feast.registry import RegistryServer_pb2, RegistryServer_pb2_grpc -from feast.registry_server import RegistryServer - - -class GrpcMockChannel: - def __init__(self, service, servicer): - self.service = service - self.test_server = grpc_testing.server_from_dictionary( - {service: servicer}, - grpc_testing.strict_real_time(), - ) - - def unary_unary( - self, method: str, request_serializer=None, response_deserializer=None - ): - method_name = method.split("/")[-1] - method_descriptor = self.service.methods_by_name[method_name] - - def handler(request): - rpc = self.test_server.invoke_unary_unary( - method_descriptor, (), request, None - ) - - response, trailing_metadata, code, details = rpc.termination() - return response - - return handler - - -@pytest.fixture -def mock_remote_registry(environment): - store: FeatureStore = environment.feature_store - registry = RemoteRegistry( - registry_config=RemoteRegistryConfig(path=""), project=None, repo_path=None - ) - mock_channel = GrpcMockChannel( - RegistryServer_pb2.DESCRIPTOR.services_by_name["RegistryServer"], - RegistryServer(store=store), - ) - registry.stub = RegistryServer_pb2_grpc.RegistryServerStub(mock_channel) - return registry - - -def test_registry_server_get_entity(environment, mock_remote_registry): - store: FeatureStore = environment.feature_store - entity = Entity(name="driver", join_keys=["driver_id"]) - store.apply(entity) - - expected = store.get_entity(entity.name) - response_entity = mock_remote_registry.get_entity(entity.name, store.project) - - assertpy.assert_that(response_entity).is_equal_to(expected) - - -def test_registry_server_proto(environment, mock_remote_registry): - store: FeatureStore = environment.feature_store - entity = Entity(name="driver", join_keys=["driver_id"]) - store.apply(entity) - - expected = store.registry.proto() - response = mock_remote_registry.proto() - - assertpy.assert_that(response).is_equal_to(expected) diff --git a/sdk/python/tests/unit/infra/test_local_registry.py b/sdk/python/tests/unit/infra/test_local_registry.py deleted file mode 100644 index c86a616c406..00000000000 --- a/sdk/python/tests/unit/infra/test_local_registry.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2022 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from tempfile import mkstemp - -import pytest - -from feast.entity import Entity -from feast.infra.registry.registry import Registry -from feast.repo_config import RegistryConfig - - -def test_commit(): - fd, registry_path = mkstemp() - registry_config = RegistryConfig(path=registry_path, cache_ttl_seconds=600) - test_registry = Registry("project", registry_config, None) - - entity = Entity( - name="driver_car_id", - description="Car driver id", - tags={"team": "matchmaking"}, - ) - - project = "project" - - # Register Entity without commiting - test_registry.apply_entity(entity, project, commit=False) - assert test_registry.cached_registry_proto - assert len(test_registry.cached_registry_proto.project_metadata) == 1 - project_metadata = test_registry.cached_registry_proto.project_metadata[0] - project_uuid = project_metadata.project_uuid - assert len(project_uuid) == 36 - validate_project_uuid(project_uuid, test_registry) - - # Retrieving the entity should still succeed - entities = test_registry.list_entities(project, allow_cache=True) - entity = entities[0] - assert ( - len(entities) == 1 - and entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" - ) - validate_project_uuid(project_uuid, test_registry) - - entity = test_registry.get_entity("driver_car_id", project, allow_cache=True) - assert ( - entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" - ) - validate_project_uuid(project_uuid, test_registry) - - # Create new registry that points to the same store - registry_with_same_store = Registry("project", registry_config, None) - - # Retrieving the entity should fail since the store is empty - entities = registry_with_same_store.list_entities(project) - assert len(entities) == 0 - validate_project_uuid(project_uuid, registry_with_same_store) - - # commit from the original registry - test_registry.commit() - - # Reconstruct the new registry in order to read the newly written store - registry_with_same_store = Registry("project", registry_config, None) - - # Retrieving the entity should now succeed - entities = registry_with_same_store.list_entities(project) - entity = entities[0] - assert ( - len(entities) == 1 - and entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" - ) - validate_project_uuid(project_uuid, registry_with_same_store) - - entity = test_registry.get_entity("driver_car_id", project) - assert ( - entity.name == "driver_car_id" - and entity.description == "Car driver id" - and "team" in entity.tags - and entity.tags["team"] == "matchmaking" - ) - - test_registry.teardown() - - # Will try to reload registry, which will fail because the file has been deleted - with pytest.raises(FileNotFoundError): - test_registry._get_registry_proto(project=project) - - -def validate_project_uuid(project_uuid, test_registry): - assert len(test_registry.cached_registry_proto.project_metadata) == 1 - project_metadata = test_registry.cached_registry_proto.project_metadata[0] - assert project_metadata.project_uuid == project_uuid diff --git a/sdk/python/tests/unit/local_feast_tests/test_feature_service.py b/sdk/python/tests/unit/local_feast_tests/test_feature_service.py index 82c1dd2a1d9..75ceb463085 100644 --- a/sdk/python/tests/unit/local_feast_tests/test_feature_service.py +++ b/sdk/python/tests/unit/local_feast_tests/test_feature_service.py @@ -6,6 +6,7 @@ create_driver_hourly_stats_df, create_global_daily_stats_df, ) +from tests.integration.feature_repos.universal.feature_views import TAGS from tests.utils.basic_read_write_test import basic_rw_test from tests.utils.cli_repo_creator import CliRunner, get_example_repo @@ -19,6 +20,9 @@ def test_apply_without_fv_inference() -> None: get_example_repo("example_feature_repo_with_feature_service_2.py"), "file" ) as store: assert len(store.list_feature_services()) == 2 + assert len(store.list_feature_services(tags={"release": "qa"})) == 1 + assert len(store.list_feature_services(tags=TAGS)) == 1 + assert len(store.list_feature_services(tags={"wrong": "tag"})) == 0 fs = store.get_feature_service("all_stats") assert len(fs.feature_view_projections) == 2 @@ -35,6 +39,7 @@ def test_apply_without_fv_inference() -> None: assert len(fs.feature_view_projections[0].desired_features) == 0 assert len(fs.feature_view_projections[0].features) == 1 assert len(fs.feature_view_projections[0].desired_features) == 0 + assert fs.tags["release"] == "qa" def test_apply_with_fv_inference() -> None: diff --git a/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py b/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py index b3e6762c17d..6b7856f347c 100644 --- a/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py +++ b/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py @@ -4,7 +4,7 @@ import pytest from pytest_lazyfixture import lazy_fixture -from feast import BatchFeatureView +from feast import BatchFeatureView, utils from feast.aggregation import Aggregation from feast.data_format import AvroFormat, ParquetFormat from feast.data_source import KafkaSource @@ -17,6 +17,7 @@ from feast.repo_config import RepoConfig from feast.stream_feature_view import stream_feature_view from feast.types import Array, Bytes, Float32, Int64, String +from tests.integration.feature_repos.universal.feature_views import TAGS from tests.utils.cli_repo_creator import CliRunner, get_example_repo from tests.utils.data_source_test_creator import prep_file_source @@ -89,7 +90,7 @@ def test_apply_feature_view(test_feature_store): Field(name="entity_id", dtype=Int64), ], entities=[entity], - tags={"team": "matchmaking"}, + tags={"team": "matchmaking", "tag": "two"}, source=batch_source, ttl=timedelta(minutes=5), ) @@ -97,11 +98,50 @@ def test_apply_feature_view(test_feature_store): # Register Feature View test_feature_store.apply([entity, fv1, bfv]) - feature_views = test_feature_store.list_feature_views() + # List Feature Views + assert len(test_feature_store.list_batch_feature_views({})) == 2 + feature_views = test_feature_store.list_batch_feature_views() + assert ( + len(feature_views) == 2 + and feature_views[0].name == "my_feature_view_1" + and feature_views[0].features[0].name == "fs1_my_feature_1" + and feature_views[0].features[0].dtype == Int64 + and feature_views[0].features[1].name == "fs1_my_feature_2" + and feature_views[0].features[1].dtype == String + and feature_views[0].features[2].name == "fs1_my_feature_3" + and feature_views[0].features[2].dtype == Array(String) + and feature_views[0].features[3].name == "fs1_my_feature_4" + and feature_views[0].features[3].dtype == Array(Bytes) + and feature_views[0].entities[0] == "fs1_my_entity_1" + ) + + assert utils.tags_str_to_dict() == {} + assert utils.tags_list_to_dict() is None + assert utils.tags_list_to_dict([]) is None + assert utils.tags_list_to_dict([""]) == {} + assert utils.tags_list_to_dict( + ( + "team : driver_performance, other:tag", + "blanktag:", + "other:two", + "other:3", + "missing", + ) + ) == {"team": "driver_performance", "other": "3", "blanktag": ""} + assert utils.has_all_tags({}) + + tags_dict = {"team": "matchmaking"} + tags_filter = utils.tags_str_to_dict("('team:matchmaking',)") + assert tags_filter == tags_dict + tags_filter = utils.tags_list_to_dict(("team:matchmaking", "test")) + assert tags_dict == tags_dict # List Feature Views + feature_views = test_feature_store.list_batch_feature_views(tags=tags_filter) assert ( len(feature_views) == 2 + and utils.has_all_tags(feature_views[0].tags, tags_filter) + and utils.has_all_tags(feature_views[1].tags, tags_filter) and feature_views[0].name == "my_feature_view_1" and feature_views[0].features[0].name == "fs1_my_feature_1" and feature_views[0].features[0].dtype == Int64 @@ -114,6 +154,34 @@ def test_apply_feature_view(test_feature_store): and feature_views[0].entities[0] == "fs1_my_entity_1" ) + tags_dict = {"team": "matchmaking", "tag": "two"} + tags_filter = utils.tags_list_to_dict((" team :matchmaking, tag: two ",)) + assert tags_filter == tags_dict + + # List Feature Views + feature_views = test_feature_store.list_batch_feature_views(tags=tags_filter) + assert ( + len(feature_views) == 1 + and utils.has_all_tags(feature_views[0].tags, tags_filter) + and feature_views[0].name == "batch_feature_view" + and feature_views[0].features[0].name == "fs1_my_feature_1" + and feature_views[0].features[0].dtype == Int64 + and feature_views[0].features[1].name == "fs1_my_feature_2" + and feature_views[0].features[1].dtype == String + and feature_views[0].features[2].name == "fs1_my_feature_3" + and feature_views[0].features[2].dtype == Array(String) + and feature_views[0].features[3].name == "fs1_my_feature_4" + and feature_views[0].features[3].dtype == Array(Bytes) + and feature_views[0].entities[0] == "fs1_my_entity_1" + ) + + tags_dict = {"missing": "tag"} + tags_filter = utils.tags_list_to_dict(("missing:tag,fdsa", "fdas")) + assert tags_filter == tags_dict + + # List Feature Views + assert len(test_feature_store.list_batch_feature_views(tags=tags_filter)) == 0 + test_feature_store.teardown() @@ -136,7 +204,7 @@ def test_apply_feature_view_with_inline_batch_source( test_feature_store.apply([entity, driver_fv]) - fvs = test_feature_store.list_feature_views() + fvs = test_feature_store.list_batch_feature_views() assert len(fvs) == 1 assert fvs[0] == driver_fv @@ -185,7 +253,7 @@ def test_apply_feature_view_with_inline_stream_source( test_feature_store.apply([entity, driver_fv]) - fvs = test_feature_store.list_feature_views() + fvs = test_feature_store.list_batch_feature_views() assert len(fvs) == 1 assert fvs[0] == driver_fv @@ -525,10 +593,12 @@ def test_apply_stream_source(test_feature_store, simple_dataset_1) -> None: topic="topic", batch_source=file_source, watermark_delay_threshold=timedelta(days=1), + tags=TAGS, ) test_feature_store.apply([stream_source]) + assert len(test_feature_store.list_data_sources(tags=TAGS)) == 1 ds = test_feature_store.list_data_sources() assert len(ds) == 2 if isinstance(ds[0], FileSource): diff --git a/sdk/python/tests/unit/online_store/__init__.py b/sdk/python/tests/unit/online_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 5368b1e11cd..1e8cf45dcc6 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -1,20 +1,27 @@ import os +import platform +import sqlite3 +import sys import time from datetime import datetime +import numpy as np import pandas as pd import pytest +import sqlite_vec from pandas.testing import assert_frame_equal from feast import FeatureStore, RepoConfig from feast.errors import FeatureViewNotFoundException from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import FloatList as FloatListProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import RegistryConfig +from tests.integration.feature_repos.universal.feature_views import TAGS from tests.utils.cli_repo_creator import CliRunner, get_example_repo -def test_online() -> None: +def test_get_online_features() -> None: """ Test reading from the online store in local mode. """ @@ -90,6 +97,9 @@ def test_online() -> None: progress=None, ) + assert len(store.list_entities()) == 3 + assert len(store.list_entities(tags=TAGS)) == 2 + # Retrieve two features using two keys, one valid one non-existing result = store.get_online_features( features=[ @@ -415,3 +425,140 @@ def test_online_to_df(): ] expected_df = pd.DataFrame({k: reversed(v) for (k, v) in df_dict.items()}) assert_frame_equal(result_df[ordered_column], expected_df) + + +@pytest.mark.skipif( + sys.version_info[0:2] != (3, 10) or platform.system() != "Darwin", + reason="Only works on Python 3.10 and MacOS", +) +def test_sqlite_get_online_documents() -> None: + """ + Test retrieving documents from the online store in local mode. + """ + n = 10 # number of samples - note: we'll actually double it + vector_length = 8 + runner = CliRunner() + with runner.local_repo( + get_example_repo("example_feature_repo_1.py"), "file" + ) as store: + store.config.online_store.vec_enabled = True + store.config.online_store.vector_len = vector_length + # Write some data to two tables + document_embeddings_fv = store.get_feature_view(name="document_embeddings") + + provider = store._get_provider() + + item_keys = [ + EntityKeyProto( + join_keys=["item_id"], entity_values=[ValueProto(int64_val=i)] + ) + for i in range(n) + ] + data = [] + for item_key in item_keys: + data.append( + ( + item_key, + { + "Embeddings": ValueProto( + float_list_val=FloatListProto( + val=np.random.random( + vector_length, + ) + ) + ) + }, + datetime.utcnow(), + datetime.utcnow(), + ) + ) + + provider.online_write_batch( + config=store.config, + table=document_embeddings_fv, + data=data, + progress=None, + ) + documents_df = pd.DataFrame( + { + "item_id": [str(i) for i in range(n)], + "Embeddings": [ + np.random.random( + vector_length, + ) + for i in range(n) + ], + "event_timestamp": [datetime.utcnow() for _ in range(n)], + } + ) + + store.write_to_online_store( + feature_view_name="document_embeddings", + df=documents_df, + ) + + document_table = store._provider._online_store._conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' and name like '%_document_embeddings';" + ).fetchall() + assert len(document_table) == 1 + document_table_name = document_table[0][0] + record_count = len( + store._provider._online_store._conn.execute( + f"select * from {document_table_name}" + ).fetchall() + ) + assert record_count == len(data) + documents_df.shape[0] + + query_embedding = np.random.random( + vector_length, + ) + result = store.retrieve_online_documents( + feature="document_embeddings:Embeddings", query=query_embedding, top_k=3 + ).to_dict() + + assert "Embeddings" in result + assert "distance" in result + assert len(result["distance"]) == 3 + + +@pytest.mark.skipif( + sys.version_info[0:2] != (3, 10) or platform.system() != "Darwin", + reason="Only works on Python 3.10 and MacOS", +) +def test_sqlite_vec_import() -> None: + db = sqlite3.connect(":memory:") + db.enable_load_extension(True) + sqlite_vec.load(db) + + db.execute(""" + create virtual table vec_examples using vec0( + sample_embedding float[8] + ); + """) + + db.execute(""" + insert into vec_examples(rowid, sample_embedding) + values + (1, '[-0.200, 0.250, 0.341, -0.211, 0.645, 0.935, -0.316, -0.924]'), + (2, '[0.443, -0.501, 0.355, -0.771, 0.707, -0.708, -0.185, 0.362]'), + (3, '[0.716, -0.927, 0.134, 0.052, -0.669, 0.793, -0.634, -0.162]'), + (4, '[-0.710, 0.330, 0.656, 0.041, -0.990, 0.726, 0.385, -0.958]'); + """) + + sqlite_version, vec_version = db.execute( + "select sqlite_version(), vec_version()" + ).fetchone() + assert vec_version == "v0.0.1-alpha.10" + print(f"sqlite_version={sqlite_version}, vec_version={vec_version}") + + result = db.execute(""" + select + rowid, + distance + from vec_examples + where sample_embedding match '[0.890, 0.544, 0.825, 0.961, 0.358, 0.0196, 0.521, 0.175]' + order by distance + limit 2; + """).fetchall() + result = [(rowid, round(distance, 2)) for rowid, distance in result] + assert result == [(2, 2.39), (1, 2.39)] diff --git a/sdk/python/tests/unit/test_feature_validation.py b/sdk/python/tests/unit/test_feature_validation.py index b349eb8ea0b..5e8e11ab912 100644 --- a/sdk/python/tests/unit/test_feature_validation.py +++ b/sdk/python/tests/unit/test_feature_validation.py @@ -1,7 +1,7 @@ import pytest from feast.errors import FeatureNameCollisionError -from feast.feature_store import _validate_feature_refs +from feast.utils import _validate_feature_refs def test_feature_name_collision_on_historical_retrieval(): diff --git a/sdk/python/tests/unit/test_offline_server.py b/sdk/python/tests/unit/test_offline_server.py new file mode 100644 index 00000000000..5991e7450d1 --- /dev/null +++ b/sdk/python/tests/unit/test_offline_server.py @@ -0,0 +1,250 @@ +import os +import tempfile +from datetime import datetime, timedelta + +import assertpy +import pandas as pd +import pyarrow as pa +import pyarrow.flight as flight +import pytest + +from feast import FeatureStore +from feast.feature_logging import FeatureServiceLoggingSource +from feast.infra.offline_stores.remote import ( + RemoteOfflineStore, + RemoteOfflineStoreConfig, +) +from feast.offline_server import OfflineServer +from feast.repo_config import RepoConfig +from tests.utils.cli_repo_creator import CliRunner + +PROJECT_NAME = "test_remote_offline" + + +@pytest.fixture +def empty_offline_server(environment): + store = environment.feature_store + + location = "grpc+tcp://localhost:0" + return OfflineServer(store=store, location=location) + + +@pytest.fixture +def arrow_client(empty_offline_server): + return flight.FlightClient(f"grpc://localhost:{empty_offline_server.port}") + + +def test_offline_server_is_alive(environment, empty_offline_server, arrow_client): + server = empty_offline_server + client = arrow_client + + assertpy.assert_that(server).is_not_none + assertpy.assert_that(server.port).is_not_equal_to(0) + + actions = list(client.list_actions()) + flights = list(client.list_flights()) + + assertpy.assert_that(actions).is_equal_to( + [ + ( + "offline_write_batch", + "Writes the specified arrow table to the data source underlying the specified feature view.", + ), + ( + "write_logged_features", + "Writes logged features to a specified destination in the offline store.", + ), + ( + "persist", + "Synchronously executes the underlying query and persists the result in the same offline store at the " + "specified destination.", + ), + ] + ) + assertpy.assert_that(flights).is_empty() + + +def default_store(temp_dir): + runner = CliRunner() + result = runner.run(["init", PROJECT_NAME], cwd=temp_dir) + repo_path = os.path.join(temp_dir, PROJECT_NAME, "feature_repo") + assert result.returncode == 0 + + result = runner.run(["--chdir", repo_path, "apply"], cwd=temp_dir) + assert result.returncode == 0 + + fs = FeatureStore(repo_path=repo_path) + return fs + + +def remote_feature_store(offline_server): + offline_config = RemoteOfflineStoreConfig( + type="remote", host="0.0.0.0", port=offline_server.port + ) + + registry_path = os.path.join( + str(offline_server.store.repo_path), + offline_server.store.config.registry.path, + ) + store = FeatureStore( + config=RepoConfig( + project=PROJECT_NAME, + registry=registry_path, + provider="local", + offline_store=offline_config, + entity_key_serialization_version=2, + ) + ) + return store + + +def test_remote_offline_store_apis(): + with tempfile.TemporaryDirectory() as temp_dir: + store = default_store(str(temp_dir)) + location = "grpc+tcp://localhost:0" + server = OfflineServer(store=store, location=location) + + assertpy.assert_that(server).is_not_none + assertpy.assert_that(server.port).is_not_equal_to(0) + + fs = remote_feature_store(server) + + _test_get_historical_features_returns_data(fs) + _test_get_historical_features_returns_nan(fs) + _test_offline_write_batch(str(temp_dir), fs) + _test_write_logged_features(str(temp_dir), fs) + _test_pull_latest_from_table_or_query(str(temp_dir), fs) + _test_pull_all_from_table_or_query(str(temp_dir), fs) + + +def _test_get_historical_features_returns_data(fs: FeatureStore): + entity_df = pd.DataFrame.from_dict( + { + "driver_id": [1001, 1002, 1003], + "event_timestamp": [ + datetime(2021, 4, 12, 10, 59, 42), + datetime(2021, 4, 12, 8, 12, 10), + datetime(2021, 4, 12, 16, 40, 26), + ], + "label_driver_reported_satisfaction": [1, 5, 3], + "val_to_add": [1, 2, 3], + "val_to_add_2": [10, 20, 30], + } + ) + + features = [ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + "transformed_conv_rate:conv_rate_plus_val1", + "transformed_conv_rate:conv_rate_plus_val2", + ] + + training_df = fs.get_historical_features(entity_df, features).to_df() + + assertpy.assert_that(training_df).is_not_none() + assertpy.assert_that(len(training_df)).is_equal_to(3) + + for index, driver_id in enumerate(entity_df["driver_id"]): + assertpy.assert_that(training_df["driver_id"][index]).is_equal_to(driver_id) + for feature in features: + column_id = feature.split(":")[1] + value = training_df[column_id][index] + assertpy.assert_that(value).is_not_nan() + + +def _test_get_historical_features_returns_nan(fs: FeatureStore): + entity_df = pd.DataFrame.from_dict( + { + "driver_id": [1, 2, 3], + "event_timestamp": [ + datetime(2021, 4, 12, 10, 59, 42), + datetime(2021, 4, 12, 8, 12, 10), + datetime(2021, 4, 12, 16, 40, 26), + ], + "label_driver_reported_satisfaction": [1, 5, 3], + "val_to_add": [1, 2, 3], + "val_to_add_2": [10, 20, 30], + } + ) + + features = [ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + "transformed_conv_rate:conv_rate_plus_val1", + "transformed_conv_rate:conv_rate_plus_val2", + ] + + training_df = fs.get_historical_features(entity_df, features).to_df() + + assertpy.assert_that(training_df).is_not_none() + assertpy.assert_that(len(training_df)).is_equal_to(3) + + for index, driver_id in enumerate(entity_df["driver_id"]): + assertpy.assert_that(training_df["driver_id"][index]).is_equal_to(driver_id) + for feature in features: + column_id = feature.split(":")[1] + value = training_df[column_id][index] + assertpy.assert_that(value).is_nan() + + +def _test_offline_write_batch(temp_dir, fs: FeatureStore): + data_file = os.path.join( + temp_dir, fs.project, "feature_repo/data/driver_stats.parquet" + ) + data_df = pd.read_parquet(data_file) + feature_view = fs.get_feature_view("driver_hourly_stats") + + RemoteOfflineStore.offline_write_batch( + fs.config, feature_view, pa.Table.from_pandas(data_df), progress=None + ) + + +def _test_write_logged_features(temp_dir, fs: FeatureStore): + data_file = os.path.join( + temp_dir, fs.project, "feature_repo/data/driver_stats.parquet" + ) + data_df = pd.read_parquet(data_file) + feature_service = fs.get_feature_service("driver_activity_v1") + + RemoteOfflineStore.write_logged_features( + config=fs.config, + data=pa.Table.from_pandas(data_df), + source=FeatureServiceLoggingSource(feature_service, fs.config.project), + logging_config=feature_service.logging_config, + registry=fs.registry, + ) + + +def _test_pull_latest_from_table_or_query(temp_dir, fs: FeatureStore): + data_source = fs.get_data_source("driver_hourly_stats_source") + + end_date = datetime.now().replace(microsecond=0, second=0, minute=0) + start_date = end_date - timedelta(days=15) + RemoteOfflineStore.pull_latest_from_table_or_query( + config=fs.config, + data_source=data_source, + join_key_columns=[], + feature_name_columns=[], + timestamp_field="event_timestamp", + created_timestamp_column="created", + start_date=start_date, + end_date=end_date, + ).to_df() + + +def _test_pull_all_from_table_or_query(temp_dir, fs: FeatureStore): + data_source = fs.get_data_source("driver_hourly_stats_source") + + end_date = datetime.now().replace(microsecond=0, second=0, minute=0) + start_date = end_date - timedelta(days=15) + RemoteOfflineStore.pull_all_from_table_or_query( + config=fs.config, + data_source=data_source, + join_key_columns=[], + feature_name_columns=[], + timestamp_field="event_timestamp", + start_date=start_date, + end_date=end_date, + ).to_df() diff --git a/sdk/python/tests/unit/test_on_demand_python_transformation.py b/sdk/python/tests/unit/test_on_demand_python_transformation.py index ebe797ffdbf..72e9b53a101 100644 --- a/sdk/python/tests/unit/test_on_demand_python_transformation.py +++ b/sdk/python/tests/unit/test_on_demand_python_transformation.py @@ -159,6 +159,10 @@ def python_singleton_view(inputs: dict[str, Any]) -> dict[str, Any]: self.store.write_to_online_store( feature_view_name="driver_hourly_stats", df=driver_df ) + assert len(self.store.list_all_feature_views()) == 4 + assert len(self.store.list_feature_views()) == 1 + assert len(self.store.list_on_demand_feature_views()) == 3 + assert len(self.store.list_stream_feature_views()) == 0 def test_python_pandas_parity(self): entity_rows = [ diff --git a/sdk/python/tests/unit/test_registry_server.py b/sdk/python/tests/unit/test_registry_server.py deleted file mode 100644 index 734bbfe19b8..00000000000 --- a/sdk/python/tests/unit/test_registry_server.py +++ /dev/null @@ -1,60 +0,0 @@ -import assertpy -import grpc_testing -import pytest -from google.protobuf.empty_pb2 import Empty - -from feast import Entity, FeatureStore -from feast.protos.feast.registry import RegistryServer_pb2 -from feast.registry_server import RegistryServer - - -def call_registry_server(server, method: str, request=None): - service = RegistryServer_pb2.DESCRIPTOR.services_by_name["RegistryServer"] - rpc = server.invoke_unary_unary( - service.methods_by_name[method], (), request if request else Empty(), None - ) - - return rpc.termination() - - -@pytest.fixture -def registry_server(environment): - store: FeatureStore = environment.feature_store - - servicer = RegistryServer(store=store) - - return grpc_testing.server_from_dictionary( - {RegistryServer_pb2.DESCRIPTOR.services_by_name["RegistryServer"]: servicer}, - grpc_testing.strict_real_time(), - ) - - -def test_registry_server_get_entity(environment, registry_server): - store: FeatureStore = environment.feature_store - entity = Entity(name="driver", join_keys=["driver_id"]) - store.apply(entity) - - expected = store.get_entity(entity.name) - - get_entity_request = RegistryServer_pb2.GetEntityRequest( - name=entity.name, project=store.project, allow_cache=False - ) - response, trailing_metadata, code, details = call_registry_server( - registry_server, "GetEntity", get_entity_request - ) - response_entity = Entity.from_proto(response) - - assertpy.assert_that(response_entity).is_equal_to(expected) - - -def test_registry_server_proto(environment, registry_server): - store: FeatureStore = environment.feature_store - entity = Entity(name="driver", join_keys=["driver_id"]) - store.apply(entity) - - expected = store.registry.proto() - response, trailing_metadata, code, details = call_registry_server( - registry_server, "Proto" - ) - - assertpy.assert_that(response).is_equal_to(expected) diff --git a/sdk/python/tests/unit/test_type_map.py b/sdk/python/tests/unit/test_type_map.py index 87e5ef0548c..39e3e7dafa5 100644 --- a/sdk/python/tests/unit/test_type_map.py +++ b/sdk/python/tests/unit/test_type_map.py @@ -1,4 +1,5 @@ import numpy as np +import pandas as pd import pytest from feast.type_map import ( @@ -79,3 +80,10 @@ def test_python_values_to_proto_values_bytes_to_list(values, value_type, expecte def test_python_values_to_proto_values_bytes_to_list_not_supported(): with pytest.raises(TypeError): _ = python_values_to_proto_values([b"[]"], ValueType.BYTES_LIST) + + +def test_python_values_to_proto_values_int_list_with_null_not_supported(): + df = pd.DataFrame({"column": [1, 2, None]}) + arr = df["column"].to_numpy() + with pytest.raises(TypeError): + _ = python_values_to_proto_values(arr, ValueType.INT32_LIST) diff --git a/sdk/python/tests/unit/test_unit_feature_store.py b/sdk/python/tests/unit/test_unit_feature_store.py index 0c13dffa629..19a133564f2 100644 --- a/sdk/python/tests/unit/test_unit_feature_store.py +++ b/sdk/python/tests/unit/test_unit_feature_store.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Dict, List -from feast import FeatureStore +from feast import utils from feast.protos.feast.types.Value_pb2 import Value @@ -36,8 +36,7 @@ def test_get_unique_entities(): projection=MockFeatureViewProjection(join_key_map={}), ) - unique_entities, indexes = FeatureStore._get_unique_entities( - FeatureStore, + unique_entities, indexes = utils._get_unique_entities( table=fv, join_key_values=entity_values, entity_name_to_join_key_map=entity_name_to_join_key_map, diff --git a/sdk/python/tests/utils/e2e_test_validation.py b/sdk/python/tests/utils/e2e_test_validation.py index 985c1661d5a..885798db109 100644 --- a/sdk/python/tests/utils/e2e_test_validation.py +++ b/sdk/python/tests/utils/e2e_test_validation.py @@ -176,7 +176,6 @@ def make_feature_store_yaml( online_store: Optional[Union[str, Dict]], ): offline_store_config = offline_creator.create_offline_store_config() - online_store = online_store config = RepoConfig( registry=str(Path(repo_dir_name) / "registry.db"), diff --git a/sdk/python/tests/utils/http_server.py b/sdk/python/tests/utils/http_server.py index 47c6cb8ac17..5bb6255d72e 100644 --- a/sdk/python/tests/utils/http_server.py +++ b/sdk/python/tests/utils/http_server.py @@ -3,9 +3,9 @@ def free_port(): - sock = socket.socket() - sock.bind(("", 0)) - return sock.getsockname()[1] + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + sock.bind(("", 0)) + return sock.getsockname()[1] def check_port_open(host, port) -> bool: diff --git a/setup.py b/setup.py index cdab69b6848..9b3d0e55e62 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ "hiredis>=2.0.0,<3", ] -AWS_REQUIRED = ["boto3>=1.17.0,<2", "docker>=5.0.2", "fsspec<=2024.1.0"] +AWS_REQUIRED = ["boto3>=1.17.0,<2", "docker>=5.0.2", "fsspec<=2024.1.0", "aiobotocore>2,<3"] KUBERNETES_REQUIRED = ["kubernetes<=20.13.0"] @@ -96,6 +96,9 @@ "pyspark>=3.0.0,<4", ] +SQLITE_VEC_REQUIRED = [ + "sqlite-vec==v0.0.1-alpha.10", +] TRINO_REQUIRED = ["trino>=0.305.0,<0.400.0", "regex"] POSTGRES_REQUIRED = [ @@ -214,6 +217,7 @@ + DUCKDB_REQUIRED + DELTA_REQUIRED + ELASTICSEARCH_REQUIRED + + SQLITE_VEC_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -381,6 +385,7 @@ def run(self): "ikv": IKV_REQUIRED, "delta": DELTA_REQUIRED, "elasticsearch": ELASTICSEARCH_REQUIRED, + "sqlite_vec": SQLITE_VEC_REQUIRED, }, include_package_data=True, license="Apache", diff --git a/ui/package.json b/ui/package.json index ec00624a823..de37f4394a0 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.38.0", + "version": "0.39.0", "private": false, "files": [ "dist" diff --git a/ui/yarn.lock b/ui/yarn.lock index 9a4338a319b..89107de0b89 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -3604,11 +3604,11 @@ brace-expansion@^2.0.1: balanced-match "^1.0.0" braces@^3.0.1, braces@~3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" - integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== dependencies: - fill-range "^7.0.1" + fill-range "^7.1.1" broadcast-channel@^3.4.1: version "3.7.0" @@ -5616,10 +5616,10 @@ filesize@^8.0.6: resolved "https://registry.yarnpkg.com/filesize/-/filesize-8.0.7.tgz#695e70d80f4e47012c132d57a059e80c6b580bd8" integrity sha512-pjmC+bkIF8XI7fWaH8KxHcZL3DPybs1roSKP4rKDvy20tAWwIObE4+JIseG2byfGKhud5ZnM4YSGKBz7Sh0ndQ== -fill-range@^7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" - integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ== +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== dependencies: to-regex-range "^5.0.1"