diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 83c52e7dbfd..d362a6bb540 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -9,7 +9,11 @@ on: jobs: lint-java: - if: github.repository == 'feast-dev/feast' + # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. + if: + ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || + (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && + github.repository == 'feast-dev/feast' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -23,7 +27,11 @@ jobs: run: make lint-java unit-test-java: - if: github.repository == 'feast-dev/feast' + # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. + if: + ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || + (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && + github.repository == 'feast-dev/feast' runs-on: ubuntu-latest needs: lint-java steps: @@ -60,7 +68,11 @@ jobs: path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/ build-docker-image-java: - if: github.repository == 'feast-dev/feast' + # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. + if: + ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || + (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && + github.repository == 'feast-dev/feast' runs-on: ubuntu-latest strategy: matrix: @@ -91,10 +103,10 @@ jobs: run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} integration-test-java-pr: - # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. + # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. if: - ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) || - (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved')))) && + ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || + (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && github.repository == 'feast-dev/feast' runs-on: ubuntu-latest needs: unit-test-java diff --git a/.github/workflows/lint_pr.yml b/.github/workflows/lint_pr.yml index f9af8b27c71..d1aa7d16a3e 100644 --- a/.github/workflows/lint_pr.yml +++ b/.github/workflows/lint_pr.yml @@ -7,9 +7,14 @@ on: - edited - synchronize +permissions: + # read-only perms specified due to use of pull_request_target in lieu of security label check + pull-requests: read + jobs: validate-title: - if: github.repository == 'feast-dev/feast' + if: + github.repository == 'feast-dev/feast' name: Validate PR title runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 73344ec2ddd..2b0e6a1056b 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -14,7 +14,7 @@ on: jobs: build-docker-image: - # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. + # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. if: ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && @@ -76,7 +76,7 @@ jobs: outputs: DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: - # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. + # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. if: ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && @@ -86,7 +86,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.8" ] + python-version: [ "3.10" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} @@ -167,4 +167,4 @@ jobs: SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} - run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread \ No newline at end of file + run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 111a9b51a9c..17ff54b1f84 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -10,16 +10,16 @@ on: jobs: integration-test-python-local: - # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. + # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. if: ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || - (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) || - github.repository != 'feast-dev/feast' + (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && + github.repository == 'feast-dev/feast' runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: [ "3.8" ] + python-version: [ "3.10" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index f03cd33346c..7e2e3b577af 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -7,13 +7,11 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.8", "3.9", "3.10" ] + python-version: [ "3.9", "3.10" ] os: [ ubuntu-latest, macOS-latest ] exclude: - os: macOS-latest python-version: "3.9" - - os: macOS-latest - python-version: "3.10" env: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} diff --git a/Makefile b/Makefile index 4b85c0e4483..1a1d3b6e920 100644 --- a/Makefile +++ b/Makefile @@ -186,6 +186,19 @@ test-python-universal-athena: not test_snowflake" \ sdk/python/tests +test-python-universal-duckdb: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.duckdb_repo_configuration \ + FEAST_USAGE=False IS_TEST=True \ + python -m pytest -n 8 --integration \ + -k "not test_nullable_online_store and \ + not gcs_registry and \ + not s3_registry and \ + not test_snowflake and \ + not bigquery and \ + not test_spark_materialization_consistency" \ + sdk/python/tests + test-python-universal-postgres-offline: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.postgres_repo_configuration \ @@ -310,7 +323,7 @@ format-python: cd ${ROOT_DIR}/sdk/python; python -m black --target-version py38 feast tests lint-python: - cd ${ROOT_DIR}/sdk/python; python -m mypy + cd ${ROOT_DIR}/sdk/python; python -m mypy feast cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/ --check-only cd ${ROOT_DIR}/sdk/python; python -m flake8 feast/ tests/ cd ${ROOT_DIR}/sdk/python; python -m black --check feast tests diff --git a/community/maintainers.md b/community/maintainers.md index 8aca48fd0db..0b3d4ab6480 100644 --- a/community/maintainers.md +++ b/community/maintainers.md @@ -9,7 +9,12 @@ In alphabetical order | Name | GitHub Username | Email | Organization | | -------------- | ---------------- |-----------------------------| ------------------ | | Achal Shah | `achals` | achals@gmail.com | Tecton | -| Willem Pienaar | `woop` | will.pienaar@gmail.com | Tecton | +| Edson Tirelli | `etirelli` | ed.tirelli@gmail.com | Red Hat | +| Francisco Javier Arceo | `franciscojavierarceo` | arceofrancisco@gmail.com | Affirm | +| Hao Xu | `HaoXuAI` | sduxuhao@gmail.com | JPMorgan | +| Jeremy Ary | `jeremyary` | jeremy.ary@gmail.com | Red Hat | +| Shuchu Han | `shuchu` | shuchu.han@gmail.com | Independent | +| Willem Pienaar | `woop` | will.pienaar@gmail.com | Cleric | | Zhiling Chen | `zhilingc` | chnzhlng@gmail.com | GetGround | ## Emeritus Maintainers diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index c80ded2adf0..8affea898ef 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -99,6 +99,7 @@ * [MySQL (contrib)](reference/online-stores/mysql.md) * [Rockset (contrib)](reference/online-stores/rockset.md) * [Hazelcast (contrib)](reference/online-stores/hazelcast.md) + * [ScyllaDB (contrib)](reference/online-stores/scylladb.md) * [Providers](reference/providers/README.md) * [Local](reference/providers/local.md) * [Google Cloud Platform](reference/providers/google-cloud-platform.md) diff --git a/docs/getting-started/concepts/registry.md b/docs/getting-started/concepts/registry.md index f7d4a5b3e11..8ac32ce87b9 100644 --- a/docs/getting-started/concepts/registry.md +++ b/docs/getting-started/concepts/registry.md @@ -57,6 +57,9 @@ registry: registry_type: sql path: postgresql://postgres:mysecretpassword@127.0.0.1:55001/feast cache_ttl_seconds: 60 + sqlalchemy_config_kwargs: + echo: false + pool_pre_ping: true ``` This supports any SQLAlchemy compatible database as a backend. The exact schema can be seen in [sql.py](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/registry/sql.py) diff --git a/docs/project/development-guide.md b/docs/project/development-guide.md index 931d0243d2b..43dae1d678d 100644 --- a/docs/project/development-guide.md +++ b/docs/project/development-guide.md @@ -154,6 +154,16 @@ pip install -e ".[dev]" This will allow the installed feast version to automatically reflect changes to your local development version of Feast without needing to reinstall everytime you make code changes. +10. Compile the protubufs +```sh +make compile-protos-python +``` + +11. Spin up Docker Image +```sh +docker build -t docker-whale -f ./sdk/python/feast/infra/feature_servers/multicloud/Dockerfile . +``` + ### Code Style & Linting Feast Python SDK / CLI codebase: - Conforms to [Black code style](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html) diff --git a/docs/reference/alpha-web-ui.md b/docs/reference/alpha-web-ui.md index 7d21a3d45dd..398c8de0aec 100644 --- a/docs/reference/alpha-web-ui.md +++ b/docs/reference/alpha-web-ui.md @@ -85,6 +85,8 @@ When you start the React app, it will look for `project-list.json` to find a lis } ``` +* **Note** - `registryPath` only supports a file location or a url. + Then start the React App ```bash diff --git a/docs/reference/data-sources/overview.md b/docs/reference/data-sources/overview.md index 112d4168d30..302c19b049c 100644 --- a/docs/reference/data-sources/overview.md +++ b/docs/reference/data-sources/overview.md @@ -19,13 +19,13 @@ Details for each specific data source can be found [here](README.md). Below is a matrix indicating which data sources support which types. | | File | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | -| :-------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | -| `bytes` | yes | yes | yes | yes | yes | yes | yes | -| `string` | yes | yes | yes | yes | yes | yes | yes | -| `int32` | yes | yes | yes | yes | yes | yes | yes | -| `int64` | yes | yes | yes | yes | yes | yes | yes | -| `float32` | yes | yes | yes | yes | yes | yes | yes | -| `float64` | yes | yes | yes | yes | yes | yes | yes | -| `bool` | yes | yes | yes | yes | yes | yes | yes | -| `timestamp` | yes | yes | yes | yes | yes | yes | yes | -| array types | yes | yes | no | no | yes | yes | no | \ No newline at end of file +| :-------------------------------- | :-- | :-- |:----------| :-- | :-- | :-- | :-- | +| `bytes` | yes | yes | yes | yes | yes | yes | yes | +| `string` | yes | yes | yes | yes | yes | yes | yes | +| `int32` | yes | yes | yes | yes | yes | yes | yes | +| `int64` | yes | yes | yes | yes | yes | yes | yes | +| `float32` | yes | yes | yes | yes | yes | yes | yes | +| `float64` | yes | yes | yes | yes | yes | yes | yes | +| `bool` | yes | yes | yes | yes | yes | yes | yes | +| `timestamp` | yes | yes | yes | yes | yes | yes | yes | +| array types | yes | yes | yes | no | yes | yes | no | \ No newline at end of file diff --git a/docs/reference/data-sources/snowflake.md b/docs/reference/data-sources/snowflake.md index 82bf5cb4d49..98a56e09f87 100644 --- a/docs/reference/data-sources/snowflake.md +++ b/docs/reference/data-sources/snowflake.md @@ -46,5 +46,5 @@ The full set of configuration options is available [here](https://rtd.feast.dev/ ## Supported Types -Snowflake data sources support all eight primitive types, but currently do not support array types. +Snowflake data sources support all eight primitive types. Array types are also supported but not with type inference. For a comparison against other batch data sources, please see [here](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/spark.md b/docs/reference/offline-stores/spark.md index ae5ea78071e..3cca2aab1af 100644 --- a/docs/reference/offline-stores/spark.md +++ b/docs/reference/offline-stores/spark.md @@ -4,7 +4,7 @@ The Spark offline store provides support for reading [SparkSources](../data-sources/spark.md). -* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be converted to a Spark dataframe and processed as a temporary view. +* Entity dataframes can be provided as a SQL query, Pandas dataframe or can be provided as a Pyspark dataframe. A Pandas dataframes will be converted to a Spark dataframe and processed as a temporary view. ## Disclaimer diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index f86e6f6a1df..d90bfcf1632 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -54,4 +54,6 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli [hazelcast.md](hazelcast.md) {% endcontent-ref %} - +{% content-ref url="scylladb.md" %} +[scylladb.md](scylladb.md) +{% endcontent-ref %} diff --git a/docs/reference/online-stores/scylladb.md b/docs/reference/online-stores/scylladb.md new file mode 100644 index 00000000000..e28e810e214 --- /dev/null +++ b/docs/reference/online-stores/scylladb.md @@ -0,0 +1,94 @@ +# ScyllaDB Cloud online store + +## Description + +ScyllaDB is a low-latency and high-performance Cassandra-compatible (uses CQL) database. You can use the existing Cassandra connector to use ScyllaDB as an online store in Feast. + +The [ScyllaDB](https://www.scylladb.com/) online store provides support for materializing feature values into a ScyllaDB or [ScyllaDB Cloud](https://www.scylladb.com/product/scylla-cloud/) cluster for serving online features real-time. + +## Getting started + +Install Feast with Cassandra support: +```bash +pip install "feast[cassandra]" +``` + +Create a new Feast project: +```bash +feast init REPO_NAME -t cassandra +``` + +### Example (ScyllaDB) + +{% code title="feature_store.yaml" %} +```yaml +project: scylla_feature_repo +registry: data/registry.db +provider: local +online_store: + type: cassandra + hosts: + - 172.17.0.2 + keyspace: feast + username: scylla + password: password +``` +{% endcode %} + +### Example (ScyllaDB Cloud) + +{% code title="feature_store.yaml" %} +```yaml +project: scylla_feature_repo +registry: data/registry.db +provider: local +online_store: + type: cassandra + hosts: + - node-0.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud + - node-1.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud + - node-2.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud + keyspace: feast + username: scylla + password: password +``` +{% endcode %} + + +The full set of configuration options is available in [CassandraOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store.CassandraOnlineStoreConfig). +For a full explanation of configuration options please look at file +`sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/README.md`. + +Storage specifications can be found at `docs/specs/online_store_format.md`. + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Cassandra plugin. + +| | Cassandra | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | yes | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). + +## Resources + +* [Sample application with ScyllaDB](https://feature-store.scylladb.com/stable/) +* [ScyllaDB website](https://www.scylladb.com/) +* [ScyllaDB Cloud documentation](https://cloud.docs.scylladb.com/stable/) diff --git a/docs/tutorials/using-scalable-registry.md b/docs/tutorials/using-scalable-registry.md index a87aedd9b9f..30b8e01ed51 100644 --- a/docs/tutorials/using-scalable-registry.md +++ b/docs/tutorials/using-scalable-registry.md @@ -29,6 +29,9 @@ registry: registry_type: sql path: postgresql://postgres:mysecretpassword@127.0.0.1:55001/feast cache_ttl_seconds: 60 + sqlalchemy_config_kwargs: + echo: false + pool_pre_ping: true ``` Specifically, the registry_type needs to be set to sql in the registry config block. On doing so, the path should refer to the [Database URL](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) for the database to be used, as expected by SQLAlchemy. No other additional commands are currently needed to configure this registry. diff --git a/examples/quickstart/quickstart.ipynb b/examples/quickstart/quickstart.ipynb index f84457ac02d..9e9a0b27ca4 100644 --- a/examples/quickstart/quickstart.ipynb +++ b/examples/quickstart/quickstart.ipynb @@ -1065,7 +1065,7 @@ "\n", "- Read the [Concepts](https://docs.feast.dev/getting-started/concepts/) page to understand the Feast data model and architecture.\n", "- Check out our [Tutorials](https://docs.feast.dev/tutorials/tutorials-overview) section for more examples on how to use Feast.\n", - "- Follow our [Running Feast with Snowflake/GCP/AWS](https://docs.feast.dev/how-to-guides/feast-snowflake-gcp-aws) guide for a more in-depth tutorial on using Feast.\n", + "- Follow our [Running Feast with Snowflake/GCP/AWS](https://docs.feast.dev/how-to-guides/feast-snowflake-gcp-aws) guide for a more in-depth tutorial on using Feast.\n" ] } ], diff --git a/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh b/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh index d7dcd03c5fb..0690b734c38 100755 --- a/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh +++ b/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh @@ -4,8 +4,8 @@ set -e # feast root directory is expected to be mounted (eg, by docker compose) cd /mnt/feast -pip install -e '.[redis]' +pip install -e '.[grpcio,redis]' cd /app python materialize.py -feast serve_transformations --port 8080 \ No newline at end of file +feast serve_transformations --port 8080 diff --git a/protos/feast/core/OnDemandFeatureView.proto b/protos/feast/core/OnDemandFeatureView.proto index 50bf8b6f557..c43b33c1d28 100644 --- a/protos/feast/core/OnDemandFeatureView.proto +++ b/protos/feast/core/OnDemandFeatureView.proto @@ -48,7 +48,10 @@ message OnDemandFeatureViewSpec { // Map of sources for this feature view. map sources = 4; - UserDefinedFunction user_defined_function = 5; + oneof transformation { + UserDefinedFunction user_defined_function = 5; + OnDemandSubstraitTransformation on_demand_substrait_transformation = 9; + } // Description of the on demand feature view. string description = 6; @@ -87,3 +90,7 @@ message UserDefinedFunction { // The string representation of the udf string body_text = 3; } + +message OnDemandSubstraitTransformation { + bytes substrait_plan = 1; +} \ No newline at end of file diff --git a/protos/feast/registry/RegistryServer.proto b/protos/feast/registry/RegistryServer.proto new file mode 100644 index 00000000000..ab324f9bd1a --- /dev/null +++ b/protos/feast/registry/RegistryServer.proto @@ -0,0 +1,229 @@ +syntax = "proto3"; + +package feast.registry; + +import "google/protobuf/empty.proto"; +import "feast/core/Registry.proto"; +import "feast/core/Entity.proto"; +import "feast/core/DataSource.proto"; +import "feast/core/FeatureView.proto"; +import "feast/core/RequestFeatureView.proto"; +import "feast/core/StreamFeatureView.proto"; +import "feast/core/OnDemandFeatureView.proto"; +import "feast/core/FeatureService.proto"; +import "feast/core/SavedDataset.proto"; +import "feast/core/ValidationProfile.proto"; +import "feast/core/InfraObject.proto"; + +service RegistryServer{ + // Entity RPCs + rpc GetEntity (GetEntityRequest) returns (feast.core.Entity) {} + rpc ListEntities (ListEntitiesRequest) returns (ListEntitiesResponse) {} + + // DataSource RPCs + rpc GetDataSource (GetDataSourceRequest) returns (feast.core.DataSource) {} + rpc ListDataSources (ListDataSourcesRequest) returns (ListDataSourcesResponse) {} + + // FeatureView RPCs + rpc GetFeatureView (GetFeatureViewRequest) returns (feast.core.FeatureView) {} + rpc ListFeatureViews (ListFeatureViewsRequest) returns (ListFeatureViewsResponse) {} + + // RequestFeatureView RPCs + rpc GetRequestFeatureView (GetRequestFeatureViewRequest) returns (feast.core.RequestFeatureView) {} + rpc ListRequestFeatureViews (ListRequestFeatureViewsRequest) returns (ListRequestFeatureViewsResponse) {} + + // StreamFeatureView RPCs + rpc GetStreamFeatureView (GetStreamFeatureViewRequest) returns (feast.core.StreamFeatureView) {} + rpc ListStreamFeatureViews (ListStreamFeatureViewsRequest) returns (ListStreamFeatureViewsResponse) {} + + // OnDemandFeatureView RPCs + rpc GetOnDemandFeatureView (GetOnDemandFeatureViewRequest) returns (feast.core.OnDemandFeatureView) {} + rpc ListOnDemandFeatureViews (ListOnDemandFeatureViewsRequest) returns (ListOnDemandFeatureViewsResponse) {} + + // FeatureService RPCs + rpc GetFeatureService (GetFeatureServiceRequest) returns (feast.core.FeatureService) {} + rpc ListFeatureServices (ListFeatureServicesRequest) returns (ListFeatureServicesResponse) {} + + // SavedDataset RPCs + rpc GetSavedDataset (GetSavedDatasetRequest) returns (feast.core.SavedDataset) {} + rpc ListSavedDatasets (ListSavedDatasetsRequest) returns (ListSavedDatasetsResponse) {} + + // ValidationReference RPCs + rpc GetValidationReference (GetValidationReferenceRequest) returns (feast.core.ValidationReference) {} + rpc ListValidationReferences (ListValidationReferencesRequest) returns (ListValidationReferencesResponse) {} + + rpc ListProjectMetadata (ListProjectMetadataRequest) returns (ListProjectMetadataResponse) {} + rpc GetInfra (GetInfraRequest) returns (feast.core.Infra) {} + rpc Refresh (RefreshRequest) returns (google.protobuf.Empty) {} + rpc Proto (google.protobuf.Empty) returns (feast.core.Registry) {} + +} + +message RefreshRequest { + string project = 1; +} + +message GetInfraRequest { + string project = 1; + bool allow_cache = 2; +} + +message ListProjectMetadataRequest { + string project = 1; + bool allow_cache = 2; +} + +message ListProjectMetadataResponse { + repeated feast.core.ProjectMetadata project_metadata = 1; +} + +message GetEntityRequest { + string name = 1; + string project = 2; + bool allow_cache = 3; +} + +message ListEntitiesRequest { + string project = 1; + bool allow_cache = 2; +} + +message ListEntitiesResponse { + repeated feast.core.Entity entities = 1; +} + +// DataSources + +message GetDataSourceRequest { + string name = 1; + string project = 2; + bool allow_cache = 3; +} + +message ListDataSourcesRequest { + string project = 1; + bool allow_cache = 2; +} + +message ListDataSourcesResponse { + repeated feast.core.DataSource data_sources = 1; +} + +// FeatureViews + +message GetFeatureViewRequest { + string name = 1; + string project = 2; + bool allow_cache = 3; +} + +message ListFeatureViewsRequest { + string project = 1; + bool allow_cache = 2; +} + +message ListFeatureViewsResponse { + repeated feast.core.FeatureView feature_views = 1; +} + +// RequestFeatureView + +message GetRequestFeatureViewRequest { + string name = 1; + string project = 2; + bool allow_cache = 3; +} + +message ListRequestFeatureViewsRequest { + string project = 1; + bool allow_cache = 2; +} + +message ListRequestFeatureViewsResponse { + repeated feast.core.RequestFeatureView request_feature_views = 1; +} + +// StreamFeatureView + +message GetStreamFeatureViewRequest { + string name = 1; + string project = 2; + bool allow_cache = 3; +} + +message ListStreamFeatureViewsRequest { + string project = 1; + bool allow_cache = 2; +} + +message ListStreamFeatureViewsResponse { + repeated feast.core.StreamFeatureView stream_feature_views = 1; +} + +// OnDemandFeatureView + +message GetOnDemandFeatureViewRequest { + string name = 1; + string project = 2; + bool allow_cache = 3; +} + +message ListOnDemandFeatureViewsRequest { + string project = 1; + bool allow_cache = 2; +} + +message ListOnDemandFeatureViewsResponse { + repeated feast.core.OnDemandFeatureView on_demand_feature_views = 1; +} + +// FeatureServices + +message GetFeatureServiceRequest { + string name = 1; + string project = 2; + bool allow_cache = 3; +} + +message ListFeatureServicesRequest { + string project = 1; + bool allow_cache = 2; +} + +message ListFeatureServicesResponse { + repeated feast.core.FeatureService feature_services = 1; +} + +// SavedDataset + +message GetSavedDatasetRequest { + string name = 1; + string project = 2; + bool allow_cache = 3; +} + +message ListSavedDatasetsRequest { + string project = 1; + bool allow_cache = 2; +} + +message ListSavedDatasetsResponse { + repeated feast.core.SavedDataset saved_datasets = 1; +} + +// ValidationReference + +message GetValidationReferenceRequest { + string name = 1; + string project = 2; + bool allow_cache = 3; +} + +message ListValidationReferencesRequest { + string project = 1; + bool allow_cache = 2; +} + +message ListValidationReferencesResponse { + repeated feast.core.ValidationReference validation_references = 1; +} diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 2eb2c27bcb7..7ce8aaef2bc 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -25,7 +25,10 @@ from pygments import formatters, highlight, lexers from feast import utils -from feast.constants import DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT +from feast.constants import ( + DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT, + DEFAULT_REGISTRY_SERVER_PORT, +) from feast.errors import FeastObjectNotFoundException, FeastProviderLoginError from feast.feature_view import FeatureView from feast.infra.contrib.grpc_server import get_grpc_server @@ -73,6 +76,7 @@ def format_options(self, ctx: click.Context, formatter: click.HelpFormatter): ) @click.option( "--feature-store-yaml", + "-f", help="Override the directory where the CLI should look for the feature_store.yaml file.", ) @click.pass_context @@ -753,6 +757,22 @@ def serve_transformations_command(ctx: click.Context, port: int): store.serve_transformations(port) +@cli.command("serve_registry") +@click.option( + "--port", + "-p", + type=click.INT, + default=DEFAULT_REGISTRY_SERVER_PORT, + help="Specify a port for the server", +) +@click.pass_context +def serve_registry_command(ctx: click.Context, port: int): + """Start a registry server locally on a given port.""" + store = create_feature_store(ctx) + + store.serve_registry(port) + + @cli.command("validate") @click.option( "--feature-service", diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 574d79f4167..c022ecba557 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -44,5 +44,8 @@ # Default FTS port DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT = 6569 +# Default registry server port +DEFAULT_REGISTRY_SERVER_PORT = 6570 + # Environment variable for feature server docker image tag DOCKER_IMAGE_TAG_ENV_NAME: str = "FEAST_SERVER_DOCKER_IMAGE_TAG" diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index b7ce19aad9b..3421fd5d309 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import enum import warnings from abc import ABC, abstractmethod @@ -485,12 +484,12 @@ def to_proto(self) -> DataSourceProto: return data_source_proto def validate(self, config: RepoConfig): - pass + raise NotImplementedError def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: - pass + raise NotImplementedError @staticmethod def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: @@ -534,12 +533,12 @@ def __init__( self.schema = schema def validate(self, config: RepoConfig): - pass + raise NotImplementedError def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: - pass + raise NotImplementedError def __eq__(self, other): if not isinstance(other, RequestSource): @@ -610,12 +609,12 @@ def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: @typechecked class KinesisSource(DataSource): def validate(self, config: RepoConfig): - pass + raise NotImplementedError def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: - pass + raise NotImplementedError @staticmethod def from_proto(data_source: DataSourceProto): @@ -639,7 +638,7 @@ def from_proto(data_source: DataSourceProto): @staticmethod def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: - pass + raise NotImplementedError def get_table_query_string(self) -> str: raise NotImplementedError @@ -772,12 +771,12 @@ def __hash__(self): return super().__hash__() def validate(self, config: RepoConfig): - pass + raise NotImplementedError def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: - pass + raise NotImplementedError @staticmethod def from_proto(data_source: DataSourceProto): diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 9097e40c94f..b7151ff0c87 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -415,3 +415,8 @@ def __init__(self): class PushSourceNotFoundException(Exception): def __init__(self, push_source_name: str): super().__init__(f"Unable to find push source '{push_source_name}'.") + + +class ReadOnlyRegistryException(Exception): + def __init__(self): + super().__init__("Registry implementation is read-only.") diff --git a/sdk/python/feast/feature_service.py b/sdk/python/feast/feature_service.py index c3037a55da2..7ec923205a3 100644 --- a/sdk/python/feast/feature_service.py +++ b/sdk/python/feast/feature_service.py @@ -56,7 +56,7 @@ def __init__( *, name: str, features: List[Union[FeatureView, OnDemandFeatureView]], - tags: Dict[str, str] = None, + tags: Optional[Dict[str, str]] = None, description: str = "", owner: str = "", logging_config: Optional[LoggingConfig] = None, diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index d3f98f80323..e38120c33d3 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -164,6 +164,10 @@ def __init__( self._registry = SnowflakeRegistry( registry_config, self.config.project, None ) + elif registry_config and registry_config.registry_type == "remote": + from feast.infra.registry.remote import RemoteRegistry + + self._registry = RemoteRegistry(registry_config, self.config.project, None) else: r = Registry(self.config.project, registry_config, repo_path=self.repo_path) r._initialize_registry(self.config.project) @@ -2278,6 +2282,13 @@ def serve_ui( root_path=root_path, ) + @log_exceptions_and_usage + def serve_registry(self, port: int) -> None: + """Start registry server locally on a given port.""" + from feast import registry_server + + registry_server.start_server(self, port) + @log_exceptions_and_usage def serve_transformations(self, port: int) -> None: """Start the feature transformation server locally on a given port.""" diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 67f9662d317..f87ae7ab132 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -101,7 +101,7 @@ def __init__( name: str, source: DataSource, schema: Optional[List[Field]] = None, - entities: List[Entity] = None, + entities: Optional[List[Entity]] = None, ttl: Optional[timedelta] = timedelta(days=0), online: bool = True, description: str = "", diff --git a/sdk/python/feast/importer.py b/sdk/python/feast/importer.py index bbd592101a6..938d29fe313 100644 --- a/sdk/python/feast/importer.py +++ b/sdk/python/feast/importer.py @@ -7,7 +7,7 @@ ) -def import_class(module_name: str, class_name: str, class_type: str = None): +def import_class(module_name: str, class_name: str, class_type: str = ""): """ Dynamically loads and returns a class from a module. diff --git a/sdk/python/feast/infra/contrib/spark_kafka_processor.py b/sdk/python/feast/infra/contrib/spark_kafka_processor.py index ea55d89988a..fc4a34f17bd 100644 --- a/sdk/python/feast/infra/contrib/spark_kafka_processor.py +++ b/sdk/python/feast/infra/contrib/spark_kafka_processor.py @@ -1,10 +1,11 @@ from types import MethodType -from typing import List, Optional +from typing import List, Optional, no_type_check import pandas as pd from pyspark.sql import DataFrame, SparkSession from pyspark.sql.avro.functions import from_avro from pyspark.sql.functions import col, from_json +from pyspark.sql.streaming import StreamingQuery from feast.data_format import AvroFormat, JsonFormat from feast.data_source import KafkaSource, PushMode @@ -63,12 +64,20 @@ def __init__( self.join_keys = [fs.get_entity(entity).join_key for entity in sfv.entities] super().__init__(fs=fs, sfv=sfv, data_source=sfv.stream_source) - def ingest_stream_feature_view(self, to: PushMode = PushMode.ONLINE) -> None: + # Type hinting for data_source type. + # data_source type has been checked to be an instance of KafkaSource. + self.data_source: KafkaSource = self.data_source # type: ignore + + def ingest_stream_feature_view( + self, to: PushMode = PushMode.ONLINE + ) -> StreamingQuery: ingested_stream_df = self._ingest_stream_data() transformed_df = self._construct_transformation_plan(ingested_stream_df) online_store_query = self._write_stream_data(transformed_df, to) return online_store_query + # In the line 64 of __init__(), the "data_source" is assigned a stream_source (and has to be KafkaSource as in line 40). + @no_type_check def _ingest_stream_data(self) -> StreamTable: """Only supports json and avro formats currently.""" if self.format == "json": @@ -122,7 +131,7 @@ def _ingest_stream_data(self) -> StreamTable: def _construct_transformation_plan(self, df: StreamTable) -> StreamTable: return self.sfv.udf.__call__(df) if self.sfv.udf else df - def _write_stream_data(self, df: StreamTable, to: PushMode): + def _write_stream_data(self, df: StreamTable, to: PushMode) -> StreamingQuery: # Validation occurs at the fs.write_to_online_store() phase against the stream feature view schema. def batch_write(row: DataFrame, batch_id: int): rows: pd.DataFrame = row.toPandas() diff --git a/sdk/python/feast/infra/contrib/stream_processor.py b/sdk/python/feast/infra/contrib/stream_processor.py index 24817c82eaa..3f1fe085109 100644 --- a/sdk/python/feast/infra/contrib/stream_processor.py +++ b/sdk/python/feast/infra/contrib/stream_processor.py @@ -1,8 +1,9 @@ -from abc import ABC +from abc import ABC, abstractmethod from types import MethodType -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Optional from pyspark.sql import DataFrame +from typing_extensions import TypeAlias from feast.data_source import DataSource, PushMode from feast.importer import import_class @@ -17,7 +18,7 @@ } # TODO: support more types other than just Spark. -StreamTable = DataFrame +StreamTable: TypeAlias = DataFrame class ProcessorConfig(FeastConfigBaseModel): @@ -49,33 +50,39 @@ def __init__( self.sfv = sfv self.data_source = data_source - def ingest_stream_feature_view(self, to: PushMode = PushMode.ONLINE) -> None: + @abstractmethod + def ingest_stream_feature_view( + self, to: PushMode = PushMode.ONLINE + ) -> Optional[Any]: """ Ingests data from the stream source attached to the stream feature view; transforms the data and then persists it to the online store and/or offline store, depending on the 'to' parameter. """ - pass + raise NotImplementedError + @abstractmethod def _ingest_stream_data(self) -> StreamTable: """ Ingests data into a StreamTable. """ - pass + raise NotImplementedError + @abstractmethod def _construct_transformation_plan(self, table: StreamTable) -> StreamTable: """ Applies transformations on top of StreamTable object. Since stream engines use lazy evaluation, the StreamTable will not be materialized until it is actually evaluated. For example: df.collect() in spark or tbl.execute() in Flink. """ - pass + raise NotImplementedError - def _write_stream_data(self, table: StreamTable, to: PushMode) -> None: + @abstractmethod + def _write_stream_data(self, table: StreamTable, to: PushMode) -> Optional[Any]: """ Launches a job to persist stream data to the online store and/or offline store, depending on the 'to' parameter, and returns a handle for the job. """ - pass + raise NotImplementedError def get_stream_processor_object( diff --git a/sdk/python/feast/infra/feature_servers/aws_lambda/config.py b/sdk/python/feast/infra/feature_servers/aws_lambda/config.py index 31dd879af6d..946831a18fb 100644 --- a/sdk/python/feast/infra/feature_servers/aws_lambda/config.py +++ b/sdk/python/feast/infra/feature_servers/aws_lambda/config.py @@ -1,5 +1,6 @@ +from typing import Literal + from pydantic import StrictBool, StrictStr -from pydantic.typing import Literal from feast.infra.feature_servers.base_config import BaseFeatureServerConfig diff --git a/sdk/python/feast/infra/feature_servers/base_config.py b/sdk/python/feast/infra/feature_servers/base_config.py index 756dd79b438..1a348032e17 100644 --- a/sdk/python/feast/infra/feature_servers/base_config.py +++ b/sdk/python/feast/infra/feature_servers/base_config.py @@ -30,5 +30,5 @@ class BaseFeatureServerConfig(FeastConfigBaseModel): enabled: StrictBool = False """Whether the feature server should be launched.""" - feature_logging: Optional[FeatureLoggingConfig] + feature_logging: Optional[FeatureLoggingConfig] = None """ Feature logging configuration """ diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py index 8d0c269cf5d..ddcbde7924a 100644 --- a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py +++ b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py @@ -1,5 +1,6 @@ +from typing import Literal + from pydantic import StrictBool -from pydantic.typing import Literal from feast.infra.feature_servers.base_config import BaseFeatureServerConfig diff --git a/sdk/python/feast/infra/feature_servers/local_process/config.py b/sdk/python/feast/infra/feature_servers/local_process/config.py index bb2e7bdf738..3d97912e4bd 100644 --- a/sdk/python/feast/infra/feature_servers/local_process/config.py +++ b/sdk/python/feast/infra/feature_servers/local_process/config.py @@ -1,4 +1,4 @@ -from pydantic.typing import Literal +from typing import Literal from feast.infra.feature_servers.base_config import BaseFeatureServerConfig diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index c95c515fb4b..4527c5b1566 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -1,15 +1,17 @@ FROM python:3.8 +# Input the feast version to install +# This requires feast package to be available in pypi before building this image +ARG VERSION + RUN apt update && \ apt install -y \ jq \ python3-dev \ - default-libmysqlclient-dev \ build-essential RUN pip install pip --upgrade -RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres]" - +RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres]==${VERSION}" RUN apt update RUN apt install -y -V ca-certificates lsb-release wget diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev index ecbc199a5b9..015e3c7ee82 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -1,20 +1,23 @@ FROM python:3.8 +# Input the feast version to install +# This requires feast package to be available in pypi before building this image +ARG VERSION + RUN apt update && \ apt install -y \ jq \ python3-dev \ - default-libmysqlclient-dev \ build-essential RUN pip install pip --upgrade COPY . . -RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres]" +RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres]==${VERSION}" RUN apt update RUN apt install -y -V ca-certificates lsb-release wget RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb RUN apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb RUN apt update -RUN apt -y install libarrow-dev \ No newline at end of file +RUN apt -y install libarrow-dev diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile b/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile index a26661ead35..a7d0af9b416 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile @@ -25,5 +25,5 @@ COPY README.md README.md # git dir to infer the version of feast we're installing. # https://github.com/pypa/setuptools_scm#usage-from-docker # I think it also assumes that this dockerfile is being built from the root of the directory. -RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir '.[aws,gcp,bytewax,snowflake]' +RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir '.[aws,gcp,bytewax,snowflake,postgres]' diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py index 5c7a719532d..060a47ce585 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py @@ -421,7 +421,7 @@ def _create_job_definition(self, job_id, namespace, pods, env, index_offset=0): } ], "image": "busybox", - "imagePullPolicy": "Always", + "imagePullPolicy": "IfNotPresent", "name": "init-hostfile", "resources": {}, "securityContext": { diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py b/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py index 9d9b328c0e9..bbc32cc1651 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py @@ -12,10 +12,10 @@ logging.basicConfig(level=logging.INFO) with open("/var/feast/feature_store.yaml") as f: - feast_config = yaml.safe_load(f) + feast_config = yaml.load(f, Loader=yaml.Loader) with open("/var/feast/bytewax_materialization_config.yaml") as b: - bytewax_config = yaml.safe_load(b) + bytewax_config = yaml.load(b, Loader=yaml.Loader) config = RepoConfig(**feast_config) store = FeatureStore(config=config) diff --git a/sdk/python/feast/infra/materialization/contrib/spark/spark_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/spark/spark_materialization_engine.py index ed4388aeb31..798d3a8e6f5 100644 --- a/sdk/python/feast/infra/materialization/contrib/spark/spark_materialization_engine.py +++ b/sdk/python/feast/infra/materialization/contrib/spark/spark_materialization_engine.py @@ -3,6 +3,7 @@ from typing import Callable, List, Literal, Optional, Sequence, Union, cast import dill +import pandas import pandas as pd import pyarrow from tqdm import tqdm @@ -178,9 +179,9 @@ def _materialize_one( self.repo_config.batch_engine.partitions ) - spark_df.foreachPartition( - lambda x: _process_by_partition(x, spark_serialized_artifacts) - ) + spark_df.mapInPandas( + lambda x: _map_by_partition(x, spark_serialized_artifacts), "status int" + ).count() # dummy action to force evaluation return SparkMaterializationJob( job_id=job_id, status=MaterializationJobStatus.SUCCEEDED @@ -225,38 +226,40 @@ def unserialize(self): return feature_view, online_store, repo_config -def _process_by_partition(rows, spark_serialized_artifacts: _SparkSerializedArtifacts): - """Load pandas df to online store""" - - # convert to pyarrow table - dicts = [] - for row in rows: - dicts.append(row.asDict()) +def _map_by_partition(iterator, spark_serialized_artifacts: _SparkSerializedArtifacts): + for pdf in iterator: + if pdf.shape[0] == 0: + print("Skipping") + return - df = pd.DataFrame.from_records(dicts) - if df.shape[0] == 0: - print("Skipping") - return + table = pyarrow.Table.from_pandas(pdf) - table = pyarrow.Table.from_pandas(df) + ( + feature_view, + online_store, + repo_config, + ) = spark_serialized_artifacts.unserialize() + + if feature_view.batch_source.field_mapping is not None: + table = _run_pyarrow_field_mapping( + table, feature_view.batch_source.field_mapping + ) - # unserialize artifacts - feature_view, online_store, repo_config = spark_serialized_artifacts.unserialize() + join_key_to_value_type = { + entity.name: entity.dtype.to_value_type() + for entity in feature_view.entity_columns + } - if feature_view.batch_source.field_mapping is not None: - table = _run_pyarrow_field_mapping( - table, feature_view.batch_source.field_mapping + rows_to_write = _convert_arrow_to_proto( + table, feature_view, join_key_to_value_type + ) + online_store.online_write_batch( + repo_config, + feature_view, + rows_to_write, + lambda x: None, ) - join_key_to_value_type = { - entity.name: entity.dtype.to_value_type() - for entity in feature_view.entity_columns - } - - rows_to_write = _convert_arrow_to_proto(table, feature_view, join_key_to_value_type) - online_store.online_write_batch( - repo_config, - feature_view, - rows_to_write, - lambda x: None, - ) + yield pd.DataFrame( + [pd.Series(range(1, 2))] + ) # dummy result because mapInPandas needs to return something diff --git a/sdk/python/feast/infra/materialization/snowflake_engine.py b/sdk/python/feast/infra/materialization/snowflake_engine.py index 36c42cd390c..28bec198a5d 100644 --- a/sdk/python/feast/infra/materialization/snowflake_engine.py +++ b/sdk/python/feast/infra/materialization/snowflake_engine.py @@ -7,14 +7,14 @@ import click import pandas as pd from colorama import Fore, Style -from pydantic import Field, StrictStr +from pydantic import ConfigDict, Field, StrictStr from pytz import utc from tqdm import tqdm import feast from feast.batch_feature_view import BatchFeatureView from feast.entity import Entity -from feast.feature_view import FeatureView +from feast.feature_view import DUMMY_ENTITY_ID, FeatureView from feast.infra.materialization.batch_materialization_engine import ( BatchMaterializationEngine, MaterializationJob, @@ -72,9 +72,7 @@ class SnowflakeMaterializationEngineConfig(FeastConfigBaseModel): schema_: Optional[str] = Field("PUBLIC", alias="schema") """ Snowflake schema name """ - - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) @dataclass @@ -276,7 +274,11 @@ def _materialize_one( fv_latest_values_sql = offline_job.to_sql() - if feature_view.entity_columns: + if ( + feature_view.entity_columns[0].name == DUMMY_ENTITY_ID + ): # entityless Feature View's placeholder entity + entities_to_write = 1 + else: join_keys = [entity.name for entity in feature_view.entity_columns] unique_entities = '"' + '", "'.join(join_keys) + '"' @@ -289,10 +291,6 @@ def _materialize_one( with GetSnowflakeConnection(self.repo_config.offline_store) as conn: entities_to_write = conn.cursor().execute(query).fetchall()[0][0] - else: - entities_to_write = ( - 1 # entityless feature view has a placeholder entity - ) if feature_view.batch_source.field_mapping is not None: fv_latest_mapped_values_sql = _run_snowflake_field_mapping( diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 0ee82a908ed..68420c06642 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -10,6 +10,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Union, @@ -19,8 +20,7 @@ import pandas as pd import pyarrow import pyarrow.parquet -from pydantic import ConstrainedStr, StrictStr, validator -from pydantic.typing import Literal +from pydantic import StrictStr, field_validator from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed from feast import flags_helper @@ -72,13 +72,6 @@ def get_http_client_info(): return http_client_info.ClientInfo(user_agent=get_user_agent()) -class BigQueryTableCreateDisposition(ConstrainedStr): - """Custom constraint for table_create_disposition. To understand more, see: - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_disposition""" - - values = {"CREATE_NEVER", "CREATE_IF_NEEDED"} - - class BigQueryOfflineStoreConfig(FeastConfigBaseModel): """Offline store config for GCP BigQuery""" @@ -102,10 +95,15 @@ class BigQueryOfflineStoreConfig(FeastConfigBaseModel): gcs_staging_location: Optional[str] = None """ (optional) GCS location used for offloading BigQuery results as parquet files.""" - table_create_disposition: Optional[BigQueryTableCreateDisposition] = None - """ (optional) Specifies whether the job is allowed to create new tables. The default value is CREATE_IF_NEEDED.""" + table_create_disposition: Literal[ + "CREATE_NEVER", "CREATE_IF_NEEDED" + ] = "CREATE_IF_NEEDED" + """ (optional) Specifies whether the job is allowed to create new tables. The default value is CREATE_IF_NEEDED. + Custom constraint for table_create_disposition. To understand more, see: + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_disposition + """ - @validator("billing_project_id") + @field_validator("billing_project_id") def project_id_exists(cls, v, values, **kwargs): if v and not values["project_id"]: raise ValueError( diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py index 85a61106aaf..ae510171db9 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena.py @@ -8,6 +8,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Union, @@ -18,7 +19,6 @@ import pyarrow import pyarrow as pa from pydantic import StrictStr -from pydantic.typing import Literal from pytz import utc from feast import OnDemandFeatureView diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena_source.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena_source.py index 8e9e3893f3a..0aca42cd682 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/athena_source.py @@ -297,9 +297,9 @@ class SavedDatasetAthenaStorage(SavedDatasetStorage): def __init__( self, table_ref: str, - query: str = None, - database: str = None, - data_source: str = None, + query: Optional[str] = None, + database: Optional[str] = None, + data_source: Optional[str] = None, ): self.athena_options = AthenaOptions( table=table_ref, query=query, database=database, data_source=data_source diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py index 384ab69e81f..6b2238830b6 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py @@ -48,10 +48,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: table_name = destination_name diff --git a/sdk/python/feast/infra/offline_stores/contrib/duckdb_offline_store/__init__.py b/sdk/python/feast/infra/offline_stores/contrib/duckdb_offline_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/offline_stores/contrib/duckdb_offline_store/duckdb.py b/sdk/python/feast/infra/offline_stores/contrib/duckdb_offline_store/duckdb.py new file mode 100644 index 00000000000..f927f2ff928 --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/contrib/duckdb_offline_store/duckdb.py @@ -0,0 +1,17 @@ +import ibis +from pydantic import StrictStr + +from feast.infra.offline_stores.contrib.ibis_offline_store.ibis import IbisOfflineStore +from feast.repo_config import FeastConfigBaseModel + + +class DuckDBOfflineStoreConfig(FeastConfigBaseModel): + type: StrictStr = "duckdb" + # """ Offline store type selector""" + + +class DuckDBOfflineStore(IbisOfflineStore): + @staticmethod + def setup_ibis_backend(): + # there's no need to call setup as duckdb is default ibis backend + ibis.set_backend("duckdb") diff --git a/sdk/python/feast/infra/offline_stores/contrib/duckdb_repo_configuration.py b/sdk/python/feast/infra/offline_stores/contrib/duckdb_repo_configuration.py new file mode 100644 index 00000000000..263ae97466f --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/contrib/duckdb_repo_configuration.py @@ -0,0 +1,19 @@ +from feast.infra.offline_stores.contrib.duckdb_offline_store.duckdb import ( + DuckDBOfflineStoreConfig, +) +from tests.integration.feature_repos.universal.data_sources.file import ( # noqa: E402 + FileDataSourceCreator, +) + + +class DuckDBDataSourceCreator(FileDataSourceCreator): + def create_offline_store_config(self): + self.duckdb_offline_store_config = DuckDBOfflineStoreConfig() + return self.duckdb_offline_store_config + + +AVAILABLE_OFFLINE_STORES = [ + ("local", DuckDBDataSourceCreator), +] + +AVAILABLE_ONLINE_STORES = {"sqlite": ({"type": "sqlite"}, None)} diff --git a/sdk/python/feast/infra/offline_stores/contrib/ibis_offline_store/__init__.py b/sdk/python/feast/infra/offline_stores/contrib/ibis_offline_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/offline_stores/contrib/ibis_offline_store/ibis.py b/sdk/python/feast/infra/offline_stores/contrib/ibis_offline_store/ibis.py new file mode 100644 index 00000000000..72e0d970c60 --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/contrib/ibis_offline_store/ibis.py @@ -0,0 +1,407 @@ +import os +import uuid +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import ibis +import ibis.selectors as s +import numpy as np +import pandas as pd +import pyarrow +from ibis.expr import datatypes as dt +from ibis.expr.types import Table +from pytz import utc + +from feast.data_source import DataSource +from feast.errors import SavedDatasetLocationAlreadyExists +from feast.feature_logging import LoggingConfig, LoggingSource +from feast.feature_view import FeatureView +from feast.infra.offline_stores import offline_utils +from feast.infra.offline_stores.file_source import ( + FileLoggingDestination, + FileSource, + SavedDatasetFileStorage, +) +from feast.infra.offline_stores.offline_store import ( + OfflineStore, + RetrievalJob, + RetrievalMetadata, +) +from feast.infra.offline_stores.offline_utils import ( + get_pyarrow_schema_from_batch_source, +) +from feast.infra.registry.base_registry import BaseRegistry +from feast.on_demand_feature_view import OnDemandFeatureView +from feast.repo_config import RepoConfig +from feast.saved_dataset import SavedDatasetStorage + + +def _get_entity_schema(entity_df: pd.DataFrame) -> Dict[str, np.dtype]: + return dict(zip(entity_df.columns, entity_df.dtypes)) + + +class IbisOfflineStore(OfflineStore): + @staticmethod + def pull_latest_from_table_or_query( + config: RepoConfig, + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + timestamp_field: str, + created_timestamp_column: Optional[str], + start_date: datetime, + end_date: datetime, + ) -> RetrievalJob: + raise NotImplementedError() + + def _get_entity_df_event_timestamp_range( + entity_df: pd.DataFrame, entity_df_event_timestamp_col: str + ) -> Tuple[datetime, datetime]: + entity_df_event_timestamp = entity_df.loc[ + :, entity_df_event_timestamp_col + ].infer_objects() + if pd.api.types.is_string_dtype(entity_df_event_timestamp): + entity_df_event_timestamp = pd.to_datetime( + entity_df_event_timestamp, utc=True + ) + entity_df_event_timestamp_range = ( + entity_df_event_timestamp.min().to_pydatetime(), + entity_df_event_timestamp.max().to_pydatetime(), + ) + + return entity_df_event_timestamp_range + + @staticmethod + def _get_historical_features_one( + feature_view: FeatureView, + entity_table: Table, + feature_refs: List[str], + full_feature_names: bool, + timestamp_range: Tuple, + acc_table: Table, + event_timestamp_col: str, + ) -> Table: + fv_table: Table = ibis.read_parquet(feature_view.batch_source.name) + + for old_name, new_name in feature_view.batch_source.field_mapping.items(): + if old_name in fv_table.columns: + fv_table = fv_table.rename({new_name: old_name}) + + timestamp_field = feature_view.batch_source.timestamp_field + + # TODO mutate only if tz-naive + fv_table = fv_table.mutate( + **{ + timestamp_field: fv_table[timestamp_field].cast( + dt.Timestamp(timezone="UTC") + ) + } + ) + + full_name_prefix = feature_view.projection.name_alias or feature_view.name + + feature_refs = [ + fr.split(":")[1] + for fr in feature_refs + if fr.startswith(f"{full_name_prefix}:") + ] + + timestamp_range_start_minus_ttl = ( + timestamp_range[0] - feature_view.ttl + if feature_view.ttl and feature_view.ttl > timedelta(0, 0, 0, 0, 0, 0, 0) + else timestamp_range[0] + ) + + timestamp_range_start_minus_ttl = ibis.literal( + timestamp_range_start_minus_ttl.strftime("%Y-%m-%d %H:%M:%S.%f") + ).cast(dt.Timestamp(timezone="UTC")) + + timestamp_range_end = ibis.literal( + timestamp_range[1].strftime("%Y-%m-%d %H:%M:%S.%f") + ).cast(dt.Timestamp(timezone="UTC")) + + fv_table = fv_table.filter( + ibis.and_( + fv_table[timestamp_field] <= timestamp_range_end, + fv_table[timestamp_field] >= timestamp_range_start_minus_ttl, + ) + ) + + # join_key_map = feature_view.projection.join_key_map or {e.name: e.name for e in feature_view.entity_columns} + # predicates = [fv_table[k] == entity_table[v] for k, v in join_key_map.items()] + + if feature_view.projection.join_key_map: + predicates = [ + fv_table[k] == entity_table[v] + for k, v in feature_view.projection.join_key_map.items() + ] + else: + predicates = [ + fv_table[e.name] == entity_table[e.name] + for e in feature_view.entity_columns + ] + + predicates.append( + fv_table[timestamp_field] <= entity_table[event_timestamp_col] + ) + + fv_table = fv_table.inner_join( + entity_table, predicates, lname="", rname="{name}_y" + ) + + fv_table = ( + fv_table.group_by(by="entity_row_id") + .order_by(ibis.desc(fv_table[timestamp_field])) + .mutate(rn=ibis.row_number()) + ) + + fv_table = fv_table.filter(fv_table["rn"] == ibis.literal(0)) + + select_cols = ["entity_row_id"] + select_cols.extend(feature_refs) + fv_table = fv_table.select(select_cols) + + if full_feature_names: + fv_table = fv_table.rename( + {f"{full_name_prefix}__{feature}": feature for feature in feature_refs} + ) + + acc_table = acc_table.left_join( + fv_table, + predicates=[fv_table.entity_row_id == acc_table.entity_row_id], + lname="", + rname="{name}_yyyy", + ) + + acc_table = acc_table.drop(s.endswith("_yyyy")) + + return acc_table + + @staticmethod + def _to_utc(entity_df: pd.DataFrame, event_timestamp_col): + entity_df_event_timestamp = entity_df.loc[ + :, event_timestamp_col + ].infer_objects() + if pd.api.types.is_string_dtype(entity_df_event_timestamp): + entity_df_event_timestamp = pd.to_datetime( + entity_df_event_timestamp, utc=True + ) + + entity_df[event_timestamp_col] = entity_df_event_timestamp + return entity_df + + @staticmethod + def _generate_row_id( + entity_table: Table, feature_views: List[FeatureView], event_timestamp_col + ) -> Table: + all_entities = [event_timestamp_col] + for fv in feature_views: + if fv.projection.join_key_map: + all_entities.extend(fv.projection.join_key_map.values()) + else: + all_entities.extend([e.name for e in fv.entity_columns]) + + r = ibis.literal("") + + for e in set(all_entities): + r = r.concat(entity_table[e].cast("string")) # type: ignore + + entity_table = entity_table.mutate(entity_row_id=r) + + return entity_table + + @staticmethod + def get_historical_features( + config: RepoConfig, + feature_views: List[FeatureView], + feature_refs: List[str], + entity_df: Union[pd.DataFrame, str], + registry: BaseRegistry, + project: str, + full_feature_names: bool = False, + ) -> RetrievalJob: + entity_schema = _get_entity_schema( + entity_df=entity_df, + ) + event_timestamp_col = offline_utils.infer_event_timestamp_from_entity_df( + entity_schema=entity_schema, + ) + + timestamp_range = IbisOfflineStore._get_entity_df_event_timestamp_range( + entity_df, event_timestamp_col + ) + entity_df = IbisOfflineStore._to_utc(entity_df, event_timestamp_col) + + entity_table = ibis.memtable(entity_df) + entity_table = IbisOfflineStore._generate_row_id( + entity_table, feature_views, event_timestamp_col + ) + + res: Table = entity_table + + for fv in feature_views: + res = IbisOfflineStore._get_historical_features_one( + fv, + entity_table, + feature_refs, + full_feature_names, + timestamp_range, + res, + event_timestamp_col, + ) + + res = res.drop("entity_row_id") + + return IbisRetrievalJob( + res, + OnDemandFeatureView.get_requested_odfvs(feature_refs, project, registry), + full_feature_names, + metadata=RetrievalMetadata( + features=feature_refs, + keys=list(set(entity_df.columns) - {event_timestamp_col}), + min_event_timestamp=timestamp_range[0], + max_event_timestamp=timestamp_range[1], + ), + ) + + @staticmethod + def pull_all_from_table_or_query( + config: RepoConfig, + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + timestamp_field: str, + start_date: datetime, + end_date: datetime, + ) -> RetrievalJob: + assert isinstance(data_source, FileSource) + + fields = join_key_columns + feature_name_columns + [timestamp_field] + start_date = start_date.astimezone(tz=utc) + end_date = end_date.astimezone(tz=utc) + + table = ibis.read_parquet(data_source.path) + + table = table.select(*fields) + + table = table.filter( + ibis.and_( + table[timestamp_field] >= ibis.literal(start_date), + table[timestamp_field] <= ibis.literal(end_date), + ) + ) + + return IbisRetrievalJob( + table=table, + on_demand_feature_views=[], + full_feature_names=False, + metadata=None, + ) + + @staticmethod + def write_logged_features( + config: RepoConfig, + data: Union[pyarrow.Table, Path], + source: LoggingSource, + logging_config: LoggingConfig, + registry: BaseRegistry, + ): + destination = logging_config.destination + assert isinstance(destination, FileLoggingDestination) + + if isinstance(data, Path): + table = ibis.read_parquet(data) + else: + table = ibis.memtable(data) + + if destination.partition_by: + kwargs = {"partition_by": destination.partition_by} + else: + kwargs = {} + + table.to_parquet( + f"{destination.path}/{uuid.uuid4().hex}-{{i}}.parquet", **kwargs + ) + + @staticmethod + def offline_write_batch( + config: RepoConfig, + feature_view: FeatureView, + table: pyarrow.Table, + progress: Optional[Callable[[int], Any]], + ): + assert isinstance(feature_view.batch_source, FileSource) + + pa_schema, column_names = get_pyarrow_schema_from_batch_source( + config, feature_view.batch_source + ) + if column_names != table.column_names: + raise ValueError( + f"The input pyarrow table has schema {table.schema} with the incorrect columns {table.column_names}. " + f"The schema is expected to be {pa_schema} with the columns (in this exact order) to be {column_names}." + ) + + file_options = feature_view.batch_source.file_options + prev_table = ibis.read_parquet(file_options.uri).to_pyarrow() + if table.schema != prev_table.schema: + table = table.cast(prev_table.schema) + new_table = pyarrow.concat_tables([table, prev_table]) + + ibis.memtable(new_table).to_parquet(file_options.uri) + + +class IbisRetrievalJob(RetrievalJob): + def __init__( + self, table, on_demand_feature_views, full_feature_names, metadata + ) -> None: + super().__init__() + self.table = table + self._on_demand_feature_views: List[ + OnDemandFeatureView + ] = on_demand_feature_views + self._full_feature_names = full_feature_names + self._metadata = metadata + + def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: + return self.table.execute() + + def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: + return self.table.to_pyarrow() + + @property + def full_feature_names(self) -> bool: + return self._full_feature_names + + @property + def on_demand_feature_views(self) -> List[OnDemandFeatureView]: + return self._on_demand_feature_views + + def persist( + self, + storage: SavedDatasetStorage, + allow_overwrite: bool = False, + timeout: Optional[int] = None, + ): + assert isinstance(storage, SavedDatasetFileStorage) + if not allow_overwrite and os.path.exists(storage.file_options.uri): + raise SavedDatasetLocationAlreadyExists(location=storage.file_options.uri) + + filesystem, path = FileSource.create_filesystem_and_path( + storage.file_options.uri, + storage.file_options.s3_endpoint_override, + ) + + if path.endswith(".parquet"): + pyarrow.parquet.write_table( + self.to_arrow(), where=path, filesystem=filesystem + ) + else: + # otherwise assume destination is directory + pyarrow.parquet.write_to_dataset( + self.to_arrow(), root_path=path, filesystem=filesystem + ) + + @property + def metadata(self) -> Optional[RetrievalMetadata]: + return self._metadata diff --git a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py index 849d5cc797f..67bae292c37 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py +++ b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/mssql.py @@ -3,7 +3,7 @@ import warnings from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Set, Tuple, Union import numpy as np import pandas @@ -11,7 +11,6 @@ import pyarrow as pa import sqlalchemy from pydantic.types import StrictStr -from pydantic.typing import Literal from sqlalchemy import create_engine from sqlalchemy.engine import Engine from sqlalchemy.orm import sessionmaker @@ -32,7 +31,7 @@ from feast.infra.provider import RetrievalJob from feast.infra.registry.base_registry import BaseRegistry from feast.on_demand_feature_view import OnDemandFeatureView -from feast.repo_config import FeastBaseModel, RepoConfig +from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.saved_dataset import SavedDatasetStorage from feast.type_map import pa_to_mssql_type from feast.usage import log_exceptions_and_usage @@ -43,7 +42,7 @@ EntitySchema = Dict[str, np.dtype] -class MsSqlServerOfflineStoreConfig(FeastBaseModel): +class MsSqlServerOfflineStoreConfig(FeastConfigBaseModel): """Offline store config for SQL Server""" type: Literal["mssql"] = "mssql" diff --git a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py index 9b751d98efe..71ce56bdefd 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/mssql_offline_store/tests/data_source.py @@ -1,4 +1,4 @@ -from typing import Dict, List +from typing import Dict, List, Optional import pandas as pd import pytest @@ -64,10 +64,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, - **kwargs, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: # Make sure the field mapping is correct and convert the datetime datasources. if timestamp_field in df: @@ -99,10 +99,10 @@ def create_data_source( ) def create_saved_dataset_destination(self) -> SavedDatasetStorage: - pass + raise NotImplementedError def get_prefixed_table_name(self, destination_name: str) -> str: return f"{self.project_name}_{destination_name}" def teardown(self): - pass + raise NotImplementedError diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py index c2e95a8648e..9b300d7bf46 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres.py @@ -9,6 +9,7 @@ Iterator, KeysView, List, + Literal, Optional, Tuple, Union, @@ -19,7 +20,6 @@ import pyarrow as pa from jinja2 import BaseLoader, Environment from psycopg2 import sql -from pydantic.typing import Literal from pytz import utc from feast.data_source import DataSource diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py index f4479501323..f50cdc4c41f 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/tests/data_source.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, Optional +from typing import Dict, Literal, Optional import pandas as pd import pytest @@ -12,6 +12,7 @@ PostgreSQLSource, ) from feast.infra.utils.postgres.connection_utils import df_to_postgres_table +from feast.infra.utils.postgres.postgres_config import PostgreSQLConfig from tests.integration.feature_repos.universal.data_source_creator import ( DataSourceCreator, ) @@ -26,6 +27,10 @@ POSTGRES_DB = "test" +class PostgreSQLOnlineStoreConfig(PostgreSQLConfig): + type: Literal["postgres"] = "postgres" + + @pytest.fixture(scope="session") def postgres_container(): container = ( @@ -82,10 +87,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) @@ -106,17 +111,17 @@ def create_offline_store_config(self) -> PostgreSQLOfflineStoreConfig: def get_prefixed_table_name(self, suffix: str) -> str: return f"{self.project_name}_{suffix}" - def create_online_store(self) -> Dict[str, str]: + def create_online_store(self) -> PostgreSQLOnlineStoreConfig: assert self.container - return { - "type": "postgres", - "host": "localhost", - "port": self.container.get_exposed_port(5432), - "database": POSTGRES_DB, - "db_schema": "feature_store", - "user": POSTGRES_USER, - "password": POSTGRES_PASSWORD, - } + return PostgreSQLOnlineStoreConfig( + type="postgres", + host="localhost", + port=self.container.get_exposed_port(5432), + database=POSTGRES_DB, + db_schema="feature_store", + user=POSTGRES_USER, + password=POSTGRES_PASSWORD, + ) def create_saved_dataset_destination(self): # FIXME: ... diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py index c9591b7c3f0..b1b1c04c7d7 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py @@ -125,7 +125,7 @@ def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], feature_refs: List[str], - entity_df: Union[pandas.DataFrame, str], + entity_df: Union[pandas.DataFrame, str, pyspark.sql.DataFrame], registry: Registry, project: str, full_feature_names: bool = False, @@ -473,15 +473,16 @@ def _get_entity_df_event_timestamp_range( entity_df_event_timestamp.min().to_pydatetime(), entity_df_event_timestamp.max().to_pydatetime(), ) - elif isinstance(entity_df, str): + elif isinstance(entity_df, str) or isinstance(entity_df, pyspark.sql.DataFrame): # If the entity_df is a string (SQL query), determine range # from table - df = spark_session.sql(entity_df).select(entity_df_event_timestamp_col) - - # Checks if executing entity sql resulted in any data - if df.rdd.isEmpty(): - raise EntitySQLEmptyResults(entity_df) - + if isinstance(entity_df, str): + df = spark_session.sql(entity_df).select(entity_df_event_timestamp_col) + # Checks if executing entity sql resulted in any data + if df.rdd.isEmpty(): + raise EntitySQLEmptyResults(entity_df) + else: + df = entity_df # TODO(kzhang132): need utc conversion here. entity_df_event_timestamp_range = ( @@ -499,8 +500,11 @@ def _get_entity_schema( ) -> Dict[str, np.dtype]: if isinstance(entity_df, pd.DataFrame): return dict(zip(entity_df.columns, entity_df.dtypes)) - elif isinstance(entity_df, str): - entity_spark_df = spark_session.sql(entity_df) + elif isinstance(entity_df, str) or isinstance(entity_df, pyspark.sql.DataFrame): + if isinstance(entity_df, str): + entity_spark_df = spark_session.sql(entity_df) + else: + entity_spark_df = entity_df return dict( zip( entity_spark_df.columns, @@ -526,6 +530,9 @@ def _upload_entity_df( elif isinstance(entity_df, str): spark_session.sql(entity_df).createOrReplaceTempView(table_name) return + elif isinstance(entity_df, pyspark.sql.DataFrame): + entity_df.createOrReplaceTempView(table_name) + return else: raise InvalidEntityType(type(entity_df)) diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py index a27065fb5ed..1ff7e6de589 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py @@ -39,7 +39,6 @@ def __init__( query: Optional[str] = None, path: Optional[str] = None, file_format: Optional[str] = None, - event_timestamp_column: Optional[str] = None, created_timestamp_column: Optional[str] = None, field_mapping: Optional[Dict[str, str]] = None, description: Optional[str] = "", diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py index 71c07b20c27..b9785218857 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/tests/data_source.py @@ -2,13 +2,14 @@ import shutil import tempfile import uuid -from typing import Dict, List +from typing import Dict, List, Optional import pandas as pd from pyspark import SparkConf from pyspark.sql import SparkSession from feast.data_source import DataSource +from feast.feature_logging import LoggingDestination from feast.infra.offline_stores.contrib.spark_offline_store.spark import ( SparkOfflineStoreConfig, ) @@ -68,10 +69,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, - **kwargs, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: if timestamp_field in df: df[timestamp_field] = pd.to_datetime(df[timestamp_field], utc=True) @@ -119,3 +120,7 @@ def create_saved_dataset_destination(self) -> SavedDatasetSparkStorage: def get_prefixed_table_name(self, suffix: str) -> str: return f"{self.project_name}_{suffix}" + + def create_logged_features_destination(self) -> LoggingDestination: + # No implementation of LoggingDestination for Spark offline store. + return None # type: ignore diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/manual_tests.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/manual_tests.py index 7d31aa90fb4..a31d368ea11 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/manual_tests.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/test_config/manual_tests.py @@ -8,6 +8,6 @@ FULL_REPO_CONFIGS = [ IntegrationTestRepoConfig( provider="local", - offline_store_creator=TrinoSourceCreator, + offline_store_creator=TrinoSourceCreator, # type: ignore ), ] diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py index a5aa53df7ab..bd3f9def8f1 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py @@ -62,10 +62,11 @@ def __init__( "must be include into pytest plugins" ) self.exposed_port = self.container.get_exposed_port("8080") + self.container_host = self.container.get_container_host_ip() self.client = Trino( user="user", catalog="memory", - host="localhost", + host=self.container_host, port=self.exposed_port, source="trino-python-client", http_scheme="http", @@ -81,10 +82,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) self.client.execute_query( @@ -123,9 +124,11 @@ def get_prefixed_table_name(self, suffix: str) -> str: def create_offline_store_config(self) -> FeastConfigBaseModel: return TrinoOfflineStoreConfig( - host="localhost", + host=self.container_host, port=self.exposed_port, catalog="memory", dataset=self.project_name, connector={"type": "memory"}, + user="test", + auth=None, ) diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py index f662cda9130..cdc94350244 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py @@ -5,7 +5,7 @@ import numpy as np import pandas as pd import pyarrow -from pydantic import Field, FilePath, SecretStr, StrictBool, StrictStr, root_validator +from pydantic import Field, FilePath, SecretStr, StrictBool, StrictStr, model_validator from trino.auth import ( BasicAuthentication, CertificateAuthentication, @@ -40,7 +40,7 @@ class BasicAuthModel(FeastConfigBaseModel): username: StrictStr - password: SecretStr + password: StrictStr class KerberosAuthModel(FeastConfigBaseModel): @@ -98,14 +98,14 @@ class AuthConfig(FeastConfigBaseModel): type: Literal["kerberos", "basic", "jwt", "oauth2", "certificate"] config: Optional[Dict[StrictStr, Any]] - @root_validator - def config_only_nullable_for_oauth2(cls, values): - auth_type = values["type"] - auth_config = values["config"] + @model_validator(mode="after") + def config_only_nullable_for_oauth2(self): + auth_type = self.type + auth_config = self.config if auth_type != "oauth2" and auth_config is None: raise ValueError(f"config cannot be null for auth type '{auth_type}'") - return values + return self def to_trino_auth(self): auth_type = self.type diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py index 5e4107545f0..0b873a2091d 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/file.py @@ -2,15 +2,15 @@ import uuid from datetime import datetime from pathlib import Path -from typing import Any, Callable, List, Optional, Tuple, Union +from typing import Any, Callable, List, Literal, Optional, Tuple, Union +import dask import dask.dataframe as dd import pandas as pd import pyarrow import pyarrow.dataset import pyarrow.parquet import pytz -from pydantic.typing import Literal from feast.data_source import DataSource from feast.errors import ( @@ -43,6 +43,11 @@ _run_dask_field_mapping, ) +# FileRetrievalJob will cast string objects to string[pyarrow] from dask version 2023.7.1 +# This is not the desired behavior for our use case, so we set the convert-string option to False +# See (https://github.com/dask/dask/issues/10881#issuecomment-1923327936) +dask.config.set({"dataframe.convert-string": False}) + class FileOfflineStoreConfig(FeastConfigBaseModel): """Offline store config for local (file-based) store""" @@ -367,8 +372,6 @@ def evaluate_offline_job(): source_df[DUMMY_ENTITY_ID] = DUMMY_ENTITY_VAL columns_to_extract.add(DUMMY_ENTITY_ID) - source_df = source_df.persist() - return source_df[list(columns_to_extract)].persist() # When materializing a single feature view, we don't need full feature names. On demand transforms aren't materialized diff --git a/sdk/python/feast/infra/offline_stores/file_source.py b/sdk/python/feast/infra/offline_stores/file_source.py index ac824b359f4..887b4100796 100644 --- a/sdk/python/feast/infra/offline_stores/file_source.py +++ b/sdk/python/feast/infra/offline_stores/file_source.py @@ -183,7 +183,7 @@ def create_filesystem_and_path( return None, path def get_table_query_string(self) -> str: - pass + raise NotImplementedError class FileOptions: diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index 6141e3c435b..30135feccb3 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import warnings -from abc import ABC, abstractmethod +from abc import ABC from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, List, Optional, Union @@ -150,9 +150,8 @@ def to_sql(self) -> str: """ Return RetrievalJob generated SQL statement if applicable. """ - pass + raise NotImplementedError - @abstractmethod def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: """ Synchronously executes the underlying query and returns the result as a pandas dataframe. @@ -162,9 +161,8 @@ def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: Does not handle on demand transformations or dataset validation. For either of those, `to_df` should be used. """ - pass + raise NotImplementedError - @abstractmethod def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: """ Synchronously executes the underlying query and returns the result as an arrow table. @@ -174,21 +172,18 @@ def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: Does not handle on demand transformations or dataset validation. For either of those, `to_arrow` should be used. """ - pass + raise NotImplementedError @property - @abstractmethod def full_feature_names(self) -> bool: """Returns True if full feature names should be applied to the results of the query.""" - pass + raise NotImplementedError @property - @abstractmethod def on_demand_feature_views(self) -> List[OnDemandFeatureView]: """Returns a list containing all the on demand feature views to be handled.""" - pass + raise NotImplementedError - @abstractmethod def persist( self, storage: SavedDatasetStorage, @@ -204,13 +199,12 @@ def persist( allow_overwrite: If True, a pre-existing location (e.g. table or file) can be overwritten. Currently not all individual offline store implementations make use of this parameter. """ - pass + raise NotImplementedError @property - @abstractmethod def metadata(self) -> Optional[RetrievalMetadata]: """Returns metadata about the retrieval job.""" - pass + raise NotImplementedError def supports_remote_storage_export(self) -> bool: """Returns True if the RetrievalJob supports `to_remote_storage`.""" @@ -226,7 +220,7 @@ def to_remote_storage(self) -> List[str]: Returns: A list of parquet file paths in remote storage. """ - raise NotImplementedError() + raise NotImplementedError class OfflineStore(ABC): @@ -239,7 +233,6 @@ class OfflineStore(ABC): """ @staticmethod - @abstractmethod def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -270,10 +263,9 @@ def pull_latest_from_table_or_query( Returns: A RetrievalJob that can be executed to get the entity rows. """ - pass + raise NotImplementedError @staticmethod - @abstractmethod def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], @@ -302,10 +294,9 @@ def get_historical_features( Returns: A RetrievalJob that can be executed to get the features. """ - pass + raise NotImplementedError @staticmethod - @abstractmethod def pull_all_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -334,7 +325,7 @@ def pull_all_from_table_or_query( Returns: A RetrievalJob that can be executed to get the entity rows. """ - pass + raise NotImplementedError @staticmethod def write_logged_features( @@ -358,7 +349,7 @@ def write_logged_features( logging_config: A LoggingConfig object that determines where the logs will be written. registry: The registry for the current feature store. """ - raise NotImplementedError() + raise NotImplementedError @staticmethod def offline_write_batch( @@ -377,4 +368,4 @@ def offline_write_batch( progress: Function to be called once a portion of the data has been written, used to show progress. """ - raise NotImplementedError() + raise NotImplementedError diff --git a/sdk/python/feast/infra/offline_stores/redshift.py b/sdk/python/feast/infra/offline_stores/redshift.py index 837cf49655d..2565a569ad1 100644 --- a/sdk/python/feast/infra/offline_stores/redshift.py +++ b/sdk/python/feast/infra/offline_stores/redshift.py @@ -9,6 +9,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Union, @@ -19,8 +20,7 @@ import pyarrow import pyarrow as pa from dateutil import parser -from pydantic import StrictStr, root_validator -from pydantic.typing import Literal +from pydantic import StrictStr, model_validator from pytz import utc from feast import OnDemandFeatureView, RedshiftSource @@ -51,13 +51,13 @@ class RedshiftOfflineStoreConfig(FeastConfigBaseModel): type: Literal["redshift"] = "redshift" """ Offline store type selector""" - cluster_id: Optional[StrictStr] + cluster_id: Optional[StrictStr] = None """ Redshift cluster identifier, for provisioned clusters """ - user: Optional[StrictStr] + user: Optional[StrictStr] = None """ Redshift user name, only required for provisioned clusters """ - workgroup: Optional[StrictStr] + workgroup: Optional[StrictStr] = None """ Redshift workgroup identifier, for serverless """ region: StrictStr @@ -72,16 +72,16 @@ class RedshiftOfflineStoreConfig(FeastConfigBaseModel): iam_role: StrictStr """ IAM Role for Redshift, granting it access to S3 """ - @root_validator - def require_cluster_and_user_or_workgroup(cls, values): + @model_validator(mode="after") + def require_cluster_and_user_or_workgroup(self): """ Provisioned Redshift clusters: Require cluster_id and user, ignore workgroup Serverless Redshift: Require workgroup, ignore cluster_id and user """ cluster_id, user, workgroup = ( - values.get("cluster_id"), - values.get("user"), - values.get("workgroup"), + self.cluster_id, + self.user, + self.workgroup, ) if not (cluster_id and user) and not workgroup: raise ValueError( @@ -90,7 +90,7 @@ def require_cluster_and_user_or_workgroup(cls, values): elif cluster_id and workgroup: raise ValueError("cannot specify both cluster_id and workgroup") - return values + return self class RedshiftOfflineStore(OfflineStore): diff --git a/sdk/python/feast/infra/offline_stores/snowflake.py b/sdk/python/feast/infra/offline_stores/snowflake.py index 38568ce79b2..14752fd8572 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake.py +++ b/sdk/python/feast/infra/offline_stores/snowflake.py @@ -1,4 +1,5 @@ import contextlib +import json import os import uuid import warnings @@ -13,6 +14,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Union, @@ -22,8 +24,7 @@ import numpy as np import pandas as pd import pyarrow -from pydantic import Field, StrictStr -from pydantic.typing import Literal +from pydantic import ConfigDict, Field, StrictStr from pytz import utc from feast import OnDemandFeatureView @@ -51,6 +52,17 @@ ) from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.saved_dataset import SavedDatasetStorage +from feast.types import ( + Array, + Bool, + Bytes, + Float32, + Float64, + Int32, + Int64, + String, + UnixTimestamp, +) from feast.usage import log_exceptions_and_usage try: @@ -107,9 +119,7 @@ class SnowflakeOfflineStoreConfig(FeastConfigBaseModel): convert_timestamp_columns: Optional[bool] = None """ Convert timestamp columns on export to a Parquet-supported format """ - - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class SnowflakeOfflineStore(OfflineStore): @@ -320,6 +330,7 @@ def query_generator() -> Iterator[str]: on_demand_feature_views=OnDemandFeatureView.get_requested_odfvs( feature_refs, project, registry ), + feature_views=feature_views, metadata=RetrievalMetadata( features=feature_refs, keys=list(entity_schema.keys() - {entity_df_event_timestamp_col}), @@ -398,9 +409,12 @@ def __init__( config: RepoConfig, full_feature_names: bool, on_demand_feature_views: Optional[List[OnDemandFeatureView]] = None, + feature_views: Optional[List[FeatureView]] = None, metadata: Optional[RetrievalMetadata] = None, ): + if feature_views is None: + feature_views = [] if not isinstance(query, str): self._query_generator = query else: @@ -416,6 +430,7 @@ def query_generator() -> Iterator[str]: self.config = config self._full_feature_names = full_feature_names self._on_demand_feature_views = on_demand_feature_views or [] + self._feature_views = feature_views self._metadata = metadata self.export_path: Optional[str] if self.config.offline_store.blob_export_location: @@ -436,12 +451,28 @@ def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: self.snowflake_conn, self.to_sql() ).fetch_pandas_all() + for feature_view in self._feature_views: + for feature in feature_view.features: + if feature.dtype in [ + Array(String), + Array(Bytes), + Array(Int32), + Array(Int64), + Array(UnixTimestamp), + Array(Float64), + Array(Float32), + Array(Bool), + ]: + df[feature.name] = [ + json.loads(x) if x else None for x in df[feature.name] + ] + return df def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: pa_table = execute_snowflake_statement( self.snowflake_conn, self.to_sql() - ).fetch_arrow_all() + ).fetch_arrow_all(force_return_table=False) if pa_table: return pa_table @@ -584,12 +615,17 @@ def to_remote_storage(self) -> List[str]: HEADER = TRUE """ cursor = execute_snowflake_statement(self.snowflake_conn, query) + # s3gov schema is used by Snowflake in AWS govcloud regions + # remove gov portion from schema and pass it to online store upload + native_export_path = self.export_path.replace("s3gov://", "s3://") + return self._get_file_names_from_copy_into(cursor, native_export_path) + def _get_file_names_from_copy_into(self, cursor, native_export_path) -> List[str]: file_name_column_index = [ idx for idx, rm in enumerate(cursor.description) if rm.name == "FILE_NAME" ][0] return [ - f"{self.export_path}/{row[file_name_column_index]}" + f"{native_export_path}/{row[file_name_column_index]}" for row in cursor.fetchall() ] diff --git a/sdk/python/feast/infra/offline_stores/snowflake_source.py b/sdk/python/feast/infra/offline_stores/snowflake_source.py index 95bd46f1ec1..9a2c6e09bc4 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake_source.py +++ b/sdk/python/feast/infra/offline_stores/snowflake_source.py @@ -1,5 +1,5 @@ import warnings -from typing import Callable, Dict, Iterable, Optional, Tuple +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, no_type_check from typeguard import typechecked @@ -202,6 +202,7 @@ def get_table_query_string(self) -> str: def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: return type_map.snowflake_type_to_feast_value_type + @no_type_check def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: @@ -223,7 +224,7 @@ def get_table_column_names_and_types( query = f"SELECT * FROM {self.get_table_query_string()} LIMIT 5" cursor = execute_snowflake_statement(conn, query) - metadata = [ + metadata: List[Dict[str, Any]] = [ { "column_name": column.name, "type_code": column.type_code, @@ -279,12 +280,12 @@ def get_table_column_names_and_types( else: row["snowflake_type"] = "NUMBERwSCALE" - elif row["type_code"] in [5, 9, 10, 12]: + elif row["type_code"] in [5, 9, 12]: error = snowflake_unsupported_map[row["type_code"]] raise NotImplementedError( f"The following Snowflake Data Type is not supported: {error}" ) - elif row["type_code"] in [1, 2, 3, 4, 6, 7, 8, 11, 13]: + elif row["type_code"] in [1, 2, 3, 4, 6, 7, 8, 10, 11, 13]: row["snowflake_type"] = snowflake_type_code_map[row["type_code"]] else: raise NotImplementedError( @@ -292,7 +293,8 @@ def get_table_column_names_and_types( ) return [ - (column["column_name"], column["snowflake_type"]) for column in metadata + (str(column["column_name"]), str(column["snowflake_type"])) + for column in metadata ] @@ -305,6 +307,7 @@ def get_table_column_names_and_types( 6: "TIMESTAMP_LTZ", 7: "TIMESTAMP_TZ", 8: "TIMESTAMP_NTZ", + 10: "ARRAY", 11: "BINARY", 13: "BOOLEAN", } @@ -312,7 +315,6 @@ def get_table_column_names_and_types( snowflake_unsupported_map = { 5: "VARIANT -- Try converting to VARCHAR", 9: "OBJECT -- Try converting to VARCHAR", - 10: "ARRAY -- Try converting to VARCHAR", 12: "TIME -- Try converting to VARCHAR", } diff --git a/sdk/python/feast/infra/online_stores/bigtable.py b/sdk/python/feast/infra/online_stores/bigtable.py index 30561d0840f..3a83d23cedb 100644 --- a/sdk/python/feast/infra/online_stores/bigtable.py +++ b/sdk/python/feast/infra/online_stores/bigtable.py @@ -2,13 +2,12 @@ import logging from concurrent import futures from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Set, Tuple import google from google.cloud import bigtable from google.cloud.bigtable import row_filters from pydantic import StrictStr -from pydantic.typing import Literal from feast import Entity, FeatureView, utils from feast.feature_view import DUMMY_ENTITY_NAME diff --git a/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py b/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py index 34a8cab036d..c672e18db03 100644 --- a/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py +++ b/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py @@ -20,7 +20,17 @@ import logging from datetime import datetime -from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Literal, + Optional, + Sequence, + Tuple, +) from cassandra.auth import PlainTextAuthProvider from cassandra.cluster import ( @@ -34,7 +44,6 @@ from cassandra.policies import DCAwareRoundRobinPolicy, TokenAwarePolicy from cassandra.query import PreparedStatement from pydantic import StrictFloat, StrictInt, StrictStr -from pydantic.typing import Literal from feast import Entity, FeatureView, RepoConfig from feast.infra.key_encoding_utils import serialize_entity_key diff --git a/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py b/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py index 1da9de89a81..4b2d8ae39c2 100644 --- a/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py +++ b/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py @@ -1,12 +1,11 @@ import calendar import struct from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple from happybase import ConnectionPool from happybase.connection import DEFAULT_PROTOCOL, DEFAULT_TRANSPORT from pydantic import StrictStr -from pydantic.typing import Literal from feast import Entity from feast.feature_view import FeatureView diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py index fa7dd2c2a49..cf07d5fef12 100644 --- a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple import pymysql import pytz @@ -23,7 +23,7 @@ class MySQLOnlineStoreConfig(FeastConfigBaseModel): NOTE: The class *must* end with the `OnlineStoreConfig` suffix. """ - type = "mysql" + type: Literal["mysql"] = "mysql" host: Optional[StrictStr] = None user: Optional[StrictStr] = None @@ -178,8 +178,11 @@ def update( # We don't create any special state for the entities in this implementation. for table in tables_to_keep: + + table_name = _table_id(project, table) + index_name = f"{table_name}_ek" cur.execute( - f"""CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key VARCHAR(512), + f"""CREATE TABLE IF NOT EXISTS {table_name} (entity_key VARCHAR(512), feature_name VARCHAR(256), value BLOB, event_ts timestamp NULL DEFAULT NULL, @@ -187,9 +190,16 @@ def update( PRIMARY KEY(entity_key, feature_name))""" ) - cur.execute( - f"ALTER TABLE {_table_id(project, table)} ADD INDEX {_table_id(project, table)}_ek (entity_key);" + index_exists = cur.execute( + f""" + SELECT 1 FROM information_schema.statistics + WHERE table_schema = DATABASE() AND table_name = '{table_name}' AND index_name = '{index_name}' + """ ) + if not index_exists: + cur.execute( + f"ALTER TABLE {table_name} ADD INDEX {index_name} (entity_key);" + ) for table in tables_to_delete: _drop_table_and_index(cur, project, table) diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index a12e66f1090..308528aaec2 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -2,14 +2,13 @@ import logging from collections import defaultdict from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple import psycopg2 import pytz from psycopg2 import sql from psycopg2.extras import execute_values from psycopg2.pool import SimpleConnectionPool -from pydantic.schema import Literal from feast import Entity from feast.feature_view import FeatureView @@ -99,6 +98,7 @@ def online_write_batch( cur_batch, page_size=batch_size, ) + conn.commit() if progress: progress(len(cur_batch)) diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py index ed4e7612ba5..ae96e16c640 100644 --- a/sdk/python/feast/infra/online_stores/datastore.py +++ b/sdk/python/feast/infra/online_stores/datastore.py @@ -17,10 +17,19 @@ from multiprocessing.pool import ThreadPool from queue import Empty, Queue from threading import Lock, Thread -from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Literal, + Optional, + Sequence, + Tuple, +) from pydantic import PositiveInt, StrictStr -from pydantic.typing import Literal from feast import Entity, utils from feast.errors import FeastProviderLoginError diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index 525978e736b..a049189de7f 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -14,10 +14,9 @@ import itertools import logging from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union from pydantic import StrictBool, StrictStr -from pydantic.typing import Literal, Union from feast import Entity, FeatureView, utils from feast.infra.infra_object import DYNAMODB_INFRA_OBJECT_CLASS_TYPE, InfraObject @@ -288,12 +287,12 @@ def _get_dynamodb_resource(self, region: str, endpoint_url: Optional[str] = None ) return self._dynamodb_resource - def _sort_dynamodb_response(self, responses: list, order: list): + def _sort_dynamodb_response(self, responses: list, order: list) -> Any: """DynamoDB Batch Get Item doesn't return items in a particular order.""" # Assign an index to order order_with_index = {value: idx for idx, value in enumerate(order)} # Sort table responses by index - table_responses_ordered = [ + table_responses_ordered: Any = [ (order_with_index[tbl_res["entity_id"]], tbl_res) for tbl_res in responses ] table_responses_ordered = sorted( diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index 9561705aaac..6f6c2fb45c6 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -21,6 +21,7 @@ Callable, Dict, List, + Literal, Optional, Sequence, Tuple, @@ -30,7 +31,6 @@ import pytz from google.protobuf.timestamp_pb2 import Timestamp from pydantic import StrictStr -from pydantic.typing import Literal from feast import Entity, FeatureView, RepoConfig, utils from feast.infra.online_stores.helpers import _mmh3, _redis_key, _redis_key_prefix @@ -106,6 +106,39 @@ def delete_entity_values(self, config: RepoConfig, join_keys: List[str]): logger.debug(f"Deleted {deleted_count} rows for entity {', '.join(join_keys)}") + def delete_table(self, config: RepoConfig, table: FeatureView): + """ + Delete all rows in Redis for a specific feature view + + Args: + config: Feast config + table: Feature view to delete + """ + client = self._get_client(config.online_store) + deleted_count = 0 + prefix = _redis_key_prefix(table.join_keys) + + redis_hash_keys = [_mmh3(f"{table.name}:{f.name}") for f in table.features] + redis_hash_keys.append(bytes(f"_ts:{table.name}", "utf8")) + + with client.pipeline(transaction=False) as pipe: + for _k in client.scan_iter( + b"".join([prefix, b"*", config.project.encode("utf8")]) + ): + _tables = { + _hk[4:] for _hk in client.hgetall(_k) if _hk.startswith(b"_ts:") + } + if bytes(table.name, "utf8") not in _tables: + continue + if len(_tables) == 1: + pipe.delete(_k) + else: + pipe.hdel(_k, *redis_hash_keys) + deleted_count += 1 + pipe.execute() + + logger.debug(f"Deleted {deleted_count} rows for feature view {table.name}") + @log_exceptions_and_usage(online_store="redis") def update( self, @@ -117,16 +150,19 @@ def update( partial: bool, ): """ - Look for join_keys (list of entities) that are not in use anymore - (usually this happens when the last feature view that was using specific compound key is deleted) - and remove all features attached to this "join_keys". + Delete data from feature views that are no longer in use. + + Args: + config: Feast config + tables_to_delete: Feature views to delete + tables_to_keep: Feature views to keep + entities_to_delete: Entities to delete + entities_to_keep: Entities to keep + partial: Whether to do a partial update """ - join_keys_to_keep = set(tuple(table.join_keys) for table in tables_to_keep) - join_keys_to_delete = set(tuple(table.join_keys) for table in tables_to_delete) - - for join_keys in join_keys_to_delete - join_keys_to_keep: - self.delete_entity_values(config, list(join_keys)) + for table in tables_to_delete: + self.delete_table(config, table) def teardown( self, diff --git a/sdk/python/feast/infra/online_stores/snowflake.py b/sdk/python/feast/infra/online_stores/snowflake.py index c1a03a2862c..f5600249c91 100644 --- a/sdk/python/feast/infra/online_stores/snowflake.py +++ b/sdk/python/feast/infra/online_stores/snowflake.py @@ -2,11 +2,10 @@ import os from binascii import hexlify from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple import pandas as pd -from pydantic import Field, StrictStr -from pydantic.schema import Literal +from pydantic import ConfigDict, Field, StrictStr from feast.entity import Entity from feast.feature_view import FeatureView @@ -57,9 +56,7 @@ class SnowflakeOnlineStoreConfig(FeastConfigBaseModel): schema_: Optional[str] = Field("PUBLIC", alias="schema") """ Snowflake schema name """ - - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class SnowflakeOnlineStore(OnlineStore): diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 6949b2bf247..4a6aa28889d 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -16,10 +16,9 @@ import sqlite3 from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple from pydantic import StrictStr -from pydantic.schema import Literal from feast import Entity from feast.feature_view import FeatureView diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 28b10c12595..aca18f4856b 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -70,7 +70,7 @@ def batch_engine(self) -> BatchMaterializationEngine: if self._batch_engine: return self._batch_engine else: - engine_config = self.repo_config._batch_engine_config + engine_config = self.repo_config.batch_engine_config config_is_dict = False if isinstance(engine_config, str): engine_config_type = engine_config @@ -180,7 +180,7 @@ def online_read( config: RepoConfig, table: FeatureView, entity_keys: List[EntityKeyProto], - requested_features: List[str] = None, + requested_features: Optional[List[str]] = None, ) -> List: set_usage_attribute("provider", self.__class__.__name__) result = [] diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 82879b264af..2a9670cacef 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -211,7 +211,7 @@ def online_read( config: RepoConfig, table: FeatureView, entity_keys: List[EntityKeyProto], - requested_features: List[str] = None, + requested_features: Optional[List[str]] = None, ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: """ Reads features values for the given entity keys. diff --git a/sdk/python/feast/infra/registry/base_registry.py b/sdk/python/feast/infra/registry/base_registry.py index 14b098bb123..9ee3bbbabcf 100644 --- a/sdk/python/feast/infra/registry/base_registry.py +++ b/sdk/python/feast/infra/registry/base_registry.py @@ -51,6 +51,7 @@ def apply_entity(self, entity: Entity, project: str, commit: bool = True): project: Feast project that this entity belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def delete_entity(self, name: str, project: str, commit: bool = True): @@ -62,6 +63,7 @@ def delete_entity(self, name: str, project: str, commit: bool = True): project: Feast project that this entity belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Entity: @@ -77,6 +79,7 @@ def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Enti Returns either the specified entity, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity]: @@ -90,6 +93,7 @@ def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity] Returns: List of entities """ + raise NotImplementedError # Data source operations @abstractmethod @@ -104,6 +108,7 @@ def apply_data_source( project: Feast project that this data source belongs to commit: Whether to immediately commit to the registry """ + raise NotImplementedError @abstractmethod def delete_data_source(self, name: str, project: str, commit: bool = True): @@ -115,6 +120,7 @@ def delete_data_source(self, name: str, project: str, commit: bool = True): project: Feast project that this data source belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def get_data_source( @@ -131,6 +137,7 @@ def get_data_source( Returns: Returns either the specified data source, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_data_sources( @@ -146,6 +153,7 @@ def list_data_sources( Returns: List of data sources """ + raise NotImplementedError # Feature service operations @abstractmethod @@ -159,6 +167,7 @@ def apply_feature_service( feature_service: A feature service that will be registered project: Feast project that this entity belongs to """ + raise NotImplementedError @abstractmethod def delete_feature_service(self, name: str, project: str, commit: bool = True): @@ -170,6 +179,7 @@ def delete_feature_service(self, name: str, project: str, commit: bool = True): project: Feast project that this feature service belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def get_feature_service( @@ -187,6 +197,7 @@ def get_feature_service( Returns either the specified feature service, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_feature_services( @@ -202,6 +213,7 @@ def list_feature_services( Returns: List of feature services """ + raise NotImplementedError # Feature view operations @abstractmethod @@ -216,6 +228,7 @@ def apply_feature_view( project: Feast project that this feature view belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def delete_feature_view(self, name: str, project: str, commit: bool = True): @@ -227,12 +240,13 @@ def delete_feature_view(self, name: str, project: str, commit: bool = True): project: Feast project that this feature view belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError # stream feature view operations @abstractmethod def get_stream_feature_view( self, name: str, project: str, allow_cache: bool = False - ): + ) -> StreamFeatureView: """ Retrieves a stream feature view. @@ -245,6 +259,7 @@ def get_stream_feature_view( Returns either the specified feature view, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_stream_feature_views( @@ -260,6 +275,7 @@ def list_stream_feature_views( Returns: List of stream feature views """ + raise NotImplementedError # on demand feature view operations @abstractmethod @@ -278,6 +294,7 @@ def get_on_demand_feature_view( Returns either the specified on demand feature view, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_on_demand_feature_views( @@ -293,6 +310,7 @@ def list_on_demand_feature_views( Returns: List of on demand feature views """ + raise NotImplementedError # regular feature view operations @abstractmethod @@ -311,6 +329,7 @@ def get_feature_view( Returns either the specified feature view, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_feature_views( @@ -326,10 +345,13 @@ def list_feature_views( Returns: List of feature views """ + raise NotImplementedError # request feature view operations @abstractmethod - def get_request_feature_view(self, name: str, project: str) -> RequestFeatureView: + def get_request_feature_view( + self, name: str, project: str, allow_cache: bool = False + ) -> RequestFeatureView: """ Retrieves a request feature view. @@ -342,6 +364,7 @@ def get_request_feature_view(self, name: str, project: str) -> RequestFeatureVie Returns either the specified feature view, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_request_feature_views( @@ -357,6 +380,7 @@ def list_request_feature_views( Returns: List of request feature views """ + raise NotImplementedError @abstractmethod def apply_materialization( @@ -377,6 +401,7 @@ def apply_materialization( end_date (datetime): End date of the materialization interval to track commit: Whether the change should be persisted immediately """ + raise NotImplementedError # Saved dataset operations @abstractmethod @@ -394,6 +419,7 @@ def apply_saved_dataset( project: Feast project that this dataset belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def get_saved_dataset( @@ -411,6 +437,7 @@ def get_saved_dataset( Returns either the specified SavedDataset, or raises an exception if none is found """ + raise NotImplementedError def delete_saved_dataset(self, name: str, project: str, allow_cache: bool = False): """ @@ -425,6 +452,7 @@ def delete_saved_dataset(self, name: str, project: str, allow_cache: bool = Fals Returns either the specified SavedDataset, or raises an exception if none is found """ + raise NotImplementedError @abstractmethod def list_saved_datasets( @@ -440,6 +468,7 @@ def list_saved_datasets( Returns: Returns the list of SavedDatasets """ + raise NotImplementedError # Validation reference operations @abstractmethod @@ -457,6 +486,7 @@ def apply_validation_reference( project: Feast project that this dataset belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def delete_validation_reference(self, name: str, project: str, commit: bool = True): @@ -468,6 +498,7 @@ def delete_validation_reference(self, name: str, project: str, commit: bool = Tr project: Feast project that this object belongs to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def get_validation_reference( @@ -485,6 +516,7 @@ def get_validation_reference( Returns either the specified ValidationReference, or raises an exception if none is found """ + raise NotImplementedError # TODO: Needs to be implemented. def list_validation_references( @@ -501,7 +533,9 @@ def list_validation_references( Returns: List of request feature views """ + raise NotImplementedError + @abstractmethod def list_project_metadata( self, project: str, allow_cache: bool = False ) -> List[ProjectMetadata]: @@ -515,6 +549,7 @@ def list_project_metadata( Returns: List of project metadata """ + raise NotImplementedError @abstractmethod def update_infra(self, infra: Infra, project: str, commit: bool = True): @@ -526,6 +561,7 @@ def update_infra(self, infra: Infra, project: str, commit: bool = True): project: Feast project that the Infra object refers to commit: Whether the change should be persisted immediately """ + raise NotImplementedError @abstractmethod def get_infra(self, project: str, allow_cache: bool = False) -> Infra: @@ -539,6 +575,7 @@ def get_infra(self, project: str, allow_cache: bool = False) -> Infra: Returns: The stored Infra object. """ + raise NotImplementedError @abstractmethod def apply_user_metadata( @@ -563,14 +600,17 @@ def proto(self) -> RegistryProto: Returns: The registry proto object. """ + raise NotImplementedError @abstractmethod def commit(self): """Commits the state of the registry cache to the remote registry store.""" + raise NotImplementedError @abstractmethod def refresh(self, project: Optional[str] = None): """Refreshes the state of the registry cache by fetching the registry state from the remote registry store.""" + raise NotImplementedError @staticmethod def _message_to_sorted_dict(message: Message) -> Dict[str, Any]: @@ -625,7 +665,7 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]: odfv_dict["spec"]["userDefinedFunction"][ "body" - ] = on_demand_feature_view.udf_string + ] = on_demand_feature_view.transformation.udf_string registry_dict["onDemandFeatureViews"].append(odfv_dict) for request_feature_view in sorted( self.list_request_feature_views(project=project), diff --git a/sdk/python/feast/infra/registry/registry.py b/sdk/python/feast/infra/registry/registry.py index 1a72cbb4a58..a9d6c44f38c 100644 --- a/sdk/python/feast/infra/registry/registry.py +++ b/sdk/python/feast/infra/registry/registry.py @@ -178,6 +178,10 @@ def __new__( from feast.infra.registry.snowflake import SnowflakeRegistry return SnowflakeRegistry(registry_config, project, repo_path) + elif registry_config and registry_config.registry_type == "remote": + from feast.infra.registry.remote import RemoteRegistry + + return RemoteRegistry(registry_config, project, repo_path) else: return super(Registry, cls).__new__(cls) @@ -528,8 +532,12 @@ def list_feature_views( ) return proto_registry_utils.list_feature_views(registry_proto, project) - def get_request_feature_view(self, name: str, project: str): - registry_proto = self._get_registry_proto(project=project, allow_cache=False) + def get_request_feature_view( + self, name: str, project: str, allow_cache: bool = False + ): + registry_proto = self._get_registry_proto( + project=project, allow_cache=allow_cache + ) return proto_registry_utils.get_request_feature_view( registry_proto, name, project ) diff --git a/sdk/python/feast/infra/registry/registry_store.py b/sdk/python/feast/infra/registry/registry_store.py index c42a55cd9d2..5151fd74b27 100644 --- a/sdk/python/feast/infra/registry/registry_store.py +++ b/sdk/python/feast/infra/registry/registry_store.py @@ -17,7 +17,7 @@ def get_registry_proto(self) -> RegistryProto: Returns: Returns either the registry proto stored at the registry path, or an empty registry proto. """ - pass + raise NotImplementedError @abstractmethod def update_registry_proto(self, registry_proto: RegistryProto): @@ -40,7 +40,7 @@ def teardown(self): class NoopRegistryStore(RegistryStore): def get_registry_proto(self) -> RegistryProto: - pass + return RegistryProto() def update_registry_proto(self, registry_proto: RegistryProto): pass diff --git a/sdk/python/feast/infra/registry/remote.py b/sdk/python/feast/infra/registry/remote.py new file mode 100644 index 00000000000..67d61ffec78 --- /dev/null +++ b/sdk/python/feast/infra/registry/remote.py @@ -0,0 +1,370 @@ +from datetime import datetime +from pathlib import Path +from typing import List, Optional, Union + +import grpc +from google.protobuf.empty_pb2 import Empty +from pydantic import StrictStr + +from feast.base_feature_view import BaseFeatureView +from feast.data_source import DataSource +from feast.entity import Entity +from feast.errors import ReadOnlyRegistryException +from feast.feature_service import FeatureService +from feast.feature_view import FeatureView +from feast.infra.infra_object import Infra +from feast.infra.registry.base_registry import BaseRegistry +from feast.on_demand_feature_view import OnDemandFeatureView +from feast.project_metadata import ProjectMetadata +from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto +from feast.protos.feast.registry import RegistryServer_pb2, RegistryServer_pb2_grpc +from feast.repo_config import RegistryConfig +from feast.request_feature_view import RequestFeatureView +from feast.saved_dataset import SavedDataset, ValidationReference +from feast.stream_feature_view import StreamFeatureView + + +class RemoteRegistryConfig(RegistryConfig): + registry_type: StrictStr = "remote" + """ str: Provider name or a class name that implements Registry.""" + + path: StrictStr = "" + """ str: Path to metadata store. + If registry_type is 'remote', then this is a URL for registry server """ + + +class RemoteRegistry(BaseRegistry): + def __init__( + self, + registry_config: Union[RegistryConfig, RemoteRegistryConfig], + project: str, + repo_path: Optional[Path], + ): + self.channel = grpc.insecure_channel(registry_config.path) + self.stub = RegistryServer_pb2_grpc.RegistryServerStub(self.channel) + + def apply_entity(self, entity: Entity, project: str, commit: bool = True): + raise ReadOnlyRegistryException() + + def delete_entity(self, name: str, project: str, commit: bool = True): + raise ReadOnlyRegistryException() + + def get_entity(self, name: str, project: str, allow_cache: bool = False) -> Entity: + request = RegistryServer_pb2.GetEntityRequest( + name=name, project=project, allow_cache=allow_cache + ) + + response = self.stub.GetEntity(request) + + return Entity.from_proto(response) + + def list_entities(self, project: str, allow_cache: bool = False) -> List[Entity]: + request = RegistryServer_pb2.ListEntitiesRequest( + project=project, allow_cache=allow_cache + ) + + response = self.stub.ListEntities(request) + + return [Entity.from_proto(entity) for entity in response.entities] + + def apply_data_source( + self, data_source: DataSource, project: str, commit: bool = True + ): + raise ReadOnlyRegistryException() + + def delete_data_source(self, name: str, project: str, commit: bool = True): + raise ReadOnlyRegistryException() + + def get_data_source( + self, name: str, project: str, allow_cache: bool = False + ) -> DataSource: + request = RegistryServer_pb2.GetDataSourceRequest( + name=name, project=project, allow_cache=allow_cache + ) + + response = self.stub.GetDataSource(request) + + return DataSource.from_proto(response) + + def list_data_sources( + self, project: str, allow_cache: bool = False + ) -> List[DataSource]: + request = RegistryServer_pb2.ListDataSourcesRequest( + project=project, allow_cache=allow_cache + ) + + response = self.stub.ListDataSources(request) + + return [ + DataSource.from_proto(data_source) for data_source in response.data_sources + ] + + def apply_feature_service( + self, feature_service: FeatureService, project: str, commit: bool = True + ): + raise ReadOnlyRegistryException() + + def delete_feature_service(self, name: str, project: str, commit: bool = True): + raise ReadOnlyRegistryException() + + def get_feature_service( + self, name: str, project: str, allow_cache: bool = False + ) -> FeatureService: + request = RegistryServer_pb2.GetFeatureServiceRequest( + name=name, project=project, allow_cache=allow_cache + ) + + response = self.stub.GetFeatureService(request) + + return FeatureService.from_proto(response) + + def list_feature_services( + self, project: str, allow_cache: bool = False + ) -> List[FeatureService]: + request = RegistryServer_pb2.ListFeatureServicesRequest( + project=project, allow_cache=allow_cache + ) + + response = self.stub.ListFeatureServices(request) + + return [ + FeatureService.from_proto(feature_service) + for feature_service in response.feature_services + ] + + def apply_feature_view( + self, feature_view: BaseFeatureView, project: str, commit: bool = True + ): + raise ReadOnlyRegistryException() + + def delete_feature_view(self, name: str, project: str, commit: bool = True): + raise ReadOnlyRegistryException() + + def get_stream_feature_view( + self, name: str, project: str, allow_cache: bool = False + ) -> StreamFeatureView: + request = RegistryServer_pb2.GetStreamFeatureViewRequest( + name=name, project=project, allow_cache=allow_cache + ) + + response = self.stub.GetStreamFeatureView(request) + + return StreamFeatureView.from_proto(response) + + def list_stream_feature_views( + self, project: str, allow_cache: bool = False + ) -> List[StreamFeatureView]: + request = RegistryServer_pb2.ListStreamFeatureViewsRequest( + project=project, allow_cache=allow_cache + ) + + response = self.stub.ListStreamFeatureViews(request) + + return [ + StreamFeatureView.from_proto(stream_feature_view) + for stream_feature_view in response.stream_feature_views + ] + + def get_on_demand_feature_view( + self, name: str, project: str, allow_cache: bool = False + ) -> OnDemandFeatureView: + request = RegistryServer_pb2.GetOnDemandFeatureViewRequest( + name=name, project=project, allow_cache=allow_cache + ) + + response = self.stub.GetOnDemandFeatureView(request) + + return OnDemandFeatureView.from_proto(response) + + def list_on_demand_feature_views( + self, project: str, allow_cache: bool = False + ) -> List[OnDemandFeatureView]: + request = RegistryServer_pb2.ListOnDemandFeatureViewsRequest( + project=project, allow_cache=allow_cache + ) + + response = self.stub.ListOnDemandFeatureViews(request) + + return [ + OnDemandFeatureView.from_proto(on_demand_feature_view) + for on_demand_feature_view in response.on_demand_feature_views + ] + + def get_feature_view( + self, name: str, project: str, allow_cache: bool = False + ) -> FeatureView: + request = RegistryServer_pb2.GetFeatureViewRequest( + name=name, project=project, allow_cache=allow_cache + ) + + response = self.stub.GetFeatureView(request) + + return FeatureView.from_proto(response) + + def list_feature_views( + self, project: str, allow_cache: bool = False + ) -> List[FeatureView]: + request = RegistryServer_pb2.ListFeatureViewsRequest( + project=project, allow_cache=allow_cache + ) + + response = self.stub.ListFeatureViews(request) + + return [ + FeatureView.from_proto(feature_view) + for feature_view in response.feature_views + ] + + def get_request_feature_view( + self, name: str, project: str, allow_cache: bool = False + ) -> RequestFeatureView: + request = RegistryServer_pb2.GetRequestFeatureViewRequest( + name=name, project=project, allow_cache=allow_cache + ) + + response = self.stub.GetRequestFeatureView(request) + + return RequestFeatureView.from_proto(response) + + def list_request_feature_views( + self, project: str, allow_cache: bool = False + ) -> List[RequestFeatureView]: + request = RegistryServer_pb2.ListRequestFeatureViewsRequest( + project=project, allow_cache=allow_cache + ) + + response = self.stub.ListRequestFeatureViews(request) + + return [ + RequestFeatureView.from_proto(request_feature_view) + for request_feature_view in response.request_feature_views + ] + + def apply_materialization( + self, + feature_view: FeatureView, + project: str, + start_date: datetime, + end_date: datetime, + commit: bool = True, + ): + raise ReadOnlyRegistryException() + + def apply_saved_dataset( + self, + saved_dataset: SavedDataset, + project: str, + commit: bool = True, + ): + raise ReadOnlyRegistryException() + + def delete_saved_dataset(self, name: str, project: str, allow_cache: bool = False): + raise ReadOnlyRegistryException() + + def get_saved_dataset( + self, name: str, project: str, allow_cache: bool = False + ) -> SavedDataset: + request = RegistryServer_pb2.GetSavedDatasetRequest( + name=name, project=project, allow_cache=allow_cache + ) + + response = self.stub.GetSavedDataset(request) + + return SavedDataset.from_proto(response) + + def list_saved_datasets( + self, project: str, allow_cache: bool = False + ) -> List[SavedDataset]: + request = RegistryServer_pb2.ListSavedDatasetsRequest( + project=project, allow_cache=allow_cache + ) + + response = self.stub.ListSavedDatasets(request) + + return [ + SavedDataset.from_proto(saved_dataset) + for saved_dataset in response.saved_datasets + ] + + def apply_validation_reference( + self, + validation_reference: ValidationReference, + project: str, + commit: bool = True, + ): + raise ReadOnlyRegistryException() + + def delete_validation_reference(self, name: str, project: str, commit: bool = True): + raise ReadOnlyRegistryException() + + def get_validation_reference( + self, name: str, project: str, allow_cache: bool = False + ) -> ValidationReference: + request = RegistryServer_pb2.GetValidationReferenceRequest( + name=name, project=project, allow_cache=allow_cache + ) + + response = self.stub.GetValidationReference(request) + + return ValidationReference.from_proto(response) + + def list_validation_references( + self, project: str, allow_cache: bool = False + ) -> List[ValidationReference]: + request = RegistryServer_pb2.ListValidationReferencesRequest( + project=project, allow_cache=allow_cache + ) + + response = self.stub.ListValidationReferences(request) + + return [ + ValidationReference.from_proto(validation_reference) + for validation_reference in response.validation_references + ] + + def list_project_metadata( + self, project: str, allow_cache: bool = False + ) -> List[ProjectMetadata]: + request = RegistryServer_pb2.ListProjectMetadataRequest( + project=project, allow_cache=allow_cache + ) + + response = self.stub.ListProjectMetadata(request) + + return [ProjectMetadata.from_proto(pm) for pm in response.project_metadata] + + def update_infra(self, infra: Infra, project: str, commit: bool = True): + raise ReadOnlyRegistryException() + + def get_infra(self, project: str, allow_cache: bool = False) -> Infra: + request = RegistryServer_pb2.GetInfraRequest( + project=project, allow_cache=allow_cache + ) + + response = self.stub.GetInfra(request) + + return Infra.from_proto(response) + + def apply_user_metadata( + self, + project: str, + feature_view: BaseFeatureView, + metadata_bytes: Optional[bytes], + ): + pass + + def get_user_metadata( + self, project: str, feature_view: BaseFeatureView + ) -> Optional[bytes]: + pass + + def proto(self) -> RegistryProto: + return self.stub.Proto(Empty()) + + def commit(self): + raise ReadOnlyRegistryException() + + def refresh(self, project: Optional[str] = None): + request = RegistryServer_pb2.RefreshRequest(project=str(project)) + + self.stub.Refresh(request) diff --git a/sdk/python/feast/infra/registry/snowflake.py b/sdk/python/feast/infra/registry/snowflake.py index 40ec27e7d9c..cdf79c78b5f 100644 --- a/sdk/python/feast/infra/registry/snowflake.py +++ b/sdk/python/feast/infra/registry/snowflake.py @@ -5,10 +5,9 @@ from datetime import datetime, timedelta from enum import Enum from threading import Lock -from typing import Any, Callable, List, Optional, Set, Union +from typing import Any, Callable, List, Literal, Optional, Set, Union -from pydantic import Field, StrictStr -from pydantic.schema import Literal +from pydantic import ConfigDict, Field, StrictStr import feast from feast import usage @@ -103,9 +102,7 @@ class SnowflakeRegistryConfig(RegistryConfig): schema_: Optional[str] = Field("PUBLIC", alias="schema") """ Snowflake schema name """ - - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class SnowflakeRegistry(BaseRegistry): @@ -124,19 +121,15 @@ def __init__( f'"{self.registry_config.database}"."{self.registry_config.schema_}"' ) - if not self._verify_registry_database(): - # Verify the existing resitry database schema from snowflake. If any table names and column types is wrong, run table recreation SQL. - with GetSnowflakeConnection(self.registry_config) as conn: - sql_function_file = f"{os.path.dirname(feast.__file__)}/infra/utils/snowflake/registry/snowflake_table_creation.sql" - with open(sql_function_file, "r") as file: - sqlFile = file.read() + with GetSnowflakeConnection(self.registry_config) as conn: + sql_function_file = f"{os.path.dirname(feast.__file__)}/infra/utils/snowflake/registry/snowflake_table_creation.sql" + with open(sql_function_file, "r") as file: + sqlFile = file.read() - sqlCommands = sqlFile.split(";") - for command in sqlCommands: - query = command.replace( - "REGISTRY_PATH", f"{self.registry_path}" - ) - execute_snowflake_statement(conn, query) + sqlCommands = sqlFile.split(";") + for command in sqlCommands: + query = command.replace("REGISTRY_PATH", f"{self.registry_path}") + execute_snowflake_statement(conn, query) self.cached_registry_proto = self.proto() proto_registry_utils.init_project_metadata(self.cached_registry_proto, project) @@ -149,55 +142,6 @@ def __init__( ) self.project = project - def _verify_registry_database( - self, - ) -> bool: - """Verify the records in registry database. To check: - 1, the 11 tables are existed. - 2, the column types are correct. - - Example return from snowflake's cursor.describe("SELECT * FROM a_table") command: - [ResultMetadata(name='ENTITY_NAME', type_code=2, display_size=None, internal_size=16777216, precision=None, scale=None, is_nullable=False), - ResultMetadata(name='PROJECT_ID', type_code=2, display_size=None, internal_size=16777216, precision=None, scale=None, is_nullable=False), - ResultMetadata(name='LAST_UPDATED_TIMESTAMP', type_code=6, display_size=None, internal_size=None, precision=0, scale=9, is_nullable=False), - ResultMetadata(name='ENTITY_PROTO', type_code=11, display_size=None, internal_size=8388608, precision=None, scale=None, is_nullable=False)] - - Returns: - True if the necessary 11 tables are existed in Snowflake and schema of each table is correct. - False if failure happens. - """ - - from feast.infra.utils.snowflake.registry.snowflake_registry_table import ( - snowflake_registry_table_names_and_column_types as expect_tables, - ) - - res = True - - try: - with GetSnowflakeConnection(self.registry_config) as conn: - for table_name in expect_tables: - result_metadata_list = conn.cursor().describe( - f"SELECT * FROM {table_name}" - ) - for col in result_metadata_list: - if ( - expect_tables[table_name][col.name]["type_code"] - != col.type_code - ): - res = False - break - except Exception as e: - res = False # Set to False for all errors. - logger.debug( - f"Failed to verify Registry tables and columns types with exception: {e}." - ) - finally: - # The implementation in snowflake_utils.py will cache the established connection without re-connection logic. - # conn.close() - pass - - return res - def refresh(self, project: Optional[str] = None): if project: project_metadata = proto_registry_utils.get_project_metadata( @@ -471,7 +415,7 @@ def _delete_object( """ cursor = execute_snowflake_statement(conn, query) - if cursor.rowcount < 1 and not_found_exception: + if cursor.rowcount < 1 and not_found_exception: # type: ignore raise not_found_exception(name, project) self._set_last_updated_metadata(datetime.utcnow(), project) diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index d57bcc7c0a3..1e5c2a87258 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -4,7 +4,7 @@ from enum import Enum from pathlib import Path from threading import Lock -from typing import Any, Callable, List, Optional, Set, Union +from typing import Any, Callable, Dict, List, Optional, Set, Union from pydantic import StrictStr from sqlalchemy import ( # type: ignore @@ -190,6 +190,9 @@ class SqlRegistryConfig(RegistryConfig): """ str: Path to metadata store. If registry_type is 'sql', then this is a database URL as expected by SQLAlchemy """ + sqlalchemy_config_kwargs: Dict[str, Any] = {"echo": False} + """ Dict[str, Any]: Extra arguments to pass to SQLAlchemy.create_engine. """ + class SqlRegistry(BaseRegistry): def __init__( @@ -199,7 +202,9 @@ def __init__( repo_path: Optional[Path], ): assert registry_config is not None, "SqlRegistry needs a valid registry_config" - self.engine: Engine = create_engine(registry_config.path, echo=False) + self.engine: Engine = create_engine( + registry_config.path, **registry_config.sqlalchemy_config_kwargs + ) metadata.create_all(self.engine) self.cached_registry_proto = self.proto() proto_registry_utils.init_project_metadata(self.cached_registry_proto, project) diff --git a/sdk/python/feast/infra/transformation_servers/Dockerfile b/sdk/python/feast/infra/transformation_servers/Dockerfile index c072ed01604..41f272c757c 100644 --- a/sdk/python/feast/infra/transformation_servers/Dockerfile +++ b/sdk/python/feast/infra/transformation_servers/Dockerfile @@ -15,7 +15,7 @@ COPY README.md README.md # Install dependencies -RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir -e '.[gcp,aws]' +RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir '.[gcp,aws]' # Start feature transformation server CMD [ "python", "app.py" ] diff --git a/sdk/python/feast/infra/utils/aws_utils.py b/sdk/python/feast/infra/utils/aws_utils.py index ef83c6d1c62..c3604ee41f0 100644 --- a/sdk/python/feast/infra/utils/aws_utils.py +++ b/sdk/python/feast/infra/utils/aws_utils.py @@ -816,7 +816,7 @@ def execute_athena_query( database: str, workgroup: str, query: str, - temp_table: str = None, + temp_table: Optional[str] = None, ) -> str: """Execute athena statement synchronously. Waits for the query to finish. diff --git a/sdk/python/feast/infra/utils/hbase_utils.py b/sdk/python/feast/infra/utils/hbase_utils.py index d44f93f1619..72afda2ef3d 100644 --- a/sdk/python/feast/infra/utils/hbase_utils.py +++ b/sdk/python/feast/infra/utils/hbase_utils.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional from happybase import ConnectionPool @@ -38,9 +38,9 @@ class HBaseConnector: def __init__( self, - pool: ConnectionPool = None, - host: str = None, - port: int = None, + pool: Optional[ConnectionPool] = None, + host: Optional[str] = None, + port: Optional[int] = None, connection_pool_size: int = 4, ): if pool is None: diff --git a/sdk/python/feast/infra/utils/snowflake/registry/snowflake_registry_table.py b/sdk/python/feast/infra/utils/snowflake/registry/snowflake_registry_table.py deleted file mode 100644 index d24fbc27ecb..00000000000 --- a/sdk/python/feast/infra/utils/snowflake/registry/snowflake_registry_table.py +++ /dev/null @@ -1,104 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -The table names and column types are following the creation detail listed -in "snowflake_table_creation.sql". - -Snowflake Reference: -1, ResultMetadata: https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-api#label-python-connector-resultmetadata-object -2, Type Codes: https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-api#label-python-connector-type-codes ----------------------------------------------- -type_code String Representation Data Type -0 FIXED NUMBER/INT -1 REAL REAL -2 TEXT VARCHAR/STRING -3 DATE DATE -4 TIMESTAMP TIMESTAMP -5 VARIANT VARIANT -6 TIMESTAMP_LTZ TIMESTAMP_LTZ -7 TIMESTAMP_TZ TIMESTAMP_TZ -8 TIMESTAMP_NTZ TIMESTAMP_TZ -9 OBJECT OBJECT -10 ARRAY ARRAY -11 BINARY BINARY -12 TIME TIME -13 BOOLEAN BOOLEAN ----------------------------------------------- - -(last update: 2023-11-30) - -""" - -snowflake_registry_table_names_and_column_types = { - "DATA_SOURCES": { - "DATA_SOURCE_NAME": {"type_code": 2, "type": "VARCHAR"}, - "PROJECT_ID": {"type_code": 2, "type": "VARCHAR"}, - "LAST_UPDATED_TIMESTAMP": {"type_code": 6, "type": "TIMESTAMP_LTZ"}, - "DATA_SOURCE_PROTO": {"type_code": 11, "type": "BINARY"}, - }, - "ENTITIES": { - "ENTITY_NAME": {"type_code": 2, "type": "VARCHAR"}, - "PROJECT_ID": {"type_code": 2, "type": "VARCHAR"}, - "LAST_UPDATED_TIMESTAMP": {"type_code": 6, "type": "TIMESTAMP_LTZ"}, - "ENTITY_PROTO": {"type_code": 11, "type": "BINARY"}, - }, - "FEAST_METADATA": { - "PROJECT_ID": {"type_code": 2, "type": "VARCHAR"}, - "METADATA_KEY": {"type_code": 2, "type": "VARCHAR"}, - "METADATA_VALUE": {"type_code": 2, "type": "VARCHAR"}, - "LAST_UPDATED_TIMESTAMP": {"type_code": 6, "type": "TIMESTAMP_LTZ"}, - }, - "FEATURE_SERVICES": { - "FEATURE_SERVICE_NAME": {"type_code": 2, "type": "VARCHAR"}, - "PROJECT_ID": {"type_code": 2, "type": "VARCHAR"}, - "LAST_UPDATED_TIMESTAMP": {"type_code": 6, "type": "TIMESTAMP_LTZ"}, - "FEATURE_SERVICE_PROTO": {"type_code": 11, "type": "BINARY"}, - }, - "FEATURE_VIEWS": { - "FEATURE_VIEW_NAME": {"type_code": 2, "type": "VARCHAR"}, - "PROJECT_ID": {"type_code": 2, "type": "VARCHAR"}, - "LAST_UPDATED_TIMESTAMP": {"type_code": 6, "type": "TIMESTAMP_LTZ"}, - "FEATURE_VIEW_PROTO": {"type_code": 11, "type": "BINARY"}, - "MATERIALIZED_INTERVALS": {"type_code": 11, "type": "BINARY"}, - "USER_METADATA": {"type_code": 11, "type": "BINARY"}, - }, - "MANAGED_INFRA": { - "INFRA_NAME": {"type_code": 2, "type": "VARCHAR"}, - "PROJECT_ID": {"type_code": 2, "type": "VARCHAR"}, - "LAST_UPDATED_TIMESTAMP": {"type_code": 6, "type": "TIMESTAMP_LTZ"}, - "INFRA_PROTO": {"type_code": 11, "type": "BINARY"}, - }, - "ON_DEMAND_FEATURE_VIEWS": { - "ON_DEMAND_FEATURE_VIEW_NAME": {"type_code": 2, "type": "VARCHAR"}, - "PROJECT_ID": {"type_code": 2, "type": "VARCHAR"}, - "LAST_UPDATED_TIMESTAMP": {"type_code": 6, "type": "TIMESTAMP_LTZ"}, - "ON_DEMAND_FEATURE_VIEW_PROTO": {"type_code": 11, "type": "BINARY"}, - "USER_METADATA": {"type_code": 11, "type": "BINARY"}, - }, - "REQUEST_FEATURE_VIEWS": { - "REQUEST_FEATURE_VIEW_NAME": {"type_code": 2, "type": "VARCHAR"}, - "PROJECT_ID": {"type_code": 2, "type": "VARCHAR"}, - "LAST_UPDATED_TIMESTAMP": {"type_code": 6, "type": "TIMESTAMP_LTZ"}, - "REQUEST_FEATURE_VIEW_PROTO": {"type_code": 11, "type": "BINARY"}, - "USER_METADATA": {"type_code": 11, "type": "BINARY"}, - }, - "SAVED_DATASETS": { - "SAVED_DATASET_NAME": {"type_code": 2, "type": "VARCHAR"}, - "PROJECT_ID": {"type_code": 2, "type": "VARCHAR"}, - "LAST_UPDATED_TIMESTAMP": {"type_code": 6, "type": "TIMESTAMP_LTZ"}, - "SAVED_DATASET_PROTO": {"type_code": 11, "type": "BINARY"}, - }, - "STREAM_FEATURE_VIEWS": { - "STREAM_FEATURE_VIEW_NAME": {"type_code": 2, "type": "VARCHAR"}, - "PROJECT_ID": {"type_code": 2, "type": "VARCHAR"}, - "LAST_UPDATED_TIMESTAMP": {"type_code": 6, "type": "TIMESTAMP_LTZ"}, - "STREAM_FEATURE_VIEW_PROTO": {"type_code": 11, "type": "BINARY"}, - "USER_METADATA": {"type_code": 11, "type": "BINARY"}, - }, - "VALIDATION_REFERENCES": { - "VALIDATION_REFERENCE_NAME": {"type_code": 2, "type": "VARCHAR"}, - "PROJECT_ID": {"type_code": 2, "type": "VARCHAR"}, - "LAST_UPDATED_TIMESTAMP": {"type_code": 6, "type": "TIMESTAMP_LTZ"}, - "VALIDATION_REFERENCE_PROTO": {"type_code": 11, "type": "BINARY"}, - }, -} diff --git a/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py b/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py index 3a56619bdb5..8548e4dbd86 100644 --- a/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py +++ b/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py @@ -43,25 +43,25 @@ class GetSnowflakeConnection: - def __init__(self, config: str, autocommit=True): + def __init__(self, config: Any, autocommit=True): self.config = config self.autocommit = autocommit def __enter__(self): - assert self.config.type in { + assert self.config.type in [ "snowflake.registry", "snowflake.offline", "snowflake.engine", "snowflake.online", - } + ] if self.config.type not in _cache: if self.config.type == "snowflake.registry": config_header = "connections.feast_registry" elif self.config.type == "snowflake.offline": config_header = "connections.feast_offline_store" - elif self.config.type == "snowflake.engine": + if self.config.type == "snowflake.engine": config_header = "connections.feast_batch_engine" elif self.config.type == "snowflake.online": config_header = "connections.feast_online_store" @@ -113,11 +113,11 @@ def __exit__(self, exc_type, exc_val, exc_tb): def assert_snowflake_feature_names(feature_view: FeatureView) -> None: for feature in feature_view.features: - assert feature.name not in { + assert feature.name not in [ "entity_key", "feature_name", "feature_value", - }, f"Feature Name: {feature.name} is a protected name to ensure query stability" + ], f"Feature Name: {feature.name} is a protected name to ensure query stability" return None diff --git a/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_python_udfs_creation.sql b/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_python_udfs_creation.sql index a197a3ee4cd..a444c0b7c5c 100644 --- a/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_python_udfs_creation.sql +++ b/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_python_udfs_creation.sql @@ -14,6 +14,62 @@ CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_varchar_to_string_pro HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_varchar_to_string_proto' IMPORTS = ('@STAGE_HOLDER/feast.zip'); +CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_bytes_to_list_bytes_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_bytes_to_list_bytes_proto' + IMPORTS = ('@STAGE_HOLDER/feast.zip'); + +CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_varchar_to_list_string_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_varchar_to_list_string_proto' + IMPORTS = ('@STAGE_HOLDER/feast.zip'); + +CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_number_to_list_int32_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_number_to_list_int32_proto' + IMPORTS = ('@STAGE_HOLDER/feast.zip'); + +CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_number_to_list_int64_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_number_to_list_int64_proto' + IMPORTS = ('@STAGE_HOLDER/feast.zip'); + +CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_float_to_list_double_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_float_to_list_double_proto' + IMPORTS = ('@STAGE_HOLDER/feast.zip'); + +CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_boolean_to_list_bool_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_boolean_to_list_bool_proto' + IMPORTS = ('@STAGE_HOLDER/feast.zip'); + +CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_array_timestamp_to_list_unix_timestamp_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_timestamp_to_list_unix_timestamp_proto' + IMPORTS = ('@STAGE_HOLDER/feast.zip'); + CREATE FUNCTION IF NOT EXISTS feast_PROJECT_NAME_snowflake_number_to_int32_proto(df NUMBER) RETURNS BINARY LANGUAGE PYTHON diff --git a/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_udfs.py b/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_udfs.py index 02311ca55d6..f5d5f10631f 100644 --- a/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_udfs.py +++ b/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_udfs.py @@ -1,6 +1,7 @@ import sys from binascii import unhexlify +import numpy as np import pandas from _snowflake import vectorized @@ -59,6 +60,180 @@ def feast_snowflake_varchar_to_string_proto(df): return df +""" +CREATE OR REPLACE FUNCTION feast_snowflake_array_bytes_to_list_bytes_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_bytes_to_list_bytes_proto' + IMPORTS = ('@feast_stage/feast.zip'); +""" +# ValueType.STRING_LIST = 12 +@vectorized(input=pandas.DataFrame) +def feast_snowflake_array_bytes_to_list_bytes_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + + # Sometimes bytes come in as strings so we need to convert back to float + numpy_arrays = np.asarray(df[0].to_list()).astype(bytes) + + df = list( + map( + ValueProto.SerializeToString, + python_values_to_proto_values(numpy_arrays, ValueType.BYTES_LIST), + ) + ) + return df + + +""" +CREATE OR REPLACE FUNCTION feast_snowflake_array_varchar_to_list_string_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_varchar_to_list_string_proto' + IMPORTS = ('@feast_stage/feast.zip'); +""" + + +@vectorized(input=pandas.DataFrame) +def feast_snowflake_array_varchar_to_list_string_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + + df = list( + map( + ValueProto.SerializeToString, + python_values_to_proto_values(df[0].to_numpy(), ValueType.STRING_LIST), + ) + ) + return df + + +""" +CREATE OR REPLACE FUNCTION feast_snowflake_array_number_to_list_int32_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_number_to_list_int32_proto' + IMPORTS = ('@feast_stage/feast.zip'); +""" + + +@vectorized(input=pandas.DataFrame) +def feast_snowflake_array_number_to_list_int32_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + + df = list( + map( + ValueProto.SerializeToString, + python_values_to_proto_values(df[0].to_numpy(), ValueType.INT32_LIST), + ) + ) + return df + + +""" +CREATE OR REPLACE FUNCTION feast_snowflake_array_number_to_list_int64_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_number_to_list_int64_proto' + IMPORTS = ('@feast_stage/feast.zip'); +""" + + +@vectorized(input=pandas.DataFrame) +def feast_snowflake_array_number_to_list_int64_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + + df = list( + map( + ValueProto.SerializeToString, + python_values_to_proto_values(df[0].to_numpy(), ValueType.INT64_LIST), + ) + ) + return df + + +""" +CREATE OR REPLACE FUNCTION feast_snowflake_array_float_to_list_double_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_float_to_list_double_proto' + IMPORTS = ('@feast_stage/feast.zip'); +""" + + +@vectorized(input=pandas.DataFrame) +def feast_snowflake_array_float_to_list_double_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + + numpy_arrays = np.asarray(df[0].to_list()).astype(float) + + df = list( + map( + ValueProto.SerializeToString, + python_values_to_proto_values(numpy_arrays, ValueType.DOUBLE_LIST), + ) + ) + return df + + +""" +CREATE OR REPLACE FUNCTION feast_snowflake_array_boolean_to_list_bool_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_boolean_to_list_bool_proto' + IMPORTS = ('@feast_stage/feast.zip'); +""" + + +@vectorized(input=pandas.DataFrame) +def feast_snowflake_array_boolean_to_list_bool_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + + df = list( + map( + ValueProto.SerializeToString, + python_values_to_proto_values(df[0].to_numpy(), ValueType.BOOL_LIST), + ) + ) + return df + + +""" +CREATE OR REPLACE FUNCTION feast_snowflake_array_timestamp_to_list_unix_timestamp_proto(df ARRAY) + RETURNS BINARY + LANGUAGE PYTHON + RUNTIME_VERSION = '3.8' + PACKAGES = ('protobuf', 'pandas') + HANDLER = 'feast.infra.utils.snowflake.snowpark.snowflake_udfs.feast_snowflake_array_timestamp_to_list_unix_timestamp_proto' + IMPORTS = ('@feast_stage/feast.zip'); +""" + + +@vectorized(input=pandas.DataFrame) +def feast_snowflake_array_timestamp_to_list_unix_timestamp_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + + numpy_arrays = np.asarray(df[0].to_list()).astype(np.datetime64) + + df = list( + map( + ValueProto.SerializeToString, + python_values_to_proto_values(numpy_arrays, ValueType.UNIX_TIMESTAMP_LIST), + ) + ) + return df + + """ CREATE OR REPLACE FUNCTION feast_snowflake_number_to_int32_proto(df NUMBER) RETURNS BINARY diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index fcafeaa2bc1..586286a3d47 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -1,5 +1,6 @@ import copy import functools +import inspect import warnings from datetime import datetime from types import FunctionType @@ -16,6 +17,8 @@ from feast.feature_view import FeatureView from feast.feature_view_projection import FeatureViewProjection from feast.field import Field, from_value_type +from feast.on_demand_pandas_transformation import OnDemandPandasTransformation +from feast.on_demand_substrait_transformation import OnDemandSubstraitTransformation from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( OnDemandFeatureView as OnDemandFeatureViewProto, ) @@ -24,9 +27,6 @@ OnDemandFeatureViewSpec, OnDemandSource, ) -from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( - UserDefinedFunction as UserDefinedFunctionProto, -) from feast.type_map import ( feast_value_type_to_pandas_type, python_type_to_feast_value_type, @@ -51,8 +51,7 @@ class OnDemandFeatureView(BaseFeatureView): sources with type FeatureViewProjection. source_request_sources: A map from input source names to the actual input sources with type RequestSource. - udf: The user defined transformation function, which must take pandas dataframes - as inputs. + transformation: The user defined transformation. description: A human-readable description. tags: A dictionary of key-value pairs to store arbitrary metadata. owner: The owner of the on demand feature view, typically the email of the primary @@ -63,8 +62,7 @@ class OnDemandFeatureView(BaseFeatureView): features: List[Field] source_feature_view_projections: Dict[str, FeatureViewProjection] source_request_sources: Dict[str, RequestSource] - udf: FunctionType - udf_string: str + transformation: Union[OnDemandPandasTransformation] description: str tags: Dict[str, str] owner: str @@ -82,8 +80,9 @@ def __init__( # noqa: C901 FeatureViewProjection, ] ], - udf: FunctionType, + udf: Optional[FunctionType] = None, udf_string: str = "", + transformation: Optional[Union[OnDemandPandasTransformation]] = None, description: str = "", tags: Optional[Dict[str, str]] = None, owner: str = "", @@ -98,9 +97,10 @@ def __init__( # noqa: C901 sources: A map from input source names to the actual input sources, which may be feature views, or request data sources. These sources serve as inputs to the udf, which will refer to them by name. - udf: The user defined transformation function, which must take pandas + udf (deprecated): The user defined transformation function, which must take pandas dataframes as inputs. - udf_string: The source code version of the udf (for diffing and displaying in Web UI) + udf_string (deprecated): The source code version of the udf (for diffing and displaying in Web UI) + transformation: The user defined transformation. description (optional): A human-readable description. tags (optional): A dictionary of key-value pairs to store arbitrary metadata. owner (optional): The owner of the on demand feature view, typically the email @@ -114,6 +114,18 @@ def __init__( # noqa: C901 owner=owner, ) + if not transformation: + if udf: + warnings.warn( + "udf and udf_string parameters are deprecated. Please use transformation=OnDemandPandasTransformation(udf, udf_string) instead.", + DeprecationWarning, + ) + transformation = OnDemandPandasTransformation(udf, udf_string) + else: + raise Exception( + "OnDemandFeatureView needs to be initialized with either transformation or udf arguments" + ) + self.source_feature_view_projections: Dict[str, FeatureViewProjection] = {} self.source_request_sources: Dict[str, RequestSource] = {} for odfv_source in sources: @@ -126,8 +138,7 @@ def __init__( # noqa: C901 odfv_source.name ] = odfv_source.projection - self.udf = udf # type: ignore - self.udf_string = udf_string + self.transformation = transformation @property def proto_class(self) -> Type[OnDemandFeatureViewProto]: @@ -139,8 +150,7 @@ def __copy__(self): schema=self.features, sources=list(self.source_feature_view_projections.values()) + list(self.source_request_sources.values()), - udf=self.udf, - udf_string=self.udf_string, + transformation=self.transformation, description=self.description, tags=self.tags, owner=self.owner, @@ -161,8 +171,7 @@ def __eq__(self, other): self.source_feature_view_projections != other.source_feature_view_projections or self.source_request_sources != other.source_request_sources - or self.udf_string != other.udf_string - or self.udf.__code__.co_code != other.udf.__code__.co_code + or self.transformation != other.transformation ): return False @@ -200,11 +209,12 @@ def to_proto(self) -> OnDemandFeatureViewProto: name=self.name, features=[feature.to_proto() for feature in self.features], sources=sources, - user_defined_function=UserDefinedFunctionProto( - name=self.udf.__name__, - body=dill.dumps(self.udf, recurse=True), - body_text=self.udf_string, - ), + user_defined_function=self.transformation.to_proto() + if type(self.transformation) == OnDemandPandasTransformation + else None, + on_demand_substrait_transformation=self.transformation.to_proto() # type: ignore + if type(self.transformation) == OnDemandSubstraitTransformation + else None, description=self.description, tags=self.tags, owner=self.owner, @@ -243,6 +253,23 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): RequestSource.from_proto(on_demand_source.request_data_source) ) + if ( + on_demand_feature_view_proto.spec.WhichOneof("transformation") + == "user_defined_function" + ): + transformation = OnDemandPandasTransformation.from_proto( + on_demand_feature_view_proto.spec.user_defined_function + ) + elif ( + on_demand_feature_view_proto.spec.WhichOneof("transformation") + == "on_demand_substrait_transformation" + ): + transformation = OnDemandSubstraitTransformation.from_proto( + on_demand_feature_view_proto.spec.on_demand_substrait_transformation + ) + else: + raise Exception("At least one transformation type needs to be provided") + on_demand_feature_view_obj = cls( name=on_demand_feature_view_proto.spec.name, schema=[ @@ -253,10 +280,7 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): for feature in on_demand_feature_view_proto.spec.features ], sources=sources, - udf=dill.loads( - on_demand_feature_view_proto.spec.user_defined_function.body - ), - udf_string=on_demand_feature_view_proto.spec.user_defined_function.body_text, + transformation=transformation, description=on_demand_feature_view_proto.spec.description, tags=dict(on_demand_feature_view_proto.spec.tags), owner=on_demand_feature_view_proto.spec.owner, @@ -315,7 +339,8 @@ def get_transformed_features_df( columns_to_cleanup.append(full_feature_ref) # Compute transformed values and apply to each result row - df_with_transformed_features = self.udf.__call__(df_with_features) + + df_with_transformed_features = self.transformation.transform(df_with_features) # Work out whether the correct columns names are used. rename_columns: Dict[str, str] = {} @@ -335,7 +360,7 @@ def get_transformed_features_df( df_with_features.drop(columns=columns_to_cleanup, inplace=True) return df_with_transformed_features.rename(columns=rename_columns) - def infer_features(self): + def infer_features(self) -> None: """ Infers the set of features associated to this feature view from the input source. @@ -365,7 +390,7 @@ def infer_features(self): dtype = feast_value_type_to_pandas_type(field.dtype.to_value_type()) sample_val = rand_df_value[dtype] if dtype in rand_df_value else None df[f"{field.name}"] = pd.Series(sample_val, dtype=dtype) - output_df: pd.DataFrame = self.udf.__call__(df) + output_df: pd.DataFrame = self.transformation.transform(df) inferred_features = [] for f, dt in zip(output_df.columns, output_df.dtypes): inferred_features.append( @@ -396,7 +421,9 @@ def infer_features(self): ) @staticmethod - def get_requested_odfvs(feature_refs, project, registry): + def get_requested_odfvs( + feature_refs, project, registry + ) -> List["OnDemandFeatureView"]: all_on_demand_feature_views = registry.list_on_demand_feature_views( project, allow_cache=True ) @@ -438,24 +465,63 @@ def on_demand_feature_view( of the primary maintainer. """ - def mainify(obj): + def mainify(obj) -> None: # Needed to allow dill to properly serialize the udf. Otherwise, clients will need to have a file with the same # name as the original file defining the ODFV. if obj.__module__ != "__main__": obj.__module__ = "__main__" def decorator(user_function): - udf_string = dill.source.getsource(user_function) - mainify(user_function) + return_annotation = inspect.signature(user_function).return_annotation + if ( + return_annotation + and return_annotation.__module__ == "ibis.expr.types.relations" + and return_annotation.__name__ == "Table" + ): + import ibis + import ibis.expr.datatypes as dt + from ibis_substrait.compiler.core import SubstraitCompiler + + compiler = SubstraitCompiler() + + input_fields: Field = [] + + for s in sources: + if type(s) == FeatureView: + fields = s.projection.features + else: + fields = s.features + + input_fields.extend( + [ + ( + f.name, + dt.dtype( + feast_value_type_to_pandas_type(f.dtype.to_value_type()) + ), + ) + for f in fields + ] + ) + + expr = user_function(ibis.table(input_fields, "t")) + + transformation = OnDemandSubstraitTransformation( + substrait_plan=compiler.compile(expr).SerializeToString() + ) + else: + udf_string = dill.source.getsource(user_function) + mainify(user_function) + transformation = OnDemandPandasTransformation(user_function, udf_string) + on_demand_feature_view_obj = OnDemandFeatureView( name=user_function.__name__, sources=sources, schema=schema, - udf=user_function, + transformation=transformation, description=description, tags=tags, owner=owner, - udf_string=udf_string, ) functools.update_wrapper( wrapper=on_demand_feature_view_obj, wrapped=user_function diff --git a/sdk/python/feast/on_demand_pandas_transformation.py b/sdk/python/feast/on_demand_pandas_transformation.py new file mode 100644 index 00000000000..52d45893c51 --- /dev/null +++ b/sdk/python/feast/on_demand_pandas_transformation.py @@ -0,0 +1,56 @@ +from types import FunctionType + +import dill +import pandas as pd + +from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( + UserDefinedFunction as UserDefinedFunctionProto, +) + + +class OnDemandPandasTransformation: + def __init__(self, udf: FunctionType, udf_string: str = ""): + """ + Creates an OnDemandPandasTransformation object. + + Args: + udf: The user defined transformation function, which must take pandas + dataframes as inputs. + udf_string: The source code version of the udf (for diffing and displaying in Web UI) + """ + self.udf = udf + self.udf_string = udf_string + + def transform(self, df: pd.DataFrame) -> pd.DataFrame: + return self.udf.__call__(df) + + def __eq__(self, other): + if not isinstance(other, OnDemandPandasTransformation): + raise TypeError( + "Comparisons should only involve OnDemandPandasTransformation class objects." + ) + + if not super().__eq__(other): + return False + + if ( + self.udf_string != other.udf_string + or self.udf.__code__.co_code != other.udf.__code__.co_code + ): + return False + + return True + + def to_proto(self) -> UserDefinedFunctionProto: + return UserDefinedFunctionProto( + name=self.udf.__name__, + body=dill.dumps(self.udf, recurse=True), + body_text=self.udf_string, + ) + + @classmethod + def from_proto(cls, user_defined_function_proto: UserDefinedFunctionProto): + return OnDemandPandasTransformation( + udf=dill.loads(user_defined_function_proto.body), + udf_string=user_defined_function_proto.body_text, + ) diff --git a/sdk/python/feast/on_demand_substrait_transformation.py b/sdk/python/feast/on_demand_substrait_transformation.py new file mode 100644 index 00000000000..4e92e77dc8a --- /dev/null +++ b/sdk/python/feast/on_demand_substrait_transformation.py @@ -0,0 +1,50 @@ +import pandas as pd +import pyarrow +import pyarrow.substrait as substrait # type: ignore # noqa + +from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( + OnDemandSubstraitTransformation as OnDemandSubstraitTransformationProto, +) + + +class OnDemandSubstraitTransformation: + def __init__(self, substrait_plan: bytes): + """ + Creates an OnDemandSubstraitTransformation object. + + Args: + substrait_plan: The user-provided substrait plan. + """ + self.substrait_plan = substrait_plan + + def transform(self, df: pd.DataFrame) -> pd.DataFrame: + def table_provider(names, schema: pyarrow.Schema): + return pyarrow.Table.from_pandas(df[schema.names]) + + table: pyarrow.Table = pyarrow.substrait.run_query( + self.substrait_plan, table_provider=table_provider + ).read_all() + return table.to_pandas() + + def __eq__(self, other): + if not isinstance(other, OnDemandSubstraitTransformation): + raise TypeError( + "Comparisons should only involve OnDemandSubstraitTransformation class objects." + ) + + if not super().__eq__(other): + return False + + return self.substrait_plan == other.substrait_plan + + def to_proto(self) -> OnDemandSubstraitTransformationProto: + return OnDemandSubstraitTransformationProto(substrait_plan=self.substrait_plan) + + @classmethod + def from_proto( + cls, + on_demand_substrait_transformation_proto: OnDemandSubstraitTransformationProto, + ): + return OnDemandSubstraitTransformation( + substrait_plan=on_demand_substrait_transformation_proto.substrait_plan + ) diff --git a/sdk/python/feast/registry_server.py b/sdk/python/feast/registry_server.py new file mode 100644 index 00000000000..221715480e5 --- /dev/null +++ b/sdk/python/feast/registry_server.py @@ -0,0 +1,202 @@ +from concurrent import futures + +import grpc +from google.protobuf.empty_pb2 import Empty + +from feast import FeatureStore +from feast.protos.feast.registry import RegistryServer_pb2, RegistryServer_pb2_grpc + + +class RegistryServer(RegistryServer_pb2_grpc.RegistryServerServicer): + def __init__(self, store: FeatureStore) -> None: + super().__init__() + self.proxied_registry = store.registry + + def GetEntity(self, request: RegistryServer_pb2.GetEntityRequest, context): + return self.proxied_registry.get_entity( + name=request.name, project=request.project, allow_cache=request.allow_cache + ).to_proto() + + def ListEntities(self, request, context): + return RegistryServer_pb2.ListEntitiesResponse( + entities=[ + entity.to_proto() + for entity in self.proxied_registry.list_entities( + project=request.project, allow_cache=request.allow_cache + ) + ] + ) + + def GetDataSource(self, request: RegistryServer_pb2.GetDataSourceRequest, context): + return self.proxied_registry.get_data_source( + name=request.name, project=request.project, allow_cache=request.allow_cache + ).to_proto() + + def ListDataSources(self, request, context): + return RegistryServer_pb2.ListDataSourcesResponse( + data_sources=[ + data_source.to_proto() + for data_source in self.proxied_registry.list_data_sources( + project=request.project, allow_cache=request.allow_cache + ) + ] + ) + + def GetFeatureView( + self, request: RegistryServer_pb2.GetFeatureViewRequest, context + ): + return self.proxied_registry.get_feature_view( + name=request.name, project=request.project, allow_cache=request.allow_cache + ).to_proto() + + def ListFeatureViews(self, request, context): + return RegistryServer_pb2.ListFeatureViewsResponse( + feature_views=[ + feature_view.to_proto() + for feature_view in self.proxied_registry.list_feature_views( + project=request.project, allow_cache=request.allow_cache + ) + ] + ) + + def GetRequestFeatureView( + self, request: RegistryServer_pb2.GetRequestFeatureViewRequest, context + ): + return self.proxied_registry.get_request_feature_view( + name=request.name, project=request.project, allow_cache=request.allow_cache + ).to_proto() + + def ListRequestFeatureViews(self, request, context): + return RegistryServer_pb2.ListRequestFeatureViewsResponse( + request_feature_views=[ + request_feature_view.to_proto() + for request_feature_view in self.proxied_registry.list_request_feature_views( + project=request.project, allow_cache=request.allow_cache + ) + ] + ) + + def GetStreamFeatureView( + self, request: RegistryServer_pb2.GetStreamFeatureViewRequest, context + ): + return self.proxied_registry.get_stream_feature_view( + name=request.name, project=request.project, allow_cache=request.allow_cache + ).to_proto() + + def ListStreamFeatureViews(self, request, context): + return RegistryServer_pb2.ListStreamFeatureViewsResponse( + stream_feature_views=[ + stream_feature_view.to_proto() + for stream_feature_view in self.proxied_registry.list_stream_feature_views( + project=request.project, allow_cache=request.allow_cache + ) + ] + ) + + def GetOnDemandFeatureView( + self, request: RegistryServer_pb2.GetOnDemandFeatureViewRequest, context + ): + return self.proxied_registry.get_on_demand_feature_view( + name=request.name, project=request.project, allow_cache=request.allow_cache + ).to_proto() + + def ListOnDemandFeatureViews(self, request, context): + return RegistryServer_pb2.ListOnDemandFeatureViewsResponse( + on_demand_feature_views=[ + on_demand_feature_view.to_proto() + for on_demand_feature_view in self.proxied_registry.list_on_demand_feature_views( + project=request.project, allow_cache=request.allow_cache + ) + ] + ) + + def GetFeatureService( + self, request: RegistryServer_pb2.GetFeatureServiceRequest, context + ): + return self.proxied_registry.get_feature_service( + name=request.name, project=request.project, allow_cache=request.allow_cache + ).to_proto() + + def ListFeatureServices( + self, request: RegistryServer_pb2.ListFeatureServicesRequest, context + ): + return RegistryServer_pb2.ListFeatureServicesResponse( + feature_services=[ + feature_service.to_proto() + for feature_service in self.proxied_registry.list_feature_services( + project=request.project, allow_cache=request.allow_cache + ) + ] + ) + + def GetSavedDataset( + self, request: RegistryServer_pb2.GetSavedDatasetRequest, context + ): + return self.proxied_registry.get_saved_dataset( + name=request.name, project=request.project, allow_cache=request.allow_cache + ).to_proto() + + def ListSavedDatasets( + self, request: RegistryServer_pb2.ListSavedDatasetsRequest, context + ): + return RegistryServer_pb2.ListSavedDatasetsResponse( + saved_datasets=[ + saved_dataset.to_proto() + for saved_dataset in self.proxied_registry.list_saved_datasets( + project=request.project, allow_cache=request.allow_cache + ) + ] + ) + + def GetValidationReference( + self, request: RegistryServer_pb2.GetValidationReferenceRequest, context + ): + return self.proxied_registry.get_validation_reference( + name=request.name, project=request.project, allow_cache=request.allow_cache + ).to_proto() + + def ListValidationReferences( + self, request: RegistryServer_pb2.ListValidationReferencesRequest, context + ): + return RegistryServer_pb2.ListValidationReferencesResponse( + validation_references=[ + validation_reference.to_proto() + for validation_reference in self.proxied_registry.list_validation_references( + project=request.project, allow_cache=request.allow_cache + ) + ] + ) + + def ListProjectMetadata( + self, request: RegistryServer_pb2.ListProjectMetadataRequest, context + ): + return RegistryServer_pb2.ListProjectMetadataResponse( + project_metadata=[ + project_metadata.to_proto() + for project_metadata in self.proxied_registry.list_project_metadata( + project=request.project, allow_cache=request.allow_cache + ) + ] + ) + + def GetInfra(self, request: RegistryServer_pb2.GetInfraRequest, context): + return self.proxied_registry.get_infra( + project=request.project, allow_cache=request.allow_cache + ).to_proto() + + def Refresh(self, request, context): + self.proxied_registry.refresh(request.project) + return Empty() + + def Proto(self, request, context): + return self.proxied_registry.proto() + + +def start_server(store: FeatureStore, port: int): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + RegistryServer_pb2_grpc.add_RegistryServerServicer_to_server( + RegistryServer(store), server + ) + server.add_insecure_port(f"[::]:{port}") + server.start() + server.wait_for_termination() diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 3461ae058bd..5708754622b 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -2,20 +2,19 @@ import os import warnings from pathlib import Path -from typing import Any +from typing import Any, Dict, Optional import yaml from pydantic import ( BaseModel, + ConfigDict, Field, StrictInt, StrictStr, ValidationError, - root_validator, - validator, + field_validator, + model_validator, ) -from pydantic.error_wrappers import ErrorWrapper -from pydantic.typing import Dict, Optional from feast.errors import ( FeastFeatureServerTypeInvalidError, @@ -40,6 +39,7 @@ "file": "feast.infra.registry.registry.Registry", "sql": "feast.infra.registry.sql.SqlRegistry", "snowflake.registry": "feast.infra.registry.snowflake.SnowflakeRegistry", + "remote": "feast.infra.registry.remote.RemoteRegistry", } BATCH_ENGINE_CLASS_FOR_TYPE = { @@ -75,6 +75,7 @@ "postgres": "feast.infra.offline_stores.contrib.postgres_offline_store.postgres.PostgreSQLOfflineStore", "athena": "feast.infra.offline_stores.contrib.athena_offline_store.athena.AthenaOfflineStore", "mssql": "feast.infra.offline_stores.contrib.mssql_offline_store.mssql.MsSqlServerOfflineStore", + "duckdb": "feast.infra.offline_stores.contrib.duckdb_offline_store.duckdb.DuckDBOfflineStore", } FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE = { @@ -93,17 +94,13 @@ class FeastBaseModel(BaseModel): """Feast Pydantic Configuration Class""" - class Config: - arbitrary_types_allowed = True - extra = "allow" + model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow") class FeastConfigBaseModel(BaseModel): """Feast Pydantic Configuration Class""" - class Config: - arbitrary_types_allowed = True - extra = "forbid" + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") class RegistryConfig(FeastBaseModel): @@ -112,7 +109,7 @@ class RegistryConfig(FeastBaseModel): registry_type: StrictStr = "file" """ str: Provider name or a class name that implements Registry.""" - registry_store_type: Optional[StrictStr] + registry_store_type: Optional[StrictStr] = None """ str: Provider name or a class name that implements RegistryStore. """ path: StrictStr = "" @@ -126,9 +123,12 @@ class RegistryConfig(FeastBaseModel): set to infinity by setting TTL to 0 seconds, which means the cache will only be loaded once and will never expire. Users can manually refresh the cache by calling feature_store.refresh_registry() """ - s3_additional_kwargs: Optional[Dict[str, str]] + s3_additional_kwargs: Optional[Dict[str, str]] = None """ Dict[str, str]: Extra arguments to pass to boto3 when writing the registry file to S3. """ + sqlalchemy_config_kwargs: Dict[str, Any] = {} + """ Dict[str, Any]: Extra arguments to pass to SQLAlchemy.create_engine. """ + class RepoConfig(FeastBaseModel): """Repo config. Typically loaded from `feature_store.yaml`""" @@ -142,7 +142,7 @@ class RepoConfig(FeastBaseModel): provider: StrictStr """ str: local or gcp or aws """ - _registry_config: Any = Field(alias="registry", default="data/registry.db") + registry_config: Any = Field(alias="registry", default="data/registry.db") """ Configures the registry. Can be: 1. str: a path to a file based registry (a local path, or remote object storage path, e.g. a GCS URI) @@ -150,19 +150,19 @@ class RepoConfig(FeastBaseModel): 3. SnowflakeRegistryConfig: Using a Snowflake table to store the registry """ - _online_config: Any = Field(alias="online_store") + online_config: Any = Field(None, alias="online_store") """ OnlineStoreConfig: Online store configuration (optional depending on provider) """ - _offline_config: Any = Field(alias="offline_store") + offline_config: Any = Field(None, alias="offline_store") """ OfflineStoreConfig: Offline store configuration (optional depending on provider) """ - _batch_engine_config: Any = Field(alias="batch_engine") + batch_engine_config: Any = Field(None, alias="batch_engine") """ BatchMaterializationEngine: Batch materialization configuration (optional depending on provider)""" - feature_server: Optional[Any] + feature_server: Optional[Any] = None """ FeatureServerConfig: Feature server configuration (optional depending on provider) """ - flags: Any + flags: Any = None """ Flags (deprecated field): Feature flags for experimental features """ repo_path: Optional[Path] = None @@ -187,42 +187,42 @@ def __init__(self, **data: Any): self._registry = None if "registry" not in data: raise FeastRegistryNotSetError() - self._registry_config = data["registry"] + self.registry_config = data["registry"] self._offline_store = None if "offline_store" in data: - self._offline_config = data["offline_store"] + self.offline_config = data["offline_store"] else: if data["provider"] == "local": - self._offline_config = "file" + self.offline_config = "file" elif data["provider"] == "gcp": - self._offline_config = "bigquery" + self.offline_config = "bigquery" elif data["provider"] == "aws": - self._offline_config = "redshift" + self.offline_config = "redshift" elif data["provider"] == "azure": - self._offline_config = "mssql" + self.offline_config = "mssql" self._online_store = None if "online_store" in data: - self._online_config = data["online_store"] + self.online_config = data["online_store"] else: if data["provider"] == "local": - self._online_config = "sqlite" + self.online_config = "sqlite" elif data["provider"] == "gcp": - self._online_config = "datastore" + self.online_config = "datastore" elif data["provider"] == "aws": - self._online_config = "dynamodb" + self.online_config = "dynamodb" elif data["provider"] == "rockset": - self._online_config = "rockset" + self.online_config = "rockset" self._batch_engine = None if "batch_engine" in data: - self._batch_engine_config = data["batch_engine"] + self.batch_engine_config = data["batch_engine"] elif "batch_engine_config" in data: - self._batch_engine_config = data["batch_engine_config"] + self.batch_engine_config = data["batch_engine_config"] else: # Defaults to using local in-process materialization engine. - self._batch_engine_config = "local" + self.batch_engine_config = "local" if isinstance(self.feature_server, Dict): self.feature_server = get_feature_server_config_from_type( @@ -242,71 +242,71 @@ def __init__(self, **data: Any): @property def registry(self): if not self._registry: - if isinstance(self._registry_config, Dict): - if "registry_type" in self._registry_config: + if isinstance(self.registry_config, Dict): + if "registry_type" in self.registry_config: self._registry = get_registry_config_from_type( - self._registry_config["registry_type"] - )(**self._registry_config) + self.registry_config["registry_type"] + )(**self.registry_config) else: # This may be a custom registry store, which does not need a 'registry_type' - self._registry = RegistryConfig(**self._registry_config) - elif isinstance(self._registry_config, str): + self._registry = RegistryConfig(**self.registry_config) + elif isinstance(self.registry_config, str): # User passed in just a path to file registry self._registry = get_registry_config_from_type("file")( - path=self._registry_config + path=self.registry_config ) - elif self._registry_config: - self._registry = self._registry_config + elif self.registry_config: + self._registry = self.registry_config return self._registry @property def offline_store(self): if not self._offline_store: - if isinstance(self._offline_config, Dict): + if isinstance(self.offline_config, Dict): self._offline_store = get_offline_config_from_type( - self._offline_config["type"] - )(**self._offline_config) - elif isinstance(self._offline_config, str): + self.offline_config["type"] + )(**self.offline_config) + elif isinstance(self.offline_config, str): self._offline_store = get_offline_config_from_type( - self._offline_config + self.offline_config )() - elif self._offline_config: - self._offline_store = self._offline_config + elif self.offline_config: + self._offline_store = self.offline_config return self._offline_store @property def online_store(self): if not self._online_store: - if isinstance(self._online_config, Dict): + if isinstance(self.online_config, Dict): self._online_store = get_online_config_from_type( - self._online_config["type"] - )(**self._online_config) - elif isinstance(self._online_config, str): - self._online_store = get_online_config_from_type(self._online_config)() - elif self._online_config: - self._online_store = self._online_config + self.online_config["type"] + )(**self.online_config) + elif isinstance(self.online_config, str): + self._online_store = get_online_config_from_type(self.online_config)() + elif self.online_config: + self._online_store = self.online_config return self._online_store @property def batch_engine(self): if not self._batch_engine: - if isinstance(self._batch_engine_config, Dict): + if isinstance(self.batch_engine_config, Dict): self._batch_engine = get_batch_engine_config_from_type( - self._batch_engine_config["type"] - )(**self._batch_engine_config) - elif isinstance(self._batch_engine_config, str): + self.batch_engine_config["type"] + )(**self.batch_engine_config) + elif isinstance(self.batch_engine_config, str): self._batch_engine = get_batch_engine_config_from_type( - self._batch_engine_config + self.batch_engine_config )() - elif self._batch_engine_config: + elif self.batch_engine_config: self._batch_engine = self._batch_engine return self._batch_engine - @root_validator(pre=True) + @model_validator(mode="before") @log_exceptions - def _validate_online_store_config(cls, values): + def _validate_online_store_config(cls, values: Any) -> Any: # This method will validate whether the online store configurations are set correctly. This explicit validation # is necessary because Pydantic Unions throw very verbose and cryptic exceptions. We also use this method to # impute the default online store type based on the selected provider. For the time being this method should be @@ -347,14 +347,12 @@ def _validate_online_store_config(cls, values): online_config_class = get_online_config_from_type(online_store_type) online_config_class(**values["online_store"]) except ValidationError as e: - raise ValidationError( - [ErrorWrapper(e, loc="online_store")], - model=RepoConfig, - ) + raise e return values - @root_validator(pre=True) - def _validate_offline_store_config(cls, values): + @model_validator(mode="before") + @classmethod + def _validate_offline_store_config(cls, values: Any) -> Any: # Set empty offline_store config if it isn't set explicitly if "offline_store" not in values: values["offline_store"] = dict() @@ -385,15 +383,13 @@ def _validate_offline_store_config(cls, values): offline_config_class = get_offline_config_from_type(offline_store_type) offline_config_class(**values["offline_store"]) except ValidationError as e: - raise ValidationError( - [ErrorWrapper(e, loc="offline_store")], - model=RepoConfig, - ) + raise e return values - @root_validator(pre=True) - def _validate_feature_server_config(cls, values): + @model_validator(mode="before") + @classmethod + def _validate_feature_server_config(cls, values: Any) -> Any: # Having no feature server is the default. if "feature_server" not in values: return values @@ -420,15 +416,13 @@ def _validate_feature_server_config(cls, values): ) feature_server_config_class(**values["feature_server"]) except ValidationError as e: - raise ValidationError( - [ErrorWrapper(e, loc="feature_server")], - model=RepoConfig, - ) + raise e return values - @validator("project") - def _validate_project_name(cls, v): + @field_validator("project") + @classmethod + def _validate_project_name(cls, v: str) -> str: from feast.repo_operations import is_valid_name if not is_valid_name(v): @@ -438,10 +432,11 @@ def _validate_project_name(cls, v): ) return v - @validator("flags") - def _validate_flags(cls, v): - if not isinstance(v, Dict): - return + @field_validator("flags") + @classmethod + def _validate_flags(cls, v: Optional[dict]) -> Optional[dict]: + if not isinstance(v, dict): + return v _logger.warning( "Flags are no longer necessary in Feast. Experimental features will log warnings instead." @@ -463,8 +458,7 @@ def write_to_path(self, repo_path: Path): sort_keys=False, ) - class Config: - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True) class FeastConfigError(Exception): diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 6a204d68136..13abbc5e284 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -284,7 +284,6 @@ def __copy__(self): fv = StreamFeatureView( name=self.name, schema=self.schema, - entities=self.entities, ttl=self.ttl, tags=self.tags, online=self.online, @@ -293,9 +292,12 @@ def __copy__(self): aggregations=self.aggregations, mode=self.mode, timestamp_field=self.timestamp_field, - source=self.source, + source=self.stream_source if self.stream_source else self.batch_source, udf=self.udf, ) + fv.entities = self.entities + fv.features = copy.copy(self.features) + fv.entity_columns = copy.copy(self.entity_columns) fv.projection = copy.copy(self.projection) return fv diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 9dbbb5a64ce..ad3e273d37b 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -51,7 +51,7 @@ import pyarrow # null timestamps get converted to -9223372036854775808 -NULL_TIMESTAMP_INT_VALUE = np.datetime64("NaT").astype(int) +NULL_TIMESTAMP_INT_VALUE: int = np.datetime64("NaT").astype(int) def feast_value_type_to_python_type(field_value_proto: ProtoValue) -> Any: @@ -114,7 +114,10 @@ def feast_value_type_to_pandas_type(value_type: ValueType) -> Any: def python_type_to_feast_value_type( - name: str, value: Any = None, recurse: bool = True, type_name: Optional[str] = None + name: str, + value: Optional[Any] = None, + recurse: bool = True, + type_name: Optional[str] = None, ) -> ValueType: """ Finds the equivalent Feast Value Type for a Python value. Both native @@ -321,7 +324,7 @@ def _python_datetime_to_int_timestamp( elif isinstance(value, Timestamp): int_timestamps.append(int(value.ToSeconds())) elif isinstance(value, np.datetime64): - int_timestamps.append(value.astype("datetime64[s]").astype(np.int_)) + int_timestamps.append(value.astype("datetime64[s]").astype(np.int_)) # type: ignore[attr-defined] elif isinstance(value, type(np.nan)): int_timestamps.append(NULL_TIMESTAMP_INT_VALUE) else: @@ -680,6 +683,14 @@ def _convert_value_name_to_snowflake_udf(value_name: str, project_name: str) -> "FLOAT": f"feast_{project_name}_snowflake_float_to_double_proto", "BOOL": f"feast_{project_name}_snowflake_boolean_to_bool_proto", "UNIX_TIMESTAMP": f"feast_{project_name}_snowflake_timestamp_to_unix_timestamp_proto", + "BYTES_LIST": f"feast_{project_name}_snowflake_array_bytes_to_list_bytes_proto", + "STRING_LIST": f"feast_{project_name}_snowflake_array_varchar_to_list_string_proto", + "INT32_LIST": f"feast_{project_name}_snowflake_array_number_to_list_int32_proto", + "INT64_LIST": f"feast_{project_name}_snowflake_array_number_to_list_int64_proto", + "DOUBLE_LIST": f"feast_{project_name}_snowflake_array_float_to_list_double_proto", + "FLOAT_LIST": f"feast_{project_name}_snowflake_array_float_to_list_double_proto", + "BOOL_LIST": f"feast_{project_name}_snowflake_array_boolean_to_list_bool_proto", + "UNIX_TIMESTAMP_LIST": f"feast_{project_name}_snowflake_array_timestamp_to_list_unix_timestamp_proto", } return name_map[value_name].upper() diff --git a/sdk/python/feast/types.py b/sdk/python/feast/types.py index 0ba1725f173..4b07c58d19e 100644 --- a/sdk/python/feast/types.py +++ b/sdk/python/feast/types.py @@ -50,7 +50,10 @@ def __hash__(self): return hash(self.to_value_type().value) def __eq__(self, other): - return self.to_value_type() == other.to_value_type() + if isinstance(other, ComplexFeastType): + return self.to_value_type() == other.to_value_type() + else: + return False class PrimitiveFeastType(Enum): diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 5e407c1a991..e41706f403d 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -4,11 +4,13 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt # -alabaster==0.7.13 +alabaster==0.7.16 # via sphinx -altair==4.2.0 +altair==4.2.2 # via great-expectations -anyio==4.0.0 +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 # via # httpx # jupyter-server @@ -32,14 +34,16 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis -attrs==23.1.0 +atpublic==4.0 + # via ibis-framework +attrs==23.2.0 # via # bowler # jsonschema # referencing avro==1.10.0 # via feast (setup.py) -azure-core==1.29.5 +azure-core==1.30.0 # via # azure-identity # azure-storage-blob @@ -47,21 +51,23 @@ azure-identity==1.15.0 # via feast (setup.py) azure-storage-blob==12.19.0 # via feast (setup.py) -babel==2.13.1 +babel==2.14.0 # via # jupyterlab-server # sphinx -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 # via nbconvert +bidict==0.23.1 + # via ibis-framework black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.29.2 +boto3==1.34.49 # via # feast (setup.py) # moto -botocore==1.32.2 +botocore==1.34.49 # via # boto3 # moto @@ -74,13 +80,13 @@ build==1.0.3 # pip-tools bytewax==0.15.1 # via feast (setup.py) -cachecontrol==0.13.1 +cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.2 # via google-auth -cassandra-driver==3.28.0 +cassandra-driver==3.29.0 # via feast (setup.py) -certifi==2023.7.22 +certifi==2024.2.2 # via # httpcore # httpx @@ -116,15 +122,13 @@ colorama==0.4.6 # via # feast (setup.py) # great-expectations -comm==0.2.0 +comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.3.2 - # via - # coverage - # pytest-cov -cryptography==41.0.6 +coverage[toml]==7.4.3 + # via pytest-cov +cryptography==42.0.4 # via # azure-identity # azure-storage-blob @@ -137,11 +141,11 @@ cryptography==41.0.6 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2023.11.0 +dask==2024.2.1 # via feast (setup.py) -db-dtypes==1.1.1 +db-dtypes==1.2.0 # via google-cloud-bigquery -debugpy==1.8.0 +debugpy==1.8.1 # via ipykernel decorator==5.1.1 # via ipython @@ -149,14 +153,14 @@ defusedxml==0.7.1 # via nbconvert deprecation==2.1.0 # via testcontainers -dill==0.3.7 +dill==0.3.8 # via # bytewax # feast (setup.py) # multiprocess -distlib==0.3.7 +distlib==0.3.8 # via virtualenv -docker==6.1.3 +docker==7.0.0 # via # feast (setup.py) # testcontainers @@ -164,7 +168,7 @@ docutils==0.19 # via sphinx entrypoints==0.4 # via altair -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # anyio # ipython @@ -173,13 +177,9 @@ execnet==2.0.2 # via pytest-xdist executing==2.0.1 # via stack-data -fastapi==0.99.1 +fastapi==0.109.2 # via feast (setup.py) -fastavro==1.9.0 - # via - # feast (setup.py) - # pandavro -fastjsonschema==2.19.0 +fastjsonschema==2.19.1 # via nbformat filelock==3.13.1 # via @@ -193,7 +193,7 @@ flake8==6.0.0 # via feast (setup.py) fqdn==1.5.1 # via jsonschema -fsspec==2023.9.2 +fsspec==2023.12.2 # via # dask # feast (setup.py) @@ -201,7 +201,7 @@ geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.14.0 +google-api-core[grpc]==2.17.1 # via # feast (setup.py) # firebase-admin @@ -213,9 +213,9 @@ google-api-core[grpc]==2.14.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.108.0 +google-api-python-client==2.119.0 # via firebase-admin -google-auth==2.23.4 +google-auth==2.28.1 # via # google-api-core # google-api-python-client @@ -223,28 +223,26 @@ google-auth==2.23.4 # google-cloud-core # google-cloud-storage # kubernetes -google-auth-httplib2==0.1.1 +google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.22.0 # via feast (setup.py) -google-cloud-bigtable==2.21.0 +google-cloud-bigquery-storage==2.24.0 + # via feast (setup.py) +google-cloud-bigtable==2.23.0 # via feast (setup.py) -google-cloud-core==2.3.3 +google-cloud-core==2.4.1 # via # google-cloud-bigquery # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.18.0 +google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.13.1 +google-cloud-firestore==2.15.0 # via firebase-admin -google-cloud-storage==2.13.0 +google-cloud-storage==2.14.0 # via # feast (setup.py) # firebase-admin @@ -252,23 +250,23 @@ google-crc32c==1.5.0 # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.6.0 +google-resumable-media==2.7.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.61.0 +googleapis-common-protos[grpc]==1.62.0 # via # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.15.50 +great-expectations==0.18.9 # via feast (setup.py) -greenlet==3.0.1 +greenlet==3.0.3 # via sqlalchemy -grpc-google-iam-v1==0.12.7 +grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.59.2 +grpcio==1.62.0 # via # feast (setup.py) # google-api-core @@ -280,15 +278,15 @@ grpcio==1.59.2 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.59.2 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.59.2 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-status==1.59.2 +grpcio-status==1.62.0 # via google-api-core -grpcio-testing==1.59.2 +grpcio-testing==1.62.0 # via feast (setup.py) -grpcio-tools==1.59.2 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -300,9 +298,9 @@ happybase==1.2.0 # via feast (setup.py) hazelcast-python-client==5.3.0 # via feast (setup.py) -hiredis==2.2.3 +hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.2 +httpcore==1.0.4 # via httpx httplib2==0.22.0 # via @@ -310,11 +308,19 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.25.1 +httpx==0.27.0 + # via + # feast (setup.py) + # jupyterlab +ibis-framework==8.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 # via feast (setup.py) -identify==2.5.31 +identify==2.5.35 # via pre-commit -idna==3.4 +idna==3.6 # via # anyio # httpx @@ -323,29 +329,28 @@ idna==3.4 # snowflake-connector-python imagesize==1.4.1 # via sphinx -importlib-metadata==6.8.0 +importlib-metadata==6.11.0 # via # dask # feast (setup.py) - # great-expectations importlib-resources==6.1.1 # via feast (setup.py) iniconfig==2.0.0 # via pytest -ipykernel==6.26.0 +ipykernel==6.29.2 # via jupyterlab -ipython==8.17.2 +ipython==8.22.1 # via # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.1 +ipywidgets==8.1.2 # via great-expectations isodate==0.6.1 # via azure-storage-blob isoduration==20.11.0 # via jsonschema -isort==5.12.0 +isort==5.13.2 # via feast (setup.py) jedi==0.19.1 # via ipython @@ -364,7 +369,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.14 +json5==0.9.17 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -372,7 +377,7 @@ jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.20.0 +jsonschema[format-nongpl]==4.21.1 # via # altair # feast (setup.py) @@ -380,14 +385,14 @@ jsonschema[format-nongpl]==4.20.0 # jupyter-events # jupyterlab-server # nbformat -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.12.1 # via jsonschema jupyter-client==8.6.0 # via # ipykernel # jupyter-server # nbclient -jupyter-core==5.5.0 +jupyter-core==5.7.1 # via # ipykernel # jupyter-client @@ -400,24 +405,24 @@ jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.2 # via jupyterlab -jupyter-server==2.11.2 +jupyter-server==2.12.5 # via # jupyter-lsp # jupyterlab # jupyterlab-server # notebook # notebook-shim -jupyter-server-terminals==0.4.4 +jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.0.8 +jupyterlab==4.1.2 # via notebook -jupyterlab-pygments==0.2.2 +jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.25.1 +jupyterlab-server==2.25.3 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.9 +jupyterlab-widgets==3.0.10 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) @@ -425,12 +430,14 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations -markupsafe==2.1.3 +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 # via # jinja2 # nbconvert # werkzeug -marshmallow==3.20.1 +marshmallow==3.20.2 # via great-expectations matplotlib-inline==0.1.6 # via @@ -438,31 +445,35 @@ matplotlib-inline==0.1.6 # ipython mccabe==0.7.0 # via flake8 +mdurl==0.1.2 + # via markdown-it-py minio==7.1.0 # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert -mmh3==4.0.1 +mmh3==4.1.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.2.9 +moto==4.2.14 # via feast (setup.py) -msal==1.25.0 +msal==1.27.0 # via # azure-identity # msal-extensions -msal-extensions==1.0.0 +msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol -multiprocess==0.70.15 +multipledispatch==1.0.0 + # via ibis-framework +multiprocess==0.70.16 # via bytewax -mypy==0.982 +mypy==1.8.0 # via # feast (setup.py) # sqlalchemy @@ -472,11 +483,9 @@ mypy-extensions==1.0.0 # mypy mypy-protobuf==3.1.0 # via feast (setup.py) -mysqlclient==2.2.0 - # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.11.0 +nbconvert==7.16.1 # via jupyter-server nbformat==5.9.2 # via @@ -484,13 +493,13 @@ nbformat==5.9.2 # jupyter-server # nbclient # nbconvert -nest-asyncio==1.5.8 +nest-asyncio==1.6.0 # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==7.0.6 +notebook==7.1.0 # via great-expectations -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via # jupyterlab # notebook @@ -500,13 +509,13 @@ numpy==1.24.4 # db-dtypes # feast (setup.py) # great-expectations + # ibis-framework # pandas - # pandavro # pyarrow # scipy oauthlib==3.2.2 # via requests-oauthlib -overrides==7.4.0 +overrides==7.7.0 # via jupyter-server packaging==23.2 # via @@ -518,39 +527,40 @@ packaging==23.2 # google-cloud-bigquery # great-expectations # gunicorn + # ibis-substrait # ipykernel # jupyter-server # jupyterlab # jupyterlab-server # marshmallow + # msal-extensions # nbconvert # pytest # snowflake-connector-python # sphinx -pandas==1.5.3 +pandas==2.2.0 ; python_version >= "3.9" # via # altair # db-dtypes # feast (setup.py) # google-cloud-bigquery # great-expectations - # pandavro # snowflake-connector-python -pandavro==1.5.2 - # via feast (setup.py) -pandocfilters==1.5.0 +pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.1 # via dask -pathspec==0.11.2 +pathspec==0.12.1 # via black pbr==6.0.0 # via mock -pexpect==4.8.0 +pexpect==4.9.0 # via ipython -pip-tools==7.3.0 +pip-tools==7.4.0 # via feast (setup.py) platformdirs==3.11.0 # via @@ -558,7 +568,7 @@ platformdirs==3.11.0 # jupyter-core # snowflake-connector-python # virtualenv -pluggy==1.3.0 +pluggy==1.4.0 # via pytest ply==3.11 # via thriftpy2 @@ -566,11 +576,11 @@ portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.18.0 +prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.41 +prompt-toolkit==3.0.43 # via ipython -proto-plus==1.22.3 +proto-plus==1.23.0 # via # feast (setup.py) # google-cloud-bigquery @@ -578,7 +588,7 @@ proto-plus==1.22.3 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # google-api-core @@ -596,6 +606,7 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus + # substrait psutil==5.9.0 # via # feast (setup.py) @@ -614,13 +625,16 @@ py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark -pyarrow==14.0.1 +pyarrow==15.0.0 # via # db-dtypes # feast (setup.py) # google-cloud-bigquery + # ibis-framework # snowflake-connector-python -pyasn1==0.5.0 +pyarrow-hotfix==0.6 + # via ibis-framework +pyasn1==0.5.1 # via # pyasn1-modules # rsa @@ -632,40 +646,45 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==1.10.13 +pydantic==2.6.2 # via # fastapi # feast (setup.py) # great-expectations +pydantic-core==2.16.3 + # via pydantic pyflakes==3.0.1 # via flake8 -pygments==2.16.1 +pygments==2.17.2 # via # feast (setup.py) # ipython # nbconvert + # rich # sphinx pyjwt[crypto]==2.8.0 # via # msal # snowflake-connector-python -pymssql==2.2.10 +pymssql==2.2.11 # via feast (setup.py) pymysql==1.1.0 # via feast (setup.py) -pyodbc==5.0.1 +pyodbc==5.1.0 # via feast (setup.py) -pyopenssl==23.3.0 +pyopenssl==24.0.0 # via snowflake-connector-python pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools pyspark==3.5.0 # via feast (setup.py) -pytest==7.4.3 +pytest==7.4.4 # via # feast (setup.py) # pytest-benchmark @@ -687,7 +706,7 @@ pytest-ordering==0.6 # via feast (setup.py) pytest-timeout==1.4.2 # via feast (setup.py) -pytest-xdist==3.4.0 +pytest-xdist==3.5.0 # via feast (setup.py) python-dateutil==2.8.2 # via @@ -695,19 +714,21 @@ python-dateutil==2.8.2 # botocore # google-cloud-bigquery # great-expectations + # ibis-framework # jupyter-client # kubernetes # moto # pandas # rockset # trino -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn python-json-logger==2.0.7 # via jupyter-events -pytz==2023.3.post1 +pytz==2024.1 # via # great-expectations + # ibis-framework # pandas # snowflake-connector-python # trino @@ -715,24 +736,25 @@ pyyaml==6.0.1 # via # dask # feast (setup.py) + # ibis-substrait # jupyter-events # kubernetes # pre-commit # responses # uvicorn -pyzmq==25.1.1 +pyzmq==25.1.2 # via # ipykernel # jupyter-client # jupyter-server redis==4.6.0 # via feast (setup.py) -referencing==0.31.0 +referencing==0.33.0 # via # jsonschema # jsonschema-specifications # jupyter-events -regex==2023.10.3 +regex==2023.12.25 # via feast (setup.py) requests==2.31.0 # via @@ -755,7 +777,7 @@ requests==2.31.0 # trino requests-oauthlib==1.3.1 # via kubernetes -responses==0.24.1 +responses==0.25.0 # via moto rfc3339-validator==0.1.4 # via @@ -765,9 +787,11 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.1.0 +rich==13.7.0 + # via ibis-framework +rockset==2.1.1 # via feast (setup.py) -rpds-py==0.13.0 +rpds-py==0.18.0 # via # jsonschema # referencing @@ -775,9 +799,9 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -s3transfer==0.7.0 +s3transfer==0.10.0 # via boto3 -scipy==1.11.3 +scipy==1.12.0 # via great-expectations send2trash==1.8.2 # via jupyter-server @@ -786,13 +810,11 @@ six==1.16.0 # asttokens # azure-core # bleach - # cassandra-driver # geomet # happybase # isodate # kubernetes # mock - # pandavro # python-dateutil # rfc3339-validator # thriftpy2 @@ -802,44 +824,40 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.5.0 - # via - # feast (setup.py) - # snowflake-connector-python +snowflake-connector-python[pandas]==3.7.1 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 - # via - # feast (setup.py) - # sphinxcontrib-applehelp - # sphinxcontrib-devhelp - # sphinxcontrib-htmlhelp - # sphinxcontrib-qthelp - # sphinxcontrib-serializinghtml -sphinxcontrib-applehelp==1.0.7 + # via feast (setup.py) +sphinxcontrib-applehelp==1.0.8 # via sphinx -sphinxcontrib-devhelp==1.0.5 +sphinxcontrib-devhelp==1.0.6 # via sphinx -sphinxcontrib-htmlhelp==2.0.4 +sphinxcontrib-htmlhelp==2.0.5 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.6 +sphinxcontrib-qthelp==1.0.7 # via sphinx -sphinxcontrib-serializinghtml==1.1.9 +sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==1.4.50 +sqlalchemy[mypy]==1.4.51 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a37 +sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +sqlglot==20.11.0 + # via ibis-framework stack-data==0.6.3 # via ipython -starlette==0.27.0 +starlette==0.36.3 # via fastapi +substrait==0.12.1 + # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) tenacity==8.2.3 @@ -868,12 +886,13 @@ tomli==2.0.1 # pytest tomlkit==0.12.3 # via snowflake-connector-python -toolz==0.12.0 +toolz==0.12.1 # via # altair # dask + # ibis-framework # partd -tornado==6.3.3 +tornado==6.4 # via # ipykernel # jupyter-client @@ -881,11 +900,11 @@ tornado==6.3.3 # jupyterlab # notebook # terminado -tqdm==4.66.1 +tqdm==4.66.2 # via # feast (setup.py) # great-expectations -traitlets==5.13.0 +traitlets==5.14.1 # via # comm # ipykernel @@ -900,9 +919,9 @@ traitlets==5.13.0 # nbclient # nbconvert # nbformat -trino==0.327.0 +trino==0.328.0 # via feast (setup.py) -typeguard==2.13.3 +typeguard==4.1.5 # via feast (setup.py) types-protobuf==3.19.22 # via @@ -910,38 +929,44 @@ types-protobuf==3.19.22 # mypy-protobuf types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==23.3.0.0 +types-pyopenssl==24.0.0.20240130 # via types-redis -types-python-dateutil==2.8.19.14 +types-python-dateutil==2.8.19.20240106 # via # arrow # feast (setup.py) -types-pytz==2023.3.1.1 +types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.10 +types-redis==4.6.0.20240218 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==68.2.0.1 +types-setuptools==69.1.0.20240223 # via feast (setup.py) -types-tabulate==0.9.0.3 +types-tabulate==0.9.0.20240106 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via + # anyio # async-lru # azure-core # azure-storage-blob # fastapi # great-expectations + # ibis-framework # mypy # pydantic + # pydantic-core # snowflake-connector-python # sqlalchemy2-stubs + # typeguard # uvicorn +tzdata==2024.1 + # via pandas tzlocal==5.2 # via # great-expectations @@ -961,11 +986,8 @@ urllib3==1.26.18 # requests # responses # rockset - # snowflake-connector-python -uvicorn[standard]==0.24.0.post1 - # via - # feast (setup.py) - # uvicorn +uvicorn[standard]==0.27.1 + # via feast (setup.py) uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 @@ -976,7 +998,7 @@ volatile==2.1.0 # via bowler watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.10 +wcwidth==0.2.13 # via prompt-toolkit webcolors==1.13 # via jsonschema @@ -984,18 +1006,17 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.6.4 +websocket-client==1.7.0 # via - # docker # jupyter-server # kubernetes websockets==12.0 # via uvicorn werkzeug==3.0.1 # via moto -wheel==0.41.3 +wheel==0.42.0 # via pip-tools -widgetsnbextension==4.0.9 +widgetsnbextension==4.0.10 # via ipywidgets wrapt==1.16.0 # via testcontainers diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 18486d7fa9a..b5dd9a78be1 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -4,21 +4,23 @@ # # pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt # -anyio==4.0.0 +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 # via # httpx # starlette # watchfiles appdirs==1.4.4 # via fissix -attrs==23.1.0 +attrs==23.2.0 # via # bowler # jsonschema # referencing bowler==0.9.0 # via feast (setup.py) -certifi==2023.7.22 +certifi==2024.2.2 # via # httpcore # httpx @@ -36,35 +38,31 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2023.11.0 +dask==2024.2.1 # via feast (setup.py) -dill==0.3.7 +dill==0.3.8 # via feast (setup.py) -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via anyio -fastapi==0.99.1 +fastapi==0.109.2 # via feast (setup.py) -fastavro==1.9.0 - # via - # feast (setup.py) - # pandavro fissix==21.11.13 # via bowler -fsspec==2023.10.0 +fsspec==2024.2.0 # via dask -greenlet==3.0.1 +greenlet==3.0.3 # via sqlalchemy -grpcio==1.59.2 +grpcio==1.62.0 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.59.2 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.59.2 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-tools==1.59.2 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -72,18 +70,18 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.2 +httpcore==1.0.4 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.25.1 +httpx==0.27.0 # via feast (setup.py) -idna==3.4 +idna==3.6 # via # anyio # httpx # requests -importlib-metadata==6.8.0 +importlib-metadata==6.11.0 # via # dask # feast (setup.py) @@ -91,19 +89,19 @@ importlib-resources==6.1.1 # via feast (setup.py) jinja2==3.1.3 # via feast (setup.py) -jsonschema==4.20.0 +jsonschema==4.21.1 # via feast (setup.py) -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 # via partd -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 -mmh3==4.0.1 +mmh3==4.1.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==1.7.0 +mypy==1.8.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy @@ -113,23 +111,18 @@ numpy==1.24.4 # via # feast (setup.py) # pandas - # pandavro # pyarrow packaging==23.2 # via # dask # gunicorn -pandas==1.5.3 - # via - # feast (setup.py) - # pandavro -pandavro==1.5.2 +pandas==2.2.0 # via feast (setup.py) partd==1.4.1 # via dask -proto-plus==1.22.3 +proto-plus==1.23.0 # via feast (setup.py) -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # grpcio-health-checking @@ -137,50 +130,50 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus -pyarrow==14.0.1 +pyarrow==15.0.0 # via feast (setup.py) -pydantic==1.10.13 +pydantic==2.6.2 # via # fastapi # feast (setup.py) -pygments==2.16.1 +pydantic-core==2.16.3 + # via pydantic +pygments==2.17.2 # via feast (setup.py) python-dateutil==2.8.2 # via pandas -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn -pytz==2023.3.post1 +pytz==2024.1 # via pandas pyyaml==6.0.1 # via # dask # feast (setup.py) # uvicorn -referencing==0.31.0 +referencing==0.33.0 # via # jsonschema # jsonschema-specifications requests==2.31.0 # via feast (setup.py) -rpds-py==0.13.0 +rpds-py==0.18.0 # via # jsonschema # referencing six==1.16.0 - # via - # pandavro - # python-dateutil + # via python-dateutil sniffio==1.3.0 # via # anyio # httpx -sqlalchemy[mypy]==1.4.50 +sqlalchemy[mypy]==1.4.51 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a37 +sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy -starlette==0.27.0 +starlette==0.36.3 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -190,29 +183,30 @@ toml==0.10.2 # via feast (setup.py) tomli==2.0.1 # via mypy -toolz==0.12.0 +toolz==0.12.1 # via # dask # partd -tqdm==4.66.1 +tqdm==4.66.2 # via feast (setup.py) -typeguard==2.13.3 +typeguard==4.1.5 # via feast (setup.py) -types-protobuf==4.24.0.4 +types-protobuf==4.24.0.20240129 # via mypy-protobuf -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via + # anyio # fastapi # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs + # typeguard # uvicorn -urllib3==2.1.0 +urllib3==2.2.1 # via requests -uvicorn[standard]==0.24.0.post1 - # via - # feast (setup.py) - # uvicorn +uvicorn[standard]==0.27.1 + # via feast (setup.py) uvloop==0.19.0 # via uvicorn volatile==2.1.0 diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 02eaf6dc307..339a6b1c497 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -6,9 +6,11 @@ # alabaster==0.7.13 # via sphinx -altair==4.2.0 +altair==4.2.2 # via great-expectations -anyio==4.0.0 +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 # via # httpx # jupyter-server @@ -32,14 +34,16 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis -attrs==23.1.0 +atpublic==3.1.2 + # via ibis-framework +attrs==23.2.0 # via # bowler # jsonschema # referencing avro==1.10.0 # via feast (setup.py) -azure-core==1.29.5 +azure-core==1.30.0 # via # azure-identity # azure-storage-blob @@ -47,7 +51,7 @@ azure-identity==1.15.0 # via feast (setup.py) azure-storage-blob==12.19.0 # via feast (setup.py) -babel==2.13.1 +babel==2.14.0 # via # jupyterlab-server # sphinx @@ -57,17 +61,17 @@ backports-zoneinfo==0.2.1 # via # trino # tzlocal -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 # via nbconvert black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.29.2 +boto3==1.34.49 # via # feast (setup.py) # moto -botocore==1.32.2 +botocore==1.34.49 # via # boto3 # moto @@ -80,13 +84,13 @@ build==1.0.3 # pip-tools bytewax==0.15.1 # via feast (setup.py) -cachecontrol==0.13.1 +cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.2 # via google-auth -cassandra-driver==3.28.0 +cassandra-driver==3.29.0 # via feast (setup.py) -certifi==2023.7.22 +certifi==2024.2.2 # via # httpcore # httpx @@ -122,15 +126,13 @@ colorama==0.4.6 # via # feast (setup.py) # great-expectations -comm==0.2.0 +comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.3.2 - # via - # coverage - # pytest-cov -cryptography==41.0.6 +coverage[toml]==7.4.3 + # via pytest-cov +cryptography==42.0.4 # via # azure-identity # azure-storage-blob @@ -145,9 +147,9 @@ cryptography==41.0.6 # types-redis dask==2023.5.0 # via feast (setup.py) -db-dtypes==1.1.1 +db-dtypes==1.2.0 # via google-cloud-bigquery -debugpy==1.8.0 +debugpy==1.8.1 # via ipykernel decorator==5.1.1 # via ipython @@ -155,14 +157,14 @@ defusedxml==0.7.1 # via nbconvert deprecation==2.1.0 # via testcontainers -dill==0.3.7 +dill==0.3.8 # via # bytewax # feast (setup.py) # multiprocess -distlib==0.3.7 +distlib==0.3.8 # via virtualenv -docker==6.1.3 +docker==7.0.0 # via # feast (setup.py) # testcontainers @@ -170,7 +172,7 @@ docutils==0.19 # via sphinx entrypoints==0.4 # via altair -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # anyio # pytest @@ -178,13 +180,9 @@ execnet==2.0.2 # via pytest-xdist executing==2.0.1 # via stack-data -fastapi==0.99.1 +fastapi==0.109.2 # via feast (setup.py) -fastavro==1.9.0 - # via - # feast (setup.py) - # pandavro -fastjsonschema==2.19.0 +fastjsonschema==2.19.1 # via nbformat filelock==3.13.1 # via @@ -198,7 +196,7 @@ flake8==6.0.0 # via feast (setup.py) fqdn==1.5.1 # via jsonschema -fsspec==2023.9.2 +fsspec==2023.12.2 # via # dask # feast (setup.py) @@ -206,7 +204,7 @@ geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.14.0 +google-api-core[grpc]==2.17.1 # via # feast (setup.py) # firebase-admin @@ -218,9 +216,9 @@ google-api-core[grpc]==2.14.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.108.0 +google-api-python-client==2.119.0 # via firebase-admin -google-auth==2.23.4 +google-auth==2.28.1 # via # google-api-core # google-api-python-client @@ -228,28 +226,26 @@ google-auth==2.23.4 # google-cloud-core # google-cloud-storage # kubernetes -google-auth-httplib2==0.1.1 +google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.22.0 # via feast (setup.py) -google-cloud-bigtable==2.21.0 +google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) -google-cloud-core==2.3.3 +google-cloud-bigtable==2.23.0 + # via feast (setup.py) +google-cloud-core==2.4.1 # via # google-cloud-bigquery # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.18.0 +google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.13.1 +google-cloud-firestore==2.15.0 # via firebase-admin -google-cloud-storage==2.13.0 +google-cloud-storage==2.14.0 # via # feast (setup.py) # firebase-admin @@ -257,23 +253,23 @@ google-crc32c==1.5.0 # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.6.0 +google-resumable-media==2.7.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.61.0 +googleapis-common-protos[grpc]==1.62.0 # via # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.15.50 +great-expectations==0.18.9 # via feast (setup.py) -greenlet==3.0.1 +greenlet==3.0.3 # via sqlalchemy -grpc-google-iam-v1==0.12.7 +grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.59.2 +grpcio==1.62.0 # via # feast (setup.py) # google-api-core @@ -285,15 +281,15 @@ grpcio==1.59.2 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.59.2 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.59.2 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-status==1.59.2 +grpcio-status==1.62.0 # via google-api-core -grpcio-testing==1.59.2 +grpcio-testing==1.62.0 # via feast (setup.py) -grpcio-tools==1.59.2 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -305,9 +301,9 @@ happybase==1.2.0 # via feast (setup.py) hazelcast-python-client==5.3.0 # via feast (setup.py) -hiredis==2.2.3 +hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.2 +httpcore==1.0.4 # via httpx httplib2==0.22.0 # via @@ -315,11 +311,19 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.25.1 +httpx==0.27.0 + # via + # feast (setup.py) + # jupyterlab +ibis-framework==4.1.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==2.29.1 # via feast (setup.py) -identify==2.5.31 +identify==2.5.35 # via pre-commit -idna==3.4 +idna==3.6 # via # anyio # httpx @@ -328,18 +332,18 @@ idna==3.4 # snowflake-connector-python imagesize==1.4.1 # via sphinx -importlib-metadata==6.8.0 +importlib-metadata==6.11.0 # via # build # dask # feast (setup.py) - # great-expectations # jupyter-client # jupyter-lsp # jupyterlab # jupyterlab-server # nbconvert # sphinx + # typeguard importlib-resources==6.1.1 # via # feast (setup.py) @@ -348,20 +352,20 @@ importlib-resources==6.1.1 # jupyterlab iniconfig==2.0.0 # via pytest -ipykernel==6.26.0 +ipykernel==6.29.2 # via jupyterlab ipython==8.12.3 # via # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.1 +ipywidgets==8.1.2 # via great-expectations isodate==0.6.1 # via azure-storage-blob isoduration==20.11.0 # via jsonschema -isort==5.12.0 +isort==5.13.2 # via feast (setup.py) jedi==0.19.1 # via ipython @@ -380,7 +384,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.14 +json5==0.9.17 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -388,7 +392,7 @@ jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.20.0 +jsonschema[format-nongpl]==4.21.1 # via # altair # feast (setup.py) @@ -396,14 +400,14 @@ jsonschema[format-nongpl]==4.20.0 # jupyter-events # jupyterlab-server # nbformat -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.12.1 # via jsonschema jupyter-client==8.6.0 # via # ipykernel # jupyter-server # nbclient -jupyter-core==5.5.0 +jupyter-core==5.7.1 # via # ipykernel # jupyter-client @@ -416,24 +420,24 @@ jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.2 # via jupyterlab -jupyter-server==2.11.2 +jupyter-server==2.12.5 # via # jupyter-lsp # jupyterlab # jupyterlab-server # notebook # notebook-shim -jupyter-server-terminals==0.4.4 +jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.0.8 +jupyterlab==4.1.2 # via notebook -jupyterlab-pygments==0.2.2 +jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.25.1 +jupyterlab-server==2.25.3 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.9 +jupyterlab-widgets==3.0.10 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) @@ -441,12 +445,14 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations -markupsafe==2.1.3 +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 # via # jinja2 # nbconvert # werkzeug -marshmallow==3.20.1 +marshmallow==3.20.2 # via great-expectations matplotlib-inline==0.1.6 # via @@ -454,31 +460,35 @@ matplotlib-inline==0.1.6 # ipython mccabe==0.7.0 # via flake8 +mdurl==0.1.2 + # via markdown-it-py minio==7.1.0 # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert -mmh3==4.0.1 +mmh3==4.1.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.2.9 +moto==4.2.14 # via feast (setup.py) -msal==1.25.0 +msal==1.27.0 # via # azure-identity # msal-extensions -msal-extensions==1.0.0 +msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol -multiprocess==0.70.15 +multipledispatch==0.6.0 + # via ibis-framework +multiprocess==0.70.16 # via bytewax -mypy==0.982 +mypy==1.8.0 # via # feast (setup.py) # sqlalchemy @@ -488,11 +498,9 @@ mypy-extensions==1.0.0 # mypy mypy-protobuf==3.1.0 # via feast (setup.py) -mysqlclient==2.2.0 - # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.11.0 +nbconvert==7.16.1 # via jupyter-server nbformat==5.9.2 # via @@ -500,13 +508,13 @@ nbformat==5.9.2 # jupyter-server # nbclient # nbconvert -nest-asyncio==1.5.8 +nest-asyncio==1.6.0 # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==7.0.6 +notebook==7.1.0 # via great-expectations -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via # jupyterlab # notebook @@ -516,13 +524,13 @@ numpy==1.24.4 # db-dtypes # feast (setup.py) # great-expectations + # ibis-framework # pandas - # pandavro # pyarrow # scipy oauthlib==3.2.2 # via requests-oauthlib -overrides==7.4.0 +overrides==7.7.0 # via jupyter-server packaging==23.2 # via @@ -534,41 +542,42 @@ packaging==23.2 # google-cloud-bigquery # great-expectations # gunicorn + # ibis-substrait # ipykernel # jupyter-server # jupyterlab # jupyterlab-server # marshmallow + # msal-extensions # nbconvert # pytest # snowflake-connector-python # sphinx -pandas==1.5.3 +pandas==1.5.3 ; python_version < "3.9" # via # altair # db-dtypes # feast (setup.py) # google-cloud-bigquery # great-expectations - # pandavro # snowflake-connector-python -pandavro==1.5.2 - # via feast (setup.py) -pandocfilters==1.5.0 +pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.1 # via dask -pathspec==0.11.2 +pathspec==0.12.1 # via black pbr==6.0.0 # via mock -pexpect==4.8.0 +pexpect==4.9.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==7.3.0 +pip-tools==7.4.0 # via feast (setup.py) pkgutil-resolve-name==1.3.10 # via jsonschema @@ -578,7 +587,7 @@ platformdirs==3.11.0 # jupyter-core # snowflake-connector-python # virtualenv -pluggy==1.3.0 +pluggy==1.4.0 # via pytest ply==3.11 # via thriftpy2 @@ -586,11 +595,11 @@ portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.18.0 +prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.41 +prompt-toolkit==3.0.43 # via ipython -proto-plus==1.22.3 +proto-plus==1.23.0 # via # feast (setup.py) # google-cloud-bigquery @@ -598,7 +607,7 @@ proto-plus==1.22.3 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # google-api-core @@ -614,6 +623,7 @@ protobuf==4.23.3 # grpcio-status # grpcio-testing # grpcio-tools + # ibis-substrait # mypy-protobuf # proto-plus psutil==5.9.0 @@ -634,13 +644,13 @@ py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark -pyarrow==14.0.1 +pyarrow==15.0.0 # via # db-dtypes # feast (setup.py) # google-cloud-bigquery # snowflake-connector-python -pyasn1==0.5.0 +pyasn1==0.5.1 # via # pyasn1-modules # rsa @@ -652,40 +662,45 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==1.10.13 +pydantic==2.6.2 # via # fastapi # feast (setup.py) # great-expectations +pydantic-core==2.16.3 + # via pydantic pyflakes==3.0.1 # via flake8 -pygments==2.16.1 +pygments==2.17.2 # via # feast (setup.py) # ipython # nbconvert + # rich # sphinx pyjwt[crypto]==2.8.0 # via # msal # snowflake-connector-python -pymssql==2.2.10 +pymssql==2.2.11 # via feast (setup.py) pymysql==1.1.0 # via feast (setup.py) -pyodbc==5.0.1 +pyodbc==5.1.0 # via feast (setup.py) -pyopenssl==23.3.0 +pyopenssl==24.0.0 # via snowflake-connector-python pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools pyspark==3.5.0 # via feast (setup.py) -pytest==7.4.3 +pytest==7.4.4 # via # feast (setup.py) # pytest-benchmark @@ -707,7 +722,7 @@ pytest-ordering==0.6 # via feast (setup.py) pytest-timeout==1.4.2 # via feast (setup.py) -pytest-xdist==3.4.0 +pytest-xdist==3.5.0 # via feast (setup.py) python-dateutil==2.8.2 # via @@ -715,20 +730,22 @@ python-dateutil==2.8.2 # botocore # google-cloud-bigquery # great-expectations + # ibis-framework # jupyter-client # kubernetes # moto # pandas # rockset # trino -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn python-json-logger==2.0.7 # via jupyter-events -pytz==2023.3.post1 +pytz==2024.1 # via # babel # great-expectations + # ibis-framework # pandas # snowflake-connector-python # trino @@ -736,24 +753,25 @@ pyyaml==6.0.1 # via # dask # feast (setup.py) + # ibis-substrait # jupyter-events # kubernetes # pre-commit # responses # uvicorn -pyzmq==25.1.1 +pyzmq==25.1.2 # via # ipykernel # jupyter-client # jupyter-server redis==4.6.0 # via feast (setup.py) -referencing==0.31.0 +referencing==0.33.0 # via # jsonschema # jsonschema-specifications # jupyter-events -regex==2023.10.3 +regex==2023.12.25 # via feast (setup.py) requests==2.31.0 # via @@ -776,7 +794,7 @@ requests==2.31.0 # trino requests-oauthlib==1.3.1 # via kubernetes -responses==0.24.1 +responses==0.25.0 # via moto rfc3339-validator==0.1.4 # via @@ -786,9 +804,11 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.1.0 +rich==13.7.0 + # via ibis-framework +rockset==2.1.1 # via feast (setup.py) -rpds-py==0.13.0 +rpds-py==0.18.0 # via # jsonschema # referencing @@ -798,7 +818,7 @@ ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.8 # via ruamel-yaml -s3transfer==0.7.0 +s3transfer==0.10.0 # via boto3 scipy==1.10.1 # via great-expectations @@ -809,12 +829,12 @@ six==1.16.0 # asttokens # azure-core # bleach - # cassandra-driver # geomet # happybase # isodate # kubernetes # mock + # multipledispatch # pandavro # python-dateutil # rfc3339-validator @@ -825,10 +845,8 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.5.0 - # via - # feast (setup.py) - # snowflake-connector-python +snowflake-connector-python[pandas]==3.7.1 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 @@ -847,15 +865,17 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.50 +sqlalchemy[mypy]==1.4.51 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a37 +sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +sqlglot==10.6.4 + # via ibis-framework stack-data==0.6.3 # via ipython -starlette==0.27.0 +starlette==0.36.3 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -885,12 +905,13 @@ tomli==2.0.1 # pytest tomlkit==0.12.3 # via snowflake-connector-python -toolz==0.12.0 +toolz==0.12.1 # via # altair # dask + # ibis-framework # partd -tornado==6.3.3 +tornado==6.4 # via # ipykernel # jupyter-client @@ -898,11 +919,11 @@ tornado==6.3.3 # jupyterlab # notebook # terminado -tqdm==4.66.1 +tqdm==4.66.2 # via # feast (setup.py) # great-expectations -traitlets==5.13.0 +traitlets==5.14.1 # via # comm # ipykernel @@ -917,9 +938,9 @@ traitlets==5.13.0 # nbclient # nbconvert # nbformat -trino==0.327.0 +trino==0.328.0 # via feast (setup.py) -typeguard==2.13.3 +typeguard==4.1.5 # via feast (setup.py) types-protobuf==3.19.22 # via @@ -927,40 +948,46 @@ types-protobuf==3.19.22 # mypy-protobuf types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==23.3.0.0 +types-pyopenssl==24.0.0.20240130 # via types-redis -types-python-dateutil==2.8.19.14 +types-python-dateutil==2.8.19.20240106 # via # arrow # feast (setup.py) -types-pytz==2023.3.1.1 +types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.10 +types-redis==4.6.0.20240218 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==68.2.0.1 +types-setuptools==69.1.0.20240223 # via feast (setup.py) -types-tabulate==0.9.0.3 +types-tabulate==0.9.0.20240106 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via + # annotated-types + # anyio # async-lru # azure-core # azure-storage-blob # black # fastapi # great-expectations + # ibis-framework # ipython # mypy # pydantic + # pydantic-core + # rich # snowflake-connector-python # sqlalchemy2-stubs # starlette + # typeguard # uvicorn tzlocal==5.2 # via @@ -982,10 +1009,8 @@ urllib3==1.26.18 # responses # rockset # snowflake-connector-python -uvicorn[standard]==0.24.0.post1 - # via - # feast (setup.py) - # uvicorn +uvicorn[standard]==0.27.1 + # via feast (setup.py) uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 @@ -996,7 +1021,7 @@ volatile==2.1.0 # via bowler watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.10 +wcwidth==0.2.13 # via prompt-toolkit webcolors==1.13 # via jsonschema @@ -1004,18 +1029,17 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.6.4 +websocket-client==1.7.0 # via - # docker # jupyter-server # kubernetes websockets==12.0 # via uvicorn werkzeug==3.0.1 # via moto -wheel==0.41.3 +wheel==0.42.0 # via pip-tools -widgetsnbextension==4.0.9 +widgetsnbextension==4.0.10 # via ipywidgets wrapt==1.16.0 # via testcontainers diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index c180c50c81e..d00a77ee6ff 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -4,21 +4,23 @@ # # pip-compile --output-file=sdk/python/requirements/py3.8-requirements.txt # -anyio==4.0.0 +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 # via # httpx # starlette # watchfiles appdirs==1.4.4 # via fissix -attrs==23.1.0 +attrs==23.2.0 # via # bowler # jsonschema # referencing bowler==0.9.0 # via feast (setup.py) -certifi==2023.7.22 +certifi==2024.2.2 # via # httpcore # httpx @@ -38,33 +40,29 @@ colorama==0.4.6 # via feast (setup.py) dask==2023.5.0 # via feast (setup.py) -dill==0.3.7 +dill==0.3.8 # via feast (setup.py) -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via anyio -fastapi==0.99.1 +fastapi==0.109.2 # via feast (setup.py) -fastavro==1.9.0 - # via - # feast (setup.py) - # pandavro fissix==21.11.13 # via bowler -fsspec==2023.10.0 +fsspec==2024.2.0 # via dask -greenlet==3.0.1 +greenlet==3.0.3 # via sqlalchemy -grpcio==1.59.2 +grpcio==1.62.0 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.59.2 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.59.2 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-tools==1.59.2 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -72,21 +70,22 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.2 +httpcore==1.0.4 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.25.1 +httpx==0.27.0 # via feast (setup.py) -idna==3.4 +idna==3.6 # via # anyio # httpx # requests -importlib-metadata==6.8.0 +importlib-metadata==6.11.0 # via # dask # feast (setup.py) + # typeguard importlib-resources==6.1.1 # via # feast (setup.py) @@ -94,19 +93,19 @@ importlib-resources==6.1.1 # jsonschema-specifications jinja2==3.1.3 # via feast (setup.py) -jsonschema==4.20.0 +jsonschema==4.21.1 # via feast (setup.py) -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 # via partd -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 -mmh3==4.0.1 +mmh3==4.1.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==1.7.0 +mypy==1.8.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy @@ -116,25 +115,20 @@ numpy==1.24.4 # via # feast (setup.py) # pandas - # pandavro # pyarrow packaging==23.2 # via # dask # gunicorn -pandas==1.5.3 - # via - # feast (setup.py) - # pandavro -pandavro==1.5.2 +pandas==2.0.3 # via feast (setup.py) partd==1.4.1 # via dask pkgutil-resolve-name==1.3.10 # via jsonschema -proto-plus==1.22.3 +proto-plus==1.23.0 # via feast (setup.py) -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # grpcio-health-checking @@ -142,50 +136,50 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus -pyarrow==14.0.1 +pyarrow==15.0.0 # via feast (setup.py) -pydantic==1.10.13 +pydantic==2.6.2 # via # fastapi # feast (setup.py) -pygments==2.16.1 +pydantic-core==2.16.3 + # via pydantic +pygments==2.17.2 # via feast (setup.py) python-dateutil==2.8.2 # via pandas -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn -pytz==2023.3.post1 +pytz==2024.1 # via pandas pyyaml==6.0.1 # via # dask # feast (setup.py) # uvicorn -referencing==0.31.0 +referencing==0.33.0 # via # jsonschema # jsonschema-specifications requests==2.31.0 # via feast (setup.py) -rpds-py==0.13.0 +rpds-py==0.18.0 # via # jsonschema # referencing six==1.16.0 - # via - # pandavro - # python-dateutil + # via python-dateutil sniffio==1.3.0 # via # anyio # httpx -sqlalchemy[mypy]==1.4.50 +sqlalchemy[mypy]==1.4.51 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a37 +sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy -starlette==0.27.0 +starlette==0.36.3 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -195,30 +189,32 @@ toml==0.10.2 # via feast (setup.py) tomli==2.0.1 # via mypy -toolz==0.12.0 +toolz==0.12.1 # via # dask # partd -tqdm==4.66.1 +tqdm==4.66.2 # via feast (setup.py) -typeguard==2.13.3 +typeguard==4.1.5 # via feast (setup.py) -types-protobuf==4.24.0.4 +types-protobuf==4.24.0.20240129 # via mypy-protobuf -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via + # annotated-types + # anyio # fastapi # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs # starlette + # typeguard # uvicorn -urllib3==2.1.0 +urllib3==2.2.1 # via requests -uvicorn[standard]==0.24.0.post1 - # via - # feast (setup.py) - # uvicorn +uvicorn[standard]==0.27.1 + # via feast (setup.py) uvloop==0.19.0 # via uvicorn volatile==2.1.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 43c49a49527..99f7ee02858 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -4,11 +4,13 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt # -alabaster==0.7.13 +alabaster==0.7.16 # via sphinx -altair==4.2.0 +altair==4.2.2 # via great-expectations -anyio==4.0.0 +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 # via # httpx # jupyter-server @@ -32,14 +34,16 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis -attrs==23.1.0 +atpublic==4.0 + # via ibis-framework +attrs==23.2.0 # via # bowler # jsonschema # referencing avro==1.10.0 # via feast (setup.py) -azure-core==1.29.5 +azure-core==1.30.0 # via # azure-identity # azure-storage-blob @@ -47,21 +51,23 @@ azure-identity==1.15.0 # via feast (setup.py) azure-storage-blob==12.19.0 # via feast (setup.py) -babel==2.13.1 +babel==2.14.0 # via # jupyterlab-server # sphinx -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 # via nbconvert +bidict==0.23.1 + # via ibis-framework black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.29.2 +boto3==1.34.49 # via # feast (setup.py) # moto -botocore==1.32.2 +botocore==1.34.49 # via # boto3 # moto @@ -74,13 +80,13 @@ build==1.0.3 # pip-tools bytewax==0.15.1 # via feast (setup.py) -cachecontrol==0.13.1 +cachecontrol==0.14.0 # via firebase-admin cachetools==5.3.2 # via google-auth -cassandra-driver==3.28.0 +cassandra-driver==3.29.0 # via feast (setup.py) -certifi==2023.7.22 +certifi==2024.2.2 # via # httpcore # httpx @@ -116,15 +122,13 @@ colorama==0.4.6 # via # feast (setup.py) # great-expectations -comm==0.2.0 +comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.3.2 - # via - # coverage - # pytest-cov -cryptography==41.0.6 +coverage[toml]==7.4.3 + # via pytest-cov +cryptography==42.0.4 # via # azure-identity # azure-storage-blob @@ -137,11 +141,11 @@ cryptography==41.0.6 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2023.11.0 +dask==2024.2.1 # via feast (setup.py) -db-dtypes==1.1.1 +db-dtypes==1.2.0 # via google-cloud-bigquery -debugpy==1.8.0 +debugpy==1.8.1 # via ipykernel decorator==5.1.1 # via ipython @@ -149,14 +153,14 @@ defusedxml==0.7.1 # via nbconvert deprecation==2.1.0 # via testcontainers -dill==0.3.7 +dill==0.3.8 # via # bytewax # feast (setup.py) # multiprocess -distlib==0.3.7 +distlib==0.3.8 # via virtualenv -docker==6.1.3 +docker==7.0.0 # via # feast (setup.py) # testcontainers @@ -164,7 +168,7 @@ docutils==0.19 # via sphinx entrypoints==0.4 # via altair -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # anyio # ipython @@ -173,13 +177,9 @@ execnet==2.0.2 # via pytest-xdist executing==2.0.1 # via stack-data -fastapi==0.99.1 +fastapi==0.109.2 # via feast (setup.py) -fastavro==1.9.0 - # via - # feast (setup.py) - # pandavro -fastjsonschema==2.19.0 +fastjsonschema==2.19.1 # via nbformat filelock==3.13.1 # via @@ -193,7 +193,7 @@ flake8==6.0.0 # via feast (setup.py) fqdn==1.5.1 # via jsonschema -fsspec==2023.9.2 +fsspec==2023.12.2 # via # dask # feast (setup.py) @@ -201,7 +201,7 @@ geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.14.0 +google-api-core[grpc]==2.17.1 # via # feast (setup.py) # firebase-admin @@ -213,9 +213,9 @@ google-api-core[grpc]==2.14.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.108.0 +google-api-python-client==2.119.0 # via firebase-admin -google-auth==2.23.4 +google-auth==2.28.1 # via # google-api-core # google-api-python-client @@ -223,28 +223,26 @@ google-auth==2.23.4 # google-cloud-core # google-cloud-storage # kubernetes -google-auth-httplib2==0.1.1 +google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.22.0 # via feast (setup.py) -google-cloud-bigtable==2.21.0 +google-cloud-bigquery-storage==2.24.0 + # via feast (setup.py) +google-cloud-bigtable==2.23.0 # via feast (setup.py) -google-cloud-core==2.3.3 +google-cloud-core==2.4.1 # via # google-cloud-bigquery # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.18.0 +google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.13.1 +google-cloud-firestore==2.15.0 # via firebase-admin -google-cloud-storage==2.13.0 +google-cloud-storage==2.14.0 # via # feast (setup.py) # firebase-admin @@ -252,23 +250,23 @@ google-crc32c==1.5.0 # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.6.0 +google-resumable-media==2.7.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.61.0 +googleapis-common-protos[grpc]==1.62.0 # via # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.15.50 +great-expectations==0.18.9 # via feast (setup.py) -greenlet==3.0.1 +greenlet==3.0.3 # via sqlalchemy -grpc-google-iam-v1==0.12.7 +grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.59.2 +grpcio==1.62.0 # via # feast (setup.py) # google-api-core @@ -280,15 +278,15 @@ grpcio==1.59.2 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.59.2 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.59.2 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-status==1.59.2 +grpcio-status==1.62.0 # via google-api-core -grpcio-testing==1.59.2 +grpcio-testing==1.62.0 # via feast (setup.py) -grpcio-tools==1.59.2 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -300,9 +298,9 @@ happybase==1.2.0 # via feast (setup.py) hazelcast-python-client==5.3.0 # via feast (setup.py) -hiredis==2.2.3 +hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.2 +httpcore==1.0.4 # via httpx httplib2==0.22.0 # via @@ -310,11 +308,19 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.25.1 +httpx==0.27.0 + # via + # feast (setup.py) + # jupyterlab +ibis-framework==8.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 # via feast (setup.py) -identify==2.5.31 +identify==2.5.35 # via pre-commit -idna==3.4 +idna==3.6 # via # anyio # httpx @@ -323,36 +329,36 @@ idna==3.4 # snowflake-connector-python imagesize==1.4.1 # via sphinx -importlib-metadata==6.8.0 +importlib-metadata==6.11.0 # via # build # dask # feast (setup.py) - # great-expectations # jupyter-client # jupyter-lsp # jupyterlab # jupyterlab-server # nbconvert # sphinx + # typeguard importlib-resources==6.1.1 # via feast (setup.py) iniconfig==2.0.0 # via pytest -ipykernel==6.26.0 +ipykernel==6.29.2 # via jupyterlab -ipython==8.17.2 +ipython==8.18.1 # via # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.1 +ipywidgets==8.1.2 # via great-expectations isodate==0.6.1 # via azure-storage-blob isoduration==20.11.0 # via jsonschema -isort==5.12.0 +isort==5.13.2 # via feast (setup.py) jedi==0.19.1 # via ipython @@ -371,7 +377,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.14 +json5==0.9.17 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -379,7 +385,7 @@ jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.20.0 +jsonschema[format-nongpl]==4.21.1 # via # altair # feast (setup.py) @@ -387,14 +393,14 @@ jsonschema[format-nongpl]==4.20.0 # jupyter-events # jupyterlab-server # nbformat -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.12.1 # via jsonschema jupyter-client==8.6.0 # via # ipykernel # jupyter-server # nbclient -jupyter-core==5.5.0 +jupyter-core==5.7.1 # via # ipykernel # jupyter-client @@ -407,24 +413,24 @@ jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.2 # via jupyterlab -jupyter-server==2.11.2 +jupyter-server==2.12.5 # via # jupyter-lsp # jupyterlab # jupyterlab-server # notebook # notebook-shim -jupyter-server-terminals==0.4.4 +jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.0.8 +jupyterlab==4.1.2 # via notebook -jupyterlab-pygments==0.2.2 +jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.25.1 +jupyterlab-server==2.25.3 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.9 +jupyterlab-widgets==3.0.10 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) @@ -432,12 +438,14 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations -markupsafe==2.1.3 +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 # via # jinja2 # nbconvert # werkzeug -marshmallow==3.20.1 +marshmallow==3.20.2 # via great-expectations matplotlib-inline==0.1.6 # via @@ -445,31 +453,35 @@ matplotlib-inline==0.1.6 # ipython mccabe==0.7.0 # via flake8 +mdurl==0.1.2 + # via markdown-it-py minio==7.1.0 # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert -mmh3==4.0.1 +mmh3==4.1.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.2.9 +moto==4.2.14 # via feast (setup.py) -msal==1.25.0 +msal==1.27.0 # via # azure-identity # msal-extensions -msal-extensions==1.0.0 +msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol -multiprocess==0.70.15 +multipledispatch==1.0.0 + # via ibis-framework +multiprocess==0.70.16 # via bytewax -mypy==0.982 +mypy==1.8.0 # via # feast (setup.py) # sqlalchemy @@ -479,11 +491,9 @@ mypy-extensions==1.0.0 # mypy mypy-protobuf==3.1.0 # via feast (setup.py) -mysqlclient==2.2.0 - # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.11.0 +nbconvert==7.16.1 # via jupyter-server nbformat==5.9.2 # via @@ -491,13 +501,13 @@ nbformat==5.9.2 # jupyter-server # nbclient # nbconvert -nest-asyncio==1.5.8 +nest-asyncio==1.6.0 # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==7.0.6 +notebook==7.1.0 # via great-expectations -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via # jupyterlab # notebook @@ -507,13 +517,13 @@ numpy==1.24.4 # db-dtypes # feast (setup.py) # great-expectations + # ibis-framework # pandas - # pandavro # pyarrow # scipy oauthlib==3.2.2 # via requests-oauthlib -overrides==7.4.0 +overrides==7.7.0 # via jupyter-server packaging==23.2 # via @@ -525,39 +535,40 @@ packaging==23.2 # google-cloud-bigquery # great-expectations # gunicorn + # ibis-substrait # ipykernel # jupyter-server # jupyterlab # jupyterlab-server # marshmallow + # msal-extensions # nbconvert # pytest # snowflake-connector-python # sphinx -pandas==1.5.3 +pandas==2.2.0 # via # altair # db-dtypes # feast (setup.py) # google-cloud-bigquery # great-expectations - # pandavro # snowflake-connector-python -pandavro==1.5.2 - # via feast (setup.py) -pandocfilters==1.5.0 +pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.1 # via dask -pathspec==0.11.2 +pathspec==0.12.1 # via black pbr==6.0.0 # via mock -pexpect==4.8.0 +pexpect==4.9.0 # via ipython -pip-tools==7.3.0 +pip-tools==7.4.0 # via feast (setup.py) platformdirs==3.11.0 # via @@ -565,7 +576,7 @@ platformdirs==3.11.0 # jupyter-core # snowflake-connector-python # virtualenv -pluggy==1.3.0 +pluggy==1.4.0 # via pytest ply==3.11 # via thriftpy2 @@ -573,11 +584,11 @@ portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.18.0 +prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.41 +prompt-toolkit==3.0.43 # via ipython -proto-plus==1.22.3 +proto-plus==1.23.0 # via # feast (setup.py) # google-cloud-bigquery @@ -585,7 +596,7 @@ proto-plus==1.22.3 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # google-api-core @@ -603,6 +614,7 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus + # substrait psutil==5.9.0 # via # feast (setup.py) @@ -621,13 +633,16 @@ py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark -pyarrow==14.0.1 +pyarrow==15.0.0 # via # db-dtypes # feast (setup.py) # google-cloud-bigquery + # ibis-framework # snowflake-connector-python -pyasn1==0.5.0 +pyarrow-hotfix==0.6 + # via ibis-framework +pyasn1==0.5.1 # via # pyasn1-modules # rsa @@ -639,40 +654,45 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==1.10.13 +pydantic==2.6.2 # via # fastapi # feast (setup.py) # great-expectations +pydantic-core==2.16.3 + # via pydantic pyflakes==3.0.1 # via flake8 -pygments==2.16.1 +pygments==2.17.2 # via # feast (setup.py) # ipython # nbconvert + # rich # sphinx pyjwt[crypto]==2.8.0 # via # msal # snowflake-connector-python -pymssql==2.2.10 +pymssql==2.2.11 # via feast (setup.py) pymysql==1.1.0 # via feast (setup.py) -pyodbc==5.0.1 +pyodbc==5.1.0 # via feast (setup.py) -pyopenssl==23.3.0 +pyopenssl==24.0.0 # via snowflake-connector-python pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools pyspark==3.5.0 # via feast (setup.py) -pytest==7.4.3 +pytest==7.4.4 # via # feast (setup.py) # pytest-benchmark @@ -694,7 +714,7 @@ pytest-ordering==0.6 # via feast (setup.py) pytest-timeout==1.4.2 # via feast (setup.py) -pytest-xdist==3.4.0 +pytest-xdist==3.5.0 # via feast (setup.py) python-dateutil==2.8.2 # via @@ -702,19 +722,21 @@ python-dateutil==2.8.2 # botocore # google-cloud-bigquery # great-expectations + # ibis-framework # jupyter-client # kubernetes # moto # pandas # rockset # trino -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn python-json-logger==2.0.7 # via jupyter-events -pytz==2023.3.post1 +pytz==2024.1 # via # great-expectations + # ibis-framework # pandas # snowflake-connector-python # trino @@ -722,24 +744,25 @@ pyyaml==6.0.1 # via # dask # feast (setup.py) + # ibis-substrait # jupyter-events # kubernetes # pre-commit # responses # uvicorn -pyzmq==25.1.1 +pyzmq==25.1.2 # via # ipykernel # jupyter-client # jupyter-server redis==4.6.0 # via feast (setup.py) -referencing==0.31.0 +referencing==0.33.0 # via # jsonschema # jsonschema-specifications # jupyter-events -regex==2023.10.3 +regex==2023.12.25 # via feast (setup.py) requests==2.31.0 # via @@ -762,7 +785,7 @@ requests==2.31.0 # trino requests-oauthlib==1.3.1 # via kubernetes -responses==0.24.1 +responses==0.25.0 # via moto rfc3339-validator==0.1.4 # via @@ -772,9 +795,11 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.1.0 +rich==13.7.0 + # via ibis-framework +rockset==2.1.1 # via feast (setup.py) -rpds-py==0.13.0 +rpds-py==0.18.0 # via # jsonschema # referencing @@ -784,9 +809,9 @@ ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.8 # via ruamel-yaml -s3transfer==0.7.0 +s3transfer==0.10.0 # via boto3 -scipy==1.11.3 +scipy==1.12.0 # via great-expectations send2trash==1.8.2 # via jupyter-server @@ -795,13 +820,11 @@ six==1.16.0 # asttokens # azure-core # bleach - # cassandra-driver # geomet # happybase # isodate # kubernetes # mock - # pandavro # python-dateutil # rfc3339-validator # thriftpy2 @@ -811,44 +834,40 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.5.0 - # via - # feast (setup.py) - # snowflake-connector-python +snowflake-connector-python[pandas]==3.7.1 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 - # via - # feast (setup.py) - # sphinxcontrib-applehelp - # sphinxcontrib-devhelp - # sphinxcontrib-htmlhelp - # sphinxcontrib-qthelp - # sphinxcontrib-serializinghtml -sphinxcontrib-applehelp==1.0.7 + # via feast (setup.py) +sphinxcontrib-applehelp==1.0.8 # via sphinx -sphinxcontrib-devhelp==1.0.5 +sphinxcontrib-devhelp==1.0.6 # via sphinx -sphinxcontrib-htmlhelp==2.0.4 +sphinxcontrib-htmlhelp==2.0.5 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.6 +sphinxcontrib-qthelp==1.0.7 # via sphinx -sphinxcontrib-serializinghtml==1.1.9 +sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy[mypy]==1.4.50 +sqlalchemy[mypy]==1.4.51 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a37 +sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +sqlglot==20.11.0 + # via ibis-framework stack-data==0.6.3 # via ipython -starlette==0.27.0 +starlette==0.36.3 # via fastapi +substrait==0.12.1 + # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) tenacity==8.2.3 @@ -877,12 +896,13 @@ tomli==2.0.1 # pytest tomlkit==0.12.3 # via snowflake-connector-python -toolz==0.12.0 +toolz==0.12.1 # via # altair # dask + # ibis-framework # partd -tornado==6.3.3 +tornado==6.4 # via # ipykernel # jupyter-client @@ -890,11 +910,11 @@ tornado==6.3.3 # jupyterlab # notebook # terminado -tqdm==4.66.1 +tqdm==4.66.2 # via # feast (setup.py) # great-expectations -traitlets==5.13.0 +traitlets==5.14.1 # via # comm # ipykernel @@ -909,9 +929,9 @@ traitlets==5.13.0 # nbclient # nbconvert # nbformat -trino==0.327.0 +trino==0.328.0 # via feast (setup.py) -typeguard==2.13.3 +typeguard==4.1.5 # via feast (setup.py) types-protobuf==3.19.22 # via @@ -919,41 +939,47 @@ types-protobuf==3.19.22 # mypy-protobuf types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==23.3.0.0 +types-pyopenssl==24.0.0.20240130 # via types-redis -types-python-dateutil==2.8.19.14 +types-python-dateutil==2.8.19.20240106 # via # arrow # feast (setup.py) -types-pytz==2023.3.1.1 +types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.10 +types-redis==4.6.0.20240218 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==68.2.0.1 +types-setuptools==69.1.0.20240223 # via feast (setup.py) -types-tabulate==0.9.0.3 +types-tabulate==0.9.0.20240106 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via + # anyio # async-lru # azure-core # azure-storage-blob # black # fastapi # great-expectations + # ibis-framework # ipython # mypy # pydantic + # pydantic-core # snowflake-connector-python # sqlalchemy2-stubs # starlette + # typeguard # uvicorn +tzdata==2024.1 + # via pandas tzlocal==5.2 # via # great-expectations @@ -974,10 +1000,8 @@ urllib3==1.26.18 # responses # rockset # snowflake-connector-python -uvicorn[standard]==0.24.0.post1 - # via - # feast (setup.py) - # uvicorn +uvicorn[standard]==0.27.1 + # via feast (setup.py) uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 @@ -988,7 +1012,7 @@ volatile==2.1.0 # via bowler watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.10 +wcwidth==0.2.13 # via prompt-toolkit webcolors==1.13 # via jsonschema @@ -996,18 +1020,17 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.6.4 +websocket-client==1.7.0 # via - # docker # jupyter-server # kubernetes websockets==12.0 # via uvicorn werkzeug==3.0.1 # via moto -wheel==0.41.3 +wheel==0.42.0 # via pip-tools -widgetsnbextension==4.0.9 +widgetsnbextension==4.0.10 # via ipywidgets wrapt==1.16.0 # via testcontainers diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 3b6f88b4e2a..4364dc62bf7 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -4,21 +4,23 @@ # # pip-compile --output-file=sdk/python/requirements/py3.9-requirements.txt # -anyio==4.0.0 +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 # via # httpx # starlette # watchfiles appdirs==1.4.4 # via fissix -attrs==23.1.0 +attrs==23.2.0 # via # bowler # jsonschema # referencing bowler==0.9.0 # via feast (setup.py) -certifi==2023.7.22 +certifi==2024.2.2 # via # httpcore # httpx @@ -36,35 +38,31 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2023.11.0 +dask==2024.2.1 # via feast (setup.py) -dill==0.3.7 +dill==0.3.8 # via feast (setup.py) -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via anyio -fastapi==0.99.1 +fastapi==0.109.2 # via feast (setup.py) -fastavro==1.9.0 - # via - # feast (setup.py) - # pandavro fissix==21.11.13 # via bowler -fsspec==2023.10.0 +fsspec==2024.2.0 # via dask -greenlet==3.0.1 +greenlet==3.0.3 # via sqlalchemy -grpcio==1.59.2 +grpcio==1.62.0 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.59.2 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.59.2 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-tools==1.59.2 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -72,38 +70,39 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.2 +httpcore==1.0.4 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.25.1 +httpx==0.27.0 # via feast (setup.py) -idna==3.4 +idna==3.6 # via # anyio # httpx # requests -importlib-metadata==6.8.0 +importlib-metadata==6.11.0 # via # dask # feast (setup.py) + # typeguard importlib-resources==6.1.1 # via feast (setup.py) jinja2==3.1.3 # via feast (setup.py) -jsonschema==4.20.0 +jsonschema==4.21.1 # via feast (setup.py) -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.12.1 # via jsonschema locket==1.0.0 # via partd -markupsafe==2.1.3 +markupsafe==2.1.5 # via jinja2 -mmh3==4.0.1 +mmh3==4.1.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==1.7.0 +mypy==1.8.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy @@ -113,23 +112,18 @@ numpy==1.24.4 # via # feast (setup.py) # pandas - # pandavro # pyarrow packaging==23.2 # via # dask # gunicorn -pandas==1.5.3 - # via - # feast (setup.py) - # pandavro -pandavro==1.5.2 +pandas==2.2.0 # via feast (setup.py) partd==1.4.1 # via dask -proto-plus==1.22.3 +proto-plus==1.23.0 # via feast (setup.py) -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # grpcio-health-checking @@ -137,50 +131,50 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus -pyarrow==14.0.1 +pyarrow==15.0.0 # via feast (setup.py) -pydantic==1.10.13 +pydantic==2.6.2 # via # fastapi # feast (setup.py) -pygments==2.16.1 +pydantic-core==2.16.3 + # via pydantic +pygments==2.17.2 # via feast (setup.py) python-dateutil==2.8.2 # via pandas -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn -pytz==2023.3.post1 +pytz==2024.1 # via pandas pyyaml==6.0.1 # via # dask # feast (setup.py) # uvicorn -referencing==0.31.0 +referencing==0.33.0 # via # jsonschema # jsonschema-specifications requests==2.31.0 # via feast (setup.py) -rpds-py==0.13.0 +rpds-py==0.18.0 # via # jsonschema # referencing six==1.16.0 - # via - # pandavro - # python-dateutil + # via python-dateutil sniffio==1.3.0 # via # anyio # httpx -sqlalchemy[mypy]==1.4.50 +sqlalchemy[mypy]==1.4.51 # via # feast (setup.py) # sqlalchemy -sqlalchemy2-stubs==0.0.2a37 +sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy -starlette==0.27.0 +starlette==0.36.3 # via fastapi tabulate==0.9.0 # via feast (setup.py) @@ -190,30 +184,31 @@ toml==0.10.2 # via feast (setup.py) tomli==2.0.1 # via mypy -toolz==0.12.0 +toolz==0.12.1 # via # dask # partd -tqdm==4.66.1 +tqdm==4.66.2 # via feast (setup.py) -typeguard==2.13.3 +typeguard==4.1.5 # via feast (setup.py) -types-protobuf==4.24.0.4 +types-protobuf==4.24.0.20240129 # via mypy-protobuf -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via + # anyio # fastapi # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs # starlette + # typeguard # uvicorn -urllib3==2.1.0 +urllib3==2.2.1 # via requests -uvicorn[standard]==0.24.0.post1 - # via - # feast (setup.py) - # uvicorn +uvicorn[standard]==0.27.1 + # via feast (setup.py) uvloop==0.19.0 # via uvicorn volatile==2.1.0 diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 728bd9b34f7..743a1ce4a0f 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -18,7 +18,7 @@ from datetime import datetime, timedelta from multiprocessing import Process from sys import platform -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Tuple, no_type_check import pandas as pd import pytest @@ -187,9 +187,10 @@ def environment(request, worker_id): e.online_store_creator.teardown() -_config_cache = {} +_config_cache: Any = {} +@no_type_check def pytest_generate_tests(metafunc: pytest.Metafunc): """ This function receives each test function (wrapped in Metafunc) diff --git a/sdk/python/tests/data/data_creator.py b/sdk/python/tests/data/data_creator.py index 2155468445a..1fc66aee845 100644 --- a/sdk/python/tests/data/data_creator.py +++ b/sdk/python/tests/data/data_creator.py @@ -9,7 +9,7 @@ def create_basic_driver_dataset( entity_type: FeastType = Int32, - feature_dtype: str = None, + feature_dtype: Optional[str] = None, feature_is_list: bool = False, list_has_empty_list: bool = False, ) -> pd.DataFrame: @@ -59,6 +59,7 @@ def get_feature_values_for_dtype( "int64": [1, 2, 3, 4, 5], "float": [1.0, None, 3.0, 4.0, 5.0], "string": ["1", None, "3", "4", "5"], + "bytes": [b"1", None, b"3", b"4", b"5"], "bool": [True, None, False, True, False], "datetime": [ datetime(1980, 1, 1), diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index d27e2645d4e..ba256a3813c 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -71,16 +71,16 @@ def get_historical_features( project: str, full_feature_names: bool = False, ) -> RetrievalJob: - pass + return RetrievalJob() def online_read( self, config: RepoConfig, table: FeatureView, entity_keys: List[EntityKeyProto], - requested_features: List[str] = None, + requested_features: Optional[List[str]] = None, ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: - pass + return [] def retrieve_saved_dataset(self, config: RepoConfig, dataset: SavedDataset): pass @@ -102,4 +102,4 @@ def retrieve_feature_service_logs( config: RepoConfig, registry: BaseRegistry, ) -> RetrievalJob: - pass + return RetrievalJob() diff --git a/sdk/python/tests/integration/e2e/test_validation.py b/sdk/python/tests/integration/e2e/test_validation.py index f49ed80a265..fdf182be573 100644 --- a/sdk/python/tests/integration/e2e/test_validation.py +++ b/sdk/python/tests/integration/e2e/test_validation.py @@ -167,7 +167,7 @@ def test_logged_features_validation(environment, universal_data_sources): { "customer_id": 2000 + i, "driver_id": 6000 + i, - "event_timestamp": datetime.datetime.now(), + "event_timestamp": make_tzaware(datetime.datetime.now()), } ] ), diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index fda5b3c11de..f745bafa132 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -83,8 +83,8 @@ "password": os.getenv("SNOWFLAKE_CI_PASSWORD", ""), "role": os.getenv("SNOWFLAKE_CI_ROLE", ""), "warehouse": os.getenv("SNOWFLAKE_CI_WAREHOUSE", ""), - "database": "FEAST", - "schema": "ONLINE", + "database": os.getenv("SNOWFLAKE_CI_DATABASE", "FEAST"), + "schema": os.getenv("SNOWFLAKE_CI_SCHEMA_ONLINE", "ONLINE"), } BIGTABLE_CONFIG = { @@ -99,7 +99,7 @@ "host": os.getenv("ROCKSET_APISERVER", "api.rs2.usw2.rockset.com"), } -OFFLINE_STORE_TO_PROVIDER_CONFIG: Dict[str, DataSourceCreator] = { +OFFLINE_STORE_TO_PROVIDER_CONFIG: Dict[str, Tuple[str, Type[DataSourceCreator]]] = { "file": ("local", FileDataSourceCreator), "bigquery": ("gcp", BigQueryDataSourceCreator), "redshift": ("aws", RedshiftDataSourceCreator), @@ -111,7 +111,7 @@ ] AVAILABLE_ONLINE_STORES: Dict[ - str, Tuple[Union[str, Dict[str, str]], Optional[Type[OnlineStoreCreator]]] + str, Tuple[Union[str, Dict[Any, Any]], Optional[Type[OnlineStoreCreator]]] ] = { "sqlite": ({"type": "sqlite"}, None), } @@ -169,7 +169,7 @@ AVAILABLE_ONLINE_STORES = { c.online_store["type"] if isinstance(c.online_store, dict) - else c.online_store: (c.online_store, c.online_store_creator) + else c.online_store: (c.online_store, c.online_store_creator) # type: ignore for c in FULL_REPO_CONFIGS } @@ -328,7 +328,7 @@ class UniversalFeatureViews: customer: FeatureView global_fv: FeatureView driver: FeatureView - driver_odfv: OnDemandFeatureView + driver_odfv: Optional[OnDemandFeatureView] order: FeatureView location: FeatureView field_mapping: FeatureView @@ -410,9 +410,7 @@ def construct_test_environment( online_creator = test_repo_config.online_store_creator( project, fixture_request=fixture_request ) - online_store = ( - test_repo_config.online_store - ) = online_creator.create_online_store() + online_store = online_creator.create_online_store() else: online_creator = None online_store = test_repo_config.online_store @@ -422,7 +420,7 @@ def construct_test_environment( AwsLambdaFeatureServerConfig, ) - feature_server = AwsLambdaFeatureServerConfig( + feature_server: Any = AwsLambdaFeatureServerConfig( enabled=True, execution_role_name=os.getenv( "AWS_LAMBDA_ROLE", @@ -465,7 +463,7 @@ def construct_test_environment( # Create feature_store.yaml out of the config with open(Path(repo_dir_name) / "feature_store.yaml", "w") as f: - yaml.safe_dump(json.loads(config.json()), f) + yaml.safe_dump(json.loads(config.model_dump_json(by_alias=True)), f) fs = FeatureStore(repo_dir_name) # We need to initialize the registry, because if nothing is applied in the test before tearing down diff --git a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py index b36af0db472..5e5062291d5 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py @@ -20,7 +20,7 @@ def create_data_source( destination_name: str, event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, timestamp_field: Optional[str] = None, ) -> DataSource: """ @@ -42,19 +42,20 @@ def create_data_source( A Data source object, pointing to a table or file that is uploaded/persisted for the purpose of the test. """ - ... + raise NotImplementedError @abstractmethod def create_offline_store_config(self) -> FeastConfigBaseModel: - ... + raise NotImplementedError @abstractmethod def create_saved_dataset_destination(self) -> SavedDatasetStorage: - ... + raise NotImplementedError + @abstractmethod def create_logged_features_destination(self) -> LoggingDestination: - pass + raise NotImplementedError @abstractmethod def teardown(self): - ... + raise NotImplementedError diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py index 384037eef14..066497a0bcd 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py @@ -64,10 +64,9 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, - **kwargs, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index 124dd4c88d6..008bb8d8815 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -39,9 +39,9 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) @@ -94,9 +94,9 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - timestamp_field="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) @@ -167,11 +167,10 @@ def _upload_parquet_file(self, df, file_name, minio_endpoint): def create_data_source( self, df: pd.DataFrame, - destination_name: Optional[str] = None, - suffix: Optional[str] = None, - timestamp_field="ts", + destination_name: str, created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: filename = f"{destination_name}.parquet" port = self.minio.get_exposed_port("9000") diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py index dfe8e3d33bf..5a4e3f10854 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py @@ -42,16 +42,17 @@ def __init__(self, project_name: str, *args, **kwargs): iam_role=os.getenv( "AWS_IAM_ROLE", "arn:aws:iam::402087665549:role/redshift_s3_access_role" ), + workgroup="", ) def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py index c7e5961a88a..1481b11a106 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py @@ -36,8 +36,8 @@ def __init__(self, project_name: str, *args, **kwargs): password=os.environ["SNOWFLAKE_CI_PASSWORD"], role=os.environ["SNOWFLAKE_CI_ROLE"], warehouse=os.environ["SNOWFLAKE_CI_WAREHOUSE"], - database="FEAST", - schema="OFFLINE", + database=os.environ.get("SNOWFLAKE_CI_DATABASE", "FEAST"), + schema=os.environ.get("SNOWFLAKE_CI_SCHEMA_OFFLINE", "OFFLINE"), storage_integration_name=os.getenv("BLOB_EXPORT_STORAGE_NAME", "FEAST_S3"), blob_export_location=os.getenv( "BLOB_EXPORT_URI", "s3://feast-snowflake-offload/export" @@ -48,10 +48,10 @@ def create_data_source( self, df: pd.DataFrame, destination_name: str, - suffix: Optional[str] = None, - timestamp_field="ts", + event_timestamp_column="ts", created_timestamp_column="created_ts", - field_mapping: Dict[str, str] = None, + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", ) -> DataSource: destination_name = self.get_prefixed_table_name(destination_name) diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 5938a0c936e..421ef416018 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -14,6 +14,8 @@ StreamFeatureView, ) from feast.data_source import DataSource, RequestSource +from feast.feature_view_projection import FeatureViewProjection +from feast.on_demand_feature_view import OnDemandPandasTransformation from feast.types import Array, FeastType, Float32, Float64, Int32, Int64 from tests.integration.feature_repos.universal.entities import ( customer, @@ -55,7 +57,7 @@ def conv_rate_plus_100(features_df: pd.DataFrame) -> pd.DataFrame: def conv_rate_plus_100_feature_view( - sources: Dict[str, Union[RequestSource, FeatureView]], + sources: List[Union[FeatureView, RequestSource, FeatureViewProjection]], infer_features: bool = False, features: Optional[List[Field]] = None, ) -> OnDemandFeatureView: @@ -69,8 +71,9 @@ def conv_rate_plus_100_feature_view( name=conv_rate_plus_100.__name__, schema=[] if infer_features else _features, sources=sources, - udf=conv_rate_plus_100, - udf_string="raw udf source", + transformation=OnDemandPandasTransformation( + udf=conv_rate_plus_100, udf_string="raw udf source" + ), ) @@ -107,8 +110,9 @@ def similarity_feature_view( name=similarity.__name__, sources=sources, schema=[] if infer_features else _fields, - udf=similarity, - udf_string="similarity raw udf", + transformation=OnDemandPandasTransformation( + udf=similarity, udf_string="similarity raw udf" + ), ) diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/redis.py b/sdk/python/tests/integration/feature_repos/universal/online_store/redis.py index 11d62d9d30a..8e18f7fb172 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/redis.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/redis.py @@ -20,7 +20,11 @@ def create_online_store(self) -> Dict[str, str]: container=self.container, predicate=log_string_to_wait_for, timeout=10 ) exposed_port = self.container.get_exposed_port("6379") - return {"type": "redis", "connection_string": f"localhost:{exposed_port},db=0"} + container_host = self.container.get_container_host_ip() + return { + "type": "redis", + "connection_string": f"{container_host}:{exposed_port},db=0", + } def teardown(self): self.container.stop() diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py b/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py index c3872ea697f..4932001e76f 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store_creator.py @@ -1,4 +1,4 @@ -from abc import ABC +from abc import ABC, abstractmethod from feast.repo_config import FeastConfigBaseModel @@ -8,7 +8,8 @@ def __init__(self, project_name: str, **kwargs): self.project_name = project_name def create_online_store(self) -> FeastConfigBaseModel: - ... + raise NotImplementedError + @abstractmethod def teardown(self): - ... + raise NotImplementedError diff --git a/sdk/python/tests/integration/materialization/test_snowflake.py b/sdk/python/tests/integration/materialization/test_snowflake.py index 0cf1471dfeb..60fa9b30aab 100644 --- a/sdk/python/tests/integration/materialization/test_snowflake.py +++ b/sdk/python/tests/integration/materialization/test_snowflake.py @@ -1,10 +1,13 @@ import os -from datetime import timedelta +from datetime import datetime, timedelta import pytest +from pytz import utc +from feast import Field from feast.entity import Entity from feast.feature_view import FeatureView +from feast.types import Array, Bool, Bytes, Float64, Int32, Int64, String, UnixTimestamp from tests.data.data_creator import create_basic_driver_dataset from tests.integration.feature_repos.integration_test_repo_config import ( IntegrationTestRepoConfig, @@ -24,8 +27,8 @@ "password": os.getenv("SNOWFLAKE_CI_PASSWORD", ""), "role": os.getenv("SNOWFLAKE_CI_ROLE", ""), "warehouse": os.getenv("SNOWFLAKE_CI_WAREHOUSE", ""), - "database": "FEAST", - "schema": "MATERIALIZATION", + "database": os.getenv("SNOWFLAKE_CI_DATABASE", "FEAST"), + "schema": os.getenv("SNOWFLAKE_CI_SCHEMA_MATERIALIZATION", "MATERIALIZATION"), } SNOWFLAKE_ONLINE_CONFIG = { @@ -35,15 +38,16 @@ "password": os.getenv("SNOWFLAKE_CI_PASSWORD", ""), "role": os.getenv("SNOWFLAKE_CI_ROLE", ""), "warehouse": os.getenv("SNOWFLAKE_CI_WAREHOUSE", ""), - "database": "FEAST", - "schema": "ONLINE", + "database": os.getenv("SNOWFLAKE_CI_DATABASE", "FEAST"), + "schema": os.getenv("SNOWFLAKE_CI_SCHEMA_ONLINE", "ONLINE"), } +@pytest.mark.parametrize("online_store", [SNOWFLAKE_ONLINE_CONFIG, "sqlite"]) @pytest.mark.integration -def test_snowflake_materialization_consistency_internal(): +def test_snowflake_materialization_consistency(online_store): snowflake_config = IntegrationTestRepoConfig( - online_store=SNOWFLAKE_ONLINE_CONFIG, + online_store=online_store, offline_store_creator=SnowflakeDataSourceCreator, batch_engine=SNOWFLAKE_ENGINE_CONFIG, ) @@ -84,15 +88,32 @@ def test_snowflake_materialization_consistency_internal(): snowflake_environment.data_source_creator.teardown() +@pytest.mark.parametrize( + "feature_dtype, feast_dtype", + [ + ("string", Array(String)), + ("bytes", Array(Bytes)), + ("int32", Array(Int32)), + ("int64", Array(Int64)), + ("float", Array(Float64)), + ("bool", Array(Bool)), + ("datetime", Array(UnixTimestamp)), + ], +) +@pytest.mark.parametrize("feature_is_empty_list", [False]) +@pytest.mark.parametrize("online_store", [SNOWFLAKE_ONLINE_CONFIG, "sqlite"]) @pytest.mark.integration -def test_snowflake_materialization_consistency_external(): +def test_snowflake_materialization_consistency_internal_with_lists( + feature_dtype, feast_dtype, feature_is_empty_list, online_store +): snowflake_config = IntegrationTestRepoConfig( + online_store=online_store, offline_store_creator=SnowflakeDataSourceCreator, batch_engine=SNOWFLAKE_ENGINE_CONFIG, ) snowflake_environment = construct_test_environment(snowflake_config, None) - df = create_basic_driver_dataset() + df = create_basic_driver_dataset(Int32, feature_dtype, True, feature_is_empty_list) ds = snowflake_environment.data_source_creator.create_data_source( df, snowflake_environment.feature_store.project, @@ -105,23 +126,124 @@ def test_snowflake_materialization_consistency_external(): join_keys=["driver_id"], ) + schema = [ + Field(name="driver_id", dtype=Int32), + Field(name="value", dtype=feast_dtype), + ] driver_stats_fv = FeatureView( name="driver_hourly_stats", entities=[driver], ttl=timedelta(weeks=52), + schema=schema, source=ds, ) try: fs.apply([driver, driver_stats_fv]) + split_dt = df["ts_1"][4].to_pydatetime() - timedelta(seconds=1) + + print(f"Split datetime: {split_dt}") + now = datetime.utcnow() + + full_feature_names = True + start_date = (now - timedelta(hours=5)).replace(tzinfo=utc) + end_date = split_dt + fs.materialize( + feature_views=[driver_stats_fv.name], + start_date=start_date, + end_date=end_date, + ) + + expected_values = { + "int32": [3] * 2, + "int64": [3] * 2, + "float": [3.0] * 2, + "string": ["3"] * 2, + "bytes": [b"3"] * 2, + "bool": [False] * 2, + "datetime": [datetime(1981, 1, 1, tzinfo=utc)] * 2, + } + expected_value = [] if feature_is_empty_list else expected_values[feature_dtype] + + response_dict = fs.get_online_features( + [f"{driver_stats_fv.name}:value"], + [{"driver_id": 1}], + full_feature_names=full_feature_names, + ).to_dict() + + actual_value = response_dict[f"{driver_stats_fv.name}__value"][0] + assert actual_value is not None, f"Response: {response_dict}" + if feature_dtype == "float": + for actual_num, expected_num in zip(actual_value, expected_value): + assert ( + abs(actual_num - expected_num) < 1e-6 + ), f"Response: {response_dict}, Expected: {expected_value}" + else: + assert actual_value == expected_value + + finally: + fs.teardown() + snowflake_environment.data_source_creator.teardown() + + +@pytest.mark.integration +def test_snowflake_materialization_entityless_fv(): + snowflake_config = IntegrationTestRepoConfig( + online_store=SNOWFLAKE_ONLINE_CONFIG, + offline_store_creator=SnowflakeDataSourceCreator, + batch_engine=SNOWFLAKE_ENGINE_CONFIG, + ) + snowflake_environment = construct_test_environment(snowflake_config, None) + + df = create_basic_driver_dataset() + entityless_df = df.drop("driver_id", axis=1) + ds = snowflake_environment.data_source_creator.create_data_source( + entityless_df, + snowflake_environment.feature_store.project, + field_mapping={"ts_1": "ts"}, + ) + + fs = snowflake_environment.feature_store + + # We include the driver entity so we can provide an entity ID when fetching features + driver = Entity( + name="driver_id", + join_keys=["driver_id"], + ) + + overall_stats_fv = FeatureView( + name="overall_hourly_stats", + entities=[], + ttl=timedelta(weeks=52), + source=ds, + ) + + try: + fs.apply([overall_stats_fv, driver]) + # materialization is run in two steps and # we use timestamp from generated dataframe as a split point split_dt = df["ts_1"][4].to_pydatetime() - timedelta(seconds=1) print(f"Split datetime: {split_dt}") - validate_offline_online_store_consistency(fs, driver_stats_fv, split_dt) + now = datetime.utcnow() + + start_date = (now - timedelta(hours=5)).replace(tzinfo=utc) + end_date = split_dt + fs.materialize( + feature_views=[overall_stats_fv.name], + start_date=start_date, + end_date=end_date, + ) + + response_dict = fs.get_online_features( + [f"{overall_stats_fv.name}:value"], + [{"driver_id": 1}], # Included because we need an entity + ).to_dict() + assert response_dict["value"] == [0.3] + finally: fs.teardown() snowflake_environment.data_source_creator.teardown() diff --git a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py index 0abb290563a..9baba2397b0 100644 --- a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py +++ b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py @@ -340,6 +340,11 @@ def test_historical_features_with_entities_from_query( table_from_sql_entities = job_from_sql.to_arrow().to_pandas() for col in table_from_sql_entities.columns: + # check if col dtype is timezone naive + if pd.api.types.is_datetime64_dtype(table_from_sql_entities[col]): + table_from_sql_entities[col] = table_from_sql_entities[col].dt.tz_localize( + "UTC" + ) expected_df_query[col] = expected_df_query[col].astype( table_from_sql_entities[col].dtype ) diff --git a/sdk/python/tests/unit/cli/test_cli_chdir.py b/sdk/python/tests/unit/cli/test_cli_chdir.py index cf1d0312272..12ca8f6b084 100644 --- a/sdk/python/tests/unit/cli/test_cli_chdir.py +++ b/sdk/python/tests/unit/cli/test_cli_chdir.py @@ -15,7 +15,7 @@ def test_cli_chdir() -> None: # Make sure the path is absolute by resolving any symlinks temp_path = Path(temp_dir).resolve() result = runner.run(["init", "my_project"], cwd=temp_path) - repo_path = temp_path / "my_project" / "feature_repo" + repo_path = str(temp_path / "my_project" / "feature_repo") assert result.returncode == 0 result = runner.run(["--chdir", repo_path, "apply"], cwd=temp_path) @@ -44,7 +44,12 @@ def test_cli_chdir() -> None: assert result.returncode == 0 result = runner.run( - ["--chdir", repo_path, "materialize-incremental", end_date.isoformat()], + [ + "--chdir", + repo_path, + "materialize-incremental", + end_date.isoformat(), + ], cwd=temp_path, ) assert result.returncode == 0 diff --git a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py index ef0cce04707..f93237fce5e 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py @@ -39,6 +39,9 @@ class MockRetrievalJob(RetrievalJob): + def to_sql(self) -> str: + return "" + def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: """ Synchronously executes the underlying query and returns the result as a pandas dataframe. @@ -46,7 +49,7 @@ def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: Does not handle on demand transformations or dataset validation. For either of those, `to_df` should be used. """ - pass + return pd.DataFrame() def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: """ @@ -55,17 +58,17 @@ def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: Does not handle on demand transformations or dataset validation. For either of those, `to_arrow` should be used. """ - pass + return pyarrow.Table() @property - def full_feature_names(self) -> bool: + def full_feature_names(self) -> bool: # type: ignore """Returns True if full feature names should be applied to the results of the query.""" - pass + return False @property - def on_demand_feature_views(self) -> List[OnDemandFeatureView]: + def on_demand_feature_views(self) -> List[OnDemandFeatureView]: # type: ignore """Returns a list containing all the on demand feature views to be handled.""" - pass + return [] def persist( self, @@ -87,7 +90,7 @@ def persist( @property def metadata(self) -> Optional[RetrievalMetadata]: """Returns metadata about the retrieval job.""" - pass + raise NotImplementedError # Since RetreivalJob are not really tested for subclasses we add some tests here. @@ -115,6 +118,7 @@ def retrieval_job(request, environment): database="feast", s3_staging_location="s3://feast-integration-tests/redshift/tests/ingestion", iam_role="arn:aws:iam::402087665549:role/redshift_s3_access_role", + workgroup="", ) environment.test_repo_config.offline_store = offline_store_config return RedshiftRetrievalJob( @@ -208,7 +212,7 @@ def retrieval_job(request, environment): def test_to_sql(): - assert MockRetrievalJob().to_sql() is None + assert MockRetrievalJob().to_sql() == "" @pytest.mark.parametrize("timeout", (None, 30)) diff --git a/sdk/python/tests/unit/infra/offline_stores/test_redshift.py b/sdk/python/tests/unit/infra/offline_stores/test_redshift.py index 049977489b9..48ee99e89ff 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_redshift.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_redshift.py @@ -31,6 +31,7 @@ def test_offline_write_batch( user="user", iam_role="abcdef", s3_staging_location="s3://bucket/path", + workgroup="", ), ) diff --git a/sdk/python/tests/unit/infra/offline_stores/test_snowflake.py b/sdk/python/tests/unit/infra/offline_stores/test_snowflake.py new file mode 100644 index 00000000000..ac55f123bbb --- /dev/null +++ b/sdk/python/tests/unit/infra/offline_stores/test_snowflake.py @@ -0,0 +1,83 @@ +import re +from unittest.mock import ANY, MagicMock, patch + +import pandas as pd +import pytest +from pytest_mock import MockFixture + +from feast import FeatureView, Field, FileSource +from feast.infra.offline_stores.snowflake import ( + SnowflakeOfflineStoreConfig, + SnowflakeRetrievalJob, +) +from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig +from feast.repo_config import RepoConfig +from feast.types import Array, String + + +@pytest.fixture(params=["s3", "s3gov"]) +def retrieval_job(request): + offline_store_config = SnowflakeOfflineStoreConfig( + type="snowflake.offline", + account="snow", + user="snow", + password="snow", + role="snow", + warehouse="snow", + database="FEAST", + schema="OFFLINE", + storage_integration_name="FEAST_S3", + blob_export_location=f"{request.param}://feast-snowflake-offload/export", + ) + retrieval_job = SnowflakeRetrievalJob( + query="SELECT * FROM snowflake", + snowflake_conn=MagicMock(), + config=RepoConfig( + registry="s3://ml-test/repo/registry.db", + project="test", + provider="snowflake.offline", + online_store=SqliteOnlineStoreConfig(type="sqlite"), + offline_store=offline_store_config, + ), + full_feature_names=True, + on_demand_feature_views=[], + ) + return retrieval_job + + +def test_to_remote_storage(retrieval_job): + stored_files = ["just a path", "maybe another"] + with patch.object( + retrieval_job, "to_snowflake", return_value=None + ) as mock_to_snowflake, patch.object( + retrieval_job, "_get_file_names_from_copy_into", return_value=stored_files + ) as mock_get_file_names_from_copy: + assert ( + retrieval_job.to_remote_storage() == stored_files + ), "should return the list of files" + mock_to_snowflake.assert_called_once() + mock_get_file_names_from_copy.assert_called_once_with(ANY, ANY) + native_path = mock_get_file_names_from_copy.call_args[0][1] + assert re.match("^s3://.*", native_path), "path should be s3://*" + + +def test_snowflake_to_df_internal( + retrieval_job: SnowflakeRetrievalJob, mocker: MockFixture +): + mock_execute = mocker.patch( + "feast.infra.offline_stores.snowflake.execute_snowflake_statement" + ) + mock_execute.return_value.fetch_pandas_all.return_value = pd.DataFrame.from_dict( + {"feature1": ['["1", "2", "3"]', None, "[]"]} # For Valid, Null, and Empty + ) + + feature_view = FeatureView( + name="my-feature-view", + entities=[], + schema=[ + Field(name="feature1", dtype=Array(String)), + ], + source=FileSource(path="dummy.path"), # Dummy value + ) + retrieval_job._feature_views = [feature_view] + retrieval_job._to_df_internal() diff --git a/sdk/python/tests/unit/infra/registry/test_remote.py b/sdk/python/tests/unit/infra/registry/test_remote.py new file mode 100644 index 00000000000..16c6f0abfb0 --- /dev/null +++ b/sdk/python/tests/unit/infra/registry/test_remote.py @@ -0,0 +1,69 @@ +import assertpy +import grpc_testing +import pytest + +from feast import Entity, FeatureStore +from feast.infra.registry.remote import RemoteRegistry, RemoteRegistryConfig +from feast.protos.feast.registry import RegistryServer_pb2, RegistryServer_pb2_grpc +from feast.registry_server import RegistryServer + + +class GrpcMockChannel: + def __init__(self, service, servicer): + self.service = service + self.test_server = grpc_testing.server_from_dictionary( + {service: servicer}, + grpc_testing.strict_real_time(), + ) + + def unary_unary( + self, method: str, request_serializer=None, response_deserializer=None + ): + method_name = method.split("/")[-1] + method_descriptor = self.service.methods_by_name[method_name] + + def handler(request): + rpc = self.test_server.invoke_unary_unary( + method_descriptor, (), request, None + ) + + response, trailing_metadata, code, details = rpc.termination() + return response + + return handler + + +@pytest.fixture +def mock_remote_registry(environment): + store: FeatureStore = environment.feature_store + registry = RemoteRegistry( + registry_config=RemoteRegistryConfig(path=""), project=None, repo_path=None + ) + mock_channel = GrpcMockChannel( + RegistryServer_pb2.DESCRIPTOR.services_by_name["RegistryServer"], + RegistryServer(store=store), + ) + registry.stub = RegistryServer_pb2_grpc.RegistryServerStub(mock_channel) + return registry + + +def test_registry_server_get_entity(environment, mock_remote_registry): + store: FeatureStore = environment.feature_store + entity = Entity(name="driver", join_keys=["driver_id"]) + store.apply(entity) + + expected = store.get_entity(entity.name) + response_entity = mock_remote_registry.get_entity(entity.name, store.project) + + assertpy.assert_that(response_entity).is_equal_to(expected) + + +def test_registry_server_proto(environment, mock_remote_registry): + store: FeatureStore = environment.feature_store + entity = Entity(name="driver", join_keys=["driver_id"]) + store.apply(entity) + + expected = store.registry.proto() + response = mock_remote_registry.proto() + + assertpy.assert_that(response).is_equal_to(expected) diff --git a/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py b/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py index 42229f8683f..ca4ed6472b9 100644 --- a/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py +++ b/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py @@ -45,8 +45,7 @@ def test_nullable_online_store_aws(): entity_key_serialization_version: 2 """ ), - expect_error="__root__ -> offline_store -> __root__\n" - " please specify either cluster_id & user if using provisioned clusters, or workgroup if using serverless (type=value_error)", + expect_error="4 validation errors for RepoConfig\nregion\n Field required", ) @@ -154,8 +153,7 @@ def test_extra_field(): path: "online_store.db" """ ), - expect_error="__root__ -> online_store -> that_field_should_not_be_here\n" - " extra fields not permitted (type=value_error.extra)", + expect_error="1 validation error for RepoConfig\nthat_field_should_not_be_here\n Extra inputs are not permitted", ) @@ -186,7 +184,7 @@ def test_bad_type(): path: 100500 """ ), - expect_error="__root__ -> online_store -> path\n str type expected", + expect_error="1 validation error for RepoConfig\npath\n Input should be a valid string", ) @@ -201,9 +199,7 @@ def test_no_project(): entity_key_serialization_version: 2 """ ), - expect_error="1 validation error for RepoConfig\n" - "project\n" - " field required (type=value_error.missing)", + expect_error="1 validation error for RepoConfig\nproject\n Field required", ) diff --git a/sdk/python/tests/unit/test_feature_views.py b/sdk/python/tests/unit/test_feature_views.py index afef332d372..2ad9680703f 100644 --- a/sdk/python/tests/unit/test_feature_views.py +++ b/sdk/python/tests/unit/test_feature_views.py @@ -1,6 +1,8 @@ +import copy from datetime import timedelta import pytest +from typeguard import TypeCheckError from feast.aggregation import Aggregation from feast.batch_feature_view import BatchFeatureView @@ -278,7 +280,7 @@ def test_hash(): def test_field_types(): - with pytest.raises(TypeError): + with pytest.raises(TypeCheckError): Field(name="name", dtype=ValueType.INT32) @@ -299,3 +301,22 @@ def test_stream_feature_view_proto_type(): aggregations=[], ) assert sfv.proto_class is StreamFeatureViewProto + + +def test_stream_feature_view_copy(): + stream_source = KafkaSource( + name="kafka", + timestamp_field="event_timestamp", + kafka_bootstrap_servers="", + message_format=AvroFormat(""), + topic="topic", + batch_source=FileSource(path="some path"), + ) + sfv = StreamFeatureView( + name="test stream featureview proto class", + entities=[], + ttl=timedelta(days=30), + source=stream_source, + aggregations=[], + ) + assert sfv == copy.copy(sfv) diff --git a/sdk/python/tests/unit/test_on_demand_feature_view.py b/sdk/python/tests/unit/test_on_demand_feature_view.py index ca8e7b25cb8..721026ea462 100644 --- a/sdk/python/tests/unit/test_on_demand_feature_view.py +++ b/sdk/python/tests/unit/test_on_demand_feature_view.py @@ -17,7 +17,10 @@ from feast.feature_view import FeatureView from feast.field import Field from feast.infra.offline_stores.file_source import FileSource -from feast.on_demand_feature_view import OnDemandFeatureView +from feast.on_demand_feature_view import ( + OnDemandFeatureView, + OnDemandPandasTransformation, +) from feast.types import Float32 @@ -54,8 +57,9 @@ def test_hash(): Field(name="output1", dtype=Float32), Field(name="output2", dtype=Float32), ], - udf=udf1, - udf_string="udf1 source code", + transformation=OnDemandPandasTransformation( + udf=udf1, udf_string="udf1 source code" + ), ) on_demand_feature_view_2 = OnDemandFeatureView( name="my-on-demand-feature-view", @@ -64,8 +68,9 @@ def test_hash(): Field(name="output1", dtype=Float32), Field(name="output2", dtype=Float32), ], - udf=udf1, - udf_string="udf1 source code", + transformation=OnDemandPandasTransformation( + udf=udf1, udf_string="udf1 source code" + ), ) on_demand_feature_view_3 = OnDemandFeatureView( name="my-on-demand-feature-view", @@ -74,10 +79,23 @@ def test_hash(): Field(name="output1", dtype=Float32), Field(name="output2", dtype=Float32), ], - udf=udf2, - udf_string="udf2 source code", + transformation=OnDemandPandasTransformation( + udf=udf2, udf_string="udf2 source code" + ), ) on_demand_feature_view_4 = OnDemandFeatureView( + name="my-on-demand-feature-view", + sources=sources, + schema=[ + Field(name="output1", dtype=Float32), + Field(name="output2", dtype=Float32), + ], + transformation=OnDemandPandasTransformation( + udf=udf2, udf_string="udf2 source code" + ), + description="test", + ) + on_demand_feature_view_5 = OnDemandFeatureView( name="my-on-demand-feature-view", sources=sources, schema=[ @@ -105,3 +123,7 @@ def test_hash(): on_demand_feature_view_4, } assert len(s4) == 3 + + assert on_demand_feature_view_5.transformation == OnDemandPandasTransformation( + udf2, "udf2 source code" + ) diff --git a/sdk/python/tests/unit/test_on_demand_substrait_transformation.py b/sdk/python/tests/unit/test_on_demand_substrait_transformation.py new file mode 100644 index 00000000000..c9d30c5b7af --- /dev/null +++ b/sdk/python/tests/unit/test_on_demand_substrait_transformation.py @@ -0,0 +1,112 @@ +import os +import tempfile +from datetime import datetime, timedelta + +import pandas as pd + +from feast import Entity, FeatureStore, FeatureView, FileSource, RepoConfig +from feast.driver_test_data import create_driver_hourly_stats_df +from feast.field import Field +from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig +from feast.on_demand_feature_view import on_demand_feature_view +from feast.types import Float32, Float64, Int64 + + +def test_ibis_pandas_parity(): + with tempfile.TemporaryDirectory() as data_dir: + store = FeatureStore( + config=RepoConfig( + project="test_on_demand_substrait_transformation", + registry=os.path.join(data_dir, "registry.db"), + provider="local", + entity_key_serialization_version=2, + online_store=SqliteOnlineStoreConfig( + path=os.path.join(data_dir, "online.db") + ), + ) + ) + + # Generate test data. + end_date = datetime.now().replace(microsecond=0, second=0, minute=0) + start_date = end_date - timedelta(days=15) + + driver_entities = [1001, 1002, 1003, 1004, 1005] + driver_df = create_driver_hourly_stats_df(driver_entities, start_date, end_date) + driver_stats_path = os.path.join(data_dir, "driver_stats.parquet") + driver_df.to_parquet(path=driver_stats_path, allow_truncated_timestamps=True) + + driver = Entity(name="driver", join_keys=["driver_id"]) + + driver_stats_source = FileSource( + name="driver_hourly_stats_source", + path=driver_stats_path, + timestamp_field="event_timestamp", + created_timestamp_column="created", + ) + + driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_source, + ) + + @on_demand_feature_view( + sources=[driver_stats_fv], + schema=[Field(name="conv_rate_plus_acc", dtype=Float64)], + ) + def pandas_view(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_acc"] = inputs["conv_rate"] + inputs["acc_rate"] + return df + + from ibis.expr.types import Table + + @on_demand_feature_view( + sources=[driver_stats_fv[["conv_rate", "acc_rate"]]], + schema=[Field(name="conv_rate_plus_acc_substrait", dtype=Float64)], + ) + def substrait_view(inputs: Table) -> Table: + return inputs.select( + (inputs["conv_rate"] + inputs["acc_rate"]).name( + "conv_rate_plus_acc_substrait" + ) + ) + + store.apply( + [driver, driver_stats_source, driver_stats_fv, substrait_view, pandas_view] + ) + + entity_df = pd.DataFrame.from_dict( + { + # entity's join key -> entity values + "driver_id": [1001, 1002, 1003], + # "event_timestamp" (reserved key) -> timestamps + "event_timestamp": [ + datetime(2021, 4, 12, 10, 59, 42), + datetime(2021, 4, 12, 8, 12, 10), + datetime(2021, 4, 12, 16, 40, 26), + ], + } + ) + + training_df = store.get_historical_features( + entity_df=entity_df, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + "substrait_view:conv_rate_plus_acc_substrait", + "pandas_view:conv_rate_plus_acc", + ], + ).to_df() + + assert training_df["conv_rate_plus_acc"].equals( + training_df["conv_rate_plus_acc_substrait"] + ) diff --git a/sdk/python/tests/unit/test_registry_server.py b/sdk/python/tests/unit/test_registry_server.py new file mode 100644 index 00000000000..734bbfe19b8 --- /dev/null +++ b/sdk/python/tests/unit/test_registry_server.py @@ -0,0 +1,60 @@ +import assertpy +import grpc_testing +import pytest +from google.protobuf.empty_pb2 import Empty + +from feast import Entity, FeatureStore +from feast.protos.feast.registry import RegistryServer_pb2 +from feast.registry_server import RegistryServer + + +def call_registry_server(server, method: str, request=None): + service = RegistryServer_pb2.DESCRIPTOR.services_by_name["RegistryServer"] + rpc = server.invoke_unary_unary( + service.methods_by_name[method], (), request if request else Empty(), None + ) + + return rpc.termination() + + +@pytest.fixture +def registry_server(environment): + store: FeatureStore = environment.feature_store + + servicer = RegistryServer(store=store) + + return grpc_testing.server_from_dictionary( + {RegistryServer_pb2.DESCRIPTOR.services_by_name["RegistryServer"]: servicer}, + grpc_testing.strict_real_time(), + ) + + +def test_registry_server_get_entity(environment, registry_server): + store: FeatureStore = environment.feature_store + entity = Entity(name="driver", join_keys=["driver_id"]) + store.apply(entity) + + expected = store.get_entity(entity.name) + + get_entity_request = RegistryServer_pb2.GetEntityRequest( + name=entity.name, project=store.project, allow_cache=False + ) + response, trailing_metadata, code, details = call_registry_server( + registry_server, "GetEntity", get_entity_request + ) + response_entity = Entity.from_proto(response) + + assertpy.assert_that(response_entity).is_equal_to(expected) + + +def test_registry_server_proto(environment, registry_server): + store: FeatureStore = environment.feature_store + entity = Entity(name="driver", join_keys=["driver_id"]) + store.apply(entity) + + expected = store.registry.proto() + response, trailing_metadata, code, details = call_registry_server( + registry_server, "Proto" + ) + + assertpy.assert_that(response).is_equal_to(expected) diff --git a/sdk/python/tests/unit/test_sql_registry.py b/sdk/python/tests/unit/test_sql_registry.py index 39896d3a9dd..4ca41423c18 100644 --- a/sdk/python/tests/unit/test_sql_registry.py +++ b/sdk/python/tests/unit/test_sql_registry.py @@ -66,10 +66,12 @@ def pg_registry(): ) logger.info("Waited for %s seconds until postgres container was up", waited) container_port = container.get_exposed_port(5432) + container_host = container.get_container_host_ip() registry_config = RegistryConfig( registry_type="sql", - path=f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@127.0.0.1:{container_port}/{POSTGRES_DB}", + path=f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{container_host}:{container_port}/{POSTGRES_DB}", + sqlalchemy_config_kwargs={"echo": False, "pool_pre_ping": True}, ) yield SqlRegistry(registry_config, "project", None) @@ -100,10 +102,12 @@ def mysql_registry(): ) logger.info("Waited for %s seconds until mysql container was up", waited) container_port = container.get_exposed_port(3306) + container_host = container.get_container_host_ip() registry_config = RegistryConfig( registry_type="sql", - path=f"mysql+mysqldb://{POSTGRES_USER}:{POSTGRES_PASSWORD}@127.0.0.1:{container_port}/{POSTGRES_DB}", + path=f"mysql+pymysql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{container_host}:{container_port}/{POSTGRES_DB}", + sqlalchemy_config_kwargs={"echo": False, "pool_pre_ping": True}, ) yield SqlRegistry(registry_config, "project", None) diff --git a/sdk/python/tests/utils/e2e_test_validation.py b/sdk/python/tests/utils/e2e_test_validation.py index bacc8c17206..d8c769f12c9 100644 --- a/sdk/python/tests/utils/e2e_test_validation.py +++ b/sdk/python/tests/utils/e2e_test_validation.py @@ -193,7 +193,7 @@ def make_feature_store_yaml( repo_path=str(Path(repo_dir_name)), entity_key_serialization_version=2, ) - config_dict = config.dict() + config_dict = config.model_dump(by_alias=True) if ( isinstance(config_dict["online_store"], dict) and "redis_type" in config_dict["online_store"] diff --git a/setup.py b/setup.py index 33bf76e1819..b601c901462 100644 --- a/setup.py +++ b/setup.py @@ -44,24 +44,17 @@ "click>=7.0.0,<9.0.0", "colorama>=0.3.9,<1", "dill~=0.3.0", - "fastavro>=1.1.0,<2", - "grpcio>=1.56.2,<2", - "grpcio-tools>=1.56.2,<2", - "grpcio-reflection>=1.56.2,<2", - "grpcio-health-checking>=1.56.2,<2", "mypy-protobuf==3.1", "Jinja2>=2,<4", "jsonschema", "mmh3", "numpy>=1.22,<1.25", - "pandas>=1.4.3,<2", - # For some reason pandavro higher than 1.5.* only support pandas less than 1.3. - "pandavro~=1.5.0", + "pandas>=1.4.3,<3", # Higher than 4.23.4 seems to cause a seg fault "protobuf<4.23.4,>3.20", "proto-plus>=1.20.0,<2", "pyarrow>=4", - "pydantic>=1,<2", + "pydantic>=2.0.0", "pygments>=2.12.0,<3", "PyYAML>=5.4.0,<7", "requests", @@ -70,8 +63,8 @@ "tenacity>=7,<9", "toml>=0.10.0,<1", "tqdm>=4,<5", - "typeguard==2.13.3", - "fastapi>=0.68.0,<0.100", + "typeguard>=4.0.0", + "fastapi>=0.68.0", "uvicorn[standard]>=0.14.0,<1", "gunicorn", "dask>=2021.1.0", @@ -90,7 +83,7 @@ "google-cloud-datastore>=2.1.0,<3", "google-cloud-storage>=1.34.0,<3", "google-cloud-bigtable>=2.11.0,<3", - "fsspec<2023.10.0", + "fsspec<=2024.1.0", ] REDIS_REQUIRED = [ @@ -98,7 +91,7 @@ "hiredis>=2.0.0,<3", ] -AWS_REQUIRED = ["boto3>=1.17.0,<2", "docker>=5.0.2", "fsspec<2023.10.0"] +AWS_REQUIRED = ["boto3>=1.17.0,<2", "docker>=5.0.2", "fsspec<=2024.1.0"] BYTEWAX_REQUIRED = ["bytewax==0.15.1", "docker>=5.0.2", "kubernetes<=20.13.0"] @@ -116,7 +109,7 @@ "psycopg2-binary>=2.8.3,<3", ] -MYSQL_REQUIRED = ["mysqlclient", "pymysql", "types-PyMySQL"] +MYSQL_REQUIRED = ["pymysql", "types-PyMySQL"] HBASE_REQUIRED = [ "happybase>=1.2.0,<3", @@ -126,7 +119,7 @@ "cassandra-driver>=3.24.0,<4", ] -GE_REQUIRED = ["great_expectations>=0.15.41,<0.16.0"] +GE_REQUIRED = ["great_expectations>=0.15.41"] AZURE_REQUIRED = [ "azure-storage-blob>=0.37.0", @@ -144,21 +137,36 @@ "hazelcast-python-client>=5.1", ] +IBIS_REQUIRED = [ + "ibis-framework", + "ibis-substrait", +] + +GRPCIO_REQUIRED = [ + "grpcio>=1.56.2,<2", + "grpcio-tools>=1.56.2,<2", + "grpcio-reflection>=1.56.2,<2", + "grpcio-health-checking>=1.56.2,<2", +] + +DUCKDB_REQUIRED = [ + "ibis-framework[duckdb]" +] + CI_REQUIRED = ( [ "build", "virtualenv==20.23.0", - "cryptography>=35.0,<42", + "cryptography>=35.0,<43", "flake8>=6.0.0,<6.1.0", "black>=22.6.0,<23", "isort>=5,<6", "grpcio-testing>=1.56.2,<2", "minio==7.1.0", "mock==2.0.0", - "moto", - "mypy>=0.981,<0.990", + "moto<5", + "mypy>=1.4.1", "avro==1.10.0", - "fsspec<2023.10.0", "urllib3>=1.25.4,<3", "psutil==5.9.0", "py>=1.11.0", # https://github.com/pytest-dev/pytest/issues/10420 @@ -186,6 +194,7 @@ "types-setuptools", "types-tabulate", "virtualenv<20.24.2", + "pandas>=1.4.3,<2; python_version < '3.9'", ] + GCP_REQUIRED + REDIS_REQUIRED @@ -202,6 +211,8 @@ + AZURE_REQUIRED + ROCKSET_REQUIRED + HAZELCAST_REQUIRED + + IBIS_REQUIRED + + GRPCIO_REQUIRED ) @@ -234,7 +245,7 @@ else: use_scm_version = None -PROTO_SUBDIRS = ["core", "serving", "types", "storage"] +PROTO_SUBDIRS = ["core", "registry", "serving", "types", "storage"] PYTHON_CODE_PREFIX = "sdk/python" @@ -368,7 +379,10 @@ def run(self): "docs": DOCS_REQUIRED, "cassandra": CASSANDRA_REQUIRED, "hazelcast": HAZELCAST_REQUIRED, + "grpcio": GRPCIO_REQUIRED, "rockset": ROCKSET_REQUIRED, + "ibis": IBIS_REQUIRED, + "duckdb": DUCKDB_REQUIRED }, include_package_data=True, license="Apache", diff --git a/ui/README.md b/ui/README.md index e91a8741ec5..12aacd329ef 100644 --- a/ui/README.md +++ b/ui/README.md @@ -46,7 +46,7 @@ ReactDOM.render( ); ``` -When you start the React app, it will look for `project-list.json` to find a list of your projects. The JSON should looks something like this. +When you start the React app, it will look for `projects-list.json` to find a list of your projects. The JSON should looks something like this. ```json { @@ -61,6 +61,8 @@ When you start the React app, it will look for `project-list.json` to find a lis } ``` +* **Note** - `registryPath` only supports a file location or a url. + ``` // Start the React App yarn start diff --git a/ui/yarn.lock b/ui/yarn.lock index becb6bbd7ba..5c9e5c17ace 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -6461,9 +6461,9 @@ invariant@^2.2.4: loose-envify "^1.0.0" ip@^1.1.0: - version "1.1.5" - resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a" - integrity sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo= + version "1.1.9" + resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.9.tgz#8dfbcc99a754d07f425310b86a99546b1151e396" + integrity sha512-cyRxvOEpNHNtchU3Ln9KC/auJgup87llfQpQ+t5ghoC/UhL16SWzbueiCsdTnWmqAWl7LadfuwhlqmtOaqMHdQ== ipaddr.js@1.9.1: version "1.9.1"