diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 841f5da87b3..aef486d601a 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -151,6 +151,27 @@ jobs: name: wheels path: dist/* + # We add this step so the docker images can be built as part of the pre-release verification steps. + build-docker-images: + runs-on: ubuntu-latest + needs: get-version + strategy: + matrix: + component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server] + env: + REGISTRY: feastdev + steps: + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Build image + run: | + make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + env: + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + verify-python-wheels: runs-on: ${{ matrix.os }} needs: [build-python-wheel, build-source-distribution, get-version] diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9587044a84d..039ab6a24e7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -46,7 +46,7 @@ jobs: build-publish-docker-images: runs-on: ubuntu-latest - needs: get-version + needs: [get-version, publish-python-sdk] strategy: matrix: component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ec56d60e4c5..0c9ac1e752c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -92,6 +92,9 @@ jobs: uses: actions/setup-node@v2 with: node-version: '16' + - name: Set up Homebrew + id: set-up-homebrew + uses: Homebrew/actions/setup-homebrew@master - name: Setup Helm-docs run: | brew install norwoodj/tap/helm-docs diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index de6d98d1400..ebc09f0080e 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -108,3 +108,23 @@ jobs: sudo apt install -y -V libarrow-dev - name: Test run: make test-go + + unit-test-ui: + runs-on: ubuntu-latest + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '17.x' + registry-url: 'https://registry.npmjs.org' + - name: Install yarn dependencies + working-directory: ./ui + run: yarn install + - name: Build yarn rollup + working-directory: ./ui + run: yarn build:lib + - name: Run yarn tests + working-directory: ./ui + run: yarn test --watchAll=false diff --git a/.releaserc.js b/.releaserc.js index 124ad8801c2..114d65d1a2a 100644 --- a/.releaserc.js +++ b/.releaserc.js @@ -67,6 +67,7 @@ module.exports = { "java/pom.xml", "infra/charts/**/*.*", "ui/package.json", + "sdk/python/feast/ui/package.json", "sdk/python/feast/ui/yarn.lock" ], message: "chore(release): release ${nextRelease.version}\n\n${nextRelease.notes}" diff --git a/CHANGELOG.md b/CHANGELOG.md index b657e9ddd1a..65aa693691e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,37 @@ # Changelog +# [0.26.0](https://github.com/feast-dev/feast/compare/v0.25.0...v0.26.0) (2022-10-06) + + +### Bug Fixes + +* Add `X-Trino-Extra-Credential` header and remove user override ([#3246](https://github.com/feast-dev/feast/issues/3246)) ([164e666](https://github.com/feast-dev/feast/commit/164e666ee8c425150903819a0035d6f6e48bcdd0)) +* Add postgres to the feature server Dockerfile to fix helm chart flow ([#3261](https://github.com/feast-dev/feast/issues/3261)) ([6f6cbb7](https://github.com/feast-dev/feast/commit/6f6cbb76a216965666719e492c5ba854ada32605)) +* Add stream feature view in the Web UI ([#3257](https://github.com/feast-dev/feast/issues/3257)) ([1f70b3a](https://github.com/feast-dev/feast/commit/1f70b3a9be738ab54f0cb59046ed1e299f92e3a2)) +* Build dockerfile correctly ([#3239](https://github.com/feast-dev/feast/issues/3239)) ([a2dc0d0](https://github.com/feast-dev/feast/commit/a2dc0d0410eb297afddfb1dd4f1f899ab70fa14f)) +* Configuration to stop coercion of tz for entity_df ([#3255](https://github.com/feast-dev/feast/issues/3255)) ([97b7ab9](https://github.com/feast-dev/feast/commit/97b7ab906081d1afe034fc017a2ec61f20818771)) +* Enable users to upgrade a batch source into a push source ([#3213](https://github.com/feast-dev/feast/issues/3213)) ([1b312fb](https://github.com/feast-dev/feast/commit/1b312fbb96a0629c0bd465ee09a01a43130c99c7)) +* Fix docker image for feature-server ([#3272](https://github.com/feast-dev/feast/issues/3272)) ([eff01d1](https://github.com/feast-dev/feast/commit/eff01d1ed3a76ff539595455f69ff1c954ad46cd)) +* Fix Feast UI release process to update the feast-ui package ([#3267](https://github.com/feast-dev/feast/issues/3267)) ([a9d48d0](https://github.com/feast-dev/feast/commit/a9d48d07f634deb0b992e36d2ae1577d9b627940)) +* Return 422 on bad push source name ([#3214](https://github.com/feast-dev/feast/issues/3214)) ([b851e01](https://github.com/feast-dev/feast/commit/b851e01ab4ae78e6714793c7efddb86b39d2a36e)) +* Stream feature view meta undefined created_timestamp issue ([#3266](https://github.com/feast-dev/feast/issues/3266)) ([12e1a8f](https://github.com/feast-dev/feast/commit/12e1a8f0e9f098cfc3ed66db6665cb5fa26240f4)) +* Stream feature view not shown in the UI ([#3251](https://github.com/feast-dev/feast/issues/3251)) ([e713dda](https://github.com/feast-dev/feast/commit/e713dda7c5a4433f5d13b4685d36978645e0cd02)) +* Udf in stream feature view UI shows pickled data ([#3268](https://github.com/feast-dev/feast/issues/3268)) ([0728117](https://github.com/feast-dev/feast/commit/07281178f8be78178b739366c90fd2074db0981b)) +* Update snowflake materialization messages ([#3230](https://github.com/feast-dev/feast/issues/3230)) ([a63d440](https://github.com/feast-dev/feast/commit/a63d440e4207c1e360630423bcda2c329673fddd)) +* Updated quickstart notebook to patch an incorrect reference to an outdated featureview name ([#3271](https://github.com/feast-dev/feast/issues/3271)) ([b9b9c54](https://github.com/feast-dev/feast/commit/b9b9c542bc0860cfb351be7d84d6461b70ad2c81)) +* Use configured user in env var instead of "user" for Trino ([#3254](https://github.com/feast-dev/feast/issues/3254)) ([532d8a1](https://github.com/feast-dev/feast/commit/532d8a1ec5dd50a959b512427fdbc3e5bb986eb4)) + + +### Features + +* Add mysql as online store ([#3190](https://github.com/feast-dev/feast/issues/3190)) ([cb8db84](https://github.com/feast-dev/feast/commit/cb8db84a1b200c7a716d23e17f150b013d5689ff)) +* Add possibility to define feature_store.yaml path with env variable ([#3231](https://github.com/feast-dev/feast/issues/3231)) ([95fdb19](https://github.com/feast-dev/feast/commit/95fdb19ed3f4b4af7fe1fb732a1fd63182cf1072)) +* Add request_timeout setting for cassandra online store adapter ([#3256](https://github.com/feast-dev/feast/issues/3256)) ([da20757](https://github.com/feast-dev/feast/commit/da20757aa1d5c5f898651295781ef61b52d7d712)) +* Add tag description in Field in the Feast UI ([#3258](https://github.com/feast-dev/feast/issues/3258)) ([086f279](https://github.com/feast-dev/feast/commit/086f2790236b776f5c2397365266d9d4f4959142)) +* Adding billing_project_id in BigQueryOfflineStoreConfig ([#3253](https://github.com/feast-dev/feast/issues/3253)) ([f80f05f](https://github.com/feast-dev/feast/commit/f80f05f5d2042c6fff533b0dc85612acda5a7417)) +* BigTable online store ([#3140](https://github.com/feast-dev/feast/issues/3140)) ([6bc91c2](https://github.com/feast-dev/feast/commit/6bc91c23545a6cf765c0e70887afb22eada2c8e1)), closes [/github.com/spulec/moto/blob/master/CHANGELOG.md#400](https://github.com//github.com/spulec/moto/blob/master/CHANGELOG.md/issues/400) +* Filter subset features in postgres [#3174](https://github.com/feast-dev/feast/issues/3174) ([#3203](https://github.com/feast-dev/feast/issues/3203)) ([b48d36b](https://github.com/feast-dev/feast/commit/b48d36bee074788626ed5c2d5cf130bd2fc8bda5)) + # [0.25.0](https://github.com/feast-dev/feast/compare/v0.24.0...v0.25.0) (2022-09-20) diff --git a/Makefile b/Makefile index 0bf0c82669f..718df6af4ab 100644 --- a/Makefile +++ b/Makefile @@ -63,11 +63,9 @@ benchmark-python-local: FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests test-python: - @(docker info > /dev/null 2>&1 && \ - FEAST_USAGE=False \ - IS_TEST=True \ - python -m pytest -n 8 sdk/python/tests \ - ) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!"; + FEAST_USAGE=False \ + IS_TEST=True \ + python -m pytest -n 8 sdk/python/tests \ test-python-integration: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests @@ -231,6 +229,27 @@ test-python-universal-postgres-online: not test_snowflake" \ sdk/python/tests + test-python-universal-mysql-online: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.mysql_repo_configuration \ + PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.mysql \ + FEAST_USAGE=False \ + IS_TEST=True \ + python -m pytest -n 8 --integration \ + -k "not test_universal_cli and \ + not test_go_feature_server and \ + not test_feature_logging and \ + not test_reorder_columns and \ + not test_logged_features_validation and \ + not test_lambda_materialization_consistency and \ + not test_offline_write and \ + not test_push_features_to_offline_store and \ + not gcs_registry and \ + not s3_registry and \ + not test_universal_types and \ + not test_snowflake" \ + sdk/python/tests + test-python-universal-cassandra: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \ @@ -448,4 +467,4 @@ build-helm-docs: # Note: requires node and yarn to be installed build-ui: - cd $(ROOT_DIR)/sdk/python/feast/ui && yarn install && npm run build --omit=dev + cd $(ROOT_DIR)/sdk/python/feast/ui && yarn upgrade @feast-dev/feast-ui --latest && yarn install && npm run build --omit=dev diff --git a/OWNERS b/OWNERS index bc7342c3d61..c34fd6baafe 100644 --- a/OWNERS +++ b/OWNERS @@ -3,16 +3,22 @@ # More info at https://www.kubernetes.dev/docs/guide/owners/ approvers: - woop - - tsotnet - achals - adchia - felixwang9817 - mavysavydav - MattDelac - kevjumba + - chhabrakadabra + - sfc-gh-madkins + - zhilingc + - whoahbot + - niklasvm + - toping4445 + - DvirDukhan + - hemidactylus reviewers: - woop - - tsotnet - achals - tedhtchang - adchia @@ -20,3 +26,12 @@ reviewers: - mavysavydav - MattDelac - kevjumba + - chhabrakadabra + - sfc-gh-madkins + - zhilingc + - whoahbot + - niklasvm + - toping4445 + - DvirDukhan + - hemidactylus + \ No newline at end of file diff --git a/README.md b/README.md index c8adfa5f22c..d57247c60b5 100644 --- a/README.md +++ b/README.md @@ -173,12 +173,12 @@ The list below contains the functionality that contributors are planning to deve * [x] [DynamoDB](https://docs.feast.dev/reference/online-stores/dynamodb) * [x] [Redis](https://docs.feast.dev/reference/online-stores/redis) * [x] [Datastore](https://docs.feast.dev/reference/online-stores/datastore) + * [x] [Bigtable](https://docs.feast.dev/reference/online-stores/bigtable) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) + * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/adding-support-for-a-new-online-store) - * [x] [Cassandra / AstraDB](https://docs.feast.dev/reference/online-stores/cassandra) - * [ ] Bigtable (in progress) * **Feature Engineering** * [x] On-demand Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) * [x] Streaming Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1UzEyETHUaGpn0ap4G82DHluiCj7zEbrQLkJJkKSv4e8/edit)) @@ -193,7 +193,8 @@ The list below contains the functionality that contributors are planning to deve * **Feature Serving** * [x] Python Client * [x] [Python feature server](https://docs.feast.dev/reference/feature-servers/python-feature-server) - * [x] [Go feature server](https://docs.feast.dev/reference/feature-servers/go-feature-server) + * [x] [Java feature server (alpha)](https://github.com/feast-dev/feast/blob/master/infra/charts/feast/README.md) + * [x] [Go feature server (alpha)](https://docs.feast.dev/reference/feature-servers/go-feature-server) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** * [x] Data profiling and validation (Great Expectations) * **Feature Discovery and Governance** diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 430f9a745d7..cb4a2664cd6 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -92,8 +92,10 @@ * [Redis](reference/online-stores/redis.md) * [Datastore](reference/online-stores/datastore.md) * [DynamoDB](reference/online-stores/dynamodb.md) + * [Bigtable](reference/online-stores/bigtable.md) * [PostgreSQL (contrib)](reference/online-stores/postgres.md) * [Cassandra + Astra DB (contrib)](reference/online-stores/cassandra.md) + * [MySQL (contrib)](reference/online-stores/mysql.md) * [Providers](reference/providers/README.md) * [Local](reference/providers/local.md) * [Google Cloud Platform](reference/providers/google-cloud-platform.md) diff --git a/docs/getting-started/concepts/feature-retrieval.md b/docs/getting-started/concepts/feature-retrieval.md index fd216fc71f5..867e17848b0 100644 --- a/docs/getting-started/concepts/feature-retrieval.md +++ b/docs/getting-started/concepts/feature-retrieval.md @@ -180,7 +180,7 @@ Feast accepts either: - [feature services](feature-retrieval.md#feature-services), which group features needed for a model version - [feature references](feature-retrieval.md#feature-references) -### Example: querying a feature service (recommended) +#### Example: querying a feature service (recommended) ```python training_df = store.get_historical_features( entity_df=entity_df, @@ -188,7 +188,7 @@ training_df = store.get_historical_features( ).to_df() ``` -### Example: querying a list of feature references +#### Example: querying a list of feature references ```python training_df = store.get_historical_features( entity_df=entity_df, @@ -204,7 +204,7 @@ Feast accepts either a **Pandas dataframe** as the entity dataframe (including e Both approaches must specify the full **entity key** needed as well as the **timestamps**. Feast then joins features onto this dataframe. -### Example: entity dataframe for generating training data +#### Example: entity dataframe for generating training data ```python entity_df = pd.DataFrame.from_dict( { @@ -228,7 +228,7 @@ training_df = store.get_historical_features( ).to_df() ``` -### Example: entity SQL query for generating training data +#### Example: entity SQL query for generating training data You can also pass a SQL string to generate the above dataframe. This is useful for getting all entities in a timeframe from some data source. ```python diff --git a/docs/getting-started/third-party-integrations.md b/docs/getting-started/third-party-integrations.md index 8e6a600aa05..ca974880ed3 100644 --- a/docs/getting-started/third-party-integrations.md +++ b/docs/getting-started/third-party-integrations.md @@ -11,7 +11,7 @@ Don't see your offline store or online store of choice here? Check out our guide ## Integrations -See [Functionality and Roadmap](../../#-functionality-and-roadmap) +See [Functionality and Roadmap](../roadmap.md) ## Standards diff --git a/docs/project/release-process.md b/docs/project/release-process.md index 2ddc697730e..0aa7d3fb5ba 100644 --- a/docs/project/release-process.md +++ b/docs/project/release-process.md @@ -19,8 +19,9 @@ After this step, you will have all the changes you need in the branch. ### 2. Pre-release verification A lot of things can go wrong. One of the most common is getting the wheels to build correctly (and not accidentally building dev wheels from improper tagging or local code changes during the release process). +Another possible failure is that the Docker images might not build correctly. -We verify the wheels building in **your fork** of Feast, not the main feast-dev/feast repo. +We verify the building the wheels and Docker images in **your fork** of Feast, not the main feast-dev/feast repo. #### For minor releases (e.g. v0.22.0) 1. Merge upstream master changes into your **fork**. Make sure you are running the workflow off of your fork! @@ -30,7 +31,7 @@ We verify the wheels building in **your fork** of Feast, not the main feast-dev/ > This is important. If you don't have a tag, then the wheels you build will be **dev wheels**, which we can't > push. The release process will automatically produce a tag for you via Semantic Release. 3. Access the `Actions` tab on your GitHub UI on your fork and click the `build_wheels` action. This workflow will - build the python sdk wheels for Python 3.8-3.10 on MacOS 10.15 and Linux and verify that these wheels are correct. + build the python sdk wheels for Python 3.8-3.10 on MacOS 10.15 and Linux and verify that these wheels are correct. It will also build the Docker images. The publish workflow uses this action to publish the python wheels for a new release to PyPI. 4. Look for the header `This workflow has a workflow_dispatch event trigger` and click `Run Workflow` on the right. 5. Run the workflow off of the tag you just created(`v0.22.0` in this case, **not** the master branch) and verify that diff --git a/docs/reference/offline-stores/bigquery.md b/docs/reference/offline-stores/bigquery.md index 0e286d78c49..b7607abf595 100644 --- a/docs/reference/offline-stores/bigquery.md +++ b/docs/reference/offline-stores/bigquery.md @@ -7,6 +7,9 @@ The BigQuery offline store provides support for reading [BigQuerySources](../dat * All joins happen within BigQuery. * Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to BigQuery as a table (marked for expiration) in order to complete join operations. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[gcp]'`. You can get started by then running `feast init -t gcp`. + ## Example {% code title="feature_store.yaml" %} diff --git a/docs/reference/offline-stores/mssql.md b/docs/reference/offline-stores/mssql.md index bec0c8deb82..e352b3dd2aa 100644 --- a/docs/reference/offline-stores/mssql.md +++ b/docs/reference/offline-stores/mssql.md @@ -6,6 +6,9 @@ The MsSQL offline store provides support for reading [MsSQL Sources](../data-sou * Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[azure]'`. You can get started by then following this [tutorial](https://github.com/feast-dev/feast/blob/master/docs/tutorials/azure/README.md). + ## Disclaimer The MsSQL offline store does not achieve full test coverage. @@ -34,26 +37,26 @@ offline_store: The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Spark offline store. -| | MsSql | -| :-------------------------------- | :-- | -| `get_historical_features` (point-in-time correct join) | yes | -| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | -| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | -| `offline_write_batch` (persist dataframes to offline store) | no | -| `write_logged_features` (persist logged features to offline store) | no | +| | MsSql | +| :----------------------------------------------------------------- | :---- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | Below is a matrix indicating which functionality is supported by `MsSqlServerRetrievalJob`. -| | MsSql | -| --------------------------------- | --- | -| export to dataframe | yes | -| export to arrow table | yes | -| export to arrow batches | no | -| export to SQL | no | -| export to data lake (S3, GCS, etc.) | no | -| export to data warehouse | no | -| local execution of Python-based on-demand transforms | no | -| remote execution of Python-based on-demand transforms | no | -| persist results in the offline store | yes | +| | MsSql | +| ----------------------------------------------------- | ----- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | no | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | no | +| local execution of Python-based on-demand transforms | no | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/postgres.md b/docs/reference/offline-stores/postgres.md index 506666fc378..094ab4885f4 100644 --- a/docs/reference/offline-stores/postgres.md +++ b/docs/reference/offline-stores/postgres.md @@ -10,6 +10,9 @@ The PostgreSQL offline store provides support for reading [PostgreSQLSources](.. The PostgreSQL offline store does not achieve full test coverage. Please do not assume complete stability. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[postgres]'`. You can get started by then running `feast init -t postgres`. + ## Example {% code title="feature_store.yaml" %} @@ -42,29 +45,29 @@ The full set of configuration options is available in [PostgreSQLOfflineStoreCon The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the PostgreSQL offline store. -| | Postgres | -| :-------------------------------- | :-- | -| `get_historical_features` (point-in-time correct join) | yes | -| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | -| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | -| `offline_write_batch` (persist dataframes to offline store) | no | -| `write_logged_features` (persist logged features to offline store) | no | +| | Postgres | +| :----------------------------------------------------------------- | :------- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | Below is a matrix indicating which functionality is supported by `PostgreSQLRetrievalJob`. -| | Postgres | -| --------------------------------- | --- | -| export to dataframe | yes | -| export to arrow table | yes | -| export to arrow batches | no | -| export to SQL | yes | -| export to data lake (S3, GCS, etc.) | yes | -| export to data warehouse | yes | -| export as Spark dataframe | no | -| local execution of Python-based on-demand transforms | yes | -| remote execution of Python-based on-demand transforms | no | -| persist results in the offline store | yes | -| preview the query plan before execution | yes | -| read partitioned data | yes | +| | Postgres | +| ----------------------------------------------------- | -------- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | yes | +| export to data warehouse | yes | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/redshift.md b/docs/reference/offline-stores/redshift.md index 2cdf49bdb9f..98092c44bec 100644 --- a/docs/reference/offline-stores/redshift.md +++ b/docs/reference/offline-stores/redshift.md @@ -7,6 +7,9 @@ The Redshift offline store provides support for reading [RedshiftSources](../dat * All joins happen within Redshift. * Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Redshift temporarily in order to complete join operations. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[aws]'`. You can get started by then running `feast init -t aws`. + ## Example {% code title="feature_store.yaml" %} @@ -32,30 +35,30 @@ The full set of configuration options is available in [RedshiftOfflineStoreConfi The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Redshift offline store. -| | Redshift | -| :-------------------------------- | :-- | -| `get_historical_features` (point-in-time correct join) | yes | -| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | -| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | -| `offline_write_batch` (persist dataframes to offline store) | yes | -| `write_logged_features` (persist logged features to offline store) | yes | +| | Redshift | +| :----------------------------------------------------------------- | :------- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | yes | +| `write_logged_features` (persist logged features to offline store) | yes | Below is a matrix indicating which functionality is supported by `RedshiftRetrievalJob`. -| | Redshift | -| --------------------------------- | --- | -| export to dataframe | yes | -| export to arrow table | yes | -| export to arrow batches | yes | -| export to SQL | yes | -| export to data lake (S3, GCS, etc.) | no | -| export to data warehouse | yes | -| export as Spark dataframe | no | -| local execution of Python-based on-demand transforms | yes | -| remote execution of Python-based on-demand transforms | no | -| persist results in the offline store | yes | -| preview the query plan before execution | yes | -| read partitioned data | yes | +| | Redshift | +| ----------------------------------------------------- | -------- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | yes | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | yes | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/snowflake.md b/docs/reference/offline-stores/snowflake.md index e40ad7cd7a2..4ac7f164363 100644 --- a/docs/reference/offline-stores/snowflake.md +++ b/docs/reference/offline-stores/snowflake.md @@ -6,6 +6,13 @@ The [Snowflake](https://trial.snowflake.com) offline store provides support for * All joins happen within Snowflake. * Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Snowflake as a temporary table in order to complete join operations. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[snowflake]'`. + +If you're using a file based registry, then you'll also need to install the relevant cloud extra (`pip install 'feast[snowflake, CLOUD]'` where `CLOUD` is one of `aws`, `gcp`, `azure`) + +You can get started by then running `feast init -t snowflake`. + ## Example {% code title="feature_store.yaml" %} @@ -31,29 +38,29 @@ The full set of configuration options is available in [SnowflakeOfflineStoreConf The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Snowflake offline store. -| | Snowflake | -| :-------------------------------- | :-- | -| `get_historical_features` (point-in-time correct join) | yes | -| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | -| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | -| `offline_write_batch` (persist dataframes to offline store) | yes | -| `write_logged_features` (persist logged features to offline store) | yes | +| | Snowflake | +| :----------------------------------------------------------------- | :-------- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | yes | +| `write_logged_features` (persist logged features to offline store) | yes | Below is a matrix indicating which functionality is supported by `SnowflakeRetrievalJob`. -| | Snowflake | -| --------------------------------- | --- | -| export to dataframe | yes | -| export to arrow table | yes | -| export to arrow batches | no | -| export to SQL | yes | -| export to data lake (S3, GCS, etc.) | yes | -| export to data warehouse | yes | -| export as Spark dataframe | no | -| local execution of Python-based on-demand transforms | yes | -| remote execution of Python-based on-demand transforms | no | -| persist results in the offline store | yes | -| preview the query plan before execution | yes | -| read partitioned data | yes | +| | Snowflake | +| ----------------------------------------------------- | --------- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | yes | +| export to data warehouse | yes | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/spark.md b/docs/reference/offline-stores/spark.md index f1ef1300bd7..ae5ea78071e 100644 --- a/docs/reference/offline-stores/spark.md +++ b/docs/reference/offline-stores/spark.md @@ -11,6 +11,9 @@ The Spark offline store provides support for reading [SparkSources](../data-sour The Spark offline store does not achieve full test coverage. Please do not assume complete stability. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[spark]'`. You can get started by then running `feast init -t spark`. + ## Example {% code title="feature_store.yaml" %} @@ -39,29 +42,29 @@ The full set of configuration options is available in [SparkOfflineStoreConfig]( The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Spark offline store. -| | Spark | -| :-------------------------------- | :-- | -| `get_historical_features` (point-in-time correct join) | yes | -| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | -| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | -| `offline_write_batch` (persist dataframes to offline store) | no | -| `write_logged_features` (persist logged features to offline store) | no | +| | Spark | +| :----------------------------------------------------------------- | :---- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | Below is a matrix indicating which functionality is supported by `SparkRetrievalJob`. -| | Spark | -| --------------------------------- | --- | -| export to dataframe | yes | -| export to arrow table | yes | -| export to arrow batches | no | -| export to SQL | no | -| export to data lake (S3, GCS, etc.) | no | -| export to data warehouse | no | -| export as Spark dataframe | yes | -| local execution of Python-based on-demand transforms | no | -| remote execution of Python-based on-demand transforms | no | -| persist results in the offline store | yes | -| preview the query plan before execution | yes | -| read partitioned data | yes | +| | Spark | +| ----------------------------------------------------- | ----- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | no | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | no | +| export as Spark dataframe | yes | +| local execution of Python-based on-demand transforms | no | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/trino.md b/docs/reference/offline-stores/trino.md index 8cc604248ff..446db620e32 100644 --- a/docs/reference/offline-stores/trino.md +++ b/docs/reference/offline-stores/trino.md @@ -10,6 +10,9 @@ The Trino offline store provides support for reading [TrinoSources](../data-sour The Trino offline store does not achieve full test coverage. Please do not assume complete stability. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[trino]'`. You can then run `feast init`, then swap out `feature_store.yaml` with the below example to connect to Trino. + ## Example {% code title="feature_store.yaml" %} @@ -36,29 +39,29 @@ The full set of configuration options is available in [TrinoOfflineStoreConfig]( The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Trino offline store. -| | Trino | -| :-------------------------------- | :-- | -| `get_historical_features` (point-in-time correct join) | yes | -| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | -| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | -| `offline_write_batch` (persist dataframes to offline store) | no | -| `write_logged_features` (persist logged features to offline store) | no | +| | Trino | +| :----------------------------------------------------------------- | :---- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | Below is a matrix indicating which functionality is supported by `TrinoRetrievalJob`. -| | Trino | -| --------------------------------- | --- | -| export to dataframe | yes | -| export to arrow table | yes | -| export to arrow batches | no | -| export to SQL | yes | -| export to data lake (S3, GCS, etc.) | no | -| export to data warehouse | no | -| export as Spark dataframe | no | -| local execution of Python-based on-demand transforms | yes | -| remote execution of Python-based on-demand transforms | no | -| persist results in the offline store | no | -| preview the query plan before execution | yes | -| read partitioned data | yes | +| | Trino | +| ----------------------------------------------------- | ----- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | no | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | no | +| preview the query plan before execution | yes | +| read partitioned data | yes | To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index 58c8705e9ee..e46fc28d162 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -26,6 +26,10 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli [dynamodb.md](dynamodb.md) {% endcontent-ref %} +{% content-ref url="bigtable.md" %} +[bigtable.md](mysql.md) +{% endcontent-ref %} + {% content-ref url="postgres.md" %} [postgres.md](postgres.md) {% endcontent-ref %} @@ -34,3 +38,7 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli [cassandra.md](cassandra.md) {% endcontent-ref %} +{% content-ref url="mysql.md" %} +[mysql.md](mysql.md) +{% endcontent-ref %} + diff --git a/docs/reference/online-stores/bigtable.md b/docs/reference/online-stores/bigtable.md new file mode 100644 index 00000000000..0d6e7cfb13b --- /dev/null +++ b/docs/reference/online-stores/bigtable.md @@ -0,0 +1,56 @@ +# Bigtable online store + +## Description + +The [Bigtable](https://cloud.google.com/bigtable) online store provides support for +materializing feature values into Cloud Bigtable. The data model used to store feature +values in Bigtable is described in more detail +[here](../../specs/online_store_format.md#google-bigtable-online-store-format). + +## Getting started + +In order to use this online store, you'll need to run `pip install 'feast[gcp]'`. You +can then get started with the command `feast init REPO_NAME -t gcp`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: gcp +online_store: + type: bigtable + project_id: my_gcp_project + instance: my_bigtable_instance +``` +{% endcode %} + +The full set of configuration options is available in +[BigtableOnlineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.bigtable.BigtableOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Bigtable online store. + +| | Bigtable | +|-----------------------------------------------------------|----------| +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/cassandra.md b/docs/reference/online-stores/cassandra.md index 48b7b73f439..e012ad250ae 100644 --- a/docs/reference/online-stores/cassandra.md +++ b/docs/reference/online-stores/cassandra.md @@ -8,7 +8,8 @@ The [Cassandra / Astra DB] online store provides support for materializing featu * Each feature view is mapped one-to-one to a specific Cassandra table * This implementation inherits all strengths of Cassandra such as high availability, fault-tolerance, and data distribution -An easy way to get started is the command `feast init REPO_NAME -t cassandra`. +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[cassandra]'`. You can then get started with the command `feast init REPO_NAME -t cassandra`. ### Example (Cassandra) @@ -66,23 +67,23 @@ Storage specifications can be found at `docs/specs/online_store_format.md`. The set of functionality supported by online stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Cassandra online store. -| | Cassandra | -| :-------------------------------------------------------- | :-- | -| write feature values to the online store | yes | -| read feature values from the online store | yes | -| update infrastructure (e.g. tables) in the online store | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | yes | -| support for on-demand transforms | yes | -| readable by Python SDK | yes | -| readable by Java | no | -| readable by Go | no | -| support for entityless feature views | yes | -| support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | -| collocated by feature view | yes | -| collocated by feature service | no | -| collocated by entity key | no | +| | Cassandra | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | yes | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/datastore.md b/docs/reference/online-stores/datastore.md index 0867853f15d..761d246ba7e 100644 --- a/docs/reference/online-stores/datastore.md +++ b/docs/reference/online-stores/datastore.md @@ -4,6 +4,9 @@ The [Datastore](https://cloud.google.com/datastore) online store provides support for materializing feature values into Cloud Datastore. The data model used to store feature values in Datastore is described in more detail [here](../../specs/online_store_format.md#google-datastore-online-store-format). +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[gcp]'`. You can then get started with the command `feast init REPO_NAME -t gcp`. + ## Example {% code title="feature_store.yaml" %} @@ -25,23 +28,23 @@ The full set of configuration options is available in [DatastoreOnlineStoreConfi The set of functionality supported by online stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Datastore online store. -| | Datastore | -| :-------------------------------------------------------- | :-- | -| write feature values to the online store | yes | -| read feature values from the online store | yes | -| update infrastructure (e.g. tables) in the online store | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | no | -| support for on-demand transforms | yes | -| readable by Python SDK | yes | -| readable by Java | no | -| readable by Go | no | -| support for entityless feature views | yes | -| support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | -| collocated by feature view | yes | -| collocated by feature service | no | -| collocated by entity key | no | +| | Datastore | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/dynamodb.md b/docs/reference/online-stores/dynamodb.md index 2f94c768199..344caccac1d 100644 --- a/docs/reference/online-stores/dynamodb.md +++ b/docs/reference/online-stores/dynamodb.md @@ -4,6 +4,9 @@ The [DynamoDB](https://aws.amazon.com/dynamodb/) online store provides support for materializing feature values into AWS DynamoDB. +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[aws]'`. You can then get started with the command `feast init REPO_NAME -t aws`. + ## Example {% code title="feature_store.yaml" %} @@ -27,7 +30,7 @@ Feast requires the following permissions in order to execute commands for Dynamo | ----------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------- | | **Apply** |

dynamodb:CreateTable

dynamodb:DescribeTable

dynamodb:DeleteTable

| arn:aws:dynamodb:\:\:table/\* | | **Materialize** | dynamodb.BatchWriteItem | arn:aws:dynamodb:\:\:table/\* | -| **Get Online Features** | dynamodb.BatchGetItem | arn:aws:dynamodb:\:\:table/\* | +| **Get Online Features** | dynamodb.BatchGetItem | arn:aws:dynamodb:\:\:table/\* | The following inline policy can be used to grant Feast the necessary permissions: @@ -59,23 +62,23 @@ Lastly, this IAM role needs to be associated with the desired Redshift cluster. The set of functionality supported by online stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the DynamoDB online store. -| | DynamoDB | -| :-------------------------------------------------------- | :-- | -| write feature values to the online store | yes | -| read feature values from the online store | yes | -| update infrastructure (e.g. tables) in the online store | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | no | -| support for on-demand transforms | yes | -| readable by Python SDK | yes | -| readable by Java | no | -| readable by Go | no | -| support for entityless feature views | yes | -| support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | -| collocated by feature view | yes | -| collocated by feature service | no | -| collocated by entity key | no | +| | DynamoDB | +| :-------------------------------------------------------- | :------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/mysql.md b/docs/reference/online-stores/mysql.md new file mode 100644 index 00000000000..cbc48457e13 --- /dev/null +++ b/docs/reference/online-stores/mysql.md @@ -0,0 +1,55 @@ +# MySQL online store (contrib) + +## Description + +The MySQL online store provides support for materializing feature values into a MySQL database for serving online features. + +* Only the latest feature values are persisted + +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[mysql]'`. You can get started by then running `feast init` and then setting the `feature_store.yaml` as described below. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: mysql + host: DB_HOST + port: DB_PORT + database: DB_NAME + user: DB_USERNAME + password: DB_PASSWORD +``` +{% endcode %} + +The full set of configuration options is available in [MySQLOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.contrib.mysql.MySQLOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Mys online store. + +| | Mys | +| :-------------------------------------------------------- | :--- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/postgres.md b/docs/reference/online-stores/postgres.md index 083c0006359..3885867dd26 100644 --- a/docs/reference/online-stores/postgres.md +++ b/docs/reference/online-stores/postgres.md @@ -8,6 +8,9 @@ The PostgreSQL online store provides support for materializing feature values in * sslmode, sslkey_path, sslcert_path, and sslrootcert_path are optional +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[postgres]'`. You can get started by then running `feast init -t postgres`. + ## Example {% code title="feature_store.yaml" %} @@ -37,23 +40,23 @@ The full set of configuration options is available in [PostgreSQLOnlineStoreConf The set of functionality supported by online stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Postgres online store. -| | Postgres | -| :-------------------------------------------------------- | :-- | -| write feature values to the online store | yes | -| read feature values from the online store | yes | -| update infrastructure (e.g. tables) in the online store | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | no | -| support for on-demand transforms | yes | -| readable by Python SDK | yes | -| readable by Java | no | -| readable by Go | no | -| support for entityless feature views | yes | -| support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | -| collocated by feature view | yes | -| collocated by feature service | no | -| collocated by entity key | no | +| | Postgres | +| :-------------------------------------------------------- | :------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/redis.md b/docs/reference/online-stores/redis.md index 80e90348c55..2078ee16b95 100644 --- a/docs/reference/online-stores/redis.md +++ b/docs/reference/online-stores/redis.md @@ -7,6 +7,15 @@ The [Redis](https://redis.io) online store provides support for materializing fe * Both Redis and Redis Cluster are supported. * The data model used to store feature values in Redis is described in more detail [here](../../specs/online\_store\_format.md). +## Getting started +In order to use this online store, you'll need to install the redis extra (along with the dependency needed for the offline store of choice). E.g. +- `pip install 'feast[gcp, redis]'` +- `pip install 'feast[snowflake, redis]'` +- `pip install 'feast[aws, redis]'` +- `pip install 'feast[azure, redis]'` + +You can get started by using any of the other templates (e.g. `feast init -t gcp` or `feast init -t snowflake` or `feast init -t aws`), and then swapping in Redis as the online store as seen below in the examples. + ## Examples Connecting to a single Redis instance: @@ -43,23 +52,23 @@ The full set of configuration options is available in [RedisOnlineStoreConfig](h The set of functionality supported by online stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Redis online store. -| | Redis | -| :-------------------------------------------------------- | :-- | -| write feature values to the online store | yes | -| read feature values from the online store | yes | -| update infrastructure (e.g. tables) in the online store | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | no | -| support for on-demand transforms | yes | -| readable by Python SDK | yes | -| readable by Java | yes | -| readable by Go | yes | -| support for entityless feature views | yes | -| support for concurrent writing to the same key | yes | -| support for ttl (time to live) at retrieval | yes | -| support for deleting expired data | yes | -| collocated by feature view | no | -| collocated by feature service | no | -| collocated by entity key | yes | +| | Redis | +| :-------------------------------------------------------- | :---- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | yes | +| readable by Go | yes | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/snowflake.md b/docs/reference/online-stores/snowflake.md index d114c87144a..6b6d107285c 100644 --- a/docs/reference/online-stores/snowflake.md +++ b/docs/reference/online-stores/snowflake.md @@ -16,6 +16,9 @@ The data model for using a Snowflake Transient Table as an online store follows (This model may be subject to change when Snowflake Hybrid Tables are released) +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[snowflake]'`. You can then get started with the command `feast init REPO_NAME -t snowflake`. + ## Example {% code title="feature_store.yaml" %} ```yaml @@ -53,23 +56,23 @@ The full set of configuration options is available in [SnowflakeOnlineStoreConfi The set of functionality supported by online stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Snowflake online store. -| | Snowflake | -| :-------------------------------------------------------- | :-- | -| write feature values to the online store | yes | -| read feature values from the online store | yes | -| update infrastructure (e.g. tables) in the online store | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | no | -| support for on-demand transforms | yes | -| readable by Python SDK | yes | -| readable by Java | no | -| readable by Go | no | -| support for entityless feature views | yes | -| support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | -| collocated by feature view | yes | -| collocated by feature service | no | -| collocated by entity key | no | +| | Snowflake | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/providers/amazon-web-services.md b/docs/reference/providers/amazon-web-services.md index 3135fedb740..68956a1be93 100644 --- a/docs/reference/providers/amazon-web-services.md +++ b/docs/reference/providers/amazon-web-services.md @@ -5,6 +5,11 @@ * Offline Store: Uses the **Redshift** offline store by default. Also supports File as the offline store. * Online Store: Uses the **DynamoDB** online store by default. Also supports Sqlite as an online store. +## Getting started +In order to use this offline store, you'll need to run (Snowflake) `pip install 'feast[aws, snowflake]'` or (Redshift) `pip install 'feast[aws]'`. + +You can get started by then running `feast init -t snowflake` or `feast init -t aws`. + ## Example {% code title="feature_store.yaml" %} diff --git a/docs/reference/providers/azure.md b/docs/reference/providers/azure.md index 123bf087635..0e7206f076e 100644 --- a/docs/reference/providers/azure.md +++ b/docs/reference/providers/azure.md @@ -10,6 +10,9 @@ The Azure provider does not achieve full test coverage. Please do not assume complete stability. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[azure]'`. You can get started by then following this [tutorial](https://github.com/feast-dev/feast/blob/master/docs/tutorials/azure/README.md). + ## Example {% code title="feature_store.yaml" %} diff --git a/docs/reference/providers/google-cloud-platform.md b/docs/reference/providers/google-cloud-platform.md index 713313d16be..96af3b6b2ff 100644 --- a/docs/reference/providers/google-cloud-platform.md +++ b/docs/reference/providers/google-cloud-platform.md @@ -5,6 +5,9 @@ * Offline Store: Uses the **BigQuery** offline store by default. Also supports File as the offline store. * Online Store: Uses the **Datastore** online store by default. Also supports Sqlite as an online store. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[gcp]'`. You can get started by then running `feast init -t gcp`. + ## Example {% code title="feature_store.yaml" %} diff --git a/docs/roadmap.md b/docs/roadmap.md index 30f4317054b..cea646a8f5f 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -31,12 +31,12 @@ The list below contains the functionality that contributors are planning to deve * [x] [DynamoDB](https://docs.feast.dev/reference/online-stores/dynamodb) * [x] [Redis](https://docs.feast.dev/reference/online-stores/redis) * [x] [Datastore](https://docs.feast.dev/reference/online-stores/datastore) + * [x] [Bigtable](https://docs.feast.dev/reference/online-stores/bigtable) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) + * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/adding-support-for-a-new-online-store) - * [x] [Cassandra / AstraDB](https://docs.feast.dev/reference/online-stores/cassandra) - * [ ] Bigtable (in progress) * **Feature Engineering** * [x] On-demand Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) * [x] Streaming Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1UzEyETHUaGpn0ap4G82DHluiCj7zEbrQLkJJkKSv4e8/edit)) diff --git a/docs/specs/online_store_format.md b/docs/specs/online_store_format.md index 5c3c545c8d6..4022138e319 100644 --- a/docs/specs/online_store_format.md +++ b/docs/specs/online_store_format.md @@ -92,6 +92,29 @@ Other types of entity keys are not supported in this version of the specificatio ![Datastore Online Example](datastore_online_example.png) +## Google Bigtable Online Store Format + +[Bigtable storage model](https://cloud.google.com/bigtable/docs/overview#storage-model) +consists of massively scalable tables, with each row keyed by a "row key". The rows in a +table are stored lexicographically sorted by this row key. + +We use the following structure to store feature data in Bigtable: + +* All feature data for an entity or a specific group of entities is stored in the same + table. The table name is derived by concatenating the lexicographically sorted names + of entities. +* This implementation only uses one column family per table, named `features`. +* Each row key is created by concatenating a hash derived from the specific entity keys + and the name of the feature view. Each row only stores feature values for a specific + feature view. This arrangement also means that feature values for a given group of + entities are colocated. +* The columns used in each row are named after the features in the feature view. + Bigtable is perfectly content being sparsely populated. +* By default, we store 1 historical value of each feature value. This can be configured + using the `max_versions` setting in `BigtableOnlineStoreConfig`. This implementation + of the online store does not have the ability to revert any given value to its old + self. To use the historical version, you'll have to use custom code. + ## Cassandra/Astra DB Online Store Format ### Overview @@ -250,4 +273,4 @@ message BoolList { repeated bool val = 1; } -``` \ No newline at end of file +``` diff --git a/examples/quickstart/quickstart.ipynb b/examples/quickstart/quickstart.ipynb index 68b9d639110..cec4df91b11 100644 --- a/examples/quickstart/quickstart.ipynb +++ b/examples/quickstart/quickstart.ipynb @@ -512,7 +512,7 @@ "\n", "# This groups features into a model version\n", "driver_stats_fs = FeatureService(\n", - " name=\"driver_activity\", features=[driver_hourly_stats_view, transformed_conv_rate]\n", + " name=\"driver_activity_v1\", features=[driver_hourly_stats_view, transformed_conv_rate]\n", ")\n", "```" ] @@ -547,7 +547,7 @@ "Created entity \u001b[1m\u001b[32mdriver\u001b[0m\n", "Created feature view \u001b[1m\u001b[32mdriver_hourly_stats\u001b[0m\n", "Created on demand feature view \u001b[1m\u001b[32mtransformed_conv_rate\u001b[0m\n", - "Created feature service \u001b[1m\u001b[32mdriver_activity\u001b[0m\n", + "Created feature service \u001b[1m\u001b[32mdriver_activity_v1\u001b[0m\n", "\n", "Created sqlite table \u001b[1m\u001b[32mfeature_repo_driver_hourly_stats\u001b[0m\n", "\n" @@ -942,11 +942,11 @@ "### Fetching features using feature services\n", "You can also use feature services to manage multiple features, and decouple feature view definitions and the features needed by end applications. The feature store can also be used to fetch either online or historical features using the same api below. More information can be found [here](https://docs.feast.dev/getting-started/concepts/feature-retrieval).\n", "\n", - " The `driver_activity` feature service pulls all features from the `driver_hourly_stats` feature view:\n", + " The `driver_activity_v1` feature service pulls all features from the `driver_hourly_stats` feature view:\n", "\n", "```python\n", "driver_stats_fs = FeatureService(\n", - " name=\"driver_activity\", features=[driver_hourly_stats_view]\n", + " name=\"driver_activity_v1\", features=[driver_hourly_stats_view]\n", ")\n", "```" ] @@ -979,7 +979,7 @@ "from feast import FeatureStore\n", "feature_store = FeatureStore('.') # Initialize the feature store\n", "\n", - "feature_service = feature_store.get_feature_service(\"driver_activity\")\n", + "feature_service = feature_store.get_feature_service(\"driver_activity_v1\")\n", "feature_vector = feature_store.get_online_features(\n", " features=feature_service,\n", " entity_rows=[\n", @@ -1101,4 +1101,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index 7095866e4f6..252dceb9f6f 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.25.0 +version: 0.26.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index 9cf5c31a631..0ae45328598 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.25.0` +Current chart version is `0.26.0` ## Installation @@ -30,7 +30,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.25.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.26.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 90954a23cb1..d897346efe4 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.25.0 + tag: 0.26.0 imagePullSecrets: [] nameOverride: "" diff --git a/infra/charts/feast-python-server/Chart.yaml b/infra/charts/feast-python-server/Chart.yaml index 30d90876182..0c7690deb27 100644 --- a/infra/charts/feast-python-server/Chart.yaml +++ b/infra/charts/feast-python-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-python-server description: Feast Feature Server in Python type: application -version: 0.25.0 +version: 0.26.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-python-server/README.md b/infra/charts/feast-python-server/README.md index c5d5393e29f..ab2de1d3a83 100644 --- a/infra/charts/feast-python-server/README.md +++ b/infra/charts/feast-python-server/README.md @@ -2,7 +2,7 @@ > Note: this helm chart is deprecated in favor of [feast-feature-server](../feast-feature-server/README.md) -Current chart version is `0.25.0` +Current chart version is `0.26.0` ## Installation Docker repository and tag are required. Helm install example: diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 02f689f5419..f0d6774a3cb 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.25.0 +version: 0.26.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 2afdaf645cc..b0dde94610b 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.25.0` +Feature store for machine learning Current chart version is `0.26.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.25.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.25.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.26.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.26.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index bdaa9ea1fcc..6e577e1fb83 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.25.0 -appVersion: v0.25.0 +version: 0.26.0 +appVersion: v0.26.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index aef8c0329a6..f13a6488c39 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.25.0](https://img.shields.io/badge/Version-0.25.0-informational?style=flat-square) ![AppVersion: v0.25.0](https://img.shields.io/badge/AppVersion-v0.25.0-informational?style=flat-square) +![Version: 0.26.0](https://img.shields.io/badge/Version-0.26.0-informational?style=flat-square) ![AppVersion: v0.26.0](https://img.shields.io/badge/AppVersion-v0.26.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.25.0"` | Image tag | +| image.tag | string | `"0.26.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index d3b8a33a645..66795aeba71 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.25.0 + tag: 0.26.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 104b2f24278..20321e687a3 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.25.0 -appVersion: v0.25.0 +version: 0.26.0 +appVersion: v0.26.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index 37be5b0f106..8b44c218df9 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.25.0](https://img.shields.io/badge/Version-0.25.0-informational?style=flat-square) ![AppVersion: v0.25.0](https://img.shields.io/badge/AppVersion-v0.25.0-informational?style=flat-square) +![Version: 0.26.0](https://img.shields.io/badge/Version-0.26.0-informational?style=flat-square) ![AppVersion: v0.26.0](https://img.shields.io/badge/AppVersion-v0.26.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.25.0"` | Image tag | +| image.tag | string | `"0.26.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index 5232ac68ca1..ecaa16e614f 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.25.0 + tag: 0.26.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index a599ac23a44..f9f23092f5d 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.25.0 + version: 0.26.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.25.0 + version: 0.26.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/java/infra/docker/feature-server/Dockerfile b/java/infra/docker/feature-server/Dockerfile index bf4e172f763..8d246ed86aa 100644 --- a/java/infra/docker/feature-server/Dockerfile +++ b/java/infra/docker/feature-server/Dockerfile @@ -12,13 +12,6 @@ COPY java/serving/pom.xml serving/pom.xml COPY java/serving-client/pom.xml serving-client/pom.xml COPY java/coverage/pom.xml coverage/pom.xml -# Setting Maven repository .m2 directory relative to /build folder gives the -# user to optionally use cached repository when building the image by copying -# the existing .m2 directory to $FEAST_REPO_ROOT/.m2 -ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3" -COPY java/pom.xml .m2/* .m2/ -RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true - COPY java/ . COPY protos/feast datatypes/src/main/proto/feast diff --git a/java/pom.xml b/java/pom.xml index 874daa27984..3b9910a7cfa 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.25.0 + 0.26.0 https://github.com/feast-dev/feast UTF-8 diff --git a/protos/feast/core/DataSource.proto b/protos/feast/core/DataSource.proto index 5258618f3bd..3992d2c247d 100644 --- a/protos/feast/core/DataSource.proto +++ b/protos/feast/core/DataSource.proto @@ -23,6 +23,7 @@ option java_outer_classname = "DataSourceProto"; option java_package = "feast.proto.core"; import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; import "feast/core/DataFormat.proto"; import "feast/types/Value.proto"; import "feast/core/Feature.proto"; @@ -89,6 +90,12 @@ message DataSource { // Optional batch source for streaming sources for historical features and materialization. DataSource batch_source = 26; + SourceMeta meta = 50; + + message SourceMeta { + google.protobuf.Timestamp earliestEventTimestamp = 1; + google.protobuf.Timestamp latestEventTimestamp = 2; + } // Defines options for DataSource that sources features from a file message FileOptions { diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.mysql_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.mysql_online_store.rst new file mode 100644 index 00000000000..036922d658f --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.mysql_online_store.rst @@ -0,0 +1,21 @@ +feast.infra.online\_stores.contrib.mysql\_online\_store package +=============================================================== + +Submodules +---------- + +feast.infra.online\_stores.contrib.mysql\_online\_store.mysql module +-------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.mysql_online_store.mysql + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.contrib.mysql_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst index 6afe9071ace..6b175f4584f 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst @@ -9,6 +9,7 @@ Subpackages feast.infra.online_stores.contrib.cassandra_online_store feast.infra.online_stores.contrib.hbase_online_store + feast.infra.online_stores.contrib.mysql_online_store Submodules ---------- @@ -29,6 +30,14 @@ feast.infra.online\_stores.contrib.hbase\_repo\_configuration module :undoc-members: :show-inheritance: +feast.infra.online\_stores.contrib.mysql\_repo\_configuration module +-------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.mysql_repo_configuration + :members: + :undoc-members: + :show-inheritance: + feast.infra.online\_stores.contrib.postgres module -------------------------------------------------- diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index af9aa511918..f677584fe5f 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -99,7 +99,7 @@ def cli( ctx.obj["FS_YAML_FILE"] = ( Path(feature_store_yaml).absolute() if feature_store_yaml - else ctx.obj["CHDIR"] / "feature_store.yaml" + else utils.get_default_yaml_file_path(ctx.obj["CHDIR"]) ) try: level = getattr(logging, log_level.upper()) diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index a2fe6f15c58..574d79f4167 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -23,6 +23,9 @@ # feature_store.yaml environment variable name for remote feature server FEATURE_STORE_YAML_ENV_NAME: str = "FEATURE_STORE_YAML_BASE64" +# feature_store.yaml path environment variable name +FEAST_FS_YAML_FILE_PATH_ENV_NAME: str = "FEAST_FS_YAML_FILE_PATH" + # Environment variable for registry REGISTRY_ENV_NAME: str = "REGISTRY_BASE64" diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 19a780b32ca..54a68ed0480 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -760,7 +760,7 @@ def __init__( def __eq__(self, other): if not isinstance(other, PushSource): - raise TypeError("Comparisons should only involve PushSource class objects.") + return False if not super().__eq__(other): return False diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 834df0e5c48..15ba86781df 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -398,3 +398,8 @@ def __init__(self): super().__init__( "The entity dataframe specified does not have the timestamp field as a datetime." ) + + +class PushSourceNotFoundException(Exception): + def __init__(self, push_source_name: str): + super().__init__(f"Unable to find push source '{push_source_name}'.") diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index c2596d411c6..7b0cfc4bed8 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -13,6 +13,7 @@ import feast from feast import proto_json from feast.data_source import PushMode +from feast.errors import PushSourceNotFoundException from feast.protos.feast.serving.ServingService_pb2 import GetOnlineFeaturesRequest @@ -98,6 +99,11 @@ def push(body=Depends(get_body)): allow_registry_cache=request.allow_registry_cache, to=to, ) + except PushSourceNotFoundException as e: + # Print the original exception on the server side + logger.exception(traceback.format_exc()) + # Raise HTTPException to return the error message to the client + raise HTTPException(status_code=422, detail=str(e)) except Exception as e: # Print the original exception on the server side logger.exception(traceback.format_exc()) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 9350220c21c..c47b06c0471 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -59,6 +59,7 @@ EntityNotFoundException, FeatureNameCollisionError, FeatureViewNotFoundException, + PushSourceNotFoundException, RequestDataNotFoundInEntityDfException, RequestDataNotFoundInEntityRowsException, ) @@ -160,7 +161,7 @@ def __init__( self.config = load_repo_config(self.repo_path, fs_yaml_file) else: self.config = load_repo_config( - self.repo_path, Path(self.repo_path) / "feature_store.yaml" + self.repo_path, utils.get_default_yaml_file_path(self.repo_path) ) registry_config = self.config.get_registry_config() @@ -1111,7 +1112,8 @@ def get_historical_features( # Check that the right request data is present in the entity_df if type(entity_df) == pd.DataFrame: - entity_df = utils.make_df_tzaware(cast(pd.DataFrame, entity_df)) + if self.config.coerce_tz_aware: + entity_df = utils.make_df_tzaware(cast(pd.DataFrame, entity_df)) for fv in request_feature_views: for feature in fv.features: if feature.name not in entity_df.columns: @@ -1444,6 +1446,9 @@ def push( ) } + if not fvs_with_push_sources: + raise PushSourceNotFoundException(push_source_name) + for fv in fvs_with_push_sources: if to == PushMode.ONLINE or to == PushMode.ONLINE_AND_OFFLINE: self.write_to_online_store( diff --git a/sdk/python/feast/infra/aws.py b/sdk/python/feast/infra/aws.py index f334998e6bc..5a045de4016 100644 --- a/sdk/python/feast/infra/aws.py +++ b/sdk/python/feast/infra/aws.py @@ -8,6 +8,7 @@ from colorama import Fore, Style +from feast import utils from feast.constants import ( AWS_LAMBDA_FEATURE_SERVER_IMAGE, AWS_LAMBDA_FEATURE_SERVER_REPOSITORY, @@ -113,7 +114,10 @@ def _deploy_feature_server(self, project: str, image_uri: str): if not self.repo_config.repo_path: raise RepoConfigPathDoesNotExist() - with open(self.repo_config.repo_path / "feature_store.yaml", "rb") as f: + + with open( + utils.get_default_yaml_file_path(self.repo_config.repo_path), "rb" + ) as f: config_bytes = f.read() config_base64 = base64.b64encode(config_bytes).decode() diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index 751a398ad4f..c95c515fb4b 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -8,9 +8,9 @@ RUN apt update && \ build-essential RUN pip install pip --upgrade -COPY . . +RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres]" + -RUN pip install -r requirements.txt RUN apt update RUN apt install -y -V ca-certificates lsb-release wget RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev index 751a398ad4f..ecbc199a5b9 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -10,7 +10,8 @@ RUN apt update && \ RUN pip install pip --upgrade COPY . . -RUN pip install -r requirements.txt +RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres]" + RUN apt update RUN apt install -y -V ca-certificates lsb-release wget RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb diff --git a/sdk/python/feast/infra/feature_servers/multicloud/requirements.txt b/sdk/python/feast/infra/feature_servers/multicloud/requirements.txt deleted file mode 100644 index 01d08a4effa..00000000000 --- a/sdk/python/feast/infra/feature_servers/multicloud/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -feast[aws,gcp,snowflake,redis,go,mysql] diff --git a/sdk/python/feast/infra/materialization/aws_lambda/lambda_engine.py b/sdk/python/feast/infra/materialization/aws_lambda/lambda_engine.py index 53a845140ec..93e33d5949d 100644 --- a/sdk/python/feast/infra/materialization/aws_lambda/lambda_engine.py +++ b/sdk/python/feast/infra/materialization/aws_lambda/lambda_engine.py @@ -10,6 +10,7 @@ from pydantic import StrictStr from tqdm import tqdm +from feast import utils from feast.batch_feature_view import BatchFeatureView from feast.constants import FEATURE_STORE_YAML_ENV_NAME from feast.entity import Entity @@ -137,7 +138,7 @@ def __init__( ) repo_path = self.repo_config.repo_path assert repo_path - feature_store_path = repo_path / "feature_store.yaml" + feature_store_path = utils.get_default_yaml_file_path(repo_path) self.feature_store_base64 = str( base64.b64encode(bytes(feature_store_path.read_text(), "UTF-8")), "UTF-8" ) diff --git a/sdk/python/feast/infra/materialization/snowflake_engine.py b/sdk/python/feast/infra/materialization/snowflake_engine.py index 0219a7923f6..b523413bcde 100644 --- a/sdk/python/feast/infra/materialization/snowflake_engine.py +++ b/sdk/python/feast/infra/materialization/snowflake_engine.py @@ -118,11 +118,6 @@ def update( entities_to_delete: Sequence[Entity], entities_to_keep: Sequence[Entity], ): - click.echo( - f"Deploying materialization functions for {Style.BRIGHT + Fore.GREEN}{project}{Style.RESET_ALL}" - ) - click.echo() - stage_context = f'"{self.repo_config.batch_engine.database}"."{self.repo_config.batch_engine.schema_}"' stage_path = f'{stage_context}."feast_{project}"' with get_snowflake_conn(self.repo_config.batch_engine) as conn: @@ -136,12 +131,13 @@ def update( # if the stage already exists, # assumes that the materialization functions have been deployed if f"feast_{project}" in stage_list["name"].tolist(): - click.echo( - f"Materialization functions for {Style.BRIGHT + Fore.GREEN}{project}{Style.RESET_ALL} already exists" - ) - click.echo() return None + click.echo( + f"Deploying materialization functions for {Style.BRIGHT + Fore.GREEN}{project}{Style.RESET_ALL}" + ) + click.echo() + query = f"CREATE STAGE {stage_path}" execute_snowflake_statement(conn, query) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index c570b8d38ab..bf010c82aaf 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -19,7 +19,7 @@ import pandas as pd import pyarrow import pyarrow.parquet -from pydantic import StrictStr +from pydantic import StrictStr, validator from pydantic.typing import Literal from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed @@ -83,7 +83,8 @@ class BigQueryOfflineStoreConfig(FeastConfigBaseModel): project_id: Optional[StrictStr] = None """ (optional) GCP project name used for the BigQuery offline store """ - + billing_project_id: Optional[StrictStr] = None + """ (optional) GCP project name used to run the bigquery jobs at """ location: Optional[StrictStr] = None """ (optional) GCP location name used for the BigQuery offline store. Examples of location names include ``US``, ``EU``, ``us-central1``, ``us-west4``. @@ -94,6 +95,14 @@ class BigQueryOfflineStoreConfig(FeastConfigBaseModel): gcs_staging_location: Optional[str] = None """ (optional) GCS location used for offloading BigQuery results as parquet files.""" + @validator("billing_project_id") + def project_id_exists(cls, v, values, **kwargs): + if v and not values["project_id"]: + raise ValueError( + "please specify project_id if billing_project_id is specified" + ) + return v + class BigQueryOfflineStore(OfflineStore): @staticmethod @@ -122,9 +131,11 @@ def pull_latest_from_table_or_query( timestamps.append(created_timestamp_column) timestamp_desc_string = " DESC, ".join(timestamps) + " DESC" field_string = ", ".join(join_key_columns + feature_name_columns + timestamps) - + project_id = ( + config.offline_store.billing_project_id or config.offline_store.project_id + ) client = _get_bigquery_client( - project=config.offline_store.project_id, + project=project_id, location=config.offline_store.location, ) query = f""" @@ -162,9 +173,11 @@ def pull_all_from_table_or_query( assert isinstance(config.offline_store, BigQueryOfflineStoreConfig) assert isinstance(data_source, BigQuerySource) from_expression = data_source.get_table_query_string() - + project_id = ( + config.offline_store.billing_project_id or config.offline_store.project_id + ) client = _get_bigquery_client( - project=config.offline_store.project_id, + project=project_id, location=config.offline_store.location, ) field_string = ", ".join( @@ -197,17 +210,22 @@ def get_historical_features( assert isinstance(config.offline_store, BigQueryOfflineStoreConfig) for fv in feature_views: assert isinstance(fv.batch_source, BigQuerySource) - + project_id = ( + config.offline_store.billing_project_id or config.offline_store.project_id + ) client = _get_bigquery_client( - project=config.offline_store.project_id, + project=project_id, location=config.offline_store.location, ) assert isinstance(config.offline_store, BigQueryOfflineStoreConfig) - + if config.offline_store.billing_project_id: + dataset_project = str(config.offline_store.project_id) + else: + dataset_project = client.project table_reference = _get_table_reference_for_new_entity( client, - client.project, + dataset_project, config.offline_store.dataset, config.offline_store.location, ) @@ -295,9 +313,11 @@ def write_logged_features( ): destination = logging_config.destination assert isinstance(destination, BigQueryLoggingDestination) - + project_id = ( + config.offline_store.billing_project_id or config.offline_store.project_id + ) client = _get_bigquery_client( - project=config.offline_store.project_id, + project=project_id, location=config.offline_store.location, ) @@ -353,9 +373,11 @@ def offline_write_batch( if table.schema != pa_schema: table = table.cast(pa_schema) - + project_id = ( + config.offline_store.billing_project_id or config.offline_store.project_id + ) client = _get_bigquery_client( - project=config.offline_store.project_id, + project=project_id, location=config.offline_store.location, ) @@ -451,7 +473,10 @@ def to_bigquery( if not job_config: today = date.today().strftime("%Y%m%d") rand_id = str(uuid.uuid4())[:7] - path = f"{self.client.project}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" + if self.config.offline_store.billing_project_id: + path = f"{self.config.offline_store.project_id}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" + else: + path = f"{self.client.project}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" job_config = bigquery.QueryJobConfig(destination=path) if not job_config.dry_run and self.on_demand_feature_views: @@ -525,7 +550,10 @@ def to_remote_storage(self) -> List[str]: bucket: str prefix: str - storage_client = StorageClient(project=self.client.project) + if self.config.offline_store.billing_project_id: + storage_client = StorageClient(project=self.config.offline_store.project_id) + else: + storage_client = StorageClient(project=self.client.project) bucket, prefix = self._gcs_path[len("gs://") :].split("/", 1) prefix = prefix.rsplit("/", 1)[0] if prefix.startswith("/"): diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py index 6c25b5768f4..a5a51311eb9 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py @@ -157,7 +157,7 @@ def pull_latest_from_table_or_query( created_timestamp_column: Optional[str], start_date: datetime, end_date: datetime, - user: str = "user", + user: Optional[str] = None, auth: Optional[Authentication] = None, http_scheme: Optional[str] = None, ) -> TrinoRetrievalJob: @@ -176,7 +176,6 @@ def pull_latest_from_table_or_query( timestamps.append(created_timestamp_column) timestamp_desc_string = " DESC, ".join(timestamps) + " DESC" field_string = ", ".join(join_key_columns + feature_name_columns + timestamps) - client = _get_trino_client( config=config, user=user, auth=auth, http_scheme=http_scheme ) @@ -212,7 +211,7 @@ def get_historical_features( registry: Registry, project: str, full_feature_names: bool = False, - user: str = "user", + user: Optional[str] = None, auth: Optional[Authentication] = None, http_scheme: Optional[str] = None, ) -> TrinoRetrievalJob: @@ -303,7 +302,7 @@ def pull_all_from_table_or_query( timestamp_field: str, start_date: datetime, end_date: datetime, - user: str = "user", + user: Optional[str] = None, auth: Optional[Authentication] = None, http_scheme: Optional[str] = None, ) -> RetrievalJob: @@ -375,7 +374,10 @@ def _upload_entity_df_and_get_entity_schema( def _get_trino_client( - config: RepoConfig, user: str, auth: Optional[Any], http_scheme: Optional[str] + config: RepoConfig, + user: Optional[str], + auth: Optional[Any], + http_scheme: Optional[str], ) -> Trino: client = Trino( user=user, diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py index 1d4b5881240..97c61f78a60 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py @@ -36,6 +36,8 @@ def __init__( catalog: Optional[str] = None, auth: Optional[Any] = None, http_scheme: Optional[str] = None, + source: Optional[str] = None, + extra_credential: Optional[str] = None, ): self.host = host or os.getenv("TRINO_HOST") self.port = port or os.getenv("TRINO_PORT") @@ -43,6 +45,8 @@ def __init__( self.catalog = catalog or os.getenv("TRINO_CATALOG") self.auth = auth or os.getenv("TRINO_AUTH") self.http_scheme = http_scheme or os.getenv("TRINO_HTTP_SCHEME") + self.source = source or os.getenv("TRINO_SOURCE") + self.extra_credential = extra_credential or os.getenv("TRINO_EXTRA_CREDENTIAL") self._cursor: Optional[Cursor] = None if self.host is None: @@ -56,6 +60,11 @@ def __init__( def _get_cursor(self) -> Cursor: if self._cursor is None: + headers = ( + {trino.constants.HEADER_EXTRA_CREDENTIAL: self.extra_credential} + if self.extra_credential + else {} + ) self._cursor = trino.dbapi.connect( host=self.host, port=self.port, @@ -63,6 +72,8 @@ def _get_cursor(self) -> Cursor: catalog=self.catalog, auth=self.auth, http_scheme=self.http_scheme, + source=self.source, + http_headers=headers, ).cursor() return self._cursor diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py index 6e989bd40ce..f09b79069ca 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py @@ -228,7 +228,6 @@ def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: client = Trino( - user="user", catalog=config.offline_store.catalog, host=config.offline_store.host, port=config.offline_store.port, diff --git a/sdk/python/feast/infra/online_stores/bigtable.py b/sdk/python/feast/infra/online_stores/bigtable.py new file mode 100644 index 00000000000..e08bc44bdbe --- /dev/null +++ b/sdk/python/feast/infra/online_stores/bigtable.py @@ -0,0 +1,338 @@ +import hashlib +import logging +from concurrent import futures +from datetime import datetime +from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple + +import google +from google.cloud import bigtable +from google.cloud.bigtable import row_filters +from pydantic import StrictStr +from pydantic.typing import Literal + +from feast import Entity, FeatureView, utils +from feast.feature_view import DUMMY_ENTITY_NAME +from feast.infra.online_stores.helpers import compute_entity_id +from feast.infra.online_stores.online_store import OnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel, RepoConfig +from feast.usage import log_exceptions_and_usage + +logger = logging.getLogger(__name__) + +# Number of mutations per Bigtable write operation we're aiming for. The official max is +# 100K; we're being conservative. +MUTATIONS_PER_OP = 50_000 +# The Bigtable client library limits the connection pool size to 10. This imposes a +# limitation to the concurrency we can get using a thread pool in each worker. +BIGTABLE_CLIENT_CONNECTION_POOL_SIZE = 10 + + +class BigtableOnlineStoreConfig(FeastConfigBaseModel): + """Online store config for GCP Bigtable""" + + type: Literal["bigtable"] = "bigtable" + """Online store typee selector""" + + project_id: Optional[StrictStr] = None + """(optional) GCP Project ID""" + + instance: StrictStr + """The Bigtable instance's ID""" + + max_versions: int = 2 + """The number of historical versions of data that will be kept around.""" + + +class BigtableOnlineStore(OnlineStore): + _client: Optional[bigtable.Client] = None + + feature_column_family: str = "features" + + @log_exceptions_and_usage(online_store="bigtable") + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + # Potential performance improvement opportunity described in + # https://github.com/feast-dev/feast/issues/3259 + feature_view = table + bt_table_name = self._get_table_name(config=config, feature_view=feature_view) + + client = self._get_client(online_config=config.online_store) + bt_instance = client.instance(instance_id=config.online_store.instance) + bt_table = bt_instance.table(bt_table_name) + row_keys = [ + self._compute_row_key( + entity_key=entity_key, + feature_view_name=feature_view.name, + config=config, + ) + for entity_key in entity_keys + ] + + row_set = bigtable.row_set.RowSet() + for row_key in row_keys: + row_set.add_row_key(row_key) + rows = bt_table.read_rows( + row_set=row_set, + filter_=( + row_filters.ColumnQualifierRegexFilter( + f"^({'|'.join(requested_features)}|event_ts)$".encode() + ) + if requested_features + else None + ), + ) + + # The BigTable client library only returns rows for keys that are found. This + # means that it's our responsibility to match the returned rows to the original + # `row_keys` and make sure that we're returning a list of the same length as + # `entity_keys`. + bt_rows_dict: Dict[bytes, bigtable.row.PartialRowData] = { + row.row_key: row for row in rows + } + return [self._process_bt_row(bt_rows_dict.get(row_key)) for row_key in row_keys] + + def _process_bt_row( + self, row: Optional[bigtable.row.PartialRowData] + ) -> Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]: + res = {} + + if row is None: + return (None, None) + + row_values = row.cells[self.feature_column_family] + event_ts = datetime.fromisoformat(row_values.pop(b"event_ts")[0].value.decode()) + for feature_name, feature_values in row_values.items(): + # We only want to retrieve the latest value for each feature + feature_value = feature_values[0] + val = ValueProto() + val.ParseFromString(feature_value.value) + res[feature_name.decode()] = val + + return (event_ts, res) + + @log_exceptions_and_usage(online_store="bigtable") + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + feature_view = table + bt_table_name = self._get_table_name(config=config, feature_view=feature_view) + + client = self._get_client(online_config=config.online_store) + bt_instance = client.instance(instance_id=config.online_store.instance) + bt_table = bt_instance.table(bt_table_name) + + # `columns_per_row` is used to calculate the number of rows we are allowed to + # mutate in one request. + columns_per_row = len(feature_view.features) + 1 # extra for event timestamp + rows_per_write = MUTATIONS_PER_OP // columns_per_row + + with futures.ThreadPoolExecutor( + max_workers=BIGTABLE_CLIENT_CONNECTION_POOL_SIZE + ) as executor: + fs = [] + while data: + rows_to_write, data = data[:rows_per_write], data[rows_per_write:] + fs.append( + executor.submit( + self._write_rows_to_bt, + rows_to_write=rows_to_write, + bt_table=bt_table, + feature_view_name=feature_view.name, + config=config, + progress=progress, + ) + ) + done_tasks, not_done_tasks = futures.wait(fs) + for task in done_tasks: + # If a task raised an exception, this will raise it here as well + task.result() + if not_done_tasks: + raise RuntimeError( + f"Not all batches were written to Bigtable: {not_done_tasks}" + ) + + def _write_rows_to_bt( + self, + rows_to_write: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + bt_table: bigtable.table.Table, + feature_view_name: str, + config: RepoConfig, + progress: Optional[Callable[[int], Any]], + ): + rows = [] + for entity_key, features, timestamp, created_ts in rows_to_write: + bt_row = bt_table.direct_row( + self._compute_row_key( + entity_key=entity_key, + feature_view_name=feature_view_name, + config=config, + ) + ) + + for feature_name, feature_value in features.items(): + bt_row.set_cell( + self.feature_column_family, + feature_name.encode(), + feature_value.SerializeToString(), + ) + bt_row.set_cell( + self.feature_column_family, + b"event_ts", + utils.make_tzaware(timestamp).isoformat().encode(), + ) + rows.append(bt_row) + bt_table.mutate_rows(rows) + + if progress: + progress(len(rows)) + + def _compute_row_key( + self, entity_key: EntityKeyProto, feature_view_name: str, config: RepoConfig + ) -> bytes: + entity_id = compute_entity_id( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + # Even though `entity_id` uniquely identifies an entity, we use the same table + # for multiple feature_views with the same set of entities. To uniquely identify + # the row for a feature_view, we suffix the name of the feature_view itself. + # This also ensures that features for entities from various feature_views are + # colocated. + return f"{entity_id}#{feature_view_name}".encode() + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + """Creates the appropriate tables and column family in Bigtable. + + We use a dedicated table for each entity combination. For example, if a + FeatureView uses the entities `shop` and `customer`, the resulting table would + be called `customer-shop` (entities are sorted lexicographically first). + """ + online_config = config.online_store + assert isinstance(online_config, BigtableOnlineStoreConfig) + client = self._get_client(online_config, admin=True) + bt_instance = client.instance(instance_id=online_config.instance) + max_versions_gc_rule = bigtable.column_family.MaxVersionsGCRule( + online_config.max_versions + ) + # The word "table" in the arguments refers to feature_views (this is for legacy + # reasons). To reduce confusion with bigtable tables, we use alternate variable + # names + feature_views_to_keep = tables_to_keep + feature_views_to_delete = tables_to_delete + # Multiple feature views can share the same tables. So just because a feature + # view has been deleted does not mean that we can just delete the table. We map + # feature views to Bigtable table names and figure out which ones to create + # and/or delete. + bt_tables_to_keep: Set[str] = { + self._get_table_name(config=config, feature_view=feature_view) + for feature_view in feature_views_to_keep + } + bt_tables_to_delete: Set[str] = { + self._get_table_name(config=config, feature_view=feature_view) + for feature_view in feature_views_to_delete + } - bt_tables_to_keep # we don't delete a table if it's in `bt_tables_to_keep` + + for bt_table_name in bt_tables_to_keep: + bt_table = bt_instance.table(bt_table_name) + if not bt_table.exists(): + logger.info(f"Creating table `{bt_table_name}` in Bigtable") + bt_table.create() + else: + logger.info(f"Table {bt_table_name} already exists in Bigtable") + + if self.feature_column_family not in bt_table.list_column_families(): + bt_table.column_family( + self.feature_column_family, gc_rule=max_versions_gc_rule + ).create() + + for bt_table_name in bt_tables_to_delete: + bt_table = bt_instance.table(bt_table_name) + logger.info(f"Deleting table {bt_table_name} in Bigtable") + bt_table.delete() + + @staticmethod + def _get_table_name(config: RepoConfig, feature_view: FeatureView) -> str: + entities_part = ( + "-".join(sorted(feature_view.entities)) + if feature_view.entities + else DUMMY_ENTITY_NAME + ) + BIGTABLE_TABLE_MAX_LENGTH = 50 + ENTITIES_PART_MAX_LENGTH = 24 + # Bigtable limits table names to 50 characters. We'll limit the max size of of + # the `entities_part` and if that's not enough, we'll just hash the + # entities_part. The remaining length is dedicated to the project name. This + # allows multiple projects to coexist in the same bigtable instance. This also + # allows multiple integration test executions to run simultaneously without + # conflicts. + if len(entities_part) > ENTITIES_PART_MAX_LENGTH: + entities_part = hashlib.md5(entities_part.encode()).hexdigest()[ + :ENTITIES_PART_MAX_LENGTH + ] + remaining_length = BIGTABLE_TABLE_MAX_LENGTH - len(entities_part) - 1 + if len(config.project) > remaining_length: + HUMAN_READABLE_PART_LENGTH = 10 + HASH_PART_LENGTH = remaining_length - HUMAN_READABLE_PART_LENGTH - 1 + project_part = ( + config.project[:HUMAN_READABLE_PART_LENGTH] + + "_" + + hashlib.md5(config.project.encode()).hexdigest()[:HASH_PART_LENGTH] + ) + else: + project_part = config.project + return f"{project_part}.{entities_part}" + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ): + # Because of historical reasons, Feast calls them tables. We use this alias for + # readability. + feature_views = tables + + bt_tables = { + self._get_table_name(config=config, feature_view=fv) for fv in feature_views + } + + online_config = config.online_store + assert isinstance(online_config, BigtableOnlineStoreConfig) + client = self._get_client(online_config, admin=True) + bt_instance = client.instance(instance_id=online_config.instance) + for table_name in bt_tables: + try: + logger.info(f"Deleting Bigtable table `{table_name}`") + bt_instance.table(table_name).delete() + except google.api_core.exceptions.NotFound: + logger.warning( + f"Table `{table_name}` was not found. Skipping deletion." + ) + + def _get_client( + self, online_config: BigtableOnlineStoreConfig, admin: bool = False + ): + return bigtable.Client(project=online_config.project_id, admin=admin) diff --git a/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py b/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py index f89517c41eb..71d3167c46f 100644 --- a/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py +++ b/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py @@ -32,7 +32,7 @@ ) from cassandra.policies import DCAwareRoundRobinPolicy, TokenAwarePolicy from cassandra.query import PreparedStatement -from pydantic import StrictInt, StrictStr +from pydantic import StrictFloat, StrictInt, StrictStr from pydantic.typing import Literal from feast import Entity, FeatureView, RepoConfig @@ -141,6 +141,9 @@ class CassandraOnlineStoreConfig(FeastConfigBaseModel): protocol_version: Optional[StrictInt] = None """Explicit specification of the CQL protocol version used.""" + request_timeout: Optional[StrictFloat] = None + """Request timeout in seconds.""" + class CassandraLoadBalancingPolicy(FeastConfigBaseModel): """ Configuration block related to the Cluster's load-balancing policy. @@ -240,7 +243,10 @@ def _get_session(self, config: RepoConfig): raise CassandraInvalidConfig(E_CASSANDRA_UNKNOWN_LB_POLICY) # wrap it up in a map of ex.profiles with a default - exe_profile = ExecutionProfile(load_balancing_policy=lb_policy) + exe_profile = ExecutionProfile( + request_timeout=online_store_config.request_timeout, + load_balancing_policy=lb_policy, + ) execution_profiles = {EXEC_PROFILE_DEFAULT: exe_profile} else: execution_profiles = None diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/README.md b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/README.md new file mode 100644 index 00000000000..ac38237cd11 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/README.md @@ -0,0 +1,81 @@ +# Mysql Online Store +Mysql is not included in current [Feast](https://github.com/feast-dev/feast) roadmap, this project intends to add Mysql support for Online Store. +We create a table _ which gets updated with data on every materialize call + + +#### Create a feature repository + +```shell +feast init feature_repo +cd feature_repo +``` + +#### Edit `feature_store.yaml` + +set `online_store` type to be `mysql` + +```yaml +project: feature_repo +registry: data/registry.db +provider: local +online_store: + type: mysql + host: 127.0.0.1 # mysql endpoint, default to 127.0.0.1 + port: 3306 # mysql port, default to 3306 + user: test # mysql user, default to test + password: test # mysql password, default to test + database: feast # mysql database, default to feast +``` + +#### Apply the feature definitions in `example.py` + +```shell +feast -c feature_repo apply +``` +##### Output +``` +Registered entity driver_id +Registered feature view driver_hourly_stats_view +Deploying infrastructure for driver_hourly_stats_view +``` + +### Materialize Latest Data to Online Feature Store (Mysql) +``` +$ CURRENT_TIME=$(date -u +"%Y-%m-%dT%H:%M:%S") +$ feast -c feature_repo materialize-incremental $CURRENT_TIME +``` +#### Output +``` +Materializing 1 feature views from 2022-04-16 15:30:39+05:30 to 2022-04-19 15:31:04+05:30 into the mysql online store. + +driver_hourly_stats_view from 2022-04-16 15:30:39+05:30 to 2022-04-19 15:31:04+05:30: +100%|████████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 120.59it/s] +``` + +### Fetch the latest features for some entity id +```python +from pprint import pprint +from feast import FeatureStore + +store = FeatureStore(repo_path=".") +feature_vector = store.get_online_features( + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + ], + entity_rows=[ + {"driver_id": 1004}, + {"driver_id": 1005}, + ], +).to_dict() +pprint(feature_vector) + +``` +#### Output +``` +{'acc_rate': [0.01390857808291912, 0.4063614010810852], + 'avg_daily_trips': [69, 706], + 'conv_rate': [0.6624961495399475, 0.7595928311347961], + 'driver_id': [1004, 1005]} +``` diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/__init__.py b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py new file mode 100644 index 00000000000..fa7dd2c2a49 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py @@ -0,0 +1,225 @@ +from __future__ import absolute_import + +from datetime import datetime +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +import pymysql +import pytz +from pydantic import StrictStr +from pymysql.connections import Connection +from pymysql.cursors import Cursor + +from feast import Entity, FeatureView, RepoConfig +from feast.infra.key_encoding_utils import serialize_entity_key +from feast.infra.online_stores.online_store import OnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel + + +class MySQLOnlineStoreConfig(FeastConfigBaseModel): + """ + Configuration for the MySQL online store. + NOTE: The class *must* end with the `OnlineStoreConfig` suffix. + """ + + type = "mysql" + + host: Optional[StrictStr] = None + user: Optional[StrictStr] = None + password: Optional[StrictStr] = None + database: Optional[StrictStr] = None + port: Optional[int] = None + + +class MySQLOnlineStore(OnlineStore): + """ + An online store implementation that uses MySQL. + NOTE: The class *must* end with the `OnlineStore` suffix. + """ + + _conn: Optional[Connection] = None + + def _get_conn(self, config: RepoConfig) -> Connection: + + online_store_config = config.online_store + assert isinstance(online_store_config, MySQLOnlineStoreConfig) + + if not self._conn: + self._conn = pymysql.connect( + host=online_store_config.host or "127.0.0.1", + user=online_store_config.user or "test", + password=online_store_config.password or "test", + database=online_store_config.database or "feast", + port=online_store_config.port or 3306, + autocommit=True, + ) + return self._conn + + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + + conn = self._get_conn(config) + cur = conn.cursor() + + project = config.project + + for entity_key, values, timestamp, created_ts in data: + entity_key_bin = serialize_entity_key( + entity_key, + entity_key_serialization_version=2, + ).hex() + timestamp = _to_naive_utc(timestamp) + if created_ts is not None: + created_ts = _to_naive_utc(created_ts) + + for feature_name, val in values.items(): + self.write_to_table( + created_ts, + cur, + entity_key_bin, + feature_name, + project, + table, + timestamp, + val, + ) + conn.commit() + if progress: + progress(1) + + @staticmethod + def write_to_table( + created_ts, cur, entity_key_bin, feature_name, project, table, timestamp, val + ) -> None: + cur.execute( + f""" + INSERT INTO {_table_id(project, table)} + (entity_key, feature_name, value, event_ts, created_ts) + values (%s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + value = %s, + event_ts = %s, + created_ts = %s; + """, + ( + # Insert + entity_key_bin, + feature_name, + val.SerializeToString(), + timestamp, + created_ts, + # Update on duplicate key + val.SerializeToString(), + timestamp, + created_ts, + ), + ) + + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + conn = self._get_conn(config) + cur = conn.cursor() + + result: List[Tuple[Optional[datetime], Optional[Dict[str, Any]]]] = [] + + project = config.project + for entity_key in entity_keys: + entity_key_bin = serialize_entity_key( + entity_key, + entity_key_serialization_version=2, + ).hex() + + cur.execute( + f"SELECT feature_name, value, event_ts FROM {_table_id(project, table)} WHERE entity_key = %s", + (entity_key_bin,), + ) + + res = {} + res_ts: Optional[datetime] = None + records = cur.fetchall() + if records: + for feature_name, val_bin, ts in records: + val = ValueProto() + val.ParseFromString(val_bin) + res[feature_name] = val + res_ts = ts + + if not res: + result.append((None, None)) + else: + result.append((res_ts, res)) + return result + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ) -> None: + conn = self._get_conn(config) + cur = conn.cursor() + project = config.project + + # We don't create any special state for the entities in this implementation. + for table in tables_to_keep: + cur.execute( + f"""CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key VARCHAR(512), + feature_name VARCHAR(256), + value BLOB, + event_ts timestamp NULL DEFAULT NULL, + created_ts timestamp NULL DEFAULT NULL, + PRIMARY KEY(entity_key, feature_name))""" + ) + + cur.execute( + f"ALTER TABLE {_table_id(project, table)} ADD INDEX {_table_id(project, table)}_ek (entity_key);" + ) + + for table in tables_to_delete: + _drop_table_and_index(cur, project, table) + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ) -> None: + conn = self._get_conn(config) + cur = conn.cursor() + project = config.project + + for table in tables: + _drop_table_and_index(cur, project, table) + + +def _drop_table_and_index(cur: Cursor, project: str, table: FeatureView) -> None: + table_name = _table_id(project, table) + cur.execute(f"DROP INDEX {table_name}_ek ON {table_name};") + cur.execute(f"DROP TABLE IF EXISTS {table_name}") + + +def _table_id(project: str, table: FeatureView) -> str: + return f"{project}_{table.name}" + + +def _to_naive_utc(ts: datetime) -> datetime: + if ts.tzinfo is None: + return ts + else: + return ts.astimezone(pytz.utc).replace(tzinfo=None) diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_repo_configuration.py b/sdk/python/feast/infra/online_stores/contrib/mysql_repo_configuration.py new file mode 100644 index 00000000000..3e92ead2d0b --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_repo_configuration.py @@ -0,0 +1,10 @@ +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) +from tests.integration.feature_repos.universal.online_store.mysql import ( + MySQLOnlineStoreCreator, +) + +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig(online_store_creator=MySQLOnlineStoreCreator), +] diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 18f3b189e7d..144b242a1d6 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -114,17 +114,30 @@ def online_read( ) ) - cur.execute( - sql.SQL( - """ - SELECT entity_key, feature_name, value, event_ts - FROM {} WHERE entity_key = ANY(%s); - """ - ).format( - sql.Identifier(_table_id(project, table)), - ), - (keys,), - ) + if not requested_features: + cur.execute( + sql.SQL( + """ + SELECT entity_key, feature_name, value, event_ts + FROM {} WHERE entity_key = ANY(%s); + """ + ).format( + sql.Identifier(_table_id(project, table)), + ), + (keys,), + ) + else: + cur.execute( + sql.SQL( + """ + SELECT entity_key, feature_name, value, event_ts + FROM {} WHERE entity_key = ANY(%s) and feature_name = ANY(%s); + """ + ).format( + sql.Identifier(_table_id(project, table)), + ), + (keys, requested_features), + ) rows = cur.fetchall() diff --git a/sdk/python/feast/infra/registry/base_registry.py b/sdk/python/feast/infra/registry/base_registry.py index 5edfae3472d..fc65932ea16 100644 --- a/sdk/python/feast/infra/registry/base_registry.py +++ b/sdk/python/feast/infra/registry/base_registry.py @@ -634,6 +634,16 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]: registry_dict["requestFeatureViews"].append( self._message_to_sorted_dict(request_feature_view.to_proto()) ) + for stream_feature_view in sorted( + self.list_stream_feature_views(project=project), + key=lambda stream_feature_view: stream_feature_view.name, + ): + sfv_dict = self._message_to_sorted_dict(stream_feature_view.to_proto()) + + sfv_dict["spec"]["userDefinedFunction"][ + "body" + ] = stream_feature_view.udf_string + registry_dict["streamFeatureViews"].append(sfv_dict) for saved_dataset in sorted( self.list_saved_datasets(project=project), key=lambda item: item.name ): diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index fab0f25b410..673d039ff0b 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -48,9 +48,11 @@ "redis": "feast.infra.online_stores.redis.RedisOnlineStore", "dynamodb": "feast.infra.online_stores.dynamodb.DynamoDBOnlineStore", "snowflake.online": "feast.infra.online_stores.snowflake.SnowflakeOnlineStore", + "bigtable": "feast.infra.online_stores.bigtable.BigtableOnlineStore", "postgres": "feast.infra.online_stores.contrib.postgres.PostgreSQLOnlineStore", "hbase": "feast.infra.online_stores.contrib.hbase_online_store.hbase.HbaseOnlineStore", "cassandra": "feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store.CassandraOnlineStore", + "mysql": "feast.infra.online_stores.contrib.mysql_online_store.mysql.MySQLOnlineStore", } OFFLINE_STORE_CLASS_FOR_TYPE = { @@ -166,6 +168,9 @@ class RepoConfig(FeastBaseModel): feature values for entities that have already been written into the online store. """ + coerce_tz_aware: Optional[bool] = True + """ If True, coerces entity_df timestamp columns to be timezone aware (to UTC by default). """ + def __init__(self, **data: Any): super().__init__(**data) diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 176f38d093f..d3a2164788f 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -71,6 +71,7 @@ class StreamFeatureView(FeatureView): timestamp_field: str materialization_intervals: List[Tuple[datetime, datetime]] udf: Optional[FunctionType] + udf_string: Optional[str] def __init__( self, @@ -88,6 +89,7 @@ def __init__( mode: Optional[str] = "spark", timestamp_field: Optional[str] = "", udf: Optional[FunctionType] = None, + udf_string: Optional[str] = "", ): if not flags_helper.is_test(): warnings.warn( @@ -114,6 +116,7 @@ def __init__( self.mode = mode or "" self.timestamp_field = timestamp_field or "" self.udf = udf + self.udf_string = udf_string super().__init__( name=name, @@ -143,6 +146,7 @@ def __eq__(self, other): self.mode != other.mode or self.timestamp_field != other.timestamp_field or self.udf.__code__.co_code != other.udf.__code__.co_code + or self.udf_string != other.udf_string or self.aggregations != other.aggregations ): return False @@ -171,6 +175,7 @@ def to_proto(self): udf_proto = UserDefinedFunctionProto( name=self.udf.__name__, body=dill.dumps(self.udf, recurse=True), + body_text=self.udf_string, ) spec = StreamFeatureViewSpecProto( name=self.name, @@ -209,6 +214,11 @@ def from_proto(cls, sfv_proto): if sfv_proto.spec.HasField("user_defined_function") else None ) + udf_string = ( + sfv_proto.spec.user_defined_function.body_text + if sfv_proto.spec.HasField("user_defined_function") + else None + ) stream_feature_view = cls( name=sfv_proto.spec.name, description=sfv_proto.spec.description, @@ -226,6 +236,7 @@ def from_proto(cls, sfv_proto): source=stream_source, mode=sfv_proto.spec.mode, udf=udf, + udf_string=udf_string, aggregations=[ Aggregation.from_proto(agg_proto) for agg_proto in sfv_proto.spec.aggregations @@ -315,6 +326,7 @@ def mainify(obj): obj.__module__ = "__main__" def decorator(user_function): + udf_string = dill.source.getsource(user_function) mainify(user_function) stream_feature_view_obj = StreamFeatureView( name=user_function.__name__, @@ -323,6 +335,7 @@ def decorator(user_function): source=source, schema=schema, udf=user_function, + udf_string=udf_string, description=description, tags=tags, online=online, diff --git a/sdk/python/feast/templates/gcp/feature_repo/feature_store.yaml b/sdk/python/feast/templates/gcp/feature_repo/feature_store.yaml index 7d4096615ac..e3d9d1d1e61 100644 --- a/sdk/python/feast/templates/gcp/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/gcp/feature_repo/feature_store.yaml @@ -13,6 +13,11 @@ online_store: # type: datastore # project_id: my_gcp_project # namespace: my_datastore_namespace +# See https://docs.feast.dev/reference/online-stores/bigtable +#online_store: +# type: bigtable +# project_id: my_gcp_project +# instance: my_bigtable_instance # See https://docs.feast.dev/reference/online-stores/redis #online_store: # type: redis diff --git a/sdk/python/feast/ui/README.md b/sdk/python/feast/ui/README.md index 0c11dcf134c..28290f0326a 100644 --- a/sdk/python/feast/ui/README.md +++ b/sdk/python/feast/ui/README.md @@ -22,4 +22,20 @@ It is used by the `feast ui` command to scaffold a local UI server. The feast py The `feast ui` command will generate the necessary `projects-list.json` file and initialize it for the UI to read. -**Note**: yarn start will not work on this because of the above dependency. +**Note**: `yarn start` will not work on this because of the above dependency. + +## Dev +To test with a locally built Feast UI package, do: +1. `yarn link` in ui/ +2. `yarn install` in ui/ +3. `yarn link` in ui/node_modules/react +4. `yarn link` in ui/node_modules/react-dom +5. and then come here to do: + ```bash + yarn link "@feast-dev/feast" + yarn link react + yarn link react-dom + yarn start + ``` + +See also https://github.com/facebook/react/issues/14257. \ No newline at end of file diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 358aa2cdd24..442cb2466e2 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -4,9 +4,9 @@ "private": true, "dependencies": { "@elastic/datemath": "^5.0.3", - "@elastic/eui": "^57.0.0", + "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "latest", + "@feast-dev/feast-ui": "0.26.0", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", @@ -16,11 +16,11 @@ "moment": "^2.29.4", "prop-types": "^15.8.1", "query-string": "^7.1.1", - "react": "^18.1.0", - "react-dom": "^18.1.0", - "react-query": "^3.39.0", - "react-router-dom": "^6.3.0", - "react-scripts": "5.0.1", + "react": "^17.0.2", + "react-dom": "^17.0.2", + "react-query": "^3.34.12", + "react-router-dom": "6", + "react-scripts": "^5.0.0", "typescript": "^4.6.4", "use-query-params": "^1.2.3", "web-vitals": "^2.1.4", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index df2bfe45ff2..66abae62440 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1202,51 +1202,6 @@ uuid "^8.3.0" vfile "^4.2.0" -"@elastic/eui@^57.0.0": - version "57.0.0" - resolved "https://registry.yarnpkg.com/@elastic/eui/-/eui-57.0.0.tgz#86d43e27196f9997ef44d2a4c701d092ce99e132" - integrity sha512-VBgW6Pr0JJB3JhJ59MV8guxb2v4Gd3SJEmsMGKGyIY+KcvSMWbVEGa44Ep12VAJYynIA05Z3OXXc/ge5dMycpA== - dependencies: - "@types/chroma-js" "^2.0.0" - "@types/lodash" "^4.14.160" - "@types/numeral" "^0.0.28" - "@types/react-beautiful-dnd" "^13.1.2" - "@types/react-input-autosize" "^2.2.1" - "@types/react-virtualized-auto-sizer" "^1.0.1" - "@types/react-window" "^1.8.5" - "@types/refractor" "^3.0.0" - "@types/resize-observer-browser" "^0.1.5" - "@types/vfile-message" "^2.0.0" - chroma-js "^2.1.0" - classnames "^2.2.6" - lodash "^4.17.21" - mdast-util-to-hast "^10.0.0" - numeral "^2.0.6" - prop-types "^15.6.0" - react-beautiful-dnd "^13.1.0" - react-dropzone "^11.5.3" - react-element-to-jsx-string "^14.3.4" - react-focus-on "^3.5.4" - react-input-autosize "^3.0.0" - react-is "^17.0.2" - react-virtualized-auto-sizer "^1.0.6" - react-window "^1.8.6" - refractor "^3.5.0" - rehype-raw "^5.0.0" - rehype-react "^6.0.0" - rehype-stringify "^8.0.0" - remark-breaks "^2.0.2" - remark-emoji "^2.1.0" - remark-parse "^8.0.3" - remark-rehype "^8.0.0" - tabbable "^5.2.1" - text-diff "^1.0.1" - unified "^9.2.0" - unist-util-visit "^2.0.3" - url-parse "^1.5.10" - uuid "^8.3.0" - vfile "^4.2.0" - "@emotion/babel-plugin@^11.7.1": version "11.9.2" resolved "https://registry.yarnpkg.com/@emotion/babel-plugin/-/babel-plugin-11.9.2.tgz#723b6d394c89fb2ef782229d92ba95a740576e95" @@ -1345,10 +1300,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@latest": - version "0.24.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.24.0.tgz#a52037247563290f92d0d993fcaf0d88e9741f36" - integrity sha512-Te27bSVFp7gCE7+p9bbCkCEQ7+nsRCzBtwWivNPBFRn8HC2ewBzmRzzasXlCHok1cXHDbh7Xj7y+2Hshp91LTg== +"@feast-dev/feast-ui@0.26.0": + version "0.26.0" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.26.0.tgz#502e61652024a2db9bc5b9b5fcfe3f3dce04161b" + integrity sha512-7TBL9zetO/7Kx+FECIaDYG+fCr7P4R6esWyZW08xJTEFEvds5VXdXvUTWCibA/sKH24dlNXwpiLpyujwqwpQgA== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^55.0.1" @@ -1362,6 +1317,7 @@ inter-ui "^3.19.3" moment "^2.29.1" prop-types "^15.8.1" + protobufjs "^7.1.1" query-string "^7.1.1" react-query "^3.34.12" react-router-dom "6" @@ -1710,6 +1666,59 @@ schema-utils "^3.0.0" source-map "^0.7.3" +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" + integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== + +"@protobufjs/eventemitter@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== + +"@protobufjs/fetch@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== + dependencies: + "@protobufjs/aspromise" "^1.1.1" + "@protobufjs/inquire" "^1.1.0" + +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== + +"@protobufjs/inquire@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" + integrity sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q== + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== + +"@protobufjs/utf8@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" + integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== + "@rollup/plugin-babel@^5.2.0": version "5.3.1" resolved "https://registry.yarnpkg.com/@rollup/plugin-babel/-/plugin-babel-5.3.1.tgz#04bc0608f4aa4b2e4b1aebf284344d0f68fda283" @@ -2373,6 +2382,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.32.tgz#51d59d7a90ef2d0ae961791e0900cad2393a0149" integrity sha512-eAIcfAvhf/BkHcf4pkLJ7ECpBAhh9kcxRBpip9cTiO+hf+aJrsxYxBeS6OXvOd9WqNAJmavXVpZvY1rBjNsXmw== +"@types/node@>=13.7.0": + version "18.8.3" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.8.3.tgz#ce750ab4017effa51aed6a7230651778d54e327c" + integrity sha512-0os9vz6BpGwxGe9LOhgP/ncvYN5Tx1fNcd2TM3rD/aCGBkysb+ZWpXEocG24h6ZzOi13+VB8HndAQFezsSOw1w== + "@types/node@^16.7.13": version "16.11.34" resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.34.tgz#520224e4be4448c279ecad09639ab460cc441a50" @@ -6947,6 +6961,11 @@ lodash@^4.17.15, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7.0: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== +long@^5.0.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/long/-/long-5.2.0.tgz#2696dadf4b4da2ce3f6f6b89186085d94d52fd61" + integrity sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w== + loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -8252,6 +8271,24 @@ property-information@^5.0.0, property-information@^5.3.0: dependencies: xtend "^4.0.0" +protobufjs@^7.1.1: + version "7.1.2" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.1.2.tgz#a0cf6aeaf82f5625bffcf5a38b7cd2a7de05890c" + integrity sha512-4ZPTPkXCdel3+L81yw3dG6+Kq3umdWKh7Dc7GW/CpNk4SX3hK58iPCWeCyhVTDrbkNeKrYNZ7EojM5WDaEWTLQ== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/node" ">=13.7.0" + long "^5.0.0" + proxy-addr@~2.0.7: version "2.0.7" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" @@ -8403,13 +8440,14 @@ react-dev-utils@^12.0.1: strip-ansi "^6.0.1" text-table "^0.2.0" -react-dom@^18.1.0: - version "18.1.0" - resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-18.1.0.tgz#7f6dd84b706408adde05e1df575b3a024d7e8a2f" - integrity sha512-fU1Txz7Budmvamp7bshe4Zi32d0ll7ect+ccxNu9FlObT605GOEB8BfO4tmRJ39R5Zj831VCpvQ05QPBW5yb+w== +react-dom@^17.0.2: + version "17.0.2" + resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-17.0.2.tgz#ecffb6845e3ad8dbfcdc498f0d0a939736502c23" + integrity sha512-s4h96KtLDUQlsENhMn1ar8t2bEa+q/YAtj8pPPdIjPDGBDIVNsrD9aXNWqspUe6AzKCIG0C1HZZLqLV7qpOBGA== dependencies: loose-envify "^1.1.0" - scheduler "^0.22.0" + object-assign "^4.1.1" + scheduler "^0.20.2" react-dropzone@^11.5.3: version "11.7.1" @@ -8481,7 +8519,7 @@ react-is@^18.0.0: resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.1.0.tgz#61aaed3096d30eacf2a2127118b5b41387d32a67" integrity sha512-Fl7FuabXsJnV5Q1qIOQwx/sagGF18kogb4gpfcG4gjLBWO0WDiiz1ko/ExayuxE7InyQkBLkxRFG5oxY6Uu3Kg== -react-query@^3.34.12, react-query@^3.39.0: +react-query@^3.34.12: version "3.39.0" resolved "https://registry.yarnpkg.com/react-query/-/react-query-3.39.0.tgz#0caca7b0da98e65008bbcd4df0d25618c2100050" integrity sha512-Od0IkSuS79WJOhzWBx/ys0x13+7wFqgnn64vBqqAAnZ9whocVhl/y1padD5uuZ6EIkXbFbInax0qvY7zGM0thA== @@ -8526,7 +8564,7 @@ react-remove-scroll@^2.5.2: use-callback-ref "^1.3.0" use-sidecar "^1.1.2" -react-router-dom@6, react-router-dom@^6.3.0: +react-router-dom@6: version "6.3.0" resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.3.0.tgz#a0216da813454e521905b5fa55e0e5176123f43d" integrity sha512-uaJj7LKytRxZNQV8+RbzJWnJ8K2nPsOOEuX7aQstlMZKQT0164C+X2w6bnkqU3sjtLvpd5ojrezAyfZ1+0sStw== @@ -8541,7 +8579,7 @@ react-router@6.3.0: dependencies: history "^5.2.0" -react-scripts@5.0.1, react-scripts@^5.0.0: +react-scripts@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/react-scripts/-/react-scripts-5.0.1.tgz#6285dbd65a8ba6e49ca8d651ce30645a6d980003" integrity sha512-8VAmEm/ZAwQzJ+GOMLbBsTdDKOpuZh7RPs0UymvBR2vRk4iZWCskjbFnxqjrzoIvlNNRZ3QJFx6/qDSi6zSnaQ== @@ -8618,12 +8656,13 @@ react-window@^1.8.6: "@babel/runtime" "^7.0.0" memoize-one ">=3.1.1 <6" -react@^18.1.0: - version "18.1.0" - resolved "https://registry.yarnpkg.com/react/-/react-18.1.0.tgz#6f8620382decb17fdc5cc223a115e2adbf104890" - integrity sha512-4oL8ivCz5ZEPyclFQXaNksK3adutVS8l2xzZU0cqEFrE9Sb7fC0EFK5uEk74wIreL1DERyjvsU915j1pcT2uEQ== +react@^17.0.2: + version "17.0.2" + resolved "https://registry.yarnpkg.com/react/-/react-17.0.2.tgz#d0b5cc516d29eb3eee383f75b62864cfb6800037" + integrity sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA== dependencies: loose-envify "^1.1.0" + object-assign "^4.1.1" readable-stream@^2.0.1: version "2.3.7" @@ -9001,12 +9040,13 @@ saxes@^5.0.1: dependencies: xmlchars "^2.2.0" -scheduler@^0.22.0: - version "0.22.0" - resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.22.0.tgz#83a5d63594edf074add9a7198b1bae76c3db01b8" - integrity sha512-6QAm1BgQI88NPYymgGQLCZgvep4FyePDWFpXVK+zNSUgHwlqpJy8VEh8Et0KxTACS4VWwMousBElAZOH9nkkoQ== +scheduler@^0.20.2: + version "0.20.2" + resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.20.2.tgz#4baee39436e34aa93b4874bddcbf0fe8b8b50e91" + integrity sha512-2eWfGgAqqWFGqtdMmcL5zCMK1U8KlXv8SQFGglL3CEtd0aDVDWgeF/YoCmvln55m5zSk3J/20hTaSBeSObsQDQ== dependencies: loose-envify "^1.1.0" + object-assign "^4.1.1" schema-utils@2.7.0: version "2.7.0" diff --git a/sdk/python/feast/ui_server.py b/sdk/python/feast/ui_server.py index 4d1fd67dc1e..f79030e8d35 100644 --- a/sdk/python/feast/ui_server.py +++ b/sdk/python/feast/ui_server.py @@ -30,14 +30,14 @@ def get_app( ) # Asynchronously refresh registry, notifying shutdown and canceling the active timer if the app is shutting down - registry_json = "" + registry_proto = None shutting_down = False active_timer: Optional[threading.Timer] = None def async_refresh(): store.refresh_registry() - nonlocal registry_json - registry_json = get_registry_dump(store.config, store.repo_path) + nonlocal registry_proto + registry_proto = store.registry.proto() if shutting_down: return nonlocal active_timer @@ -70,7 +70,10 @@ def shutdown_event(): @app.get("/registry") def read_registry(): - return json.loads(registry_json) + return Response( + content=registry_proto.SerializeToString(), + media_type="application/octet-stream", + ) # For all other paths (such as paths that would otherwise be handled by react router), pass to React @app.api_route("/p/{path_name:path}", methods=["GET"]) diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 1b999341597..50b1e73c86c 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -1,6 +1,8 @@ +import os import typing from collections import defaultdict from datetime import datetime +from pathlib import Path from typing import Dict, List, Optional, Tuple, Union import pandas as pd @@ -9,6 +11,7 @@ from dateutil.tz import tzlocal from pytz import utc +from feast.constants import FEAST_FS_YAML_FILE_PATH_ENV_NAME from feast.entity import Entity from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto @@ -51,6 +54,14 @@ def maybe_local_tz(t: datetime) -> datetime: return t +def get_default_yaml_file_path(repo_path: Path) -> Path: + if FEAST_FS_YAML_FILE_PATH_ENV_NAME in os.environ: + yaml_path = os.environ[FEAST_FS_YAML_FILE_PATH_ENV_NAME] + return Path(yaml_path) + else: + return repo_path / "feature_store.yaml" + + def _get_requested_feature_views_to_features_dict( feature_refs: List[str], feature_views: List["FeatureView"], diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 9d10b2c3132..93729efc518 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt # -absl-py==1.2.0 - # via tensorflow-metadata adal==1.2.7 # via # azure-datalake-store @@ -14,13 +12,13 @@ adlfs==0.5.9 # via feast (setup.py) aiobotocore==2.1.2 # via s3fs -aiohttp==3.8.1 +aiohttp==3.8.3 # via # adlfs # aiobotocore # gcsfs # s3fs -aioitertools==0.10.0 +aioitertools==0.11.0 # via aiobotocore aiosignal==1.2.0 # via aiohttp @@ -56,7 +54,7 @@ attrs==22.1.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.25.0 +azure-core==1.25.1 # via # adlfs # azure-identity @@ -64,7 +62,7 @@ azure-core==1.25.0 # msrest azure-datalake-store==0.0.52 # via adlfs -azure-identity==1.10.0 +azure-identity==1.11.0 # via # adlfs # feast (setup.py) @@ -76,7 +74,7 @@ babel==2.10.3 # via sphinx backcall==0.2.0 # via ipython -black==22.6.0 +black==22.8.0 # via feast (setup.py) boto3==1.20.23 # via @@ -102,7 +100,7 @@ cachetools==5.2.0 # via google-auth cassandra-driver==3.25.0 # via feast (setup.py) -certifi==2022.6.15 +certifi==2022.9.24 # via # kubernetes # minio @@ -116,7 +114,7 @@ cffi==1.15.1 # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via # aiohttp # requests @@ -131,13 +129,13 @@ click==8.1.3 # moreorless # pip-tools # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via # feast (setup.py) # great-expectations -coverage[toml]==6.4.4 +coverage[toml]==6.5.0 # via pytest-cov cryptography==35.0.0 # via @@ -155,7 +153,7 @@ dask==2022.1.1 # via feast (setup.py) dataclasses==0.6 # via great-expectations -db-dtypes==1.0.3 +db-dtypes==1.0.4 # via google-cloud-bigquery decorator==5.1.1 # via @@ -169,7 +167,7 @@ dill==0.3.5.1 # via # feast (setup.py) # multiprocess -distlib==0.3.5 +distlib==0.3.6 # via virtualenv docker==6.0.0 # via @@ -183,19 +181,21 @@ entrypoints==0.4 # via altair execnet==1.9.0 # via pytest-xdist -executing==0.10.0 +executing==1.1.0 # via stack-data -fastapi==0.79.1 +fastapi==0.85.0 # via feast (setup.py) -fastavro==1.6.0 +fastavro==1.6.1 # via # feast (setup.py) # pandavro -fastjsonschema==2.16.1 +fastjsonschema==2.16.2 # via nbformat filelock==3.8.0 - # via virtualenv -firebase-admin==5.2.0 + # via + # snowflake-connector-python + # virtualenv +firebase-admin==5.4.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -215,20 +215,21 @@ gcsfs==2022.1.0 # via feast (setup.py) geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.8.2 +google-api-core[grpc]==2.10.1 # via # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-core # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.57.0 +google-api-python-client==2.63.0 # via firebase-admin -google-auth==2.10.0 +google-auth==2.12.0 # via # gcsfs # google-api-core @@ -240,63 +241,68 @@ google-auth==2.10.0 # kubernetes google-auth-httplib2==0.1.0 # via google-api-python-client -google-auth-oauthlib==0.5.2 +google-auth-oauthlib==0.5.3 # via gcsfs -google-cloud-bigquery[pandas]==3.3.2 +google-cloud-bigquery[pandas]==3.3.3 # via feast (setup.py) -google-cloud-bigquery-storage==2.14.2 +google-cloud-bigquery-storage==2.16.1 # via # feast (setup.py) # google-cloud-bigquery +google-cloud-bigtable==2.12.0 + # via feast (setup.py) google-cloud-core==2.3.2 # via # google-cloud-bigquery + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.8.1 # via feast (setup.py) -google-cloud-firestore==2.6.1 +google-cloud-firestore==2.7.0 # via firebase-admin google-cloud-storage==2.5.0 # via # feast (setup.py) # firebase-admin # gcsfs -google-crc32c==1.3.0 +google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.3.3 +google-resumable-media==2.4.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos==1.56.4 +googleapis-common-protos[grpc]==1.56.4 # via # feast (setup.py) # google-api-core + # grpc-google-iam-v1 # grpcio-status - # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.47.0 +grpc-google-iam-v1==0.12.4 + # via google-cloud-bigtable +grpcio==1.49.1 # via # feast (setup.py) # google-api-core # google-cloud-bigquery + # googleapis-common-protos + # grpc-google-iam-v1 # grpcio-reflection # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.47.0 +grpcio-reflection==1.49.1 # via feast (setup.py) -grpcio-status==1.47.0 +grpcio-status==1.49.1 # via google-api-core -grpcio-testing==1.47.0 +grpcio-testing==1.49.1 # via feast (setup.py) -grpcio-tools==1.47.0 +grpcio-tools==1.49.1 # via feast (setup.py) -h11==0.13.0 +h11==0.14.0 # via uvicorn happybase==1.2.0 # via feast (setup.py) @@ -306,11 +312,11 @@ httplib2==0.20.4 # via # google-api-python-client # google-auth-httplib2 -httptools==0.4.0 +httptools==0.5.0 # via uvicorn -identify==2.5.3 +identify==2.5.5 # via pre-commit -idna==3.3 +idna==3.4 # via # anyio # requests @@ -322,7 +328,7 @@ importlib-metadata==4.12.0 # via great-expectations iniconfig==1.1.1 # via pytest -ipython==8.4.0 +ipython==8.5.0 # via great-expectations isodate==0.6.1 # via msrest @@ -345,7 +351,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.13.0 +jsonschema==4.16.0 # via # altair # feast (setup.py) @@ -377,7 +383,7 @@ moreorless==0.4.0 # via bowler moto==3.1.18 # via feast (setup.py) -msal==1.18.0 +msal==1.19.0 # via # azure-identity # msal-extensions @@ -397,7 +403,7 @@ multidict==6.0.2 # yarl multiprocess==0.70.13 # via bytewax -mypy==0.971 +mypy==0.981 # via # feast (setup.py) # sqlalchemy @@ -409,11 +415,11 @@ mypy-protobuf==3.1 # via feast (setup.py) mysqlclient==2.1.1 # via feast (setup.py) -nbformat==5.4.0 +nbformat==5.6.1 # via great-expectations nodeenv==1.7.0 # via pre-commit -numpy==1.23.2 +numpy==1.23.3 # via # altair # db-dtypes @@ -423,7 +429,7 @@ numpy==1.23.2 # pandavro # pyarrow # scipy -oauthlib==3.2.0 +oauthlib==3.2.1 # via requests-oauthlib oscrypto==1.3.0 # via snowflake-connector-python @@ -439,7 +445,7 @@ packaging==21.3 # pytest # redis # sphinx -pandas==1.4.3 +pandas==1.4.4 # via # altair # db-dtypes @@ -454,7 +460,7 @@ parso==0.8.3 # via jedi partd==1.3.0 # via dask -pathspec==0.9.0 +pathspec==0.10.1 # via black pbr==5.10.0 # via mock @@ -478,21 +484,23 @@ portalocker==2.5.1 # via msal-extensions pre-commit==2.20.0 # via feast (setup.py) -prompt-toolkit==3.0.30 +prompt-toolkit==3.0.31 # via ipython -proto-plus==1.22.0 +proto-plus==1.22.1 # via # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==3.20.1 +protobuf==4.21.7 # via # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # googleapis-common-protos @@ -502,7 +510,6 @@ protobuf==3.20.1 # grpcio-tools # mypy-protobuf # proto-plus - # tensorflow-metadata psutil==5.9.0 # via feast (setup.py) psycopg2-binary==2.9.3 @@ -519,7 +526,7 @@ py-cpuinfo==8.0.0 # via pytest-benchmark py4j==0.10.9.5 # via pyspark -pyarrow==6.0.1 +pyarrow==8.0.0 # via # db-dtypes # feast (setup.py) @@ -539,7 +546,7 @@ pycparser==2.21 # via cffi pycryptodomex==3.15.0 # via snowflake-connector-python -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -550,17 +557,21 @@ pygments==2.13.0 # feast (setup.py) # ipython # sphinx -pyjwt[crypto]==2.4.0 +pyjwt[crypto]==2.5.0 # via # adal # msal # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) +pymysql==1.0.2 + # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 - # via snowflake-connector-python + # via + # feast (setup.py) + # snowflake-connector-python pyparsing==2.4.7 # via # great-expectations @@ -570,7 +581,7 @@ pyrsistent==0.18.1 # via jsonschema pyspark==3.3.0 # via feast (setup.py) -pytest==7.1.2 +pytest==7.1.3 # via # feast (setup.py) # pytest-benchmark @@ -583,7 +594,7 @@ pytest==7.1.2 # pytest-xdist pytest-benchmark==3.4.1 # via feast (setup.py) -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via feast (setup.py) pytest-forked==1.4.0 # via pytest-xdist @@ -606,7 +617,7 @@ python-dateutil==2.8.2 # kubernetes # moto # pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via @@ -664,7 +675,7 @@ s3fs==2022.1.0 # via feast (setup.py) s3transfer==0.5.2 # via boto3 -scipy==1.9.0 +scipy==1.9.1 # via great-expectations six==1.16.0 # via @@ -682,11 +693,11 @@ six==1.16.0 # msrestazure # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==2.7.8 +snowflake-connector-python[pandas]==2.8.0 # via feast (setup.py) sphinx==4.3.2 # via @@ -706,23 +717,21 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -stack-data==0.4.0 +stack-data==0.5.1 # via ipython -starlette==0.19.1 +starlette==0.20.4 # via fastapi tabulate==0.8.10 # via feast (setup.py) -tenacity==8.0.1 +tenacity==8.1.0 # via feast (setup.py) -tensorflow-metadata==1.9.0 - # via feast (setup.py) -termcolor==1.1.0 +termcolor==2.0.1 # via great-expectations -testcontainers==3.6.1 +testcontainers==3.7.0 # via feast (setup.py) thriftpy2==0.4.14 # via happybase @@ -743,39 +752,43 @@ toolz==0.12.0 # altair # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via # feast (setup.py) # great-expectations -traitlets==5.3.0 +traitlets==5.4.0 # via # ipython # jupyter-core # matplotlib-inline # nbformat -trino==0.315.0 +trino==0.316.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==3.19.22 +types-cryptography==3.3.23 + # via pyjwt +types-protobuf==3.20.4 # via # feast (setup.py) # mypy-protobuf +types-pymysql==1.0.19 + # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 # via feast (setup.py) -types-pyyaml==6.0.11 +types-pyyaml==6.0.12 # via feast (setup.py) -types-redis==4.3.14 +types-redis==4.3.21 # via feast (setup.py) -types-requests==2.28.9 +types-requests==2.28.11 # via feast (setup.py) -types-setuptools==65.1.0 +types-setuptools==65.4.0.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.23 +types-urllib3==1.26.25 # via types-requests typing-extensions==4.3.0 # via @@ -783,14 +796,15 @@ typing-extensions==4.3.0 # great-expectations # mypy # pydantic + # snowflake-connector-python # sqlalchemy2-stubs -tzdata==2022.2 +tzdata==2022.4 # via pytz-deprecation-shim tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.11 +urllib3==1.26.12 # via # botocore # docker @@ -800,19 +814,20 @@ urllib3==1.26.11 # minio # requests # responses -uvicorn[standard]==0.18.2 + # snowflake-connector-python +uvicorn[standard]==0.18.3 # via feast (setup.py) -uvloop==0.16.0 +uvloop==0.17.0 # via uvicorn -virtualenv==20.16.3 +virtualenv==20.16.5 # via pre-commit volatile==2.1.0 # via bowler -watchfiles==0.16.1 +watchfiles==0.17.0 # via uvicorn wcwidth==0.2.5 # via prompt-toolkit -websocket-client==1.3.3 +websocket-client==1.4.1 # via # docker # kubernetes diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index ac12befb87c..4770b765e50 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt # -absl-py==1.2.0 - # via tensorflow-metadata anyio==3.6.1 # via # starlette @@ -20,9 +18,9 @@ bowler==0.9.0 # via feast (setup.py) cachetools==5.2.0 # via google-auth -certifi==2022.6.15 +certifi==2022.9.24 # via requests -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via requests click==8.1.3 # via @@ -30,7 +28,7 @@ click==8.1.3 # feast (setup.py) # moreorless # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via feast (setup.py) @@ -38,44 +36,41 @@ dask==2022.1.1 # via feast (setup.py) dill==0.3.5.1 # via feast (setup.py) -fastapi==0.79.1 +fastapi==0.85.0 # via feast (setup.py) -fastavro==1.6.0 +fastavro==1.6.1 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2022.7.1 +fsspec==2022.8.2 # via dask -google-api-core==2.8.2 +google-api-core==2.10.1 # via feast (setup.py) -google-auth==2.10.0 +google-auth==2.12.0 # via google-api-core googleapis-common-protos==1.56.4 # via # feast (setup.py) # google-api-core - # tensorflow-metadata -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.47.0 +grpcio==1.49.1 # via # feast (setup.py) # grpcio-reflection -grpcio-reflection==1.47.0 +grpcio-reflection==1.49.1 # via feast (setup.py) -h11==0.13.0 +h11==0.14.0 # via uvicorn -httptools==0.4.0 +httptools==0.5.0 # via uvicorn -idna==3.3 +idna==3.4 # via # anyio # requests jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.13.0 +jsonschema==4.16.0 # via feast (setup.py) locket==1.0.0 # via partd @@ -85,11 +80,11 @@ mmh3==3.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==0.971 +mypy==0.981 # via sqlalchemy mypy-extensions==0.4.3 # via mypy -numpy==1.23.2 +numpy==1.23.3 # via # feast (setup.py) # pandas @@ -97,7 +92,7 @@ numpy==1.23.2 # pyarrow packaging==21.3 # via dask -pandas==1.4.3 +pandas==1.5.0 # via # feast (setup.py) # pandavro @@ -105,16 +100,15 @@ pandavro==1.5.2 # via feast (setup.py) partd==1.3.0 # via dask -proto-plus==1.22.0 +proto-plus==1.22.1 # via feast (setup.py) -protobuf==3.20.1 +protobuf==4.21.7 # via # feast (setup.py) # google-api-core # googleapis-common-protos # grpcio-reflection # proto-plus - # tensorflow-metadata pyarrow==8.0.0 # via feast (setup.py) pyasn1==0.4.8 @@ -123,7 +117,7 @@ pyasn1==0.4.8 # rsa pyasn1-modules==0.2.8 # via google-auth -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -135,7 +129,7 @@ pyrsistent==0.18.1 # via jsonschema python-dateutil==2.8.2 # via pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via pandas @@ -154,19 +148,17 @@ six==1.16.0 # grpcio # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -starlette==0.19.1 +starlette==0.20.4 # via fastapi tabulate==0.8.10 # via feast (setup.py) -tenacity==8.0.1 - # via feast (setup.py) -tensorflow-metadata==1.9.0 +tenacity==8.1.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) @@ -176,7 +168,7 @@ toolz==0.12.0 # via # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) @@ -185,15 +177,15 @@ typing-extensions==4.3.0 # mypy # pydantic # sqlalchemy2-stubs -urllib3==1.26.11 +urllib3==1.26.12 # via requests -uvicorn[standard]==0.18.2 +uvicorn[standard]==0.18.3 # via feast (setup.py) -uvloop==0.16.0 +uvloop==0.17.0 # via uvicorn volatile==2.1.0 # via bowler -watchfiles==0.16.1 +watchfiles==0.17.0 # via uvicorn websockets==10.3 # via uvicorn diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 93011cfdcf4..e1b99210aa5 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.8-ci-requirements.txt # -absl-py==1.2.0 - # via tensorflow-metadata adal==1.2.7 # via # azure-datalake-store @@ -14,13 +12,13 @@ adlfs==0.5.9 # via feast (setup.py) aiobotocore==2.1.2 # via s3fs -aiohttp==3.8.1 +aiohttp==3.8.3 # via # adlfs # aiobotocore # gcsfs # s3fs -aioitertools==0.10.0 +aioitertools==0.11.0 # via aiobotocore aiosignal==1.2.0 # via aiohttp @@ -56,7 +54,7 @@ attrs==22.1.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.25.0 +azure-core==1.25.1 # via # adlfs # azure-identity @@ -64,7 +62,7 @@ azure-core==1.25.0 # msrest azure-datalake-store==0.0.52 # via adlfs -azure-identity==1.10.0 +azure-identity==1.11.0 # via # adlfs # feast (setup.py) @@ -80,7 +78,7 @@ backports-zoneinfo==0.2.1 # via # pytz-deprecation-shim # tzlocal -black==22.6.0 +black==22.8.0 # via feast (setup.py) boto3==1.20.23 # via @@ -106,7 +104,7 @@ cachetools==5.2.0 # via google-auth cassandra-driver==3.25.0 # via feast (setup.py) -certifi==2022.6.15 +certifi==2022.9.24 # via # kubernetes # minio @@ -120,7 +118,7 @@ cffi==1.15.1 # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via # aiohttp # requests @@ -135,13 +133,13 @@ click==8.1.3 # moreorless # pip-tools # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via # feast (setup.py) # great-expectations -coverage[toml]==6.4.4 +coverage[toml]==6.5.0 # via pytest-cov cryptography==35.0.0 # via @@ -159,7 +157,7 @@ dask==2022.1.1 # via feast (setup.py) dataclasses==0.6 # via great-expectations -db-dtypes==1.0.3 +db-dtypes==1.0.4 # via google-cloud-bigquery decorator==5.1.1 # via @@ -173,7 +171,7 @@ dill==0.3.5.1 # via # feast (setup.py) # multiprocess -distlib==0.3.5 +distlib==0.3.6 # via virtualenv docker==6.0.0 # via @@ -187,19 +185,21 @@ entrypoints==0.4 # via altair execnet==1.9.0 # via pytest-xdist -executing==0.10.0 +executing==1.1.0 # via stack-data -fastapi==0.79.1 +fastapi==0.85.0 # via feast (setup.py) -fastavro==1.6.0 +fastavro==1.6.1 # via # feast (setup.py) # pandavro -fastjsonschema==2.16.1 +fastjsonschema==2.16.2 # via nbformat filelock==3.8.0 - # via virtualenv -firebase-admin==5.2.0 + # via + # snowflake-connector-python + # virtualenv +firebase-admin==5.4.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -219,20 +219,21 @@ gcsfs==2022.1.0 # via feast (setup.py) geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.8.2 +google-api-core[grpc]==2.10.1 # via # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-core # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.57.0 +google-api-python-client==2.63.0 # via firebase-admin -google-auth==2.10.0 +google-auth==2.12.0 # via # gcsfs # google-api-core @@ -244,63 +245,68 @@ google-auth==2.10.0 # kubernetes google-auth-httplib2==0.1.0 # via google-api-python-client -google-auth-oauthlib==0.5.2 +google-auth-oauthlib==0.5.3 # via gcsfs -google-cloud-bigquery[pandas]==3.3.2 +google-cloud-bigquery[pandas]==3.3.3 # via feast (setup.py) -google-cloud-bigquery-storage==2.14.2 +google-cloud-bigquery-storage==2.16.1 # via # feast (setup.py) # google-cloud-bigquery +google-cloud-bigtable==2.12.0 + # via feast (setup.py) google-cloud-core==2.3.2 # via # google-cloud-bigquery + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.8.1 # via feast (setup.py) -google-cloud-firestore==2.6.1 +google-cloud-firestore==2.7.0 # via firebase-admin google-cloud-storage==2.5.0 # via # feast (setup.py) # firebase-admin # gcsfs -google-crc32c==1.3.0 +google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.3.3 +google-resumable-media==2.4.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos==1.56.4 +googleapis-common-protos[grpc]==1.56.4 # via # feast (setup.py) # google-api-core + # grpc-google-iam-v1 # grpcio-status - # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.47.0 +grpc-google-iam-v1==0.12.4 + # via google-cloud-bigtable +grpcio==1.49.1 # via # feast (setup.py) # google-api-core # google-cloud-bigquery + # googleapis-common-protos + # grpc-google-iam-v1 # grpcio-reflection # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.47.0 +grpcio-reflection==1.49.1 # via feast (setup.py) -grpcio-status==1.47.0 +grpcio-status==1.49.1 # via google-api-core -grpcio-testing==1.47.0 +grpcio-testing==1.49.1 # via feast (setup.py) -grpcio-tools==1.47.0 +grpcio-tools==1.49.1 # via feast (setup.py) -h11==0.13.0 +h11==0.14.0 # via uvicorn happybase==1.2.0 # via feast (setup.py) @@ -310,11 +316,11 @@ httplib2==0.20.4 # via # google-api-python-client # google-auth-httplib2 -httptools==0.4.0 +httptools==0.5.0 # via uvicorn -identify==2.5.3 +identify==2.5.5 # via pre-commit -idna==3.3 +idna==3.4 # via # anyio # requests @@ -328,7 +334,7 @@ importlib-resources==5.9.0 # via jsonschema iniconfig==1.1.1 # via pytest -ipython==8.4.0 +ipython==8.5.0 # via great-expectations isodate==0.6.1 # via msrest @@ -351,7 +357,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.13.0 +jsonschema==4.16.0 # via # altair # feast (setup.py) @@ -383,7 +389,7 @@ moreorless==0.4.0 # via bowler moto==3.1.18 # via feast (setup.py) -msal==1.18.0 +msal==1.19.0 # via # azure-identity # msal-extensions @@ -403,7 +409,7 @@ multidict==6.0.2 # yarl multiprocess==0.70.13 # via bytewax -mypy==0.971 +mypy==0.981 # via # feast (setup.py) # sqlalchemy @@ -415,11 +421,11 @@ mypy-protobuf==3.1 # via feast (setup.py) mysqlclient==2.1.1 # via feast (setup.py) -nbformat==5.4.0 +nbformat==5.6.1 # via great-expectations nodeenv==1.7.0 # via pre-commit -numpy==1.23.2 +numpy==1.23.3 # via # altair # db-dtypes @@ -429,7 +435,7 @@ numpy==1.23.2 # pandavro # pyarrow # scipy -oauthlib==3.2.0 +oauthlib==3.2.1 # via requests-oauthlib oscrypto==1.3.0 # via snowflake-connector-python @@ -445,7 +451,7 @@ packaging==21.3 # pytest # redis # sphinx -pandas==1.4.3 +pandas==1.4.4 # via # altair # db-dtypes @@ -460,7 +466,7 @@ parso==0.8.3 # via jedi partd==1.3.0 # via dask -pathspec==0.9.0 +pathspec==0.10.1 # via black pbr==5.10.0 # via mock @@ -486,21 +492,23 @@ portalocker==2.5.1 # via msal-extensions pre-commit==2.20.0 # via feast (setup.py) -prompt-toolkit==3.0.30 +prompt-toolkit==3.0.31 # via ipython -proto-plus==1.22.0 +proto-plus==1.22.1 # via # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==3.20.1 +protobuf==4.21.7 # via # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # googleapis-common-protos @@ -510,7 +518,6 @@ protobuf==3.20.1 # grpcio-tools # mypy-protobuf # proto-plus - # tensorflow-metadata psutil==5.9.0 # via feast (setup.py) psycopg2-binary==2.9.3 @@ -527,7 +534,7 @@ py-cpuinfo==8.0.0 # via pytest-benchmark py4j==0.10.9.5 # via pyspark -pyarrow==6.0.1 +pyarrow==8.0.0 # via # db-dtypes # feast (setup.py) @@ -547,7 +554,7 @@ pycparser==2.21 # via cffi pycryptodomex==3.15.0 # via snowflake-connector-python -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -558,17 +565,21 @@ pygments==2.13.0 # feast (setup.py) # ipython # sphinx -pyjwt[crypto]==2.4.0 +pyjwt[crypto]==2.5.0 # via # adal # msal # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) +pymysql==1.0.2 + # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 - # via snowflake-connector-python + # via + # feast (setup.py) + # snowflake-connector-python pyparsing==2.4.7 # via # great-expectations @@ -578,7 +589,7 @@ pyrsistent==0.18.1 # via jsonschema pyspark==3.3.0 # via feast (setup.py) -pytest==7.1.2 +pytest==7.1.3 # via # feast (setup.py) # pytest-benchmark @@ -591,7 +602,7 @@ pytest==7.1.2 # pytest-xdist pytest-benchmark==3.4.1 # via feast (setup.py) -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via feast (setup.py) pytest-forked==1.4.0 # via pytest-xdist @@ -614,7 +625,7 @@ python-dateutil==2.8.2 # kubernetes # moto # pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via @@ -674,7 +685,7 @@ s3fs==2022.1.0 # via feast (setup.py) s3transfer==0.5.2 # via boto3 -scipy==1.9.0 +scipy==1.9.1 # via great-expectations six==1.16.0 # via @@ -692,11 +703,11 @@ six==1.16.0 # msrestazure # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==2.7.8 +snowflake-connector-python[pandas]==2.8.0 # via feast (setup.py) sphinx==4.3.2 # via @@ -716,23 +727,21 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -stack-data==0.4.0 +stack-data==0.5.1 # via ipython -starlette==0.19.1 +starlette==0.20.4 # via fastapi tabulate==0.8.10 # via feast (setup.py) -tenacity==8.0.1 +tenacity==8.1.0 # via feast (setup.py) -tensorflow-metadata==1.9.0 - # via feast (setup.py) -termcolor==1.1.0 +termcolor==2.0.1 # via great-expectations -testcontainers==3.6.1 +testcontainers==3.7.0 # via feast (setup.py) thriftpy2==0.4.14 # via happybase @@ -753,39 +762,43 @@ toolz==0.12.0 # altair # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via # feast (setup.py) # great-expectations -traitlets==5.3.0 +traitlets==5.4.0 # via # ipython # jupyter-core # matplotlib-inline # nbformat -trino==0.315.0 +trino==0.316.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==3.19.22 +types-cryptography==3.3.23 + # via pyjwt +types-protobuf==3.20.4 # via # feast (setup.py) # mypy-protobuf +types-pymysql==1.0.19 + # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 # via feast (setup.py) -types-pyyaml==6.0.11 +types-pyyaml==6.0.12 # via feast (setup.py) -types-redis==4.3.14 +types-redis==4.3.21 # via feast (setup.py) -types-requests==2.28.9 +types-requests==2.28.11 # via feast (setup.py) -types-setuptools==65.1.0 +types-setuptools==65.4.0.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.23 +types-urllib3==1.26.25 # via types-requests typing-extensions==4.3.0 # via @@ -795,15 +808,16 @@ typing-extensions==4.3.0 # great-expectations # mypy # pydantic + # snowflake-connector-python # sqlalchemy2-stubs # starlette -tzdata==2022.2 +tzdata==2022.4 # via pytz-deprecation-shim tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.11 +urllib3==1.26.12 # via # botocore # docker @@ -813,19 +827,20 @@ urllib3==1.26.11 # minio # requests # responses -uvicorn[standard]==0.18.2 + # snowflake-connector-python +uvicorn[standard]==0.18.3 # via feast (setup.py) -uvloop==0.16.0 +uvloop==0.17.0 # via uvicorn -virtualenv==20.16.3 +virtualenv==20.16.5 # via pre-commit volatile==2.1.0 # via bowler -watchfiles==0.16.1 +watchfiles==0.17.0 # via uvicorn wcwidth==0.2.5 # via prompt-toolkit -websocket-client==1.3.3 +websocket-client==1.4.1 # via # docker # kubernetes diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index c2aef636733..9a321bf9a50 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --output-file=sdk/python/requirements/py3.8-requirements.txt # -absl-py==1.2.0 - # via tensorflow-metadata anyio==3.6.1 # via # starlette @@ -20,9 +18,9 @@ bowler==0.9.0 # via feast (setup.py) cachetools==5.2.0 # via google-auth -certifi==2022.6.15 +certifi==2022.9.24 # via requests -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via requests click==8.1.3 # via @@ -30,7 +28,7 @@ click==8.1.3 # feast (setup.py) # moreorless # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via feast (setup.py) @@ -38,38 +36,35 @@ dask==2022.1.1 # via feast (setup.py) dill==0.3.5.1 # via feast (setup.py) -fastapi==0.79.1 +fastapi==0.85.0 # via feast (setup.py) -fastavro==1.6.0 +fastavro==1.6.1 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2022.7.1 +fsspec==2022.8.2 # via dask -google-api-core==2.8.2 +google-api-core==2.10.1 # via feast (setup.py) -google-auth==2.10.0 +google-auth==2.12.0 # via google-api-core googleapis-common-protos==1.56.4 # via # feast (setup.py) # google-api-core - # tensorflow-metadata -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.47.0 +grpcio==1.49.1 # via # feast (setup.py) # grpcio-reflection -grpcio-reflection==1.47.0 +grpcio-reflection==1.49.1 # via feast (setup.py) -h11==0.13.0 +h11==0.14.0 # via uvicorn -httptools==0.4.0 +httptools==0.5.0 # via uvicorn -idna==3.3 +idna==3.4 # via # anyio # requests @@ -77,7 +72,7 @@ importlib-resources==5.9.0 # via jsonschema jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.13.0 +jsonschema==4.16.0 # via feast (setup.py) locket==1.0.0 # via partd @@ -87,11 +82,11 @@ mmh3==3.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==0.971 +mypy==0.981 # via sqlalchemy mypy-extensions==0.4.3 # via mypy -numpy==1.23.2 +numpy==1.23.3 # via # feast (setup.py) # pandas @@ -99,7 +94,7 @@ numpy==1.23.2 # pyarrow packaging==21.3 # via dask -pandas==1.4.3 +pandas==1.5.0 # via # feast (setup.py) # pandavro @@ -109,16 +104,15 @@ partd==1.3.0 # via dask pkgutil-resolve-name==1.3.10 # via jsonschema -proto-plus==1.22.0 +proto-plus==1.22.1 # via feast (setup.py) -protobuf==3.20.1 +protobuf==4.21.7 # via # feast (setup.py) # google-api-core # googleapis-common-protos # grpcio-reflection # proto-plus - # tensorflow-metadata pyarrow==8.0.0 # via feast (setup.py) pyasn1==0.4.8 @@ -127,7 +121,7 @@ pyasn1==0.4.8 # rsa pyasn1-modules==0.2.8 # via google-auth -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -139,7 +133,7 @@ pyrsistent==0.18.1 # via jsonschema python-dateutil==2.8.2 # via pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via pandas @@ -158,19 +152,17 @@ six==1.16.0 # grpcio # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -starlette==0.19.1 +starlette==0.20.4 # via fastapi tabulate==0.8.10 # via feast (setup.py) -tenacity==8.0.1 - # via feast (setup.py) -tensorflow-metadata==1.9.0 +tenacity==8.1.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) @@ -180,7 +172,7 @@ toolz==0.12.0 # via # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) @@ -190,15 +182,15 @@ typing-extensions==4.3.0 # pydantic # sqlalchemy2-stubs # starlette -urllib3==1.26.11 +urllib3==1.26.12 # via requests -uvicorn[standard]==0.18.2 +uvicorn[standard]==0.18.3 # via feast (setup.py) -uvloop==0.16.0 +uvloop==0.17.0 # via uvicorn volatile==2.1.0 # via bowler -watchfiles==0.16.1 +watchfiles==0.17.0 # via uvicorn websockets==10.3 # via uvicorn diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index e13eee056bc..af7b79ea229 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt # -absl-py==1.2.0 - # via tensorflow-metadata adal==1.2.7 # via # azure-datalake-store @@ -14,13 +12,13 @@ adlfs==0.5.9 # via feast (setup.py) aiobotocore==2.1.2 # via s3fs -aiohttp==3.8.1 +aiohttp==3.8.3 # via # adlfs # aiobotocore # gcsfs # s3fs -aioitertools==0.10.0 +aioitertools==0.11.0 # via aiobotocore aiosignal==1.2.0 # via aiohttp @@ -56,7 +54,7 @@ attrs==22.1.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.25.0 +azure-core==1.25.1 # via # adlfs # azure-identity @@ -64,7 +62,7 @@ azure-core==1.25.0 # msrest azure-datalake-store==0.0.52 # via adlfs -azure-identity==1.10.0 +azure-identity==1.11.0 # via # adlfs # feast (setup.py) @@ -76,7 +74,7 @@ babel==2.10.3 # via sphinx backcall==0.2.0 # via ipython -black==22.6.0 +black==22.8.0 # via feast (setup.py) boto3==1.20.23 # via @@ -102,7 +100,7 @@ cachetools==5.2.0 # via google-auth cassandra-driver==3.25.0 # via feast (setup.py) -certifi==2022.6.15 +certifi==2022.9.24 # via # kubernetes # minio @@ -116,7 +114,7 @@ cffi==1.15.1 # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via # aiohttp # requests @@ -131,13 +129,13 @@ click==8.1.3 # moreorless # pip-tools # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via # feast (setup.py) # great-expectations -coverage[toml]==6.4.4 +coverage[toml]==6.5.0 # via pytest-cov cryptography==35.0.0 # via @@ -155,7 +153,7 @@ dask==2022.1.1 # via feast (setup.py) dataclasses==0.6 # via great-expectations -db-dtypes==1.0.3 +db-dtypes==1.0.4 # via google-cloud-bigquery decorator==5.1.1 # via @@ -169,7 +167,7 @@ dill==0.3.5.1 # via # feast (setup.py) # multiprocess -distlib==0.3.5 +distlib==0.3.6 # via virtualenv docker==6.0.0 # via @@ -183,19 +181,21 @@ entrypoints==0.4 # via altair execnet==1.9.0 # via pytest-xdist -executing==0.10.0 +executing==1.1.0 # via stack-data -fastapi==0.79.1 +fastapi==0.85.0 # via feast (setup.py) -fastavro==1.6.0 +fastavro==1.6.1 # via # feast (setup.py) # pandavro -fastjsonschema==2.16.1 +fastjsonschema==2.16.2 # via nbformat filelock==3.8.0 - # via virtualenv -firebase-admin==5.2.0 + # via + # snowflake-connector-python + # virtualenv +firebase-admin==5.4.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -215,20 +215,21 @@ gcsfs==2022.1.0 # via feast (setup.py) geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.8.2 +google-api-core[grpc]==2.10.1 # via # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-core # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.57.0 +google-api-python-client==2.63.0 # via firebase-admin -google-auth==2.10.0 +google-auth==2.12.0 # via # gcsfs # google-api-core @@ -240,63 +241,68 @@ google-auth==2.10.0 # kubernetes google-auth-httplib2==0.1.0 # via google-api-python-client -google-auth-oauthlib==0.5.2 +google-auth-oauthlib==0.5.3 # via gcsfs -google-cloud-bigquery[pandas]==3.3.2 +google-cloud-bigquery[pandas]==3.3.3 # via feast (setup.py) -google-cloud-bigquery-storage==2.14.2 +google-cloud-bigquery-storage==2.16.1 # via # feast (setup.py) # google-cloud-bigquery +google-cloud-bigtable==2.12.0 + # via feast (setup.py) google-cloud-core==2.3.2 # via # google-cloud-bigquery + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.8.1 # via feast (setup.py) -google-cloud-firestore==2.6.1 +google-cloud-firestore==2.7.0 # via firebase-admin google-cloud-storage==2.5.0 # via # feast (setup.py) # firebase-admin # gcsfs -google-crc32c==1.3.0 +google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.3.3 +google-resumable-media==2.4.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos==1.56.4 +googleapis-common-protos[grpc]==1.56.4 # via # feast (setup.py) # google-api-core + # grpc-google-iam-v1 # grpcio-status - # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.47.0 +grpc-google-iam-v1==0.12.4 + # via google-cloud-bigtable +grpcio==1.49.1 # via # feast (setup.py) # google-api-core # google-cloud-bigquery + # googleapis-common-protos + # grpc-google-iam-v1 # grpcio-reflection # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.47.0 +grpcio-reflection==1.49.1 # via feast (setup.py) -grpcio-status==1.47.0 +grpcio-status==1.49.1 # via google-api-core -grpcio-testing==1.47.0 +grpcio-testing==1.49.1 # via feast (setup.py) -grpcio-tools==1.47.0 +grpcio-tools==1.49.1 # via feast (setup.py) -h11==0.13.0 +h11==0.14.0 # via uvicorn happybase==1.2.0 # via feast (setup.py) @@ -306,11 +312,11 @@ httplib2==0.20.4 # via # google-api-python-client # google-auth-httplib2 -httptools==0.4.0 +httptools==0.5.0 # via uvicorn -identify==2.5.3 +identify==2.5.5 # via pre-commit -idna==3.3 +idna==3.4 # via # anyio # requests @@ -322,7 +328,7 @@ importlib-metadata==4.12.0 # via great-expectations iniconfig==1.1.1 # via pytest -ipython==8.4.0 +ipython==8.5.0 # via great-expectations isodate==0.6.1 # via msrest @@ -345,7 +351,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.13.0 +jsonschema==4.16.0 # via # altair # feast (setup.py) @@ -377,7 +383,7 @@ moreorless==0.4.0 # via bowler moto==3.1.18 # via feast (setup.py) -msal==1.18.0 +msal==1.19.0 # via # azure-identity # msal-extensions @@ -397,7 +403,7 @@ multidict==6.0.2 # yarl multiprocess==0.70.13 # via bytewax -mypy==0.971 +mypy==0.981 # via # feast (setup.py) # sqlalchemy @@ -409,11 +415,11 @@ mypy-protobuf==3.1 # via feast (setup.py) mysqlclient==2.1.1 # via feast (setup.py) -nbformat==5.4.0 +nbformat==5.6.1 # via great-expectations nodeenv==1.7.0 # via pre-commit -numpy==1.23.2 +numpy==1.23.3 # via # altair # db-dtypes @@ -423,7 +429,7 @@ numpy==1.23.2 # pandavro # pyarrow # scipy -oauthlib==3.2.0 +oauthlib==3.2.1 # via requests-oauthlib oscrypto==1.3.0 # via snowflake-connector-python @@ -439,7 +445,7 @@ packaging==21.3 # pytest # redis # sphinx -pandas==1.4.3 +pandas==1.4.4 # via # altair # db-dtypes @@ -454,7 +460,7 @@ parso==0.8.3 # via jedi partd==1.3.0 # via dask -pathspec==0.9.0 +pathspec==0.10.1 # via black pbr==5.10.0 # via mock @@ -478,21 +484,23 @@ portalocker==2.5.1 # via msal-extensions pre-commit==2.20.0 # via feast (setup.py) -prompt-toolkit==3.0.30 +prompt-toolkit==3.0.31 # via ipython -proto-plus==1.22.0 +proto-plus==1.22.1 # via # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==3.20.1 +protobuf==4.21.7 # via # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # googleapis-common-protos @@ -502,7 +510,6 @@ protobuf==3.20.1 # grpcio-tools # mypy-protobuf # proto-plus - # tensorflow-metadata psutil==5.9.0 # via feast (setup.py) psycopg2-binary==2.9.3 @@ -519,7 +526,7 @@ py-cpuinfo==8.0.0 # via pytest-benchmark py4j==0.10.9.5 # via pyspark -pyarrow==6.0.1 +pyarrow==8.0.0 # via # db-dtypes # feast (setup.py) @@ -539,7 +546,7 @@ pycparser==2.21 # via cffi pycryptodomex==3.15.0 # via snowflake-connector-python -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -550,17 +557,21 @@ pygments==2.13.0 # feast (setup.py) # ipython # sphinx -pyjwt[crypto]==2.4.0 +pyjwt[crypto]==2.5.0 # via # adal # msal # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) +pymysql==1.0.2 + # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 - # via snowflake-connector-python + # via + # feast (setup.py) + # snowflake-connector-python pyparsing==2.4.7 # via # great-expectations @@ -570,7 +581,7 @@ pyrsistent==0.18.1 # via jsonschema pyspark==3.3.0 # via feast (setup.py) -pytest==7.1.2 +pytest==7.1.3 # via # feast (setup.py) # pytest-benchmark @@ -583,7 +594,7 @@ pytest==7.1.2 # pytest-xdist pytest-benchmark==3.4.1 # via feast (setup.py) -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via feast (setup.py) pytest-forked==1.4.0 # via pytest-xdist @@ -606,7 +617,7 @@ python-dateutil==2.8.2 # kubernetes # moto # pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via @@ -658,15 +669,15 @@ responses==0.21.0 # via moto rsa==4.9 # via google-auth -ruamel.yaml==0.17.17 +ruamel-yaml==0.17.17 # via great-expectations -ruamel.yaml.clib==0.2.6 - # via ruamel.yaml +ruamel-yaml-clib==0.2.6 + # via ruamel-yaml s3fs==2022.1.0 # via feast (setup.py) s3transfer==0.5.2 # via boto3 -scipy==1.9.0 +scipy==1.9.1 # via great-expectations six==1.16.0 # via @@ -684,11 +695,11 @@ six==1.16.0 # msrestazure # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==2.7.8 +snowflake-connector-python[pandas]==2.8.0 # via feast (setup.py) sphinx==4.3.2 # via @@ -708,23 +719,21 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -stack-data==0.4.0 +stack-data==0.5.1 # via ipython -starlette==0.19.1 +starlette==0.20.4 # via fastapi tabulate==0.8.10 # via feast (setup.py) -tenacity==8.0.1 +tenacity==8.1.0 # via feast (setup.py) -tensorflow-metadata==1.9.0 - # via feast (setup.py) -termcolor==1.1.0 +termcolor==2.0.1 # via great-expectations -testcontainers==3.6.1 +testcontainers==3.7.0 # via feast (setup.py) thriftpy2==0.4.14 # via happybase @@ -745,39 +754,43 @@ toolz==0.12.0 # altair # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via # feast (setup.py) # great-expectations -traitlets==5.3.0 +traitlets==5.4.0 # via # ipython # jupyter-core # matplotlib-inline # nbformat -trino==0.315.0 +trino==0.316.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==3.19.22 +types-cryptography==3.3.23 + # via pyjwt +types-protobuf==3.20.4 # via # feast (setup.py) # mypy-protobuf +types-pymysql==1.0.19 + # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 # via feast (setup.py) -types-pyyaml==6.0.11 +types-pyyaml==6.0.12 # via feast (setup.py) -types-redis==4.3.14 +types-redis==4.3.21 # via feast (setup.py) -types-requests==2.28.9 +types-requests==2.28.11 # via feast (setup.py) -types-setuptools==65.1.0 +types-setuptools==65.4.0.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.23 +types-urllib3==1.26.25 # via types-requests typing-extensions==4.3.0 # via @@ -787,15 +800,16 @@ typing-extensions==4.3.0 # great-expectations # mypy # pydantic + # snowflake-connector-python # sqlalchemy2-stubs # starlette -tzdata==2022.2 +tzdata==2022.4 # via pytz-deprecation-shim tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.11 +urllib3==1.26.12 # via # botocore # docker @@ -805,19 +819,20 @@ urllib3==1.26.11 # minio # requests # responses -uvicorn[standard]==0.18.2 + # snowflake-connector-python +uvicorn[standard]==0.18.3 # via feast (setup.py) -uvloop==0.16.0 +uvloop==0.17.0 # via uvicorn -virtualenv==20.16.3 +virtualenv==20.16.5 # via pre-commit volatile==2.1.0 # via bowler -watchfiles==0.16.1 +watchfiles==0.17.0 # via uvicorn wcwidth==0.2.5 # via prompt-toolkit -websocket-client==1.3.3 +websocket-client==1.4.1 # via # docker # kubernetes diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 0d3cb22bbca..3b5135b602d 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --output-file=sdk/python/requirements/py3.9-requirements.txt # -absl-py==1.2.0 - # via tensorflow-metadata anyio==3.6.1 # via # starlette @@ -20,9 +18,9 @@ bowler==0.9.0 # via feast (setup.py) cachetools==5.2.0 # via google-auth -certifi==2022.6.15 +certifi==2022.9.24 # via requests -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via requests click==8.1.3 # via @@ -30,7 +28,7 @@ click==8.1.3 # feast (setup.py) # moreorless # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via feast (setup.py) @@ -38,44 +36,41 @@ dask==2022.1.1 # via feast (setup.py) dill==0.3.5.1 # via feast (setup.py) -fastapi==0.79.1 +fastapi==0.85.0 # via feast (setup.py) -fastavro==1.6.0 +fastavro==1.6.1 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2022.7.1 +fsspec==2022.8.2 # via dask -google-api-core==2.8.2 +google-api-core==2.10.1 # via feast (setup.py) -google-auth==2.10.0 +google-auth==2.12.0 # via google-api-core googleapis-common-protos==1.56.4 # via # feast (setup.py) # google-api-core - # tensorflow-metadata -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.47.0 +grpcio==1.49.1 # via # feast (setup.py) # grpcio-reflection -grpcio-reflection==1.47.0 +grpcio-reflection==1.49.1 # via feast (setup.py) -h11==0.13.0 +h11==0.14.0 # via uvicorn -httptools==0.4.0 +httptools==0.5.0 # via uvicorn -idna==3.3 +idna==3.4 # via # anyio # requests jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.13.0 +jsonschema==4.16.0 # via feast (setup.py) locket==1.0.0 # via partd @@ -85,11 +80,11 @@ mmh3==3.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==0.971 +mypy==0.981 # via sqlalchemy mypy-extensions==0.4.3 # via mypy -numpy==1.23.2 +numpy==1.23.3 # via # feast (setup.py) # pandas @@ -97,7 +92,7 @@ numpy==1.23.2 # pyarrow packaging==21.3 # via dask -pandas==1.4.3 +pandas==1.5.0 # via # feast (setup.py) # pandavro @@ -105,16 +100,15 @@ pandavro==1.5.2 # via feast (setup.py) partd==1.3.0 # via dask -proto-plus==1.22.0 +proto-plus==1.22.1 # via feast (setup.py) -protobuf==3.20.1 +protobuf==4.21.7 # via # feast (setup.py) # google-api-core # googleapis-common-protos # grpcio-reflection # proto-plus - # tensorflow-metadata pyarrow==8.0.0 # via feast (setup.py) pyasn1==0.4.8 @@ -123,7 +117,7 @@ pyasn1==0.4.8 # rsa pyasn1-modules==0.2.8 # via google-auth -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -135,7 +129,7 @@ pyrsistent==0.18.1 # via jsonschema python-dateutil==2.8.2 # via pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via pandas @@ -154,19 +148,17 @@ six==1.16.0 # grpcio # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -starlette==0.19.1 +starlette==0.20.4 # via fastapi tabulate==0.8.10 # via feast (setup.py) -tenacity==8.0.1 - # via feast (setup.py) -tensorflow-metadata==1.9.0 +tenacity==8.1.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) @@ -176,7 +168,7 @@ toolz==0.12.0 # via # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) @@ -186,15 +178,15 @@ typing-extensions==4.3.0 # pydantic # sqlalchemy2-stubs # starlette -urllib3==1.26.11 +urllib3==1.26.12 # via requests -uvicorn[standard]==0.18.2 +uvicorn[standard]==0.18.3 # via feast (setup.py) -uvloop==0.16.0 +uvloop==0.17.0 # via uvicorn volatile==2.1.0 # via bowler -watchfiles==0.16.1 +watchfiles==0.17.0 # via uvicorn websockets==10.3 # via uvicorn diff --git a/sdk/python/tests/integration/e2e/test_python_feature_server.py b/sdk/python/tests/integration/e2e/test_python_feature_server.py index 9c61f6fa198..089efd7a562 100644 --- a/sdk/python/tests/integration/e2e/test_python_feature_server.py +++ b/sdk/python/tests/integration/e2e/test_python_feature_server.py @@ -84,6 +84,29 @@ def test_push(python_fs_client): ) == [initial_temp * 100] +@pytest.mark.integration +@pytest.mark.universal_online_stores +def test_push_source_does_not_exist(python_fs_client): + initial_temp = _get_temperatures_from_feature_server( + python_fs_client, location_ids=[1] + )[0] + response = python_fs_client.post( + "/push", + data=json.dumps( + { + "push_source_name": "push_source_does_not_exist", + "df": { + "location_id": [1], + "temperature": [initial_temp * 100], + "event_timestamp": [str(datetime.utcnow())], + "created": [str(datetime.utcnow())], + }, + } + ), + ) + assert response.status_code == 422 + + def _get_temperatures_from_feature_server(client, location_ids: List[int]): get_request_data = { "features": ["pushable_location_stats:temperature"], diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 708d9c0a142..174b0b91ad1 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -51,6 +51,9 @@ create_order_feature_view, create_pushable_feature_view, ) +from tests.integration.feature_repos.universal.online_store.bigtable import ( + BigtableOnlineStoreCreator, +) from tests.integration.feature_repos.universal.online_store.datastore import ( DatastoreOnlineStoreCreator, ) @@ -84,6 +87,12 @@ "schema": "ONLINE", } +BIGTABLE_CONFIG = { + "type": "bigtable", + "project_id": os.getenv("GCLOUD_PROJECT", "kf-feast"), + "instance": os.getenv("BIGTABLE_INSTANCE_ID", "feast-integration-tests"), +} + OFFLINE_STORE_TO_PROVIDER_CONFIG: Dict[str, DataSourceCreator] = { "file": ("local", FileDataSourceCreator), "bigquery": ("gcp", BigQueryDataSourceCreator), @@ -115,6 +124,7 @@ AVAILABLE_ONLINE_STORES["dynamodb"] = (DYNAMO_CONFIG, None) AVAILABLE_ONLINE_STORES["datastore"] = ("datastore", None) AVAILABLE_ONLINE_STORES["snowflake"] = (SNOWFLAKE_CONFIG, None) + AVAILABLE_ONLINE_STORES["bigtable"] = (BIGTABLE_CONFIG, None) full_repo_configs_module = os.environ.get(FULL_REPO_CONFIGS_MODULE_ENV_NAME) @@ -161,6 +171,7 @@ "redis": (REDIS_CONFIG, RedisOnlineStoreCreator), "dynamodb": (DYNAMO_CONFIG, DynamoDBOnlineStoreCreator), "datastore": ("datastore", DatastoreOnlineStoreCreator), + "bigtable": ("bigtable", BigtableOnlineStoreCreator), } for key, replacement in replacements.items(): diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/bigtable.py b/sdk/python/tests/integration/feature_repos/universal/online_store/bigtable.py new file mode 100644 index 00000000000..c06143e245b --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/bigtable.py @@ -0,0 +1,47 @@ +import os +from typing import Dict + +from google.cloud import bigtable +from testcontainers.core.container import DockerContainer +from testcontainers.core.waiting_utils import wait_for_logs + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class BigtableOnlineStoreCreator(OnlineStoreCreator): + gcp_project = "test-project" + host = "0.0.0.0" + port = "8086" + bt_instance = "test-instance" + + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.container = ( + DockerContainer( + "gcr.io/google.com/cloudsdktool/cloud-sdk:380.0.0-emulators" + ) + .with_command( + f"gcloud beta emulators bigtable start --project {self.gcp_project} --host-port {self.host}:{self.port}" + ) + .with_exposed_ports(self.port) + ) + + def create_online_store(self) -> Dict[str, str]: + self.container.start() + log_string_to_wait_for = r"\[bigtable\] Cloud Bigtable emulator running" + wait_for_logs( + container=self.container, predicate=log_string_to_wait_for, timeout=10 + ) + exposed_port = self.container.get_exposed_port(self.port) + os.environ[bigtable.client.BIGTABLE_EMULATOR] = f"{self.host}:{exposed_port}" + return { + "type": "bigtable", + "project_id": self.gcp_project, + "instance": self.bt_instance, + } + + def teardown(self): + del os.environ[bigtable.client.BIGTABLE_EMULATOR] + self.container.stop() diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py b/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py new file mode 100644 index 00000000000..093295c86ba --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py @@ -0,0 +1,33 @@ +from typing import Dict + +from testcontainers.mysql import MySqlContainer + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class MySQLOnlineStoreCreator(OnlineStoreCreator): + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.container = ( + MySqlContainer("mysql:latest", platform="linux/amd64") + .with_exposed_ports(3306) + .with_env("MYSQL_USER", "root") + .with_env("MYSQL_PASSWORD", "test") + .with_env("MYSQL_DATABASE", "test") + ) + + def create_online_store(self) -> Dict[str, str]: + self.container.start() + exposed_port = self.container.get_exposed_port(3306) + return { + "type": "mysql", + "user": "root", + "password": "test", + "database": "test", + "port": exposed_port, + } + + def teardown(self): + self.container.stop() diff --git a/sdk/python/tests/unit/cli/test_cli.py b/sdk/python/tests/unit/cli/test_cli.py index f55e5ffc068..25a1dfed349 100644 --- a/sdk/python/tests/unit/cli/test_cli.py +++ b/sdk/python/tests/unit/cli/test_cli.py @@ -1,7 +1,9 @@ +import os import tempfile from contextlib import contextmanager from pathlib import Path from textwrap import dedent +from unittest import mock from assertpy import assertpy @@ -85,6 +87,21 @@ def test_3rd_party_registry_store_with_fs_yaml_override() -> None: assertpy.assert_that(return_code).is_equal_to(0) +def test_3rd_party_registry_store_with_fs_yaml_override_by_env_var() -> None: + runner = CliRunner() + + fs_yaml_file = "test_fs.yaml" + with setup_third_party_registry_store_repo( + "foo.registry_store.FooRegistryStore", fs_yaml_file_name=fs_yaml_file + ) as repo_path: + custom_yaml_path = os.path.join(repo_path, fs_yaml_file) + with mock.patch.dict( + "os.environ", {"FEAST_FS_YAML_FILE_PATH": custom_yaml_path}, clear=True + ): + return_code, output = runner.run_with_output(["apply"], cwd=repo_path) + assertpy.assert_that(return_code).is_equal_to(0) + + @contextmanager def setup_third_party_provider_repo(provider_name: str): with tempfile.TemporaryDirectory() as repo_dir_name: diff --git a/sdk/python/tests/unit/diff/test_registry_diff.py b/sdk/python/tests/unit/diff/test_registry_diff.py index 8af6c50a13d..ce40295f8b6 100644 --- a/sdk/python/tests/unit/diff/test_registry_diff.py +++ b/sdk/python/tests/unit/diff/test_registry_diff.py @@ -1,6 +1,6 @@ import pandas as pd -from feast import Field +from feast import Field, PushSource from feast.diff.registry_diff import ( diff_registry_objects, tag_objects_for_keep_delete_update_add, @@ -145,3 +145,27 @@ def post_changed(inputs: pd.DataFrame) -> pd.DataFrame: feast_object_diffs.feast_object_property_diffs[2].property_name == "user_defined_function.body_text" ) + + +def test_diff_registry_objects_batch_to_push_source(simple_dataset_1): + with prep_file_source(df=simple_dataset_1, timestamp_field="ts_1") as file_source: + entity = Entity(name="id", join_keys=["id"]) + pre_changed = FeatureView( + name="fv2", + entities=[entity], + source=file_source, + ) + post_changed = FeatureView( + name="fv2", + entities=[entity], + source=PushSource(name="push_source", batch_source=file_source), + ) + + feast_object_diffs = diff_registry_objects( + pre_changed, post_changed, "feature view" + ) + assert len(feast_object_diffs.feast_object_property_diffs) == 1 + assert ( + feast_object_diffs.feast_object_property_diffs[0].property_name + == "stream_source" + ) diff --git a/setup.py b/setup.py index 37ed471cfa6..2b09bcbe946 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ "mmh3", "numpy>=1.22,<3", "pandas>=1.4.3,<2", - "pandavro==1.5.*", # For some reason pandavro higher than 1.5.* only support pandas less than 1.3. + "pandavro==1.5.*", # For some reason pandavro higher than 1.5.* only support pandas less than 1.3. "protobuf<5,>3", "proto-plus>=1.20.0,<2", "pyarrow>=4,<9", @@ -74,7 +74,6 @@ "typeguard", "fastapi>=0.68.0,<1", "uvicorn[standard]>=0.14.0,<1", - "tensorflow-metadata>=1.0.0,<2.0.0", "dask>=2021.*,<2022.02.0", "bowler", # Needed for automatic repo upgrades ] @@ -84,6 +83,7 @@ "google-cloud-bigquery-storage >= 2.0.0,<3", "google-cloud-datastore>=2.1.*,<3", "google-cloud-storage>=1.34.*,<3", + "google-cloud-bigtable>=2.11.*,<3", ] REDIS_REQUIRED = [ @@ -96,7 +96,11 @@ BYTEWAX_REQUIRED = ["bytewax==0.10.0", "docker>=5.0.2", "kubernetes<=20.13.0"] SNOWFLAKE_REQUIRED = [ - "snowflake-connector-python[pandas]>=2.7.3,<=2.7.8", + "snowflake-connector-python[pandas]>=2.7.3,<3", + # `pyOpenSSL==22.1.0` requires `cryptography<39,>=38.0.0`, which is incompatible + # with `snowflake-connector-python[pandas]==2.8.0`, which depends on + # `cryptography<37.0.0,>=3.1.0`. + "pyOpenSSL<22.1.0", ] SPARK_REQUIRED = [ @@ -111,9 +115,7 @@ "psycopg2-binary>=2.8.3,<3", ] -MYSQL_REQUIRED = [ - "mysqlclient", -] +MYSQL_REQUIRED = ["mysqlclient", "pymysql", "types-PyMySQL"] HBASE_REQUIRED = [ "happybase>=1.2.0,<3", @@ -129,15 +131,13 @@ "cffi==1.15.*,<2", ] -AZURE_REQUIRED = ( - [ - "azure-storage-blob>=0.37.0", - "azure-identity>=1.6.1", - "SQLAlchemy>=1.4.19", - "pyodbc>=4.0.30", - "pymssql", - ] -) +AZURE_REQUIRED = [ + "azure-storage-blob>=0.37.0", + "azure-identity>=1.6.1", + "SQLAlchemy>=1.4.19", + "pyodbc>=4.0.30", + "pymssql", +] CI_REQUIRED = ( [ @@ -150,7 +150,7 @@ "grpcio-testing>=1.47.0", "minio==7.1.0", "mock==2.0.0", - "moto", + "moto<4", "mypy>=0.931", "mypy-protobuf==3.1", "avro==1.10.0", diff --git a/ui/.gitignore b/ui/.gitignore new file mode 100644 index 00000000000..728f2aab717 --- /dev/null +++ b/ui/.gitignore @@ -0,0 +1,2 @@ +src/protos.d.ts +src/protos.js diff --git a/ui/feature_repo/test_get_features.py b/ui/feature_repo/test_get_features.py deleted file mode 100644 index 722c5a3dd46..00000000000 --- a/ui/feature_repo/test_get_features.py +++ /dev/null @@ -1,86 +0,0 @@ -import pandas as pd -from great_expectations.core.expectation_suite import ExpectationSuite -from great_expectations.dataset import PandasDataset - -from feast import FeatureStore -from feast.dqm.profilers.ge_profiler import ge_profiler -from feast.infra.offline_stores.file import SavedDatasetFileStorage - -DELTA = 0.1 # controlling allowed window in fraction of the value on scale [0, 1] -# Note: the GE integration allows asserting differences between datasets. The "ds" below is the reference dataset to check and this generates the expectation suite which can be used against future datasets. -# It's used via ge.validate(new_dataset, ExpectationSuite) -# For this demo though, we ignore this and - - -@ge_profiler -def credit_profiler(ds: PandasDataset) -> ExpectationSuite: - # simple checks on data consistency - ds.expect_column_values_to_be_between( - "credit_card_due", min_value=0, mostly=0.99, # allow some outliers - ) - - ds.expect_column_values_to_be_between( - "missed_payments_1y", - min_value=0, - max_value=5, - mostly=0.99, # allow some outliers - ) - - return ds.get_expectation_suite() - - -def generate_saved_dataset(): - store = FeatureStore(repo_path=".") - entity_df = pd.read_parquet(path="data/loan_table.parquet") - - fs = store.get_feature_service("credit_score_v1") - job = store.get_historical_features(entity_df=entity_df, features=fs,) - store.create_saved_dataset( - from_=job, - name="my_training_ds", - storage=SavedDatasetFileStorage(path="my_training_ds.parquet"), - feature_service=fs, - profiler=credit_profiler, - ) - - -def get_latest_timestamps(): - store = FeatureStore(repo_path=".") - feature_views = store.list_feature_views() - for fv in feature_views: - print( - f"Data source latest event for {fv.name} is {fv.batch_source._meta.latest_event_timestamp}" - ) - - -def test_ge(): - store = FeatureStore(repo_path=".") - - print("--- Historical features (from saved dataset) ---") - ds = store.get_saved_dataset("my_training_ds") - print(ds._profile) - - -def run_demo(): - store = FeatureStore(repo_path=".") - - print("--- Historical features (from saved dataset) ---") - ds = store.get_saved_dataset("my_training_ds") - print(ds.to_df()) - - print("\n--- Online features ---") - features = store.get_online_features( - features=store.get_feature_service("credit_score_v3"), - entity_rows=[ - {"zipcode": 30721, "dob_ssn": "19530219_5179", "transaction_amt": 1023} - ], - ).to_dict() - for key, value in sorted(features.items()): - print(key, " : ", value) - - -if __name__ == "__main__": - generate_saved_dataset() - get_latest_timestamps() - # test_ge() - run_demo() diff --git a/ui/package.json b/ui/package.json index 73165523252..6cb2fef5c3f 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.25.0", + "version": "0.26.0", "private": false, "files": [ "dist" @@ -41,6 +41,7 @@ "inter-ui": "^3.19.3", "moment": "^2.29.1", "prop-types": "^15.8.1", + "protobufjs": "^7.1.1", "query-string": "^7.1.1", "react-query": "^3.34.12", "react-router-dom": "6", @@ -49,12 +50,13 @@ "zod": "^3.11.6" }, "scripts": { - "start": "react-scripts start", - "build": "react-scripts build", - "build:lib": "rimraf ./dist && tsc && rollup -c", - "build:lib-dev": "rimraf ./dist && tsc && rollup -c && yalc publish -f", - "test": "react-scripts test", - "eject": "react-scripts eject" + "start": "npm run generate-protos && react-scripts start", + "build": "npm run generate-protos && react-scripts build", + "build:lib": "npm run generate-protos && rimraf ./dist && tsc && rollup -c", + "build:lib-dev": "npm run generate-protos && rimraf ./dist && tsc && rollup -c && yalc publish -f", + "test": "npm run generate-protos && react-scripts test", + "eject": "react-scripts eject", + "generate-protos": "pbjs --no-encode -o src/protos.js -w commonjs -t static-module `find ../protos/feast/ -iname *.proto` && pbts -n protos -o src/protos.d.ts src/protos.js" }, "eslintConfig": { "extends": [ @@ -92,6 +94,7 @@ "@testing-library/react": "^12.0.0", "@testing-library/user-event": "^13.2.1", "msw": "^0.36.8", + "protobufjs-cli": "^1.0.2", "react": "^17.0.2", "react-dom": "^17.0.2", "rimraf": "^3.0.2", diff --git a/ui/public/projects-list.json b/ui/public/projects-list.json index d3d7c3b7d91..8e2bc3c616a 100644 --- a/ui/public/projects-list.json +++ b/ui/public/projects-list.json @@ -4,7 +4,7 @@ "name": "Credit Score Project", "description": "Project for credit scoring team and associated models.", "id": "credit_score_project", - "registryPath": "/registry.json" + "registryPath": "/registry.db" }, { "name": "Empty Registry", diff --git a/ui/public/registry.db b/ui/public/registry.db new file mode 100644 index 00000000000..617771999c7 Binary files /dev/null and b/ui/public/registry.db differ diff --git a/ui/public/registry.json b/ui/public/registry.json deleted file mode 100644 index 2d5c93c9620..00000000000 --- a/ui/public/registry.json +++ /dev/null @@ -1,634 +0,0 @@ -{ - "dataSources": [ - { - "createdTimestampColumn": "created_timestamp", - "fileOptions": { - "uri": "data/credit_history.parquet" - }, - "name": "credit_history", - "timestampField": "event_timestamp", - "type": "BATCH_FILE" - }, - { - "name": "transaction", - "requestDataOptions": { - "schema": [ - { - "name": "transaction_amt", - "valueType": "INT64" - } - ] - }, - "type": "REQUEST_SOURCE" - }, - { - "createdTimestampColumn": "created_timestamp", - "fileOptions": { - "uri": "data/zipcode_table.parquet" - }, - "name": "zipcode", - "timestampField": "event_timestamp", - "type": "BATCH_FILE" - } - ], - "entities": [ - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171062Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.171062Z" - }, - "spec": { - "joinKey": "__dummy_id", - "name": "__dummy", - "valueType": "STRING" - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171112Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.171112Z" - }, - "spec": { - "description": "Date of birth and last four digits of social security number", - "joinKey": "dob_ssn", - "name": "dob_ssn", - "tags": { - "owner": "tony@tecton.ai", - "team": "hack week" - }, - "valueType": "STRING" - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171153Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.171153Z" - }, - "spec": { - "description": "A zipcode", - "joinKey": "zipcode", - "name": "zipcode", - "tags": { - "owner": "danny@tecton.ai", - "team": "hack week" - }, - "valueType": "INT64" - } - } - ], - "featureServices": [ - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.172623Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.172623Z" - }, - "spec": { - "description": "Credit scoring model", - "features": [ - { - "featureColumns": [ - { - "name": "credit_card_due", - "valueType": "INT64" - }, - { - "name": "missed_payments_1y", - "valueType": "INT64" - } - ], - "featureViewName": "credit_history" - }, - { - "featureColumns": [ - { - "name": "city", - "valueType": "STRING" - }, - { - "name": "state", - "valueType": "STRING" - }, - { - "name": "location_type", - "valueType": "STRING" - }, - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "population", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "featureViewName": "zipcode_features" - } - ], - "name": "credit_score_v1", - "tags": { - "owner": "tony@tecton.ai", - "stage": "staging" - } - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.172405Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.172405Z" - }, - "spec": { - "description": "Credit scoring model", - "features": [ - { - "featureColumns": [ - { - "name": "credit_card_due", - "valueType": "INT64" - }, - { - "name": "mortgage_due", - "valueType": "INT64" - }, - { - "name": "missed_payments_1y", - "valueType": "INT64" - } - ], - "featureViewName": "credit_history" - }, - { - "featureColumns": [ - { - "name": "city", - "valueType": "STRING" - }, - { - "name": "state", - "valueType": "STRING" - }, - { - "name": "location_type", - "valueType": "STRING" - }, - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "population", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "featureViewName": "zipcode_features" - } - ], - "name": "credit_score_v2", - "tags": { - "owner": "tony@tecton.ai", - "stage": "prod" - } - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.172264Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.172264Z" - }, - "spec": { - "description": "Credit scoring model", - "features": [ - { - "featureColumns": [ - { - "name": "credit_card_due", - "valueType": "INT64" - }, - { - "name": "mortgage_due", - "valueType": "INT64" - }, - { - "name": "missed_payments_1y", - "valueType": "INT64" - } - ], - "featureViewName": "credit_history" - }, - { - "featureColumns": [ - { - "name": "city", - "valueType": "STRING" - }, - { - "name": "state", - "valueType": "STRING" - }, - { - "name": "location_type", - "valueType": "STRING" - }, - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "population", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "featureViewName": "zipcode_features" - }, - { - "featureColumns": [ - { - "name": "transaction_gt_last_credit_card_due", - "valueType": "BOOL" - } - ], - "featureViewName": "transaction_gt_last_credit_card_due" - } - ], - "name": "credit_score_v3", - "tags": { - "owner": "tony@tecton.ai", - "stage": "dev" - } - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.172530Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.172530Z" - }, - "spec": { - "description": "Location model", - "features": [ - { - "featureColumns": [ - { - "name": "city", - "valueType": "STRING" - }, - { - "name": "state", - "valueType": "STRING" - }, - { - "name": "location_type", - "valueType": "STRING" - }, - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "population", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "featureViewName": "zipcode_features" - } - ], - "name": "zipcode_model", - "tags": { - "owner": "amanda@tecton.ai", - "stage": "dev" - } - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.172188Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.172188Z" - }, - "spec": { - "description": "Location model", - "features": [ - { - "featureColumns": [ - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "featureViewName": "zipcode_money_features" - } - ], - "name": "zipcode_model_v2", - "tags": { - "owner": "amanda@tecton.ai", - "stage": "dev" - } - } - } - ], - "featureViews": [ - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171421Z", - "lastUpdatedTimestamp": "2022-05-11T19:28:21.444739Z", - "materializationIntervals": [ - { - "endTime": "2019-05-11T19:27:05Z", - "startTime": "1997-09-19T19:27:13.273753Z" - }, - { - "endTime": "2022-05-11T19:27:43Z", - "startTime": "2019-05-11T19:27:05Z" - } - ] - }, - "spec": { - "batchSource": { - "createdTimestampColumn": "created_timestamp", - "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", - "fileOptions": { - "uri": "data/credit_history.parquet" - }, - "name": "credit_history", - "timestampField": "event_timestamp", - "type": "BATCH_FILE" - }, - "entities": [ - "dob_ssn" - ], - "features": [ - { - "name": "credit_card_due", - "valueType": "INT64" - }, - { - "name": "mortgage_due", - "valueType": "INT64" - }, - { - "name": "student_loan_due", - "valueType": "INT64" - }, - { - "name": "vehicle_loan_due", - "valueType": "INT64" - }, - { - "name": "hard_pulls", - "valueType": "INT64" - }, - { - "name": "missed_payments_2y", - "valueType": "INT64" - }, - { - "name": "missed_payments_1y", - "valueType": "INT64" - }, - { - "name": "missed_payments_6m", - "valueType": "INT64" - }, - { - "name": "bankruptcies", - "valueType": "INT64" - } - ], - "name": "credit_history", - "online": true, - "tags": { - "access_group": "feast-team@tecton.ai", - "date_added": "2022-02-6", - "experiments": "experiment-A" - }, - "ttl": "777600000s" - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171300Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:46.002348Z", - "materializationIntervals": [ - { - "endTime": "2019-05-11T19:27:05Z", - "startTime": "2012-05-13T19:27:08.483036Z" - }, - { - "endTime": "2022-05-11T19:27:43Z", - "startTime": "2019-05-11T19:27:05Z" - } - ] - }, - "spec": { - "batchSource": { - "createdTimestampColumn": "created_timestamp", - "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", - "fileOptions": { - "uri": "data/zipcode_table.parquet" - }, - "name": "zipcode", - "timestampField": "event_timestamp", - "type": "BATCH_FILE" - }, - "entities": [ - "zipcode" - ], - "features": [ - { - "name": "city", - "valueType": "STRING" - }, - { - "name": "state", - "valueType": "STRING" - }, - { - "name": "location_type", - "valueType": "STRING" - }, - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "population", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "name": "zipcode_features", - "online": true, - "tags": { - "access_group": "feast-team@tecton.ai", - "date_added": "2022-02-7", - "experiments": "experiment-A,experiment-B,experiment-C" - }, - "ttl": "315360000s" - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171195Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:45.942549Z", - "materializationIntervals": [ - { - "endTime": "2019-05-11T19:27:05Z", - "startTime": "2012-05-13T19:27:06.493847Z" - }, - { - "endTime": "2022-05-11T19:27:43Z", - "startTime": "2019-05-11T19:27:05Z" - } - ] - }, - "spec": { - "batchSource": { - "createdTimestampColumn": "created_timestamp", - "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", - "fileOptions": { - "uri": "data/zipcode_table.parquet" - }, - "name": "zipcode", - "timestampField": "event_timestamp", - "type": "BATCH_FILE" - }, - "entities": [ - "zipcode" - ], - "features": [ - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "name": "zipcode_money_features", - "online": true, - "tags": { - "access_group": "feast-team@tecton.ai", - "date_added": "2022-02-7", - "experiments": "experiment-A,experiment-B,experiment-C" - }, - "ttl": "315360000s" - } - } - ], - "infra": [ - { - "name": "credit_scoring_aws_credit_history", - "path": "/Users/dannychiao/GitHub/feast/ui/feature_repo/data/online.db" - }, - { - "name": "credit_scoring_aws_zipcode_features", - "path": "/Users/dannychiao/GitHub/feast/ui/feature_repo/data/online.db" - }, - { - "name": "credit_scoring_aws_zipcode_money_features", - "path": "/Users/dannychiao/GitHub/feast/ui/feature_repo/data/online.db" - } - ], - "onDemandFeatureViews": [ - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171556Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.171556Z" - }, - "spec": { - "features": [ - { - "name": "transaction_gt_last_credit_card_due", - "valueType": "BOOL" - } - ], - "name": "transaction_gt_last_credit_card_due", - "sources": { - "credit_history": { - "featureViewProjection": { - "featureColumns": [ - { - "name": "credit_card_due", - "valueType": "INT64" - }, - { - "name": "mortgage_due", - "valueType": "INT64" - }, - { - "name": "student_loan_due", - "valueType": "INT64" - }, - { - "name": "vehicle_loan_due", - "valueType": "INT64" - }, - { - "name": "hard_pulls", - "valueType": "INT64" - }, - { - "name": "missed_payments_2y", - "valueType": "INT64" - }, - { - "name": "missed_payments_1y", - "valueType": "INT64" - }, - { - "name": "missed_payments_6m", - "valueType": "INT64" - }, - { - "name": "bankruptcies", - "valueType": "INT64" - } - ], - "featureViewName": "credit_history" - } - }, - "transaction": { - "requestDataSource": { - "name": "transaction", - "requestDataOptions": { - "schema": [ - { - "name": "transaction_amt", - "valueType": "INT64" - } - ] - }, - "type": "REQUEST_SOURCE" - } - } - }, - "userDefinedFunction": { - "body": "@on_demand_feature_view(\n sources=[credit_history, input_request],\n schema=[\n Field(name=\"transaction_gt_last_credit_card_due\", dtype=Bool),\n ],\n)\ndef transaction_gt_last_credit_card_due(inputs: pd.DataFrame) -> pd.DataFrame:\n df = pd.DataFrame()\n df[\"transaction_gt_last_credit_card_due\"] = (\n inputs[\"transaction_amt\"] > inputs[\"credit_card_due\"]\n )\n return df\n", - "name": "transaction_gt_last_credit_card_due" - } - } - } - ], - "project": "credit_scoring_aws" -} diff --git a/ui/src/FeastUISansProviders.test.tsx b/ui/src/FeastUISansProviders.test.tsx index 09985bc1338..46702328090 100644 --- a/ui/src/FeastUISansProviders.test.tsx +++ b/ui/src/FeastUISansProviders.test.tsx @@ -15,13 +15,17 @@ import { creditHistoryRegistry, } from "./mocks/handlers"; -import registry from "../public/registry.json"; +import { readFileSync } from "fs"; +import { feast } from "./protos"; +import path from "path"; // declare which API requests to mock const server = setupServer( projectsListWithDefaultProject, creditHistoryRegistry ); +const registry = readFileSync(path.resolve(__dirname, "../public/registry.db")); +const parsedRegistry = feast.core.Registry.decode(registry); // establish API mocking before all tests beforeAll(() => server.listen()); @@ -50,7 +54,10 @@ test("full app rendering", async () => { // Explore Panel Should Appear expect(screen.getByText(/Explore this Project/i)).toBeInTheDocument(); - const projectNameRegExp = new RegExp(registry.project, "i"); + const projectNameRegExp = new RegExp( + parsedRegistry.projectMetadata[0].project!, + "i" + ); // It should load the default project, which is credit_scoring_aws await waitFor(() => { @@ -95,9 +102,9 @@ test("routes are reachable", async () => { } }); - -const featureViewName = registry.featureViews[0].spec.name; -const featureName = registry.featureViews[0].spec.features[0].name; +const spec = parsedRegistry.featureViews[0].spec!; +const featureViewName = spec.name!; +const featureName = spec.features![0]!.name!; test("features are reachable", async () => { render(); @@ -106,10 +113,7 @@ test("features are reachable", async () => { await screen.findByText(/Explore this Project/i); const routeRegExp = new RegExp("Feature Views", "i"); - userEvent.click( - screen.getByRole("button", { name: routeRegExp }), - leftClick - ); + userEvent.click(screen.getByRole("button", { name: routeRegExp }), leftClick); screen.getByRole("heading", { name: "Feature Views", @@ -118,18 +122,12 @@ test("features are reachable", async () => { await screen.findAllByText(/Feature Views/i); const fvRegExp = new RegExp(featureViewName, "i"); - userEvent.click( - screen.getByRole("link", { name: fvRegExp }), - leftClick - ) + userEvent.click(screen.getByRole("link", { name: fvRegExp }), leftClick); await screen.findByText(featureName); const fRegExp = new RegExp(featureName, "i"); - userEvent.click( - screen.getByRole("link", { name: fRegExp }), - leftClick - ) + userEvent.click(screen.getByRole("link", { name: fRegExp }), leftClick); // Should land on a page with the heading // await screen.findByText("Feature: " + featureName); screen.getByRole("heading", { diff --git a/ui/src/components/FeaturesInServiceDisplay.tsx b/ui/src/components/FeaturesInServiceDisplay.tsx index 39091b81dec..6ca2fcc8b09 100644 --- a/ui/src/components/FeaturesInServiceDisplay.tsx +++ b/ui/src/components/FeaturesInServiceDisplay.tsx @@ -1,36 +1,26 @@ import React from "react"; -import { z } from "zod"; import { EuiBasicTable } from "@elastic/eui"; -import { FeastFeatureInServiceType } from "../parsers/feastFeatureServices"; import EuiCustomLink from "./EuiCustomLink"; -import { FEAST_FEATURE_VALUE_TYPES } from "../parsers/types"; import { useParams } from "react-router-dom"; +import { feast } from "../protos"; interface FeatureViewsListInterace { - featureViews: FeastFeatureInServiceType[]; + featureViews: feast.core.IFeatureViewProjection[]; +} + +interface IFeatureColumnInService { + featureViewName: string; + name: string; + valueType: feast.types.ValueType.Enum; } const FeaturesInServiceList = ({ featureViews }: FeatureViewsListInterace) => { const { projectName } = useParams(); - - const FeatureInService = z.object({ - featureViewName: z.string(), - featureColumnName: z.string(), - valueType: z.nativeEnum(FEAST_FEATURE_VALUE_TYPES), - }); - type FeatureInServiceType = z.infer; - - var items: FeatureInServiceType[] = []; - featureViews.forEach((featureView) => { - featureView.featureColumns.forEach((featureColumn) => { - const row: FeatureInServiceType = { - featureViewName: featureView.featureViewName, - featureColumnName: featureColumn.name, - valueType: featureColumn.valueType, - }; - items.push(row); - }); - }); + const items: IFeatureColumnInService[] = featureViews.flatMap(featureView => featureView.featureColumns!.map(featureColumn => ({ + featureViewName: featureView.featureViewName!, + name: featureColumn.name!, + valueType: featureColumn.valueType!, + }))); const columns = [ { @@ -49,15 +39,18 @@ const FeaturesInServiceList = ({ featureViews }: FeatureViewsListInterace) => { }, { name: "Feature Column", - field: "featureColumnName", + field: "name", }, { name: "Value Type", field: "valueType", + render: (valueType: feast.types.ValueType.Enum) => { + return feast.types.ValueType.Enum[valueType]; + }, }, ]; - const getRowProps = (item: FeatureInServiceType) => { + const getRowProps = (item: IFeatureColumnInService) => { return { "data-test-subj": `row-${item.featureViewName}`, }; diff --git a/ui/src/components/FeaturesListDisplay.tsx b/ui/src/components/FeaturesListDisplay.tsx index dcb6ba81eb1..a40730c6873 100644 --- a/ui/src/components/FeaturesListDisplay.tsx +++ b/ui/src/components/FeaturesListDisplay.tsx @@ -1,38 +1,39 @@ -import React, { useContext } from "react"; -import { EuiBasicTable, EuiLoadingSpinner, EuiBadge } from "@elastic/eui"; -import { FeastFeatureColumnType } from "../parsers/feastFeatureViews"; -import useLoadFeatureViewSummaryStatistics from "../queries/useLoadFeatureViewSummaryStatistics"; -import SparklineHistogram from "./SparklineHistogram"; -import FeatureFlagsContext from "../contexts/FeatureFlagsContext"; +import { EuiBasicTable } from "@elastic/eui"; import EuiCustomLink from "./EuiCustomLink"; +import { feast } from "../protos"; interface FeaturesListProps { projectName: string; featureViewName: string; - features: FeastFeatureColumnType[]; + features: feast.core.IFeatureSpecV2[]; link: boolean; } -const FeaturesList = ({ projectName, featureViewName, features, link }: FeaturesListProps) => { - const { enabledFeatureStatistics } = useContext(FeatureFlagsContext); - const { isLoading, isError, isSuccess, data } = - useLoadFeatureViewSummaryStatistics(featureViewName); - +const FeaturesList = ({ + projectName, + featureViewName, + features, + link, +}: FeaturesListProps) => { let columns: { name: string; render?: any; field: any }[] = [ - { + { name: "Name", field: "name", - render: (item: string) => ( - ( + + to={`/p/${projectName}/feature-view/${featureViewName}/feature/${item}`} + > {item} - ) + ), }, { name: "Value Type", field: "valueType", + render: (valueType: feast.types.ValueType.Enum) => { + return feast.types.ValueType.Enum[valueType]; + }, }, ]; @@ -40,50 +41,7 @@ const FeaturesList = ({ projectName, featureViewName, features, link }: Features columns[0].render = undefined; } - if (enabledFeatureStatistics) { - columns.push( - ...[ - { - name: "Sample", - field: "", - render: (item: FeastFeatureColumnType) => { - const statistics = - isSuccess && data && data.columnsSummaryStatistics[item.name]; - - return ( - - {isLoading && } - {isError && ( - error loading samples - )} - {statistics && statistics.sampleValues.join(",")} - - ); - }, - }, - { - name: "Sparklines", - field: "", - render: (item: FeastFeatureColumnType) => { - const statistics = - isSuccess && data && data.columnsSummaryStatistics[item.name]; - - if ( - statistics && - statistics.valueType === "INT64" && - statistics.histogram - ) { - return ; - } else { - return ""; - } - }, - }, - ] - ); - } - - const getRowProps = (item: FeastFeatureColumnType) => { + const getRowProps = (item: feast.core.IFeatureSpecV2) => { return { "data-test-subj": `row-${item.name}`, }; diff --git a/ui/src/components/NumericFeaturesTable.tsx b/ui/src/components/NumericFeaturesTable.tsx deleted file mode 100644 index 7c55f5ddbae..00000000000 --- a/ui/src/components/NumericFeaturesTable.tsx +++ /dev/null @@ -1,60 +0,0 @@ -import { EuiBasicTable } from "@elastic/eui"; -import React from "react"; -import { NumericColumnSummaryStatisticType } from "../parsers/featureViewSummaryStatistics"; -import SparklineHistogram from "./SparklineHistogram"; - -interface NumericFeaturesTableProps { - data: NumericColumnSummaryStatisticType[]; -} - -const NumericFeaturesTable = ({ data }: NumericFeaturesTableProps) => { - const columns = [ - { name: "Name", field: "name" }, - { - name: "Value Type", - field: "valueType", - }, - { - name: "Sample", - render: (statistics: NumericColumnSummaryStatisticType) => { - return ( - - {statistics && statistics.sampleValues.join(",")} - - ); - }, - }, - { - name: "Min/Max", - render: (statistics: NumericColumnSummaryStatisticType) => { - return statistics.min !== undefined && statistics.max !== undefined - ? `${statistics.min}/${statistics.max}` - : undefined; - }, - }, - { name: "zeros", field: "proportionOfZeros" }, - { name: "missing", field: "proportionMissing" }, - { - name: "Sparklines", - render: (statistics: NumericColumnSummaryStatisticType) => { - if (statistics && statistics.histogram) { - return ; - } else { - return ""; - } - }, - }, - ]; - - const getRowProps = (item: NumericColumnSummaryStatisticType) => { - return { - "data-test-subj": `row-${item.name}`, - }; - }; - - return ( - - ); -}; - -export default NumericFeaturesTable; diff --git a/ui/src/components/SparklineHistogram.tsx b/ui/src/components/SparklineHistogram.tsx deleted file mode 100644 index bd632ec20d1..00000000000 --- a/ui/src/components/SparklineHistogram.tsx +++ /dev/null @@ -1,58 +0,0 @@ -import React from "react"; -import { HistogramDataType } from "../parsers/featureViewSummaryStatistics"; -import { extent } from "d3-array"; -import { scaleLinear } from "d3"; -import { EuiBadge, useEuiTheme } from "@elastic/eui"; - -interface SparklineHistogramProps { - data: HistogramDataType; -} - -const SparklineHistogram = ({ data }: SparklineHistogramProps) => { - const width = 100; - const height = 24; - - const yMax = height - 2; - - const { euiTheme } = useEuiTheme(); - - if (data.length > 0) { - const x0Extent = extent(data, (d) => d.x0) as [number, number]; - const xScale = scaleLinear() - .domain(x0Extent) - .range([0, width - width / data.length]); - - const yExtent = extent(data, (d) => d.count) as [number, number]; - const yScale = scaleLinear().domain(yExtent).range([0, yMax]); - - return ( - - - {data.map((d) => { - const barHeight = yScale(d.count); - - return ( - - ); - })} - - ); - } else { - return histogram n/a; - } -}; - -export default SparklineHistogram; diff --git a/ui/src/custom-tabs/TabsRegistryContext.tsx b/ui/src/custom-tabs/TabsRegistryContext.tsx index 9f493e6d11b..83820de1535 100644 --- a/ui/src/custom-tabs/TabsRegistryContext.tsx +++ b/ui/src/custom-tabs/TabsRegistryContext.tsx @@ -10,6 +10,7 @@ import { import RegularFeatureViewCustomTabLoadingWrapper from "../utils/custom-tabs/RegularFeatureViewCustomTabLoadingWrapper"; import OnDemandFeatureViewCustomTabLoadingWrapper from "../utils/custom-tabs/OnDemandFeatureViewCustomTabLoadingWrapper"; +import StreamFeatureViewCustomTabLoadingWrapper from "../utils/custom-tabs/StreamFeatureViewCustomTabLoadingWrapper"; import FeatureServiceCustomTabLoadingWrapper from "../utils/custom-tabs/FeatureServiceCustomTabLoadingWrapper"; import FeatureCustomTabLoadingWrapper from "../utils/custom-tabs/FeatureCustomTabLoadingWrapper"; import DataSourceCustomTabLoadingWrapper from "../utils/custom-tabs/DataSourceCustomTabLoadingWrapper"; @@ -19,6 +20,7 @@ import DatasetCustomTabLoadingWrapper from "../utils/custom-tabs/DatasetCustomTa import { RegularFeatureViewCustomTabRegistrationInterface, OnDemandFeatureViewCustomTabRegistrationInterface, + StreamFeatureViewCustomTabRegistrationInterface, FeatureServiceCustomTabRegistrationInterface, FeatureCustomTabRegistrationInterface, DataSourceCustomTabRegistrationInterface, @@ -30,6 +32,7 @@ import { interface FeastTabsRegistryInterface { RegularFeatureViewCustomTabs?: RegularFeatureViewCustomTabRegistrationInterface[]; OnDemandFeatureViewCustomTabs?: OnDemandFeatureViewCustomTabRegistrationInterface[]; + StreamFeatureViewCustomTabs?: StreamFeatureViewCustomTabRegistrationInterface[]; FeatureServiceCustomTabs?: FeatureServiceCustomTabRegistrationInterface[]; FeatureCustomTabs?: FeatureCustomTabRegistrationInterface[]; DataSourceCustomTabs?: DataSourceCustomTabRegistrationInterface[]; @@ -148,6 +151,16 @@ const useOnDemandFeatureViewCustomTabs = (navigate: NavigateFunction) => { ); }; +const useStreamFeatureViewCustomTabs = (navigate: NavigateFunction) => { + const { StreamFeatureViewCustomTabs } = + React.useContext(TabsRegistryContext); + + return useGenericCustomTabsNavigation( + StreamFeatureViewCustomTabs || [], + navigate + ); +}; + const useFeatureServiceCustomTabs = (navigate: NavigateFunction) => { const { FeatureServiceCustomTabs } = React.useContext(TabsRegistryContext); @@ -214,6 +227,16 @@ const useOnDemandFeatureViewCustomTabRoutes = () => { ); }; +const useStreamFeatureViewCustomTabRoutes = () => { + const { StreamFeatureViewCustomTabs } = + React.useContext(TabsRegistryContext); + + return genericCustomTabRoutes( + StreamFeatureViewCustomTabs || [], + StreamFeatureViewCustomTabLoadingWrapper + ); +}; + const useFeatureServiceCustomTabRoutes = () => { const { FeatureServiceCustomTabs } = React.useContext(TabsRegistryContext); @@ -264,6 +287,7 @@ export { // Navigation useRegularFeatureViewCustomTabs, useOnDemandFeatureViewCustomTabs, + useStreamFeatureViewCustomTabs, useFeatureServiceCustomTabs, useFeatureCustomTabs, useDataSourceCustomTabs, @@ -272,6 +296,7 @@ export { // Routes useRegularFeatureViewCustomTabRoutes, useOnDemandFeatureViewCustomTabRoutes, + useStreamFeatureViewCustomTabRoutes, useFeatureServiceCustomTabRoutes, useFeatureCustomTabRoutes, useDataSourceCustomTabRoutes, diff --git a/ui/src/custom-tabs/stream-fv-demo-tab/DemoCustomTab.tsx b/ui/src/custom-tabs/stream-fv-demo-tab/DemoCustomTab.tsx new file mode 100644 index 00000000000..86e59d10c71 --- /dev/null +++ b/ui/src/custom-tabs/stream-fv-demo-tab/DemoCustomTab.tsx @@ -0,0 +1,85 @@ +import React from "react"; + +import { + // Feature View Custom Tabs will get these props + StreamFeatureViewCustomTabProps, +} from "../types"; + +import { + EuiLoadingContent, + EuiEmptyPrompt, + EuiFlexGroup, + EuiFlexItem, + EuiCode, + EuiSpacer, +} from "@elastic/eui"; + +// Separating out the query is not required, +// but encouraged for code readability +import useDemoQuery from "./useDemoQuery"; + +const DemoCustomTab = ({ + id, + feastObjectQuery, +}: StreamFeatureViewCustomTabProps) => { + // Use React Query to fetch data + // that is custom to this tab. + // See: https://react-query.tanstack.com/guides/queries + const { isLoading, isError, isSuccess, data } = useDemoQuery({ + featureView: id, + }); + + if (isLoading) { + // Handle Loading State + // https://elastic.github.io/eui/#/display/loading + return ; + } + + if (isError) { + // Handle Data Fetching Error + // https://elastic.github.io/eui/#/display/empty-prompt + return ( + Unable to load your demo page} + body={ +

+ There was an error loading the Dashboard application. Contact your + administrator for help. +

+ } + /> + ); + } + + // Feast UI uses the Elastic UI component system. + // and are particularly + // useful for layouts. + return ( + + + +

Hello World. The following is fetched data.

+ + {isSuccess && data && ( + +
{JSON.stringify(data, null, 2)}
+
+ )} +
+ +

... and this is data from Feast UI’s own query.

+ + {feastObjectQuery.isSuccess && feastObjectQuery.data && ( + +
{JSON.stringify(feastObjectQuery.data, null, 2)}
+
+ )} +
+
+
+ ); +}; + +export default DemoCustomTab; diff --git a/ui/src/custom-tabs/stream-fv-demo-tab/useDemoQuery.tsx b/ui/src/custom-tabs/stream-fv-demo-tab/useDemoQuery.tsx new file mode 100644 index 00000000000..b93602dbe3b --- /dev/null +++ b/ui/src/custom-tabs/stream-fv-demo-tab/useDemoQuery.tsx @@ -0,0 +1,44 @@ +import { useQuery } from "react-query"; +import { z } from "zod"; + +// Use Zod to check the shape of the +// json object being loaded +const demoSchema = z.object({ + hello: z.string(), + name: z.string().optional(), +}); + +// Make the type of the object available +type DemoDataType = z.infer; + +interface DemoQueryInterface { + featureView: string | undefined; +} + +const useDemoQuery = ({ featureView }: DemoQueryInterface) => { + // React Query manages caching for you based on query keys + // See: https://react-query.tanstack.com/guides/query-keys + const queryKey = `demo-tab-namespace:${featureView}`; + + // Pass the type to useQuery + // so that components consuming the + // result gets nice type hints + // on the other side. + return useQuery( + queryKey, + () => { + // Customizing the URL based on your needs + const url = `/demo-custom-tabs/demo.json`; + + return fetch(url) + .then((res) => res.json()) + .then((data) => demoSchema.parse(data)); // Use zod to parse results + }, + { + enabled: !!featureView, // Only start the query when the variable is not undefined + } + ); +}; + +export default useDemoQuery; +export type { DemoDataType }; diff --git a/ui/src/custom-tabs/types.ts b/ui/src/custom-tabs/types.ts index 1e555d6185c..ea1dbc8757b 100644 --- a/ui/src/custom-tabs/types.ts +++ b/ui/src/custom-tabs/types.ts @@ -1,5 +1,6 @@ import { useLoadOnDemandFeatureView, + useLoadStreamFeatureView, useLoadRegularFeatureView, } from "../pages/feature-views/useLoadFeatureView"; import useLoadFeature from "../pages/features/useLoadFeature"; @@ -48,6 +49,23 @@ interface OnDemandFeatureViewCustomTabRegistrationInterface }: OnDemandFeatureViewCustomTabProps) => JSX.Element; } +// Type for Stream Feature View Custom Tabs +type StreamFeatureViewQueryReturnType = ReturnType< + typeof useLoadStreamFeatureView +>; +interface StreamFeatureViewCustomTabProps { + id: string | undefined; + feastObjectQuery: StreamFeatureViewQueryReturnType; +} +interface StreamFeatureViewCustomTabRegistrationInterface + extends CustomTabRegistrationInterface { + Component: ({ + id, + feastObjectQuery, + ...args + }: StreamFeatureViewCustomTabProps) => JSX.Element; +} + // Type for Entity Custom Tabs interface EntityCustomTabProps { id: string | undefined; @@ -127,6 +145,9 @@ export type { OnDemandFeatureViewQueryReturnType, OnDemandFeatureViewCustomTabProps, OnDemandFeatureViewCustomTabRegistrationInterface, + StreamFeatureViewQueryReturnType, + StreamFeatureViewCustomTabProps, + StreamFeatureViewCustomTabRegistrationInterface, FeatureServiceCustomTabRegistrationInterface, FeatureServiceCustomTabProps, DataSourceCustomTabRegistrationInterface, diff --git a/ui/src/hooks/useFCOExploreSuggestions.ts b/ui/src/hooks/useFCOExploreSuggestions.ts index 5767f73beda..822642daf4b 100644 --- a/ui/src/hooks/useFCOExploreSuggestions.ts +++ b/ui/src/hooks/useFCOExploreSuggestions.ts @@ -1,9 +1,9 @@ import { encodeSearchQueryString } from "./encodeSearchQueryString"; import { FEAST_FCO_TYPES } from "../parsers/types"; -import { FeastFeatureViewType } from "../parsers/feastFeatureViews"; import { useParams } from "react-router-dom"; import { useFeatureViewTagsAggregation } from "./useTagsAggregation"; +import { feast } from "../protos"; interface ExplorationSuggestionItem { name: string; @@ -66,14 +66,14 @@ const sortTagsByTotalUsage = ( }; const generateExplorationSuggestions = ( - tagAggregation: Record>, + tagAggregation: Record>, projectName: string ) => { const suggestions: ExplorationSuggestion[] = []; if (tagAggregation) { const SortedCandidates = - sortTagByUniqueValues(tagAggregation); + sortTagByUniqueValues(tagAggregation); SortedCandidates.slice(0, NUMBER_OF_SUGGESTION_GROUPS).forEach( ([selectedTag, selectedTagValuesMap]) => { diff --git a/ui/src/hooks/useTagsAggregation.ts b/ui/src/hooks/useTagsAggregation.ts index 21480fa8b0c..13d0c2f9669 100644 --- a/ui/src/hooks/useTagsAggregation.ts +++ b/ui/src/hooks/useTagsAggregation.ts @@ -1,8 +1,7 @@ import { useContext, useMemo } from "react"; import RegistryPathContext from "../contexts/RegistryPathContext"; -import { FeastFeatureServiceType } from "../parsers/feastFeatureServices"; -import { FeastFeatureViewType } from "../parsers/feastFeatureViews"; import useLoadRegistry from "../queries/useLoadRegistry"; +import { feast } from "../protos"; // Usage of generic type parameter T // https://stackoverflow.com/questions/53203409/how-to-tell-typescript-that-im-returning-an-array-of-arrays-of-the-input-type @@ -44,12 +43,12 @@ const useFeatureViewTagsAggregation = () => { const data = useMemo(() => { return query.data && query.data.objects && query.data.objects.featureViews - ? buildTagCollection( - query.data.objects.featureViews, - (fv) => { - return fv.spec.tags; - } - ) + ? buildTagCollection( + query.data.objects.featureViews!, + (fv) => { + return fv.spec?.tags!; + } + ) : undefined; }, [query.data]); @@ -67,12 +66,12 @@ const useFeatureServiceTagsAggregation = () => { return query.data && query.data.objects && query.data.objects.featureServices - ? buildTagCollection( - query.data.objects.featureServices, - (fs) => { - return fs.spec.tags; - } - ) + ? buildTagCollection( + query.data.objects.featureServices, + (fs) => { + return fs.spec?.tags!; + } + ) : undefined; }, [query.data]); diff --git a/ui/src/index.tsx b/ui/src/index.tsx index 2233b90c9e6..e38570929d4 100644 --- a/ui/src/index.tsx +++ b/ui/src/index.tsx @@ -18,6 +18,7 @@ import FeastUI from "./FeastUI"; import DataTab from "./custom-tabs/data-tab/DataTab"; import RFVDemoCustomTab from "./custom-tabs/reguar-fv-demo-tab/DemoCustomTab"; import ODFVDemoCustomTab from "./custom-tabs/ondemand-fv-demo-tab/DemoCustomTab"; +import SFVDemoCustomTab from "./custom-tabs/stream-fv-demo-tab/DemoCustomTab"; import FSDemoCustomTab from "./custom-tabs/feature-service-demo-tab/DemoCustomTab"; import DSDemoCustomTab from "./custom-tabs/data-source-demo-tab/DemoCustomTab"; import EntDemoCustomTab from "./custom-tabs/entity-demo-tab/DemoCustomTab"; @@ -46,6 +47,13 @@ const tabsRegistry = { Component: ODFVDemoCustomTab, }, ], + StreamFeatureViewCustomTabs: [ + { + label: "Custom Tab Demo", + path: "demo-tab", + Component: SFVDemoCustomTab, + }, + ], FeatureServiceCustomTabs: [ { label: "Custom Tab Demo", @@ -93,4 +101,4 @@ ReactDOM.render( /> , document.getElementById("root") -); \ No newline at end of file +); diff --git a/ui/src/mocks/handlers.ts b/ui/src/mocks/handlers.ts index e7b0040f0dc..39f30b62a6d 100644 --- a/ui/src/mocks/handlers.ts +++ b/ui/src/mocks/handlers.ts @@ -1,5 +1,8 @@ import { rest } from "msw"; -import registry from "../../public/registry.json"; +import {readFileSync} from 'fs'; +import path from "path"; + +const registry = readFileSync(path.resolve(__dirname, "../../public/registry.db")); const projectsListWithDefaultProject = rest.get( "/projects-list.json", @@ -14,7 +17,7 @@ const projectsListWithDefaultProject = rest.get( description: "Project for credit scoring team and associated models.", id: "credit_score_project", - registryPath: "/registry.json", + registryPath: "/registry.pb", }, ], }) @@ -22,8 +25,11 @@ const projectsListWithDefaultProject = rest.get( } ); -const creditHistoryRegistry = rest.get("/registry.json", (req, res, ctx) => { - return res(ctx.status(200), ctx.json(registry)); +const creditHistoryRegistry = rest.get("/registry.pb", (req, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/octet-stream'), + ctx.body(registry)); }); export { projectsListWithDefaultProject, creditHistoryRegistry }; diff --git a/ui/src/pages/data-sources/BatchSourcePropertiesView.tsx b/ui/src/pages/data-sources/BatchSourcePropertiesView.tsx index 97f9f58d274..c19e4ff50f8 100644 --- a/ui/src/pages/data-sources/BatchSourcePropertiesView.tsx +++ b/ui/src/pages/data-sources/BatchSourcePropertiesView.tsx @@ -1,38 +1,16 @@ import React from "react"; import { - EuiCodeBlock, EuiDescriptionList, EuiDescriptionListDescription, EuiDescriptionListTitle, EuiFlexGroup, EuiFlexItem, - EuiSpacer, - EuiTitle, } from "@elastic/eui"; +import { feast } from "../../protos"; +import { toDate } from "../../utils/timestamp"; interface BatchSourcePropertiesViewProps { - batchSource: { - type?: string | undefined; - owner?: string | undefined; - description?: string | undefined; - dataSourceClassType?: string | undefined; - fileOptions?: - | { - uri?: string | undefined; - } - | undefined; - meta?: - | { - latestEventTimestamp?: Date | undefined; - earliestEventTimestamp?: Date | undefined; - } - | undefined; - bigqueryOptions?: - | { - dbtModelSerialized?: string | undefined; - } - | undefined; - }; + batchSource: feast.core.IDataSource; } const BatchSourcePropertiesView = (props: BatchSourcePropertiesViewProps) => { @@ -49,7 +27,7 @@ const BatchSourcePropertiesView = (props: BatchSourcePropertiesViewProps) => { {batchSource.dataSourceClassType.split(".").at(-1)} - ) : batchSource.type ? ( + ) : feast.core.DataSource.SourceType[batchSource.type!] ? ( {batchSource.type} @@ -87,9 +65,9 @@ const BatchSourcePropertiesView = (props: BatchSourcePropertiesViewProps) => { Latest Event - {batchSource.meta.latestEventTimestamp.toLocaleDateString( - "en-CA" - )} + {toDate( + batchSource.meta.latestEventTimestamp + ).toLocaleDateString("en-CA")} )} @@ -99,35 +77,14 @@ const BatchSourcePropertiesView = (props: BatchSourcePropertiesViewProps) => { Earliest Event - {batchSource.meta.earliestEventTimestamp.toLocaleDateString( - "en-CA" - )} + {toDate( + batchSource.meta?.earliestEventTimestamp + ).toLocaleDateString("en-CA")} )}
- - {batchSource.bigqueryOptions?.dbtModelSerialized && ( - - - - )} - {batchSource.bigqueryOptions?.dbtModelSerialized && ( - - -

Dbt Transformation

-
- - {batchSource.bigqueryOptions.dbtModelSerialized} - -
- )}
); diff --git a/ui/src/pages/data-sources/DataSourceDbt.tsx b/ui/src/pages/data-sources/DataSourceDbt.tsx deleted file mode 100644 index 4e61ba80266..00000000000 --- a/ui/src/pages/data-sources/DataSourceDbt.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import React from "react"; -import { - EuiCodeBlock, - EuiPanel, - EuiHorizontalRule, - EuiTitle, -} from "@elastic/eui"; -import { useParams } from "react-router-dom"; -import useLoadDataSource from "./useLoadDataSource"; - -const DataSourceDbt = () => { - let { dataSourceName } = useParams(); - - const dsName = dataSourceName === undefined ? "" : dataSourceName; - - const { isSuccess, data } = useLoadDataSource(dsName); - - return isSuccess && data && data.bigqueryOptions ? ( - - -

Dbt Transformation

-
- - - {data.bigqueryOptions.dbtModelSerialized} - -
- ) : ( - - No data so sad - - ); -}; - -export default DataSourceDbt; diff --git a/ui/src/pages/data-sources/DataSourceInstance.tsx b/ui/src/pages/data-sources/DataSourceInstance.tsx index 332f51e0235..d6db4c87472 100644 --- a/ui/src/pages/data-sources/DataSourceInstance.tsx +++ b/ui/src/pages/data-sources/DataSourceInstance.tsx @@ -7,12 +7,10 @@ import { } from "@elastic/eui"; import { DataSourceIcon32 } from "../../graphics/DataSourceIcon"; -import { useMatchExact, useMatchSubpath } from "../../hooks/useMatchSubpath"; +import { useMatchExact } from "../../hooks/useMatchSubpath"; import { useDocumentTitle } from "../../hooks/useDocumentTitle"; import DataSourceRawData from "./DataSourceRawData"; import DataSourceOverviewTab from "./DataSourceOverviewTab"; -import DataSourceDbt from "./DataSourceDbt"; -import useLoadDataSource from "./useLoadDataSource"; import { useDataSourceCustomTabs, @@ -24,8 +22,6 @@ const DataSourceInstance = () => { let { dataSourceName } = useParams(); useDocumentTitle(`${dataSourceName} | Data Source | Feast`); - const dsName = dataSourceName === undefined ? "" : dataSourceName; - const { isSuccess, data } = useLoadDataSource(dsName); let tabs = [ { @@ -37,17 +33,6 @@ const DataSourceInstance = () => { }, ]; - const dbtTab = { - label: "Dbt Definition", - isSelected: useMatchSubpath("dbt"), - onClick: () => { - navigate("dbt"); - }, - }; - if (isSuccess && data?.bigqueryOptions?.dbtModelSerialized) { - tabs.push(dbtTab); - } - const { customNavigationTabs } = useDataSourceCustomTabs(navigate); tabs = tabs.concat(customNavigationTabs); @@ -72,7 +57,6 @@ const DataSourceInstance = () => { } /> } /> - } /> {CustomTabRoutes} diff --git a/ui/src/pages/data-sources/DataSourceOverviewTab.tsx b/ui/src/pages/data-sources/DataSourceOverviewTab.tsx index 124a0e6ab92..6eadfc42d0a 100644 --- a/ui/src/pages/data-sources/DataSourceOverviewTab.tsx +++ b/ui/src/pages/data-sources/DataSourceOverviewTab.tsx @@ -19,6 +19,7 @@ import BatchSourcePropertiesView from "./BatchSourcePropertiesView"; import FeatureViewEdgesList from "../entities/FeatureViewEdgesList"; import RequestDataSourceSchemaTable from "./RequestDataSourceSchemaTable"; import useLoadDataSource from "./useLoadDataSource"; +import { feast } from "../../protos"; const DataSourceOverviewTab = () => { let { dataSourceName } = useParams(); @@ -57,7 +58,7 @@ const DataSourceOverviewTab = () => { Source Type - {data.type} + {feast.core.DataSource.SourceType[data.type]} @@ -77,12 +78,12 @@ const DataSourceOverviewTab = () => { { + fields={data?.requestDataOptions?.schema!.map((obj) => { return { - fieldName: obj.name, - valueType: obj.valueType, + fieldName: obj.name!, + valueType: obj.valueType!, }; - })} + })!} /> ) : ( diff --git a/ui/src/pages/data-sources/DataSourcesListingTable.tsx b/ui/src/pages/data-sources/DataSourcesListingTable.tsx index 661d18c7e9d..50c1f933a9c 100644 --- a/ui/src/pages/data-sources/DataSourcesListingTable.tsx +++ b/ui/src/pages/data-sources/DataSourcesListingTable.tsx @@ -1,11 +1,11 @@ import React from "react"; import { EuiBasicTable } from "@elastic/eui"; import EuiCustomLink from "../../components/EuiCustomLink"; -import { FeastDatasourceType } from "../../parsers/feastDatasources"; import { useParams } from "react-router-dom"; +import { feast } from "../../protos"; interface DatasourcesListingTableProps { - dataSources: FeastDatasourceType[]; + dataSources: feast.core.IDataSource[]; } const DatasourcesListingTable = ({ @@ -33,10 +33,13 @@ const DatasourcesListingTable = ({ name: "Type", field: "type", sortable: true, + render: (valueType: feast.types.ValueType.Enum) => { + return feast.types.ValueType.Enum[valueType]; + }, }, ]; - const getRowProps = (item: FeastDatasourceType) => { + const getRowProps = (item: feast.core.IDataSource) => { return { "data-test-subj": `row-${item.name}`, }; diff --git a/ui/src/pages/data-sources/RequestDataSourceSchemaTable.tsx b/ui/src/pages/data-sources/RequestDataSourceSchemaTable.tsx index 60ef4c406a4..a55aeac0280 100644 --- a/ui/src/pages/data-sources/RequestDataSourceSchemaTable.tsx +++ b/ui/src/pages/data-sources/RequestDataSourceSchemaTable.tsx @@ -1,10 +1,10 @@ import React from "react"; import { EuiBasicTable } from "@elastic/eui"; -import { FEAST_FEATURE_VALUE_TYPES } from "../../parsers/types"; +import { feast } from "../../protos"; interface RequestDataSourceSchemaField { fieldName: string; - valueType: FEAST_FEATURE_VALUE_TYPES; + valueType: feast.types.ValueType.Enum; } interface RequestDataSourceSchema { @@ -12,7 +12,6 @@ interface RequestDataSourceSchema { } const RequestDataSourceSchemaTable = ({ fields }: RequestDataSourceSchema) => { - console.log(fields); const columns = [ { name: "Field", @@ -21,6 +20,9 @@ const RequestDataSourceSchemaTable = ({ fields }: RequestDataSourceSchema) => { { name: "Value Type", field: "valueType", + render: (valueType: feast.types.ValueType.Enum) => { + return feast.types.ValueType.Enum[valueType]; + }, }, ]; diff --git a/ui/src/pages/entities/EntitiesListingTable.tsx b/ui/src/pages/entities/EntitiesListingTable.tsx index 2e608e37692..2a017b18aac 100644 --- a/ui/src/pages/entities/EntitiesListingTable.tsx +++ b/ui/src/pages/entities/EntitiesListingTable.tsx @@ -1,12 +1,12 @@ import React from "react"; import { EuiBasicTable } from "@elastic/eui"; import EuiCustomLink from "../../components/EuiCustomLink"; -import { FeastEntityType } from "../../parsers/feastEntities"; import useFeatureViewEdgesByEntity from "./useFeatureViewEdgesByEntity"; import { useParams } from "react-router-dom"; +import { feast } from "../../protos"; interface EntitiesListingTableProps { - entities: FeastEntityType[]; + entities: feast.core.IEntity[]; } const EntitiesListingTable = ({ entities }: EntitiesListingTableProps) => { @@ -33,15 +33,15 @@ const EntitiesListingTable = ({ entities }: EntitiesListingTableProps) => { name: "Type", field: "spec.valueType", sortable: true, - render: (valueType: string) => { - return valueType; + render: (valueType: feast.types.ValueType.Enum) => { + return feast.types.ValueType.Enum[valueType]; }, }, { name: "# of FVs", - render: (item: FeastEntityType) => { + render: (item: feast.core.IEntity) => { if (isSuccess && data) { - return data[item.spec.name] ? data[item.spec.name].length : "0"; + return data[item?.spec?.name!] ? data[item?.spec?.name!].length : "0"; } else { return "."; } @@ -49,9 +49,9 @@ const EntitiesListingTable = ({ entities }: EntitiesListingTableProps) => { }, ]; - const getRowProps = (item: FeastEntityType) => { + const getRowProps = (item: feast.core.IEntity) => { return { - "data-test-subj": `row-${item.spec.name}`, + "data-test-subj": `row-${item?.spec?.name}`, }; }; diff --git a/ui/src/pages/entities/EntityOverviewTab.tsx b/ui/src/pages/entities/EntityOverviewTab.tsx index dc649c2a9fb..b4df67bba49 100644 --- a/ui/src/pages/entities/EntityOverviewTab.tsx +++ b/ui/src/pages/entities/EntityOverviewTab.tsx @@ -19,6 +19,8 @@ import TagsDisplay from "../../components/TagsDisplay"; import FeatureViewEdgesList from "./FeatureViewEdgesList"; import useFeatureViewEdgesByEntity from "./useFeatureViewEdgesByEntity"; import useLoadEntity from "./useLoadEntity"; +import { toDate } from "../../utils/timestamp"; +import { feast } from "../../protos"; const EntityOverviewTab = () => { let { entityName } = useParams(); @@ -52,17 +54,17 @@ const EntityOverviewTab = () => { Join Key - {data.spec.joinKey} + {data?.spec?.joinKey} Description - {data.spec.description} + {data?.spec?.description} Value Type - {data.spec.valueType} + {feast.types.ValueType.Enum[data?.spec?.valueType!]} @@ -71,8 +73,8 @@ const EntityOverviewTab = () => { Created - {data.meta.createdTimestamp ? ( - data.meta.createdTimestamp.toLocaleDateString("en-CA") + {data?.meta?.createdTimestamp ? ( + toDate(data.meta.createdTimestamp).toLocaleDateString("en-CA") ) : ( No createdTimestamp specified on this entity. )} @@ -80,8 +82,8 @@ const EntityOverviewTab = () => { Updated - {data.meta.lastUpdatedTimestamp ? ( - data.meta.lastUpdatedTimestamp.toLocaleDateString("en-CA") + {data?.meta?.lastUpdatedTimestamp ? ( + toDate(data.meta.lastUpdatedTimestamp).toLocaleDateString("en-CA") ) : ( No lastUpdatedTimestamp specified on this entity. )} @@ -117,8 +119,8 @@ const EntityOverviewTab = () => {

Labels

- {data.spec.labels ? ( - + {data?.spec?.tags ? ( + ) : ( No labels specified on this entity. )} diff --git a/ui/src/pages/entities/useLoadEntity.ts b/ui/src/pages/entities/useLoadEntity.ts index a1ca6d55c16..6f7bb1f59cf 100644 --- a/ui/src/pages/entities/useLoadEntity.ts +++ b/ui/src/pages/entities/useLoadEntity.ts @@ -10,8 +10,8 @@ const useLoadEntity = (entityName: string) => { registryQuery.data === undefined ? undefined : registryQuery.data.objects.entities?.find( - (fv) => fv.spec.name === entityName - ); + (fv) => fv?.spec?.name === entityName + ); return { ...registryQuery, diff --git a/ui/src/pages/feature-services/FeatureServiceListingTable.tsx b/ui/src/pages/feature-services/FeatureServiceListingTable.tsx index b865da6e23c..c81edeaeb58 100644 --- a/ui/src/pages/feature-services/FeatureServiceListingTable.tsx +++ b/ui/src/pages/feature-services/FeatureServiceListingTable.tsx @@ -5,20 +5,18 @@ import { EuiTableFieldDataColumnType, } from "@elastic/eui"; import EuiCustomLink from "../../components/EuiCustomLink"; -import { - FeastFeatureInServiceType, - FeastFeatureServiceType, -} from "../../parsers/feastFeatureServices"; import { useParams } from "react-router-dom"; +import { feast } from "../../protos"; +import { toDate } from "../../utils/timestamp"; interface FeatureServiceListingTableProps { tagKeysSet: Set; - featureServices: FeastFeatureServiceType[]; + featureServices: feast.core.IFeatureService[]; } type FeatureServiceTypeColumn = - | EuiTableFieldDataColumnType - | EuiTableComputedColumnType; + | EuiTableFieldDataColumnType + | EuiTableComputedColumnType; const FeatureServiceListingTable = ({ tagKeysSet, @@ -44,10 +42,10 @@ const FeatureServiceListingTable = ({ { name: "# of Features", field: "spec.features", - render: (featureViews: FeastFeatureInServiceType[]) => { + render: (featureViews: feast.core.IFeatureViewProjection[]) => { var numFeatures = 0; featureViews.forEach((featureView) => { - numFeatures += featureView.featureColumns.length; + numFeatures += featureView.featureColumns!.length; }); return numFeatures; }, @@ -55,8 +53,8 @@ const FeatureServiceListingTable = ({ { name: "Last updated", field: "meta.lastUpdatedTimestamp", - render: (date: Date) => { - return date ? date.toLocaleDateString("en-CA") : "n/a"; + render: (date: any) => { + return date ? toDate(date).toLocaleDateString("en-CA") : "n/a"; }, }, ]; @@ -64,10 +62,10 @@ const FeatureServiceListingTable = ({ tagKeysSet.forEach((key) => { columns.push({ name: key, - render: (item: FeastFeatureServiceType) => { + render: (item: feast.core.IFeatureService) => { let tag = n/a; - const value = item.spec.tags ? item.spec.tags[key] : undefined; + const value = item?.spec?.tags ? item.spec.tags[key] : undefined; if (value) { tag = {value}; @@ -78,9 +76,9 @@ const FeatureServiceListingTable = ({ }); }); - const getRowProps = (item: FeastFeatureServiceType) => { + const getRowProps = (item: feast.core.IFeatureService) => { return { - "data-test-subj": `row-${item.spec.name}`, + "data-test-subj": `row-${item?.spec?.name}`, }; }; diff --git a/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx b/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx index ea62b3b3a70..387320778ff 100644 --- a/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx +++ b/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx @@ -19,6 +19,7 @@ import TagsDisplay from "../../components/TagsDisplay"; import { encodeSearchQueryString } from "../../hooks/encodeSearchQueryString"; import FeatureViewEdgesList from "../entities/FeatureViewEdgesList"; import useLoadFeatureService from "./useLoadFeatureService"; +import { toDate } from "../../utils/timestamp"; const FeatureServiceOverviewTab = () => { let { featureServiceName, projectName } = useParams(); @@ -32,9 +33,9 @@ const FeatureServiceOverviewTab = () => { let numFeatures = 0; let numFeatureViews = 0; if (data) { - data.spec.features.forEach((featureView) => { + data?.spec?.features?.forEach((featureView) => { numFeatureViews += 1; - numFeatures += featureView.featureColumns.length; + numFeatures += featureView?.featureColumns!.length; }); } @@ -66,10 +67,10 @@ const FeatureServiceOverviewTab = () => { description="Feature Views" /> - {data.meta.lastUpdatedTimestamp ? ( + {data?.meta?.lastUpdatedTimestamp ? ( {

Features

- {data.spec.features ? ( - + {data?.spec?.features ? ( + ) : ( No features specified for this feature service. @@ -103,7 +104,7 @@ const FeatureServiceOverviewTab = () => {

Tags

- {data.spec.tags ? ( + {data?.spec?.tags ? ( { @@ -154,11 +155,11 @@ const FeatureServiceOverviewTab = () => {

All Feature Views

- {data.spec.features.length > 0 ? ( + {data?.spec?.features?.length! > 0 ? ( { - return f.featureViewName; - })} + fvNames={data?.spec?.features?.map((f) => { + return f.featureViewName!; + })!} /> ) : ( No feature views in this feature service diff --git a/ui/src/pages/feature-services/Index.tsx b/ui/src/pages/feature-services/Index.tsx index 441f3cf82c9..d034a24bd7c 100644 --- a/ui/src/pages/feature-services/Index.tsx +++ b/ui/src/pages/feature-services/Index.tsx @@ -22,12 +22,12 @@ import { filterInputInterface, tagTokenGroupsType, } from "../../hooks/useSearchInputWithTags"; -import { FeastFeatureServiceType } from "../../parsers/feastFeatureServices"; import { useDocumentTitle } from "../../hooks/useDocumentTitle"; import RegistryPathContext from "../../contexts/RegistryPathContext"; import FeatureServiceIndexEmptyState from "./FeatureServiceIndexEmptyState"; import TagSearch from "../../components/TagSearch"; import { useFeatureServiceTagsAggregation } from "../../hooks/useTagsAggregation"; +import { feast } from "../../protos"; const useLoadFeatureServices = () => { const registryUrl = useContext(RegistryPathContext); @@ -45,11 +45,11 @@ const useLoadFeatureServices = () => { }; const shouldIncludeFSsGivenTokenGroups = ( - entry: FeastFeatureServiceType, + entry: feast.core.IFeatureService, tagTokenGroups: tagTokenGroupsType ) => { return Object.entries(tagTokenGroups).every(([key, values]) => { - const entryTagValue = entry.spec.tags ? entry.spec.tags[key] : undefined; + const entryTagValue = entry?.spec?.tags ? entry.spec.tags[key] : undefined; if (entryTagValue) { return values.every((value) => { @@ -62,7 +62,7 @@ const shouldIncludeFSsGivenTokenGroups = ( }; const filterFn = ( - data: FeastFeatureServiceType[], + data: feast.core.IFeatureService[], filterInput: filterInputInterface ) => { let filteredByTags = data; @@ -79,7 +79,7 @@ const filterFn = ( if (filterInput.searchTokens.length) { return filteredByTags.filter((entry) => { return filterInput.searchTokens.find((token) => { - return token.length >= 3 && entry.spec.name.indexOf(token) >= 0; + return token.length >= 3 && entry?.spec?.name?.indexOf(token)! >= 0; }); }); } diff --git a/ui/src/pages/feature-services/useLoadFeatureService.ts b/ui/src/pages/feature-services/useLoadFeatureService.ts index be2242eae04..7d991533a1d 100644 --- a/ui/src/pages/feature-services/useLoadFeatureService.ts +++ b/ui/src/pages/feature-services/useLoadFeatureService.ts @@ -13,23 +13,23 @@ const useLoadFeatureService = (featureServiceName: string) => { registryQuery.data === undefined ? undefined : registryQuery.data.objects.featureServices?.find( - (fs) => fs.spec.name === featureServiceName - ); + (fs) => fs?.spec?.name === featureServiceName + ); let entities = data === undefined ? undefined : registryQuery.data?.indirectRelationships - .filter((relationship) => { - return ( - relationship.target.type === FEAST_FCO_TYPES.featureService && - relationship.target.name === data.spec.name && - relationship.source.type === FEAST_FCO_TYPES.entity - ); - }) - .map((relationship) => { - return relationship.source; - }); + .filter((relationship) => { + return ( + relationship.target.type === FEAST_FCO_TYPES.featureService && + relationship.target.name === data?.spec?.name && + relationship.source.type === FEAST_FCO_TYPES.entity + ); + }) + .map((relationship) => { + return relationship.source; + }); // Deduplicate on name of entity if (entities) { let entityToName: { [key: string]: EntityReference } = {}; diff --git a/ui/src/pages/feature-views/FeatureViewInstance.tsx b/ui/src/pages/feature-views/FeatureViewInstance.tsx index b0fa7c32b03..dbe4dad6ec1 100644 --- a/ui/src/pages/feature-views/FeatureViewInstance.tsx +++ b/ui/src/pages/feature-views/FeatureViewInstance.tsx @@ -3,12 +3,13 @@ import React from "react"; import { useParams } from "react-router-dom"; import { EuiLoadingSpinner } from "@elastic/eui"; -import { FeastFeatureViewType } from "../../parsers/feastFeatureViews"; import RegularFeatureInstance from "./RegularFeatureViewInstance"; import { FEAST_FV_TYPES } from "../../parsers/mergedFVTypes"; -import { FeastODFVType } from "../../parsers/feastODFVS"; + import useLoadFeatureView from "./useLoadFeatureView"; import OnDemandFeatureInstance from "./OnDemandFeatureViewInstance"; +import StreamFeatureInstance from "./StreamFeatureViewInstance"; +import { feast } from "../../protos"; const FeatureViewInstance = () => { const { featureViewName } = useParams(); @@ -35,16 +36,21 @@ const FeatureViewInstance = () => { if (isSuccess && !isEmpty) { if (data.type === FEAST_FV_TYPES.regular) { - const fv: FeastFeatureViewType = data.object; + const fv: feast.core.IFeatureView = data.object; return ; } if (data.type === FEAST_FV_TYPES.ondemand) { - const odfv: FeastODFVType = data.object; + const odfv: feast.core.IOnDemandFeatureView = data.object; return ; } + if (data.type === FEAST_FV_TYPES.stream) { + const sfv: feast.core.IStreamFeatureView = data.object; + + return ; + } } return

No Data So Sad

; diff --git a/ui/src/pages/feature-views/FeatureViewListingTable.tsx b/ui/src/pages/feature-views/FeatureViewListingTable.tsx index 59f8b1ed7aa..e4eccecc975 100644 --- a/ui/src/pages/feature-views/FeatureViewListingTable.tsx +++ b/ui/src/pages/feature-views/FeatureViewListingTable.tsx @@ -35,7 +35,7 @@ const FeatureViewListingTable = ({ href={`/p/${projectName}/feature-view/${name}`} to={`/p/${projectName}/feature-view/${name}`} > - {name} {item.type === "ondemand" && ondemand} + {name} {(item.type === "ondemand" && ondemand) || (item.type === "stream" && stream)}
); }, @@ -58,7 +58,7 @@ const FeatureViewListingTable = ({ let tag = n/a; if (item.type === "regular") { - const value = item.object.spec.tags + const value = item?.object?.spec!.tags ? item.object.spec.tags[key] : undefined; diff --git a/ui/src/pages/feature-views/FeatureViewSummaryStatisticsTab.tsx b/ui/src/pages/feature-views/FeatureViewSummaryStatisticsTab.tsx deleted file mode 100644 index 7371d4a73bf..00000000000 --- a/ui/src/pages/feature-views/FeatureViewSummaryStatisticsTab.tsx +++ /dev/null @@ -1,84 +0,0 @@ -import React from "react"; - -import { EuiEmptyPrompt, EuiLoadingContent, EuiTitle } from "@elastic/eui"; -import { useParams } from "react-router-dom"; -import useLoadFeatureViewSummaryStatistics from "../../queries/useLoadFeatureViewSummaryStatistics"; -import { - NumericColumnSummaryStatisticType, - StringColumnSummaryStatisticType, -} from "../../parsers/featureViewSummaryStatistics"; -import NumericFeaturesTable from "../../components/NumericFeaturesTable"; - -interface ColumnsByGroup { - INT64?: NumericColumnSummaryStatisticType[]; - STRING?: StringColumnSummaryStatisticType[]; -} - -const FeatureViewSummaryStatisticsTab = () => { - let { featureViewName } = useParams(); - - if (!featureViewName) { - throw new Error("Unable to get Feature View Name"); - } - - const { isError, data } = - useLoadFeatureViewSummaryStatistics(featureViewName); - - if (isError) { - return ( - Error loading Statistics} - body={ -

- There was an error loading statistics for{" "} - {featureViewName}. Please check that statistics - have been generated. -

- } - /> - ); - } - - if (data) { - const columnsByGroup = Object.entries( - data.columnsSummaryStatistics - ).reduce((memo, [key, columnStatistics]) => { - if (columnStatistics.valueType === "INT64") { - if (!memo["INT64"]) { - memo[columnStatistics.valueType] = [columnStatistics]; - } else { - memo["INT64"].push(columnStatistics); - } - } - - if (columnStatistics.valueType === "STRING") { - if (!memo["STRING"]) { - memo[columnStatistics.valueType] = [columnStatistics]; - } else { - memo["STRING"].push(columnStatistics); - } - } - - return memo; - }, {}); - - return ( - - {columnsByGroup["INT64"] && ( - <> - -

Numeric Columns

-
- - - )} -
- ); - } - - return ; -}; - -export default FeatureViewSummaryStatisticsTab; diff --git a/ui/src/pages/feature-views/Index.tsx b/ui/src/pages/feature-views/Index.tsx index 3abd42a22b2..b874a532362 100644 --- a/ui/src/pages/feature-views/Index.tsx +++ b/ui/src/pages/feature-views/Index.tsx @@ -48,7 +48,7 @@ const shouldIncludeFVsGivenTokenGroups = ( tagTokenGroups: Record ) => { return Object.entries(tagTokenGroups).every(([key, values]) => { - const entryTagValue = entry.object.spec.tags + const entryTagValue = entry?.object?.spec!.tags ? entry.object.spec.tags[key] : undefined; diff --git a/ui/src/pages/feature-views/OnDemandFeatureViewInstance.tsx b/ui/src/pages/feature-views/OnDemandFeatureViewInstance.tsx index 1db2f5194fd..166746df220 100644 --- a/ui/src/pages/feature-views/OnDemandFeatureViewInstance.tsx +++ b/ui/src/pages/feature-views/OnDemandFeatureViewInstance.tsx @@ -9,16 +9,16 @@ import { import { FeatureViewIcon32 } from "../../graphics/FeatureViewIcon"; import { useMatchExact } from "../../hooks/useMatchSubpath"; -import { FeastODFVType } from "../../parsers/feastODFVS"; import OnDemandFeatureViewOverviewTab from "./OnDemandFeatureViewOverviewTab"; import { useOnDemandFeatureViewCustomTabs, useOnDemandFeatureViewCustomTabRoutes, } from "../../custom-tabs/TabsRegistryContext"; +import { feast } from "../../protos"; interface OnDemandFeatureInstanceProps { - data: FeastODFVType; + data: feast.core.IOnDemandFeatureView; } const OnDemandFeatureInstance = ({ data }: OnDemandFeatureInstanceProps) => { diff --git a/ui/src/pages/feature-views/OnDemandFeatureViewOverviewTab.tsx b/ui/src/pages/feature-views/OnDemandFeatureViewOverviewTab.tsx index 0922f62102b..ee8e41bbf6c 100644 --- a/ui/src/pages/feature-views/OnDemandFeatureViewOverviewTab.tsx +++ b/ui/src/pages/feature-views/OnDemandFeatureViewOverviewTab.tsx @@ -10,11 +10,6 @@ import { } from "@elastic/eui"; import React from "react"; import FeaturesListDisplay from "../../components/FeaturesListDisplay"; -import { - FeastODFVType, - RequestDataSourceType, - FeatureViewProjectionType, -} from "../../parsers/feastODFVS"; import { useParams } from "react-router-dom"; import { EntityRelation } from "../../parsers/parseEntityRelationships"; import { FEAST_FCO_TYPES } from "../../parsers/types"; @@ -22,9 +17,10 @@ import useLoadRelationshipData from "../../queries/useLoadRelationshipsData"; import FeatureViewProjectionDisplayPanel from "./components/FeatureViewProjectionDisplayPanel"; import RequestDataDisplayPanel from "./components/RequestDataDisplayPanel"; import ConsumingFeatureServicesList from "./ConsumingFeatureServicesList"; +import { feast } from "../../protos"; interface OnDemandFeatureViewOverviewTabProps { - data: FeastODFVType; + data: feast.core.IOnDemandFeatureView; } const whereFSconsumesThisFv = (fvName: string) => { @@ -39,13 +35,13 @@ const whereFSconsumesThisFv = (fvName: string) => { const OnDemandFeatureViewOverviewTab = ({ data, }: OnDemandFeatureViewOverviewTabProps) => { - const inputs = Object.entries(data.spec.sources); + const inputs = Object.entries(data?.spec?.sources!); const { projectName } = useParams(); const relationshipQuery = useLoadRelationshipData(); const fsNames = relationshipQuery.data ? relationshipQuery.data - .filter(whereFSconsumesThisFv(data.spec.name)) + .filter(whereFSconsumesThisFv(data?.spec?.name!)) .map((fs) => { return fs.target.name; }) @@ -61,7 +57,7 @@ const OnDemandFeatureViewOverviewTab = ({ - {data.spec.userDefinedFunction.body} + {data?.spec?.userDefinedFunction?.bodyText} @@ -70,13 +66,13 @@ const OnDemandFeatureViewOverviewTab = ({ -

Features ({data.spec.features.length})

+

Features ({data?.spec?.features!.length})

- {projectName && data.spec.features ? ( + {projectName && data?.spec?.features ? ( @@ -93,21 +89,26 @@ const OnDemandFeatureViewOverviewTab = ({ {inputs.map(([key, inputGroup]) => { - if ((inputGroup as RequestDataSourceType).requestDataSource) { + if ( + (inputGroup as feast.core.IOnDemandSource).requestDataSource + ) { return ( ); } - if (inputGroup as FeatureViewProjectionType) { + if ( + (inputGroup as feast.core.IOnDemandSource) + .featureViewProjection + ) { return ( ); diff --git a/ui/src/pages/feature-views/RegularFeatureViewInstance.tsx b/ui/src/pages/feature-views/RegularFeatureViewInstance.tsx index 7200163614b..6d34745d7cb 100644 --- a/ui/src/pages/feature-views/RegularFeatureViewInstance.tsx +++ b/ui/src/pages/feature-views/RegularFeatureViewInstance.tsx @@ -9,18 +9,17 @@ import { import { FeatureViewIcon32 } from "../../graphics/FeatureViewIcon"; import { useMatchExact, useMatchSubpath } from "../../hooks/useMatchSubpath"; -import { FeastFeatureViewType } from "../../parsers/feastFeatureViews"; import RegularFeatureViewOverviewTab from "./RegularFeatureViewOverviewTab"; -import FeatureViewSummaryStatisticsTab from "./FeatureViewSummaryStatisticsTab"; import { useRegularFeatureViewCustomTabs, useRegularFeatureViewCustomTabRoutes, } from "../../custom-tabs/TabsRegistryContext"; import FeatureFlagsContext from "../../contexts/FeatureFlagsContext"; +import { feast } from "../../protos"; interface RegularFeatureInstanceProps { - data: FeastFeatureViewType; + data: feast.core.IFeatureView; } const RegularFeatureInstance = ({ data }: RegularFeatureInstanceProps) => { @@ -58,7 +57,7 @@ const RegularFeatureInstance = ({ data }: RegularFeatureInstanceProps) => { { path="/" element={} /> - } - /> {TabRoutes} diff --git a/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx b/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx index 689bc6b9024..3bbb906e05b 100644 --- a/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx +++ b/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx @@ -15,12 +15,13 @@ import { useNavigate, useParams } from "react-router-dom"; import FeaturesListDisplay from "../../components/FeaturesListDisplay"; import TagsDisplay from "../../components/TagsDisplay"; import { encodeSearchQueryString } from "../../hooks/encodeSearchQueryString"; -import { FeastFeatureViewType } from "../../parsers/feastFeatureViews"; import { EntityRelation } from "../../parsers/parseEntityRelationships"; import { FEAST_FCO_TYPES } from "../../parsers/types"; import useLoadRelationshipData from "../../queries/useLoadRelationshipsData"; import BatchSourcePropertiesView from "../data-sources/BatchSourcePropertiesView"; import ConsumingFeatureServicesList from "./ConsumingFeatureServicesList"; +import { feast } from "../../protos"; +import { toDate } from "../../utils/timestamp"; const whereFSconsumesThisFv = (fvName: string) => { return (r: EntityRelation) => { @@ -32,7 +33,7 @@ const whereFSconsumesThisFv = (fvName: string) => { }; interface RegularFeatureViewOverviewTabProps { - data: FeastFeatureViewType; + data: feast.core.IFeatureView; } const RegularFeatureViewOverviewTab = ({ @@ -49,8 +50,8 @@ const RegularFeatureViewOverviewTab = ({ const fsNames = relationshipQuery.data ? relationshipQuery.data.filter(whereFSconsumesThisFv(fvName)).map((fs) => { - return fs.target.name; - }) + return fs.target.name; + }) : []; const numOfFs = fsNames.length; @@ -66,13 +67,13 @@ const RegularFeatureViewOverviewTab = ({ -

Features ({data.spec.features.length})

+

Features ({data?.spec?.features?.length})

- {projectName && data.spec.features ? ( + {projectName && data?.spec?.features ? ( @@ -87,7 +88,7 @@ const RegularFeatureViewOverviewTab = ({

Entities

- {data.spec.entities ? ( + {data?.spec?.entities ? ( {data.spec.entities.map((entity) => { return ( @@ -128,7 +129,7 @@ const RegularFeatureViewOverviewTab = ({

Tags

- {data.spec.tags ? ( + {data?.spec?.tags ? ( { @@ -137,8 +138,8 @@ const RegularFeatureViewOverviewTab = ({ encodeSearchQueryString(`${key}:${value}`) ); }} - owner={data.spec.owner} - description={data.spec.description} + owner={data?.spec?.owner!} + description={data?.spec?.description!} /> ) : ( No Tags specified on this feature view. @@ -154,7 +155,7 @@ const RegularFeatureViewOverviewTab = ({

Batch Source

- +
@@ -164,11 +165,11 @@ const RegularFeatureViewOverviewTab = ({

Materialization Intervals

- {data.meta.materializationIntervals?.map((interval, i) => { + {data?.meta?.materializationIntervals?.map((interval, i) => { return (

- {interval.startTime.toLocaleDateString("en-CA")} to{" "} - {interval.endTime.toLocaleDateString("en-CA")} + {toDate(interval.startTime!).toLocaleDateString("en-CA")} to{" "} + {toDate(interval.endTime!).toLocaleDateString("en-CA")}

); })} diff --git a/ui/src/pages/feature-views/StreamFeatureViewInstance.tsx b/ui/src/pages/feature-views/StreamFeatureViewInstance.tsx new file mode 100644 index 00000000000..52bc06bc5e1 --- /dev/null +++ b/ui/src/pages/feature-views/StreamFeatureViewInstance.tsx @@ -0,0 +1,69 @@ +import React from "react"; +import { Route, Routes, useNavigate } from "react-router-dom"; +import { useParams } from "react-router-dom"; +import { + EuiPageHeader, + EuiPageContent, + EuiPageContentBody, +} from "@elastic/eui"; + +import { FeatureViewIcon32 } from "../../graphics/FeatureViewIcon"; +import { useMatchExact } from "../../hooks/useMatchSubpath"; +import StreamFeatureViewOverviewTab from "./StreamFeatureViewOverviewTab"; + +import { + useStreamFeatureViewCustomTabs, + useStreamFeatureViewCustomTabRoutes, +} from "../../custom-tabs/TabsRegistryContext"; +import { feast } from "../../protos"; + +interface StreamFeatureInstanceProps { + data: feast.core.IStreamFeatureView; +} + +const StreamFeatureInstance = ({ data }: StreamFeatureInstanceProps) => { + const navigate = useNavigate(); + let { featureViewName } = useParams(); + + const { customNavigationTabs } = useStreamFeatureViewCustomTabs(navigate); + const CustomTabRoutes = useStreamFeatureViewCustomTabRoutes(); + + return ( + + { + navigate(""); + }, + }, + ...customNavigationTabs, + ]} + /> + + + + } + /> + {CustomTabRoutes} + + + + + ); +}; + +export default StreamFeatureInstance; diff --git a/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx b/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx new file mode 100644 index 00000000000..b6cf12d3725 --- /dev/null +++ b/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx @@ -0,0 +1,133 @@ +import { + EuiFlexGroup, + EuiFlexItem, + EuiHorizontalRule, + EuiText, + EuiTitle, + EuiPanel, + EuiCodeBlock, + EuiSpacer, +} from "@elastic/eui"; +import React from "react"; +import FeaturesListDisplay from "../../components/FeaturesListDisplay"; +import { useParams } from "react-router-dom"; +import { EntityRelation } from "../../parsers/parseEntityRelationships"; +import { FEAST_FCO_TYPES } from "../../parsers/types"; +import useLoadRelationshipData from "../../queries/useLoadRelationshipsData"; +import ConsumingFeatureServicesList from "./ConsumingFeatureServicesList"; +import EuiCustomLink from "../../components/EuiCustomLink"; +import { feast } from "../../protos"; + +interface StreamFeatureViewOverviewTabProps { + data: feast.core.IStreamFeatureView; +} + +const whereFSconsumesThisFv = (fvName: string) => { + return (r: EntityRelation) => { + return ( + r.source.name === fvName && + r.target.type === FEAST_FCO_TYPES.featureService + ); + }; +}; + +const StreamFeatureViewOverviewTab = ({ + data, +}: StreamFeatureViewOverviewTabProps) => { + const inputs = Object.entries([data.spec?.streamSource]); + const { projectName } = useParams(); + + const relationshipQuery = useLoadRelationshipData(); + const fsNames = relationshipQuery.data + ? relationshipQuery.data + .filter(whereFSconsumesThisFv(data.spec?.name!)) + .map((fs) => { + return fs.target.name; + }) + : []; + + return ( + + + + + +

Transformation

+
+ + + {data.spec?.userDefinedFunction?.body} + +
+
+
+ + + + +

Features ({data.spec?.features?.length})

+
+ + {projectName && data.spec?.features ? ( + + ) : ( + No Tags sepcified on this feature view. + )} +
+
+ + + +

Inputs ({inputs.length})

+
+ + + {inputs.map(([key, inputGroup]) => { + + return ( + + + Stream Source + + + + {inputGroup?.name} + + + + + {JSON.stringify(inputGroup, null, 2)} + + + + ); + })} + +
+ + + +

Consuming Feature Services

+
+ + {fsNames.length > 0 ? ( + + ) : ( + No services consume this feature view + )} +
+
+
+
+ ); +}; + +export default StreamFeatureViewOverviewTab; diff --git a/ui/src/pages/feature-views/components/FeatureViewProjectionDisplayPanel.tsx b/ui/src/pages/feature-views/components/FeatureViewProjectionDisplayPanel.tsx index 7b110f326d7..156f6db1ec6 100644 --- a/ui/src/pages/feature-views/components/FeatureViewProjectionDisplayPanel.tsx +++ b/ui/src/pages/feature-views/components/FeatureViewProjectionDisplayPanel.tsx @@ -1,25 +1,26 @@ import React from "react"; import { EuiBasicTable, EuiPanel, EuiText, EuiTitle } from "@elastic/eui"; -import { FeatureViewProjectionType } from "../../../parsers/feastODFVS"; import { useParams } from "react-router-dom"; import EuiCustomLink from "../../../components/EuiCustomLink"; +import { feast } from "../../../protos"; -interface RequestDataDisplayPanelProps extends FeatureViewProjectionType {} +interface RequestDataDisplayPanelProps extends feast.core.IFeatureViewProjection { } -const FeatureViewProjectionDisplayPanel = ({ - featureViewProjection, -}: RequestDataDisplayPanelProps) => { +const FeatureViewProjectionDisplayPanel = (featureViewProjection: RequestDataDisplayPanelProps) => { const { projectName } = useParams(); const columns = [ { name: "Column Name", - field: "name", + field: "name" }, { name: "Type", field: "valueType", + render: (valueType: any) => { + return feast.types.ValueType.Enum[valueType]; + }, }, ]; @@ -33,12 +34,12 @@ const FeatureViewProjectionDisplayPanel = ({ href={`/p/${projectName}/feature-view/${featureViewProjection.featureViewName}`} to={`/p/${projectName}/feature-view/${featureViewProjection.featureViewName}`} > - {featureViewProjection.featureViewName} + {featureViewProjection?.featureViewName}
); diff --git a/ui/src/pages/feature-views/components/RequestDataDisplayPanel.tsx b/ui/src/pages/feature-views/components/RequestDataDisplayPanel.tsx index a6e546d9d89..e8e6854389a 100644 --- a/ui/src/pages/feature-views/components/RequestDataDisplayPanel.tsx +++ b/ui/src/pages/feature-views/components/RequestDataDisplayPanel.tsx @@ -1,17 +1,17 @@ import React from "react"; import { EuiBasicTable, EuiPanel, EuiText, EuiTitle } from "@elastic/eui"; import { useParams } from "react-router-dom"; -import { RequestDataSourceType } from "../../../parsers/feastODFVS"; import EuiCustomLink from "../../../components/EuiCustomLink"; +import { feast } from "../../../protos"; -interface RequestDataDisplayPanelProps extends RequestDataSourceType {} +interface RequestDataDisplayPanelProps extends feast.core.IOnDemandSource { } const RequestDataDisplayPanel = ({ requestDataSource, }: RequestDataDisplayPanelProps) => { const { projectName } = useParams(); - const items = Object.entries(requestDataSource.requestDataOptions.schema).map( + const items = Object.entries(requestDataSource?.requestDataOptions?.schema!).map( ([key, type]) => { return { key, @@ -38,10 +38,10 @@ const RequestDataDisplayPanel = ({ - {requestDataSource.name} + {requestDataSource?.name} diff --git a/ui/src/pages/feature-views/useLoadFeatureView.ts b/ui/src/pages/feature-views/useLoadFeatureView.ts index ded7900ea94..14970360f24 100644 --- a/ui/src/pages/feature-views/useLoadFeatureView.ts +++ b/ui/src/pages/feature-views/useLoadFeatureView.ts @@ -25,8 +25,8 @@ const useLoadRegularFeatureView = (featureViewName: string) => { registryQuery.data === undefined ? undefined : registryQuery.data.objects.featureViews?.find((fv) => { - return fv.spec.name === featureViewName; - }); + return fv?.spec?.name === featureViewName; + }); return { ...registryQuery, @@ -42,7 +42,24 @@ const useLoadOnDemandFeatureView = (featureViewName: string) => { registryQuery.data === undefined ? undefined : registryQuery.data.objects.onDemandFeatureViews?.find((fv) => { - return fv.spec.name === featureViewName; + return fv?.spec?.name === featureViewName; + }); + + return { + ...registryQuery, + data, + }; +}; + +const useLoadStreamFeatureView = (featureViewName: string) => { + const registryUrl = useContext(RegistryPathContext); + const registryQuery = useLoadRegistry(registryUrl); + + const data = + registryQuery.data === undefined + ? undefined + : registryQuery.data.objects.streamFeatureViews?.find((fv) => { + return fv.spec?.name === featureViewName; }); return { @@ -52,4 +69,4 @@ const useLoadOnDemandFeatureView = (featureViewName: string) => { }; export default useLoadFeatureView; -export { useLoadRegularFeatureView, useLoadOnDemandFeatureView }; +export { useLoadRegularFeatureView, useLoadOnDemandFeatureView, useLoadStreamFeatureView }; diff --git a/ui/src/pages/features/FeatureOverviewTab.tsx b/ui/src/pages/features/FeatureOverviewTab.tsx index 0a1c48509c2..92adfe1501d 100644 --- a/ui/src/pages/features/FeatureOverviewTab.tsx +++ b/ui/src/pages/features/FeatureOverviewTab.tsx @@ -3,16 +3,20 @@ import { EuiHorizontalRule, EuiLoadingSpinner, EuiTitle, + EuiSpacer, EuiPanel, EuiFlexItem, + EuiText, EuiDescriptionList, EuiDescriptionListTitle, EuiDescriptionListDescription, } from "@elastic/eui"; import EuiCustomLink from "../../components/EuiCustomLink"; +import TagsDisplay from "../../components/TagsDisplay"; import React from "react"; import { useParams } from "react-router-dom"; import useLoadFeature from "./useLoadFeature"; +import { feast } from "../../protos"; const FeatureOverviewTab = () => { let { projectName, FeatureViewName, FeatureName } = useParams(); @@ -48,19 +52,33 @@ const FeatureOverviewTab = () => { Value Type - {featureData?.valueType} + {feast.types.ValueType.Enum[featureData?.valueType!]} FeatureView - - {FeatureViewName} + {FeatureViewName} + + + +

Tags

+
+ + {featureData?.tags ? ( + + ) : ( + No Tags specified on this field. + )} +
diff --git a/ui/src/pages/features/useLoadFeature.ts b/ui/src/pages/features/useLoadFeature.ts index 5ddaf282043..f1918dd4d59 100644 --- a/ui/src/pages/features/useLoadFeature.ts +++ b/ui/src/pages/features/useLoadFeature.ts @@ -10,15 +10,15 @@ const useLoadFeature = (featureViewName: string, featureName: string) => { registryQuery.data === undefined ? undefined : registryQuery.data.objects.featureViews?.find((fv) => { - return fv.spec.name === featureViewName; - }); + return fv?.spec?.name === featureViewName; + }); - const featureData = + const featureData = data === undefined ? undefined - : data?.spec.features.find((f) => { - return f.name === featureName; - }); + : data?.spec?.features?.find((f) => { + return f.name === featureName; + }); return { ...registryQuery, diff --git a/ui/src/pages/saved-data-sets/DatasetExpectationsTab.tsx b/ui/src/pages/saved-data-sets/DatasetExpectationsTab.tsx index 10ebb87297b..dc49355c28f 100644 --- a/ui/src/pages/saved-data-sets/DatasetExpectationsTab.tsx +++ b/ui/src/pages/saved-data-sets/DatasetExpectationsTab.tsx @@ -11,7 +11,7 @@ const DatasetExpectationsTab = () => { } const { isSuccess, data } = useLoadDataset(datasetName); - if (!data || !data.spec.profile) { + if (!data || !data.spec?.name) { return ( No data so sad @@ -21,15 +21,9 @@ const DatasetExpectationsTab = () => { let expectationsData; - try { - expectationsData = JSON.parse(data.spec.profile); - } catch (e) { - throw new Error(`Unable to parse Expectations Profile: ${e}`); - } - - return isSuccess && expectationsData ? ( + return isSuccess ? ( -
{JSON.stringify(expectationsData, null, 2)}
+
{JSON.stringify(data.spec, null, 2)}
) : ( diff --git a/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx b/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx index a20c83b1e21..5f6ffa2101c 100644 --- a/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx +++ b/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx @@ -15,6 +15,7 @@ import { useParams } from "react-router-dom"; import DatasetFeaturesTable from "./DatasetFeaturesTable"; import DatasetJoinKeysTable from "./DatasetJoinKeysTable"; import useLoadDataset from "./useLoadDataset"; +import { toDate } from "../../utils/timestamp"; const EntityOverviewTab = () => { let { datasetName } = useParams(); @@ -47,7 +48,7 @@ const EntityOverviewTab = () => { { + features={data.spec?.features!.map((joinedName: string) => { const [featureViewName, featureName] = joinedName.split(":"); @@ -55,7 +56,7 @@ const EntityOverviewTab = () => { featureViewName, featureName, }; - })} + })!} /> @@ -65,9 +66,9 @@ const EntityOverviewTab = () => { { + joinKeys={data?.spec?.joinKeys!.map((joinKey) => { return { name: joinKey }; - })} + })!} />
@@ -82,7 +83,7 @@ const EntityOverviewTab = () => { Source Feature Service - {data.spec.featureService} + {data?.spec?.featureServiceName!} @@ -91,7 +92,7 @@ const EntityOverviewTab = () => { Created - {data.meta.createdTimestamp.toLocaleDateString("en-CA")} + {toDate(data?.meta?.createdTimestamp!).toLocaleDateString("en-CA")} diff --git a/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx b/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx index 97d11b0b24c..a1a97084171 100644 --- a/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx +++ b/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx @@ -2,10 +2,11 @@ import React from "react"; import { EuiBasicTable } from "@elastic/eui"; import EuiCustomLink from "../../components/EuiCustomLink"; import { useParams } from "react-router-dom"; -import { FeastSavedDatasetType } from "../../parsers/feastSavedDataset"; +import { feast } from "../../protos"; +import { toDate } from "../../utils/timestamp"; interface DatasetsListingTableProps { - datasets: FeastSavedDatasetType[]; + datasets: feast.core.ISavedDataset[]; } const DatasetsListingTable = ({ datasets }: DatasetsListingTableProps) => { @@ -33,15 +34,15 @@ const DatasetsListingTable = ({ datasets }: DatasetsListingTableProps) => { }, { name: "Created", - render: (item: FeastSavedDatasetType) => { - return item.meta.createdTimestamp.toLocaleDateString("en-CA"); + render: (item: feast.core.ISavedDataset) => { + return toDate(item?.meta?.createdTimestamp!).toLocaleString("en-CA")!; }, }, ]; - const getRowProps = (item: FeastSavedDatasetType) => { + const getRowProps = (item: feast.core.ISavedDataset) => { return { - "data-test-subj": `row-${item.spec.name}`, + "data-test-subj": `row-${item.spec?.name}`, }; }; diff --git a/ui/src/pages/saved-data-sets/useLoadDataset.ts b/ui/src/pages/saved-data-sets/useLoadDataset.ts index a3dbd3225d8..17a77d97993 100644 --- a/ui/src/pages/saved-data-sets/useLoadDataset.ts +++ b/ui/src/pages/saved-data-sets/useLoadDataset.ts @@ -10,8 +10,8 @@ const useLoadEntity = (entityName: string) => { registryQuery.data === undefined ? undefined : registryQuery.data.objects.savedDatasets?.find( - (fv) => fv.spec.name === entityName - ); + (fv) => fv.spec?.name === entityName + ); return { ...registryQuery, diff --git a/ui/src/parsers/feastDatasources.ts b/ui/src/parsers/feastDatasources.ts deleted file mode 100644 index 3e1dca72d1f..00000000000 --- a/ui/src/parsers/feastDatasources.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { z } from "zod"; -import { FeastFeatureColumnSchema } from "./feastFeatureViews"; - -const FeastDatasourceSchema = z.object({ - type: z.string(), - eventTimestampColumn: z.string().optional(), - createdTimestampColumn: z.string().optional(), - fileOptions: z.object({ - uri: z.string().optional(), - }).optional(), - name: z.string(), - description: z.string().optional(), - owner: z.string().optional(), - meta: z.object({ - latestEventTimestamp: z.string().transform((val) => new Date(val)), - earliestEventTimestamp: z.string().transform((val) => new Date(val)), - }).optional(), - requestDataOptions: z.object({ - schema: z.array(FeastFeatureColumnSchema), - }).optional(), - bigqueryOptions: z.object({ - tableRef: z.string().optional(), - dbtModelSerialized: z.string().optional() - }).optional(), -}); - -type FeastDatasourceType = z.infer; - -export { FeastDatasourceSchema }; -export type { FeastDatasourceType }; diff --git a/ui/src/parsers/feastEntities.ts b/ui/src/parsers/feastEntities.ts deleted file mode 100644 index 09057c6fe92..00000000000 --- a/ui/src/parsers/feastEntities.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { z } from "zod"; -import { FEAST_FEATURE_VALUE_TYPES } from "./types"; - -const FeastEntitySchema = z.object({ - spec: z.object({ - name: z.string(), - valueType: z.nativeEnum(FEAST_FEATURE_VALUE_TYPES).optional(), - joinKey: z.string(), - description: z.string().optional(), - labels: z.record(z.string()).optional(), - }), - meta: z.object({ - createdTimestamp: z.string().transform((val) => new Date(val)).optional(), - lastUpdatedTimestamp: z.string().transform((val) => new Date(val)).optional(), - }), -}); - -type FeastEntityType = z.infer; - -export { FeastEntitySchema }; -export type { FeastEntityType }; diff --git a/ui/src/parsers/feastFeatureServices.ts b/ui/src/parsers/feastFeatureServices.ts deleted file mode 100644 index 6812b7e02cb..00000000000 --- a/ui/src/parsers/feastFeatureServices.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { z } from "zod"; -import { FEAST_FEATURE_VALUE_TYPES } from "./types"; - -const FeatureColumnInService = z.object({ - name: z.string(), - valueType: z.nativeEnum(FEAST_FEATURE_VALUE_TYPES), -}); - -const FeatureInServiceSchema = z.object({ - featureViewName: z.string(), - featureColumns: z.array(FeatureColumnInService), -}); - -const FeastFeatureServiceSchema = z.object({ - spec: z.object({ - name: z.string(), - features: z.array(FeatureInServiceSchema), - tags: z.record(z.string()).optional(), - description: z.string().optional(), - }), - meta: z.object({ - createdTimestamp: z.string().transform((val) => new Date(val)).optional(), - lastUpdatedTimestamp: z.string().transform((val) => new Date(val)).optional(), - }), -}); - -type FeastFeatureServiceType = z.infer; -type FeastFeatureInServiceType = z.infer; - -export { FeastFeatureServiceSchema }; -export type { FeastFeatureServiceType, FeastFeatureInServiceType }; diff --git a/ui/src/parsers/feastFeatureViews.ts b/ui/src/parsers/feastFeatureViews.ts deleted file mode 100644 index cbf15d280e6..00000000000 --- a/ui/src/parsers/feastFeatureViews.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { z } from "zod"; -import { FEAST_FEATURE_VALUE_TYPES } from "./types"; - -const FeastFeatureColumnSchema = z.object({ - name: z.string(), - valueType: z.nativeEnum(FEAST_FEATURE_VALUE_TYPES), - tags: z.record(z.string()).optional(), -}); - -const FeastBatchSourceSchema = z.object({ - type: z.string(), - eventTimestampColumn: z.string().optional(), - createdTimestampColumn: z.string().optional(), - fileOptions: z.object({ - uri: z.string().optional(), - }).optional(), - name: z.string().optional(), - description: z.string().optional(), - owner: z.string().optional(), - meta: z.object({ - earliestEventTimestamp: z.string().transform((val) => new Date(val)), - latestEventTimestamp: z.string().transform((val) => new Date(val)), - }).optional(), - requestDataOptions: z.object({ - schema: z.record(z.nativeEnum(FEAST_FEATURE_VALUE_TYPES)), - }).optional(), - bigqueryOptions: z.object({ - tableRef: z.string().optional(), - dbtModelSerialized: z.string().optional() - }).optional(), - dataSourceClassType: z.string(), -}); - -const FeastFeatureViewSchema = z.object({ - spec: z.object({ - description: z.string().optional(), - name: z.string(), - entities: z.array(z.string()), - features: z.array(FeastFeatureColumnSchema), - ttl: z.string().transform((val) => parseInt(val)), - batchSource: FeastBatchSourceSchema, - online: z.boolean().optional(), - owner: z.string().optional(), - tags: z.record(z.string()).optional(), - }), - meta: z.object({ - createdTimestamp: z.string().transform((val) => new Date(val)).optional(), - lastUpdatedTimestamp: z.string().transform((val) => new Date(val)).optional(), - materializationIntervals: z - .array( - z.object({ - startTime: z.string().transform((val) => new Date(val)), - endTime: z.string().transform((val) => new Date(val)), - }) - ) - .optional(), - }), -}); - -type FeastFeatureViewType = z.infer; -type FeastFeatureColumnType = z.infer; - -export { FeastFeatureViewSchema, FeastFeatureColumnSchema }; -export type { FeastFeatureViewType, FeastFeatureColumnType }; diff --git a/ui/src/parsers/feastFeatures.ts b/ui/src/parsers/feastFeatures.ts deleted file mode 100644 index 129120c168d..00000000000 --- a/ui/src/parsers/feastFeatures.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { z } from "zod"; -import { FEAST_FEATURE_VALUE_TYPES } from "./types"; -import { jsonSchema } from "./jsonType" - -const FeastFeatureSchema = z.object({ - name: z.string(), - valueType: z.nativeEnum(FEAST_FEATURE_VALUE_TYPES), - metadata: jsonSchema.optional(), -}); - -export { FeastFeatureSchema }; diff --git a/ui/src/parsers/feastODFVS.ts b/ui/src/parsers/feastODFVS.ts deleted file mode 100644 index 4d09cc72dfa..00000000000 --- a/ui/src/parsers/feastODFVS.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { z } from "zod"; -import { FeastFeatureColumnSchema } from "./feastFeatureViews"; - -const FeatureViewProjectionSchema = z.object({ - featureViewProjection: z.object({ - featureViewName: z.string(), - featureColumns: z.array(FeastFeatureColumnSchema), - }), -}); - -const RequestDataSourceSchema = z.object({ - requestDataSource: z.object({ - type: z.string(), - name: z.string(), - requestDataOptions: z.object({ - schema: z.array(FeastFeatureColumnSchema), - }), - }), -}); - -const ODFVInputsSchema = z.union([ - FeatureViewProjectionSchema, - RequestDataSourceSchema, -]); - -const FeastODFVSchema = z.object({ - spec: z.object({ - name: z.string(), - features: z.array(FeastFeatureColumnSchema), - sources: z.record(ODFVInputsSchema), - userDefinedFunction: z.object({ - name: z.string(), - body: z.string(), - }), - }), - meta: z.object({ - createdTimestamp: z.string().transform((val) => new Date(val)), - lastUpdatedTimestamp: z.string().transform((val) => new Date(val)), - }), -}); - -type FeastODFVType = z.infer; -type RequestDataSourceType = z.infer; -type FeatureViewProjectionType = z.infer; - -export { FeastODFVSchema }; -export type { FeastODFVType, RequestDataSourceType, FeatureViewProjectionType }; diff --git a/ui/src/parsers/feastRegistry.ts b/ui/src/parsers/feastRegistry.ts deleted file mode 100644 index 98e4fccca2a..00000000000 --- a/ui/src/parsers/feastRegistry.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { z } from "zod"; -import { FeastDatasourceSchema } from "./feastDatasources"; -import { FeastEntitySchema } from "./feastEntities"; -import { FeastFeatureServiceSchema } from "./feastFeatureServices"; -import { FeastFeatureViewSchema } from "./feastFeatureViews"; -import { FeastSavedDatasetSchema } from "./feastSavedDataset"; -import { FeastODFVSchema } from "./feastODFVS"; - -const FeastRegistrySchema = z.object({ - project: z.string(), - dataSources: z.array(FeastDatasourceSchema).optional(), - entities: z.array(FeastEntitySchema).optional(), - featureViews: z.array(FeastFeatureViewSchema).optional(), - onDemandFeatureViews: z.array(FeastODFVSchema).optional(), - featureServices: z.array(FeastFeatureServiceSchema).optional(), - savedDatasets: z.array(FeastSavedDatasetSchema).optional(), -}); - -type FeastRegistryType = z.infer; - -export { FeastRegistrySchema }; -export type { FeastRegistryType }; diff --git a/ui/src/parsers/feastSavedDataset.ts b/ui/src/parsers/feastSavedDataset.ts deleted file mode 100644 index ce1d39b4e73..00000000000 --- a/ui/src/parsers/feastSavedDataset.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { z } from "zod"; - -const FeastSavedDatasetSchema = z.object({ - spec: z.object({ - name: z.string(), - features: z.array(z.string()), - joinKeys: z.array(z.string()), - storage: z.object({ - fileStorage: z.object({ - fileFormat: z.object({ - parquetFormat: z.object({}).optional(), - }).optional(), - fileUrl: z.string(), - }).optional(), - }).optional(), - featureService: z - .object({ - spec: z.object({ - name: z.string(), - }), - }) - .transform((obj) => { - return obj.spec.name; - }).optional(), - profile: z.string().optional(), - }), - meta: z.object({ - createdTimestamp: z.string().transform((val) => new Date(val)), - minEventTimestamp: z.string().transform((val) => new Date(val)), - maxEventTimestamp: z.string().transform((val) => new Date(val)), - }), -}); - -type FeastSavedDatasetType = z.infer; - -export { FeastSavedDatasetSchema }; -export type { FeastSavedDatasetType }; diff --git a/ui/src/parsers/featureViewSummaryStatistics.ts b/ui/src/parsers/featureViewSummaryStatistics.ts deleted file mode 100644 index f8eca669d37..00000000000 --- a/ui/src/parsers/featureViewSummaryStatistics.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { z } from "zod"; - -const histogramSchema = z.array( - z.object({ - x0: z.number(), - x1: z.number(), - count: z.number(), - }) -); - -const numericColumnSummaryStaticsSchema = z.object({ - name: z.string(), - valueType: z.literal("INT64"), - sampleValues: z.array(z.number()), - histogram: histogramSchema.optional(), - proportionOfZeros: z.number().optional(), - proportionMissing: z.number().optional(), - min: z.number().optional(), - max: z.number().optional(), -}); - -const stringColumnSummaryStaticsSchema = z.object({ - name: z.string(), - valueType: z.literal("STRING"), - sampleValues: z.array(z.string()), -}); - -const columnsSummaryStatisticsSchema = z.union([ - numericColumnSummaryStaticsSchema, - stringColumnSummaryStaticsSchema, -]); - -const featureViewSummaryStatisticsSchema = z.object({ - columnsSummaryStatistics: z.record(columnsSummaryStatisticsSchema), -}); - -type FeatureViewSummaryStatisticsType = z.infer< - typeof featureViewSummaryStatisticsSchema ->; - -type NumericColumnSummaryStatisticType = z.infer< - typeof numericColumnSummaryStaticsSchema ->; -type StringColumnSummaryStatisticType = z.infer< - typeof stringColumnSummaryStaticsSchema ->; - -type HistogramDataType = z.infer; - -export { featureViewSummaryStatisticsSchema }; -export type { - FeatureViewSummaryStatisticsType, - HistogramDataType, - NumericColumnSummaryStatisticType, - StringColumnSummaryStatisticType, -}; diff --git a/ui/src/parsers/jsonType.ts b/ui/src/parsers/jsonType.ts deleted file mode 100644 index be484b5477f..00000000000 --- a/ui/src/parsers/jsonType.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { z } from "zod"; - -// Taken from the zod documentation code - accepts any JSON object. -const literalSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]); -type Literal = z.infer; -type Json = Literal | { [key: string]: Json } | Json[]; -const jsonSchema: z.ZodType = z.lazy(() => - z.union([literalSchema, z.array(jsonSchema), z.record(jsonSchema)]) -); - -export { jsonSchema }; diff --git a/ui/src/parsers/mergedFVTypes.ts b/ui/src/parsers/mergedFVTypes.ts index 6a53b18e94d..7e23e095209 100644 --- a/ui/src/parsers/mergedFVTypes.ts +++ b/ui/src/parsers/mergedFVTypes.ts @@ -1,57 +1,72 @@ -import { - FeastFeatureColumnType, - FeastFeatureViewType, -} from "./feastFeatureViews"; -import { FeastODFVType } from "./feastODFVS"; -import { FeastRegistryType } from "./feastRegistry"; +import { feast } from "../protos"; enum FEAST_FV_TYPES { regular = "regular", ondemand = "ondemand", + stream = "stream" } interface regularFVInterface { name: string; type: FEAST_FV_TYPES.regular; - features: FeastFeatureColumnType[]; - object: FeastFeatureViewType; + features: feast.core.IFeatureSpecV2[]; + object: feast.core.IFeatureView; } interface ODFVInterface { name: string; type: FEAST_FV_TYPES.ondemand; - features: FeastFeatureColumnType[]; - object: FeastODFVType; + features: feast.core.IOnDemandFeatureViewSpec[]; + object: feast.core.IOnDemandFeatureView; } -type genericFVType = regularFVInterface | ODFVInterface; +interface SFVInterface { + name: string; + type: FEAST_FV_TYPES.stream; + features: feast.core.IFeatureSpecV2[]; + object: feast.core.IStreamFeatureView; +} + +type genericFVType = regularFVInterface | ODFVInterface | SFVInterface; -const mergedFVTypes = (objects: FeastRegistryType) => { +const mergedFVTypes = (objects: feast.core.Registry) => { const mergedFVMap: Record = {}; const mergedFVList: genericFVType[] = []; objects.featureViews?.forEach((fv) => { const obj: genericFVType = { - name: fv.spec.name, + name: fv.spec?.name!, type: FEAST_FV_TYPES.regular, - features: fv.spec.features, + features: fv.spec?.features!, object: fv, }; - mergedFVMap[fv.spec.name] = obj; + mergedFVMap[fv.spec?.name!] = obj; mergedFVList.push(obj); }); objects.onDemandFeatureViews?.forEach((odfv) => { const obj: genericFVType = { - name: odfv.spec.name, + name: odfv.spec?.name!, type: FEAST_FV_TYPES.ondemand, - features: odfv.spec.features, + features: odfv.spec?.features!, object: odfv, }; - mergedFVMap[odfv.spec.name] = obj; + mergedFVMap[odfv.spec?.name!] = obj; + mergedFVList.push(obj); + }); + + objects.streamFeatureViews?.forEach((sfv) => { + const obj: genericFVType = { + name: sfv.spec?.name!, + type: FEAST_FV_TYPES.stream, + features: sfv.spec?.features!, + object: sfv, + }; + + mergedFVMap[sfv.spec?.name!] = obj; mergedFVList.push(obj); }); @@ -60,4 +75,4 @@ const mergedFVTypes = (objects: FeastRegistryType) => { export default mergedFVTypes; export { FEAST_FV_TYPES }; -export type { genericFVType, regularFVInterface, ODFVInterface }; +export type { genericFVType, regularFVInterface, ODFVInterface, SFVInterface }; diff --git a/ui/src/parsers/parseEntityRelationships.ts b/ui/src/parsers/parseEntityRelationships.ts index f54bff63a1c..7a791fd0a15 100644 --- a/ui/src/parsers/parseEntityRelationships.ts +++ b/ui/src/parsers/parseEntityRelationships.ts @@ -1,5 +1,5 @@ -import { FeastRegistryType } from "./feastRegistry"; import { FEAST_FCO_TYPES } from "./types"; +import { feast } from "../protos"; interface EntityReference { type: FEAST_FCO_TYPES; @@ -11,26 +11,26 @@ interface EntityRelation { target: EntityReference; } -const parseEntityRelationships = (objects: FeastRegistryType) => { +const parseEntityRelationships = (objects: feast.core.Registry) => { const links: EntityRelation[] = []; objects.featureServices?.forEach((fs) => { - fs.spec.features.forEach((feature) => { + fs.spec?.features!.forEach((feature) => { links.push({ source: { type: FEAST_FCO_TYPES["featureView"], - name: feature.featureViewName, + name: feature?.featureViewName!, }, target: { type: FEAST_FCO_TYPES["featureService"], - name: fs.spec.name, + name: fs.spec?.name!, }, }); }); }); objects.featureViews?.forEach((fv) => { - fv.spec.entities.forEach((ent) => { + fv.spec?.entities?.forEach((ent) => { links.push({ source: { type: FEAST_FCO_TYPES["entity"], @@ -38,11 +38,11 @@ const parseEntityRelationships = (objects: FeastRegistryType) => { }, target: { type: FEAST_FCO_TYPES["featureView"], - name: fv.spec.name, + name: fv.spec?.name!, }, }); }); - if (fv.spec.batchSource) { + if (fv.spec?.batchSource) { links.push({ source: { type: FEAST_FCO_TYPES["dataSource"], @@ -50,42 +50,68 @@ const parseEntityRelationships = (objects: FeastRegistryType) => { }, target: { type: FEAST_FCO_TYPES["featureView"], - name: fv.spec.name, + name: fv.spec?.name!, } }) } }); objects.onDemandFeatureViews?.forEach((fv) => { - Object.values(fv.spec.sources).forEach((input: { [key: string]: any }) => { - if (input.requestDataSource) { - links.push({ - source: { - type: FEAST_FCO_TYPES["dataSource"], - name: input.requestDataSource.name, - }, - target: { - type: FEAST_FCO_TYPES["featureView"], - name: fv.spec.name, - }, - }); - } else if (input.featureViewProjection?.featureViewName) { - const source_fv = objects.featureViews?.find(el => el.spec.name === input.featureViewProjection.featureViewName); - if (!source_fv) { - return; - } - links.push({ - source: { - type: FEAST_FCO_TYPES["dataSource"], - name: source_fv?.spec.batchSource.name || '', - }, - target: { - type: FEAST_FCO_TYPES["featureView"], - name: fv.spec.name, - }, - }); - } - }); + Object.values(fv.spec?.sources!).forEach((input: { [key: string]: any }) => { + if (input.requestDataSource) { + links.push({ + source: { + type: FEAST_FCO_TYPES["dataSource"], + name: input.requestDataSource.name, + }, + target: { + type: FEAST_FCO_TYPES["featureView"], + name: fv.spec?.name!, + }, + }); + } else if (input.featureViewProjection?.featureViewName) { + const source_fv = objects.featureViews?.find(el => el.spec?.name === input.featureViewProjection.featureViewName); + if (!source_fv) { + return; + } + links.push({ + source: { + type: FEAST_FCO_TYPES["dataSource"], + name: source_fv.spec?.batchSource?.name || '', + }, + target: { + type: FEAST_FCO_TYPES["featureView"], + name: fv.spec?.name!, + }, + }); + } + }); + }); + + objects.streamFeatureViews?.forEach((fv) => { + // stream source + links.push({ + source: { + type: FEAST_FCO_TYPES["dataSource"], + name: fv.spec?.streamSource?.name!, + }, + target: { + type: FEAST_FCO_TYPES["featureView"], + name: fv.spec?.name!, + }, + }); + + // batch source + links.push({ + source: { + type: FEAST_FCO_TYPES["dataSource"], + name: fv.spec?.batchSource?.name!, + }, + target: { + type: FEAST_FCO_TYPES["featureView"], + name: fv.spec?.name!, + }, + }); }); return links; diff --git a/ui/src/parsers/parseIndirectRelationships.ts b/ui/src/parsers/parseIndirectRelationships.ts index d7d532ad3e7..092176c9b62 100644 --- a/ui/src/parsers/parseIndirectRelationships.ts +++ b/ui/src/parsers/parseIndirectRelationships.ts @@ -1,16 +1,16 @@ -import { FeastRegistryType } from "./feastRegistry"; import { EntityRelation } from "./parseEntityRelationships"; import { FEAST_FCO_TYPES } from "./types"; +import { feast } from "../protos"; const parseIndirectRelationships = ( relationships: EntityRelation[], - objects: FeastRegistryType + objects: feast.core.Registry ) => { const indirectLinks: EntityRelation[] = []; // Only contains Entity -> FS or DS -> FS relationships objects.featureServices?.forEach((featureService) => { - featureService.spec.features.forEach((featureView) => { + featureService.spec?.features?.forEach((featureView) => { relationships .filter( (relationship) => @@ -21,7 +21,7 @@ const parseIndirectRelationships = ( source: relationship.source, target: { type: FEAST_FCO_TYPES["featureService"], - name: featureService.spec.name, + name: featureService.spec?.name!, }, }); }); diff --git a/ui/src/parsers/types.ts b/ui/src/parsers/types.ts index 2f88eea4f06..1e515f23f34 100644 --- a/ui/src/parsers/types.ts +++ b/ui/src/parsers/types.ts @@ -5,25 +5,4 @@ enum FEAST_FCO_TYPES { featureService = "featureService", } -enum FEAST_FEATURE_VALUE_TYPES { - FLOAT = "FLOAT", - INT64 = "INT64", - STRING = "STRING", - BOOL = "BOOL", - BYTES = "BYTES", - INT32 = "INT32", - DOUBLE = "DOUBLE", - UNIX_TIMESTAMP = "UNIX_TIMESTAMP", - INVALID = "INVALID", - BYTES_LIST = "BYTES_LIST", - STRING_LIST = "STRING_LIST", - INT32_LIST = "INT32_LIST", - INT64_LIST = "INT64_LIST", - DOUBLE_LIST = "DOUBLE_LIST", - FLOAT_LIST = "FLOAT_LIST", - BOOL_LIST = "BOOL_LIST", - UNIX_TIMESTAMP_LIST = "UNIX_TIMESTAMP_LIST", - NULL = "NULL" -} - -export { FEAST_FCO_TYPES, FEAST_FEATURE_VALUE_TYPES }; +export { FEAST_FCO_TYPES }; diff --git a/ui/src/queries/useLoadFeatureViewSummaryStatistics.ts b/ui/src/queries/useLoadFeatureViewSummaryStatistics.ts deleted file mode 100644 index fea0bd9d816..00000000000 --- a/ui/src/queries/useLoadFeatureViewSummaryStatistics.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { useQuery } from "react-query"; -import { useParams } from "react-router-dom"; -import { - featureViewSummaryStatisticsSchema, - FeatureViewSummaryStatisticsType, -} from "../parsers/featureViewSummaryStatistics"; - -const useLoadFeatureViewSummaryStatistics = (featureViewName: string) => { - const { projectName } = useParams(); - - const queryKey = `featureViewSummaryStatistics:${featureViewName}`; - const url = `/data/${projectName}/featureView/${featureViewName}.json`; - - return useQuery( - queryKey, - () => { - return fetch(url, { - headers: { - "Content-Type": "application/json", - }, - }) - .then((res) => { - return res.json(); - }) - .then((json) => { - const summary = featureViewSummaryStatisticsSchema.parse(json); - - return summary; - }); - }, - { - staleTime: 15 * 60 * 1000, // Given that we are reading from a registry dump, this seems reasonable for now. - } - ); -}; - -export default useLoadFeatureViewSummaryStatistics; diff --git a/ui/src/queries/useLoadRegistry.ts b/ui/src/queries/useLoadRegistry.ts index ffb06756437..be8ab65a8cd 100644 --- a/ui/src/queries/useLoadRegistry.ts +++ b/ui/src/queries/useLoadRegistry.ts @@ -1,18 +1,15 @@ import { useQuery } from "react-query"; -import { - FeastRegistrySchema, - FeastRegistryType, -} from "../parsers/feastRegistry"; import mergedFVTypes, { genericFVType } from "../parsers/mergedFVTypes"; import parseEntityRelationships, { EntityRelation, } from "../parsers/parseEntityRelationships"; import parseIndirectRelationships from "../parsers/parseIndirectRelationships"; +import { feast } from "../protos"; interface FeatureStoreAllData { project: string; description?: string; - objects: FeastRegistryType; + objects: feast.core.Registry; relationships: EntityRelation[]; mergedFVMap: Record; mergedFVList: genericFVType[]; @@ -29,10 +26,12 @@ const useLoadRegistry = (url: string) => { }, }) .then((res) => { - return res.json(); + return res.arrayBuffer(); }) - .then((json) => { - const objects = FeastRegistrySchema.parse(json); + .then((arrayBuffer) => { + + const objects = feast.core.Registry.decode(new Uint8Array(arrayBuffer)); + // const objects = FeastRegistrySchema.parse(json); const { mergedFVMap, mergedFVList } = mergedFVTypes(objects); @@ -53,7 +52,7 @@ const useLoadRegistry = (url: string) => { // }); return { - project: objects.project, + project: objects.projectMetadata[0].project!, objects, mergedFVMap, mergedFVList, diff --git a/ui/src/utils/custom-tabs/StreamFeatureViewCustomTabLoadingWrapper.tsx b/ui/src/utils/custom-tabs/StreamFeatureViewCustomTabLoadingWrapper.tsx new file mode 100644 index 00000000000..098ab848a55 --- /dev/null +++ b/ui/src/utils/custom-tabs/StreamFeatureViewCustomTabLoadingWrapper.tsx @@ -0,0 +1,46 @@ +import React from "react"; + +import { useParams } from "react-router-dom"; +import useLoadFeatureView from "../../pages/feature-views/useLoadFeatureView"; +import { + StreamFeatureViewCustomTabProps, + StreamFeatureViewQueryReturnType, +} from "../../custom-tabs/types"; +import { FEAST_FV_TYPES } from "../../parsers/mergedFVTypes"; + +interface StreamFeatureViewCustomTabLoadingWrapperProps { + Component: (props: StreamFeatureViewCustomTabProps) => JSX.Element; +} + +const StreamFeatureViewCustomTabLoadingWrapper = ({ + Component, +}: StreamFeatureViewCustomTabLoadingWrapperProps) => { + const { featureViewName } = useParams(); + + if (!featureViewName) { + throw new Error( + `This route has no 'featureViewName' part. This route is likely not supposed to render this component.` + ); + } + + const feastObjectQuery = useLoadFeatureView(featureViewName); + + if ( + feastObjectQuery.isSuccess && + feastObjectQuery.data && + feastObjectQuery.data.type !== FEAST_FV_TYPES.stream + ) { + throw new Error( + `This should not happen. Somehow a custom tab on a SFV page received data that does not have the shape?` + ); + } + + return ( + + ); +}; + +export default StreamFeatureViewCustomTabLoadingWrapper; diff --git a/ui/src/utils/timestamp.ts b/ui/src/utils/timestamp.ts new file mode 100644 index 00000000000..869d24870f0 --- /dev/null +++ b/ui/src/utils/timestamp.ts @@ -0,0 +1,13 @@ +import long from 'long'; +import { google } from '../protos'; + +export function toDate(ts: google.protobuf.ITimestamp) { + var seconds: number; + if (ts.seconds instanceof long) { + seconds = ts.seconds.low + } else { + seconds = ts.seconds!; + } + + return new Date(seconds * 1000); +} \ No newline at end of file diff --git a/ui/yarn.lock b/ui/yarn.lock index ad31cbeac51..948eb78796e 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -334,6 +334,11 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.17.3.tgz#b07702b982990bf6fdc1da5049a23fece4c5c3d0" integrity sha512-7yJPvPV+ESz2IUTPbOL+YkIGyCqOyNIzdguKQuJGnH7bg1WTIifuM21YqokFt/THWh1AkCRn9IgoykTRCBVpzA== +"@babel/parser@^7.9.4": + version "7.19.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.19.0.tgz#497fcafb1d5b61376959c1c338745ef0577aa02c" + integrity sha512-74bEXKX2h+8rrfQUfsBfuZZHzsEs6Eql4pqy/T4Nn6Y9wNPggQOqD6z6pn5Bl8ZfysKouFZT/UXEH94ummEeQw== + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.16.7.tgz#4eda6d6c2a0aa79c70fa7b6da67763dfe2141050" @@ -1592,6 +1597,59 @@ schema-utils "^3.0.0" source-map "^0.7.3" +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" + integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== + +"@protobufjs/eventemitter@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== + +"@protobufjs/fetch@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== + dependencies: + "@protobufjs/aspromise" "^1.1.1" + "@protobufjs/inquire" "^1.1.0" + +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== + +"@protobufjs/inquire@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" + integrity sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q== + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== + +"@protobufjs/utf8@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" + integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== + "@rollup/plugin-babel@^5.2.0": version "5.3.0" resolved "https://registry.yarnpkg.com/@rollup/plugin-babel/-/plugin-babel-5.3.0.tgz#9cb1c5146ddd6a4968ad96f209c50c62f92f9879" @@ -2316,11 +2374,24 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4= +"@types/linkify-it@*": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@types/linkify-it/-/linkify-it-3.0.2.tgz#fd2cd2edbaa7eaac7e7f3c1748b52a19143846c9" + integrity sha512-HZQYqbiFVWufzCwexrvh694SOim8z2d+xJl5UNamcvQFejLY/2YUtzXHYi3cHdI7PMlS8ejH2slRAOJQ32aNbA== + "@types/lodash@^4.14.160": version "4.14.178" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.178.tgz#341f6d2247db528d4a13ddbb374bcdc80406f4f8" integrity sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw== +"@types/markdown-it@^12.2.3": + version "12.2.3" + resolved "https://registry.yarnpkg.com/@types/markdown-it/-/markdown-it-12.2.3.tgz#0d6f6e5e413f8daaa26522904597be3d6cd93b51" + integrity sha512-GKMHFfv3458yYy+v/N8gjufHO6MSZKCOXpZc5GXIWWy8uldwfmPn98vp81gZ5f9SVw8YYBctgfJ22a2d7AOMeQ== + dependencies: + "@types/linkify-it" "*" + "@types/mdurl" "*" + "@types/mdast@^3.0.0": version "3.0.10" resolved "https://registry.yarnpkg.com/@types/mdast/-/mdast-3.0.10.tgz#4724244a82a4598884cbbe9bcfd73dff927ee8af" @@ -2328,6 +2399,11 @@ dependencies: "@types/unist" "*" +"@types/mdurl@*": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@types/mdurl/-/mdurl-1.0.2.tgz#e2ce9d83a613bacf284c7be7d491945e39e1f8e9" + integrity sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA== + "@types/mime@^1": version "1.3.2" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" @@ -2343,6 +2419,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.13.tgz#5ed7ed7c662948335fcad6c412bb42d99ea754e3" integrity sha512-Y86MAxASe25hNzlDbsviXl8jQHb0RDvKt4c40ZJQ1Don0AAL0STLZSs4N+6gLEO55pedy7r2cLwS+ZDxPm/2Bw== +"@types/node@>=13.7.0": + version "18.7.18" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.7.18.tgz#633184f55c322e4fb08612307c274ee6d5ed3154" + integrity sha512-m+6nTEOadJZuTPkKR/SYK3A2d7FZrgElol9UP1Kae90VVU4a6mxnPuLiIW1m4Cq4gZ/nWb9GrdVXJCoCazDAbg== + "@types/node@^16.7.13": version "16.11.21" resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.21.tgz#474d7589a30afcf5291f59bd49cca9ad171ffde4" @@ -2809,7 +2890,7 @@ acorn-import-assertions@^1.7.6: resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz#ba2b5939ce62c238db6d93d81c9b111b29b855e9" integrity sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw== -acorn-jsx@^5.3.1: +acorn-jsx@^5.3.1, acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== @@ -2838,6 +2919,11 @@ acorn@^8.2.4, acorn@^8.4.1, acorn@^8.5.0, acorn@^8.7.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30" integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A== +acorn@^8.8.0: + version "8.8.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.0.tgz#88c0187620435c7f6015803f5539dae05a9dbea8" + integrity sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w== + address@^1.0.1, address@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/address/-/address-1.1.2.tgz#bf1116c9c758c51b7a933d296b72c221ed9428b6" @@ -3296,7 +3382,7 @@ bl@^4.1.0: inherits "^2.0.4" readable-stream "^3.4.0" -bluebird@^3.5.5: +bluebird@^3.5.5, bluebird@^3.7.2: version "3.7.2" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== @@ -3473,15 +3559,22 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001286, caniuse-lite@^1.0.30001297, caniuse-lite@^1.0.30001299: - version "1.0.30001303" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001303.tgz#9b168e4f43ccfc372b86f4bc5a551d9b909c95c9" - integrity sha512-/Mqc1oESndUNszJP0kx0UaQU9kEv9nNtJ7Kn8AdA0mNnH8eR1cj0kG+NbNuC1Wq/b21eA8prhKRA3bbkjONegQ== + version "1.0.30001416" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001416.tgz" + integrity sha512-06wzzdAkCPZO+Qm4e/eNghZBDfVNDsCgw33T27OwBH9unE9S478OYw//Q2L7Npf/zBzs7rjZOszIFQkwQKAEqA== case-sensitive-paths-webpack-plugin@^2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.4.0.tgz#db64066c6422eed2e08cc14b986ca43796dbc6d4" integrity sha512-roIFONhcxog0JSSWbvVAh3OocukmSgpqOH6YpMkCvav/ySIV3JKg4Dc8vYtQjYi/UxpNE36r/9v+VqTQqgkYmw== +catharsis@^0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/catharsis/-/catharsis-0.9.0.tgz#40382a168be0e6da308c277d3a2b3eb40c7d2121" + integrity sha512-prMTQVpcns/tzFgFVkVp6ak6RykZyWb3gu8ckUpd6YkTlacOd3DXGJjIpD4Q6zJirizvaiAjSSHlOsA+6sNh2A== + dependencies: + lodash "^4.17.15" + ccount@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/ccount/-/ccount-1.1.0.tgz#246687debb6014735131be8abab2d93898f8d043" @@ -4736,6 +4829,11 @@ entities@^2.0.0: resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== +entities@~2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-2.1.0.tgz#992d3129cf7df6870b96c57858c249a120f8b8b5" + integrity sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w== + error-ex@^1.3.1: version "1.3.2" resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" @@ -4815,6 +4913,18 @@ escape-string-regexp@^4.0.0: resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== +escodegen@^1.13.0: + version "1.14.3" + resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-1.14.3.tgz#4e7b81fba61581dc97582ed78cab7f0e8d63f503" + integrity sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw== + dependencies: + esprima "^4.0.1" + estraverse "^4.2.0" + esutils "^2.0.2" + optionator "^0.8.1" + optionalDependencies: + source-map "~0.6.1" + escodegen@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-2.0.0.tgz#5e32b12833e8aa8fa35e1bf0befa89380484c7dd" @@ -4980,6 +5090,11 @@ eslint-visitor-keys@^3.0.0, eslint-visitor-keys@^3.1.0, eslint-visitor-keys@^3.2 resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.2.0.tgz#6fbb166a6798ee5991358bc2daa1ba76cc1254a1" integrity sha512-IOzT0X126zn7ALX0dwFiUQEdsfzrm4+ISsQS8nukaJXwEyYKRSnEIIDULYg1mCtGp7UUXgfGl7BIolXREQK+XQ== +eslint-visitor-keys@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz#f6480fa6b1f30efe2d1968aa8ac745b862469826" + integrity sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA== + eslint-webpack-plugin@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/eslint-webpack-plugin/-/eslint-webpack-plugin-3.1.1.tgz#83dad2395e5f572d6f4d919eedaa9cf902890fcb" @@ -5032,6 +5147,15 @@ eslint@^8.3.0: text-table "^0.2.0" v8-compile-cache "^2.0.3" +espree@^9.0.0: + version "9.4.0" + resolved "https://registry.yarnpkg.com/espree/-/espree-9.4.0.tgz#cd4bc3d6e9336c433265fc0aa016fc1aaf182f8a" + integrity sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw== + dependencies: + acorn "^8.8.0" + acorn-jsx "^5.3.2" + eslint-visitor-keys "^3.3.0" + espree@^9.2.0, espree@^9.3.0: version "9.3.0" resolved "https://registry.yarnpkg.com/espree/-/espree-9.3.0.tgz#c1240d79183b72aaee6ccfa5a90bc9111df085a8" @@ -5060,7 +5184,7 @@ esrecurse@^4.3.0: dependencies: estraverse "^5.2.0" -estraverse@^4.1.1: +estraverse@^4.1.1, estraverse@^4.2.0: version "4.3.0" resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.3.0.tgz#398ad3f3c5a24948be7725e83d11a7de28cdbd1d" integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw== @@ -5536,6 +5660,17 @@ glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^8.0.0: + version "8.0.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-8.0.3.tgz#415c6eb2deed9e502c68fa44a272e6da6eeca42e" + integrity sha512-ull455NHSHI/Y1FqGaaYFaLGkNMMJbavMrEGFXG/PGrg6y7sutWHUHrz6gy6WEBH6akM1M414dWKCNs+IhKdiQ== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^5.0.1" + once "^1.3.0" + global-modules@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780" @@ -5595,6 +5730,11 @@ graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4, resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.9.tgz#041b05df45755e587a24942279b9d113146e1c96" integrity sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ== +graceful-fs@^4.1.9: + version "4.2.10" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" + integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== + graphql@^15.5.1: version "15.8.0" resolved "https://registry.yarnpkg.com/graphql/-/graphql-15.8.0.tgz#33410e96b012fa3bdb1091cc99a94769db212b38" @@ -6862,6 +7002,34 @@ js-yaml@^4.1.0: dependencies: argparse "^2.0.1" +js2xmlparser@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/js2xmlparser/-/js2xmlparser-4.0.2.tgz#2a1fdf01e90585ef2ae872a01bc169c6a8d5e60a" + integrity sha512-6n4D8gLlLf1n5mNLQPRfViYzu9RATblzPEtm1SthMX1Pjao0r9YI9nw7ZIfRxQMERS87mcswrg+r/OYrPRX6jA== + dependencies: + xmlcreate "^2.0.4" + +jsdoc@^3.6.3: + version "3.6.11" + resolved "https://registry.yarnpkg.com/jsdoc/-/jsdoc-3.6.11.tgz#8bbb5747e6f579f141a5238cbad4e95e004458ce" + integrity sha512-8UCU0TYeIYD9KeLzEcAu2q8N/mx9O3phAGl32nmHlE0LpaJL71mMkP4d+QE5zWfNt50qheHtOZ0qoxVrsX5TUg== + dependencies: + "@babel/parser" "^7.9.4" + "@types/markdown-it" "^12.2.3" + bluebird "^3.7.2" + catharsis "^0.9.0" + escape-string-regexp "^2.0.0" + js2xmlparser "^4.0.2" + klaw "^3.0.0" + markdown-it "^12.3.2" + markdown-it-anchor "^8.4.1" + marked "^4.0.10" + mkdirp "^1.0.4" + requizzle "^0.2.3" + strip-json-comments "^3.1.0" + taffydb "2.6.2" + underscore "~1.13.2" + jsdom@^16.6.0: version "16.7.0" resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-16.7.0.tgz#918ae71965424b197c819f8183a754e18977b710" @@ -6983,6 +7151,13 @@ kind-of@^6.0.2: resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== +klaw@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/klaw/-/klaw-3.0.0.tgz#b11bec9cf2492f06756d6e809ab73a2910259146" + integrity sha512-0Fo5oir+O9jnXu5EefYbVK+mHMBeEVEy2cmctR1O1NECcCkPRreJKrS6Qt/j3KC2C148Dfo9i3pCmCMsdqGr0g== + dependencies: + graceful-fs "^4.1.9" + kleur@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" @@ -7036,6 +7211,13 @@ lines-and-columns@^1.1.6: resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== +linkify-it@^3.0.1: + version "3.0.3" + resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-3.0.3.tgz#a98baf44ce45a550efb4d49c769d07524cc2fa2e" + integrity sha512-ynTsyrFSdE5oZ/O9GEf00kPngmOfVwazR5GKDq6EYfhlpFug3J2zybX56a2PRRpc9P+FuSoGNAwjlbDs9jJBPQ== + dependencies: + uc.micro "^1.0.1" + loader-runner@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.2.0.tgz#d7022380d66d14c5fb1d496b89864ebcfd478384" @@ -7132,6 +7314,11 @@ log-symbols@^4.1.0: chalk "^4.1.0" is-unicode-supported "^0.1.0" +long@^5.0.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/long/-/long-5.2.0.tgz#2696dadf4b4da2ce3f6f6b89186085d94d52fd61" + integrity sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w== + loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -7184,6 +7371,27 @@ markdown-escapes@^1.0.0: resolved "https://registry.yarnpkg.com/markdown-escapes/-/markdown-escapes-1.0.4.tgz#c95415ef451499d7602b91095f3c8e8975f78535" integrity sha512-8z4efJYk43E0upd0NbVXwgSTQs6cT3T06etieCMEg7dRbzCbxUCK/GHlX8mhHRDcp+OLlHkPKsvqQTCvsRl2cg== +markdown-it-anchor@^8.4.1: + version "8.6.5" + resolved "https://registry.yarnpkg.com/markdown-it-anchor/-/markdown-it-anchor-8.6.5.tgz#30c4bc5bbff327f15ce3c429010ec7ba75e7b5f8" + integrity sha512-PI1qEHHkTNWT+X6Ip9w+paonfIQ+QZP9sCeMYi47oqhH+EsW8CrJ8J7CzV19QVOj6il8ATGbK2nTECj22ZHGvQ== + +markdown-it@^12.3.2: + version "12.3.2" + resolved "https://registry.yarnpkg.com/markdown-it/-/markdown-it-12.3.2.tgz#bf92ac92283fe983fe4de8ff8abfb5ad72cd0c90" + integrity sha512-TchMembfxfNVpHkbtriWltGWc+m3xszaRD0CZup7GFFhzIgQqxIfn3eGj1yZpfuflzPvfkt611B2Q/Bsk1YnGg== + dependencies: + argparse "^2.0.1" + entities "~2.1.0" + linkify-it "^3.0.1" + mdurl "^1.0.1" + uc.micro "^1.0.5" + +marked@^4.0.10: + version "4.1.0" + resolved "https://registry.yarnpkg.com/marked/-/marked-4.1.0.tgz#3fc6e7485f21c1ca5d6ec4a39de820e146954796" + integrity sha512-+Z6KDjSPa6/723PQYyc1axYZpYYpDnECDaU6hkaf5gqBieBkMKYReL5hteF2QizhlMbgbo8umXl/clZ67+GlsA== + match-sorter@^6.0.2: version "6.3.1" resolved "https://registry.yarnpkg.com/match-sorter/-/match-sorter-6.3.1.tgz#98cc37fda756093424ddf3cbc62bfe9c75b92bda" @@ -7228,7 +7436,7 @@ mdn-data@~1.1.0: resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-1.1.4.tgz#50b5d4ffc4575276573c4eedb8780812a8419f01" integrity sha512-FSYbp3lyKjyj3E7fMl6rYvUdX0FBXaluGqlFoYESWQlyUTq8R+wp0rkFxoYFqZlHCvsUXGjyJmLQSnXToYhOSA== -mdurl@^1.0.0: +mdurl@^1.0.0, mdurl@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e" integrity sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4= @@ -7355,6 +7563,11 @@ mkdirp@^0.5.5, mkdirp@~0.5.1: dependencies: minimist "^1.2.5" +mkdirp@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== + moment@^2.29.1: version "2.29.4" resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108" @@ -8502,6 +8715,40 @@ property-information@^5.0.0, property-information@^5.3.0: dependencies: xtend "^4.0.0" +protobufjs-cli@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/protobufjs-cli/-/protobufjs-cli-1.0.2.tgz#905fc49007cf4aaf3c45d5f250eb294eedeea062" + integrity sha512-cz9Pq9p/Zs7okc6avH20W7QuyjTclwJPgqXG11jNaulfS3nbVisID8rC+prfgq0gbZE0w9LBFd1OKFF03kgFzg== + dependencies: + chalk "^4.0.0" + escodegen "^1.13.0" + espree "^9.0.0" + estraverse "^5.1.0" + glob "^8.0.0" + jsdoc "^3.6.3" + minimist "^1.2.0" + semver "^7.1.2" + tmp "^0.2.1" + uglify-js "^3.7.7" + +protobufjs@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.1.1.tgz#0117befb4b0f5a49d028e93f2ca62c3c1f5e7c65" + integrity sha512-d0nMQqS/aT3lfV8bKi9Gbg73vPd2LcDdTDOu6RE/M+h9DY8g1EmDzk3ADPccthEWfTBjkR2oxNdx9Gs8YubT+g== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/node" ">=13.7.0" + long "^5.0.0" + proxy-addr@~2.0.7: version "2.0.7" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" @@ -9104,6 +9351,13 @@ requires-port@^1.0.0: resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" integrity sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8= +requizzle@^0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/requizzle/-/requizzle-0.2.3.tgz#4675c90aacafb2c036bd39ba2daa4a1cb777fded" + integrity sha512-YanoyJjykPxGHii0fZP0uUPEXpvqfBDxWV7s6GKAiiOsiqhX6vHNyW3Qzdmqp/iq/ExbhaGbVrjB4ruEVSM4GQ== + dependencies: + lodash "^4.17.14" + resolve-cwd@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d" @@ -9379,6 +9633,13 @@ semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +semver@^7.1.2: + version "7.3.7" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f" + integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g== + dependencies: + lru-cache "^6.0.0" + semver@^7.3.2, semver@^7.3.5: version "7.3.5" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" @@ -9929,6 +10190,11 @@ tabbable@^5.2.1: resolved "https://registry.yarnpkg.com/tabbable/-/tabbable-5.3.2.tgz#66d6119ee8a533634c3f17deb0caa1c379e36ac7" integrity sha512-6G/8EWRFx8CiSe2++/xHhXkmCRq2rHtDtZbQFHx34cvDfZzIBfvwG9zGUNTWMXWLCYvDj3aQqOzdl3oCxKuBkQ== +taffydb@2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/taffydb/-/taffydb-2.6.2.tgz#7cbcb64b5a141b6a2efc2c5d2c67b4e150b2a268" + integrity sha512-y3JaeRSplks6NYQuCOj3ZFMO3j60rTwbuKCvZxsAraGYH2epusatvZ0baZYA01WsGqJBq/Dl6vOrMUJqyMj8kA== + tailwindcss@^3.0.2: version "3.0.18" resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-3.0.18.tgz#ea4825e6496d77dc21877b6b61c7cc56cda3add5" @@ -10060,6 +10326,13 @@ tmp@^0.0.33: dependencies: os-tmpdir "~1.0.2" +tmp@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.1.tgz#8457fc3037dcf4719c251367a1af6500ee1ccf14" + integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ== + dependencies: + rimraf "^3.0.0" + tmpl@1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc" @@ -10221,6 +10494,16 @@ typescript@^4.4.2: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3" integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA== +uc.micro@^1.0.1, uc.micro@^1.0.5: + version "1.0.6" + resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac" + integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA== + +uglify-js@^3.7.7: + version "3.17.0" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.17.0.tgz#55bd6e9d19ce5eef0d5ad17cd1f587d85b180a85" + integrity sha512-aTeNPVmgIMPpm1cxXr2Q/nEbvkmV8yq66F3om7X3P/cvOXQ0TMQ64Wk63iyT1gPlmdmGzjGpyLh1f3y8MZWXGg== + unbox-primitive@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.1.tgz#085e215625ec3162574dc8859abee78a59b14471" @@ -10231,6 +10514,11 @@ unbox-primitive@^1.0.1: has-symbols "^1.0.2" which-boxed-primitive "^1.0.2" +underscore@~1.13.2: + version "1.13.4" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.4.tgz#7886b46bbdf07f768e0052f1828e1dcab40c0dee" + integrity sha512-BQFnUDuAQ4Yf/cYY5LNrK9NCJFKriaRbD9uR1fTeXnBeoa97W0i41qkZfGO9pSo8I5KzjAcSY2XYtdf0oKd7KQ== + unherit@^1.0.4: version "1.1.3" resolved "https://registry.yarnpkg.com/unherit/-/unherit-1.1.3.tgz#6c9b503f2b41b262330c80e91c8614abdaa69c22" @@ -10988,6 +11276,11 @@ xmlchars@^2.2.0: resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw== +xmlcreate@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/xmlcreate/-/xmlcreate-2.0.4.tgz#0c5ab0f99cdd02a81065fa9cd8f8ae87624889be" + integrity sha512-nquOebG4sngPmGPICTS5EnxqhKbCmz5Ox5hsszI2T6U5qdrJizBc+0ilYSEjTSzU0yZcmvppztXe/5Al5fUwdg== + xtend@^4.0.0, xtend@^4.0.1, xtend@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" @@ -11050,9 +11343,9 @@ yocto-queue@^0.1.0: integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== zod@^3.11.6: - version "3.11.6" - resolved "https://registry.yarnpkg.com/zod/-/zod-3.11.6.tgz#e43a5e0c213ae2e02aefe7cb2b1a6fa3d7f1f483" - integrity sha512-daZ80A81I3/9lIydI44motWe6n59kRBfNzTuS2bfzVh1nAXi667TOTWWtatxyG+fwgNUiagSj/CWZwRRbevJIg== + version "3.19.1" + resolved "https://registry.yarnpkg.com/zod/-/zod-3.19.1.tgz#112f074a97b50bfc4772d4ad1576814bd8ac4473" + integrity sha512-LYjZsEDhCdYET9ikFu6dVPGp2YH9DegXjdJToSzD9rO6fy4qiRYFoyEYwps88OseJlPyl2NOe2iJuhEhL7IpEA== zwitch@^1.0.0: version "1.0.5"