From a63d440e4207c1e360630423bcda2c329673fddd Mon Sep 17 00:00:00 2001 From: sfc-gh-madkins <82121043+sfc-gh-madkins@users.noreply.github.com> Date: Wed, 21 Sep 2022 10:57:25 -0500 Subject: [PATCH 01/33] fix: Update snowflake materialization messages (#3230) Signed-off-by: miles.adkins Signed-off-by: miles.adkins --- .../infra/materialization/snowflake_engine.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/sdk/python/feast/infra/materialization/snowflake_engine.py b/sdk/python/feast/infra/materialization/snowflake_engine.py index 0219a7923f6..b523413bcde 100644 --- a/sdk/python/feast/infra/materialization/snowflake_engine.py +++ b/sdk/python/feast/infra/materialization/snowflake_engine.py @@ -118,11 +118,6 @@ def update( entities_to_delete: Sequence[Entity], entities_to_keep: Sequence[Entity], ): - click.echo( - f"Deploying materialization functions for {Style.BRIGHT + Fore.GREEN}{project}{Style.RESET_ALL}" - ) - click.echo() - stage_context = f'"{self.repo_config.batch_engine.database}"."{self.repo_config.batch_engine.schema_}"' stage_path = f'{stage_context}."feast_{project}"' with get_snowflake_conn(self.repo_config.batch_engine) as conn: @@ -136,12 +131,13 @@ def update( # if the stage already exists, # assumes that the materialization functions have been deployed if f"feast_{project}" in stage_list["name"].tolist(): - click.echo( - f"Materialization functions for {Style.BRIGHT + Fore.GREEN}{project}{Style.RESET_ALL} already exists" - ) - click.echo() return None + click.echo( + f"Deploying materialization functions for {Style.BRIGHT + Fore.GREEN}{project}{Style.RESET_ALL}" + ) + click.echo() + query = f"CREATE STAGE {stage_path}" execute_snowflake_statement(conn, query) From 357f3e9630af2fcb6720733fe83d9bbc4a36986c Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Wed, 21 Sep 2022 21:45:50 -0400 Subject: [PATCH 02/33] chore: Fix header levels in feature retrieval page (#3225) Signed-off-by: Danny Chiao Signed-off-by: Danny Chiao --- docs/getting-started/concepts/feature-retrieval.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/getting-started/concepts/feature-retrieval.md b/docs/getting-started/concepts/feature-retrieval.md index fd216fc71f5..867e17848b0 100644 --- a/docs/getting-started/concepts/feature-retrieval.md +++ b/docs/getting-started/concepts/feature-retrieval.md @@ -180,7 +180,7 @@ Feast accepts either: - [feature services](feature-retrieval.md#feature-services), which group features needed for a model version - [feature references](feature-retrieval.md#feature-references) -### Example: querying a feature service (recommended) +#### Example: querying a feature service (recommended) ```python training_df = store.get_historical_features( entity_df=entity_df, @@ -188,7 +188,7 @@ training_df = store.get_historical_features( ).to_df() ``` -### Example: querying a list of feature references +#### Example: querying a list of feature references ```python training_df = store.get_historical_features( entity_df=entity_df, @@ -204,7 +204,7 @@ Feast accepts either a **Pandas dataframe** as the entity dataframe (including e Both approaches must specify the full **entity key** needed as well as the **timestamps**. Feast then joins features onto this dataframe. -### Example: entity dataframe for generating training data +#### Example: entity dataframe for generating training data ```python entity_df = pd.DataFrame.from_dict( { @@ -228,7 +228,7 @@ training_df = store.get_historical_features( ).to_df() ``` -### Example: entity SQL query for generating training data +#### Example: entity SQL query for generating training data You can also pass a SQL string to generate the above dataframe. This is useful for getting all entities in a timeframe from some data source. ```python From 6acc948d6b9faebfef890fa5847b231846beb13f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Sep 2022 18:46:47 -0700 Subject: [PATCH 03/33] chore: Bump oauthlib from 3.2.0 to 3.2.1 in /sdk/python/requirements (#3229) chore(deps): bump oauthlib in /sdk/python/requirements Bumps [oauthlib](https://github.com/oauthlib/oauthlib) from 3.2.0 to 3.2.1. - [Release notes](https://github.com/oauthlib/oauthlib/releases) - [Changelog](https://github.com/oauthlib/oauthlib/blob/master/CHANGELOG.rst) - [Commits](https://github.com/oauthlib/oauthlib/compare/v3.2.0...v3.2.1) --- updated-dependencies: - dependency-name: oauthlib dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.8-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 9d10b2c3132..07507ad0a3c 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -423,7 +423,7 @@ numpy==1.23.2 # pandavro # pyarrow # scipy -oauthlib==3.2.0 +oauthlib==3.2.1 # via requests-oauthlib oscrypto==1.3.0 # via snowflake-connector-python diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 93011cfdcf4..3e4bdefd4f8 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -429,7 +429,7 @@ numpy==1.23.2 # pandavro # pyarrow # scipy -oauthlib==3.2.0 +oauthlib==3.2.1 # via requests-oauthlib oscrypto==1.3.0 # via snowflake-connector-python diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index e13eee056bc..b9441fb461b 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -423,7 +423,7 @@ numpy==1.23.2 # pandavro # pyarrow # scipy -oauthlib==3.2.0 +oauthlib==3.2.1 # via requests-oauthlib oscrypto==1.3.0 # via snowflake-connector-python From 2624879cbc845930a6c46fbac444d4584af74c8d Mon Sep 17 00:00:00 2001 From: sfc-gh-madkins <82121043+sfc-gh-madkins@users.noreply.github.com> Date: Wed, 21 Sep 2022 22:35:25 -0500 Subject: [PATCH 04/33] chore: Update snowflake dependencies (#3238) chore: Update snowflake python dependencies Signed-off-by: miles.adkins Signed-off-by: miles.adkins --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 37ed471cfa6..cddfb63b414 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ BYTEWAX_REQUIRED = ["bytewax==0.10.0", "docker>=5.0.2", "kubernetes<=20.13.0"] SNOWFLAKE_REQUIRED = [ - "snowflake-connector-python[pandas]>=2.7.3,<=2.7.8", + "snowflake-connector-python[pandas]>=2.7.3,<3", ] SPARK_REQUIRED = [ From a2dc0d0410eb297afddfb1dd4f1f899ab70fa14f Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Wed, 21 Sep 2022 20:35:42 -0700 Subject: [PATCH 05/33] fix: Build dockerfile correctly (#3239) * Build dockerfile correctly Signed-off-by: Felix Wang * Update Signed-off-by: Felix Wang * Fix Signed-off-by: Felix Wang Signed-off-by: Felix Wang --- sdk/python/feast/infra/feature_servers/multicloud/Dockerfile | 2 +- .../feast/infra/feature_servers/multicloud/Dockerfile.dev | 2 +- .../feast/infra/feature_servers/multicloud/requirements.txt | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 sdk/python/feast/infra/feature_servers/multicloud/requirements.txt diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index 751a398ad4f..7dbe5fd0c49 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -9,8 +9,8 @@ RUN apt update && \ RUN pip install pip --upgrade COPY . . +RUN pip install ".[aws,gcp,snowflake,redis,go,mysql]" -RUN pip install -r requirements.txt RUN apt update RUN apt install -y -V ca-certificates lsb-release wget RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev index 751a398ad4f..7dbe5fd0c49 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -9,8 +9,8 @@ RUN apt update && \ RUN pip install pip --upgrade COPY . . +RUN pip install ".[aws,gcp,snowflake,redis,go,mysql]" -RUN pip install -r requirements.txt RUN apt update RUN apt install -y -V ca-certificates lsb-release wget RUN wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb diff --git a/sdk/python/feast/infra/feature_servers/multicloud/requirements.txt b/sdk/python/feast/infra/feature_servers/multicloud/requirements.txt deleted file mode 100644 index 01d08a4effa..00000000000 --- a/sdk/python/feast/infra/feature_servers/multicloud/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -feast[aws,gcp,snowflake,redis,go,mysql] From cb8db84a1b200c7a716d23e17f150b013d5689ff Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Thu, 22 Sep 2022 13:37:39 -0700 Subject: [PATCH 06/33] feat: Add mysql as online store (#3190) * update remove debug Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * add interface Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * fix lint Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * fix lint Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * update sql Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * update ci reqs Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * remove pip index Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * format Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * fix unit test issue Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * fix lint issue Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * add entity_key_serialization_version Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * update doc Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * format Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * format Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * fix makefile typo Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * move to func Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * format Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- Makefile | 21 ++ docs/SUMMARY.md | 1 + docs/reference/online-stores/README.md | 4 + docs/reference/online-stores/mysql.md | 53 +++++ ...line_stores.contrib.mysql_online_store.rst | 21 ++ .../feast.infra.online_stores.contrib.rst | 9 + .../contrib/mysql_online_store/README.md | 81 +++++++ .../contrib/mysql_online_store/__init__.py | 0 .../contrib/mysql_online_store/mysql.py | 225 ++++++++++++++++++ .../contrib/mysql_repo_configuration.py | 10 + sdk/python/feast/repo_config.py | 1 + .../requirements/py3.10-ci-requirements.txt | 4 + .../requirements/py3.8-ci-requirements.txt | 4 + .../requirements/py3.9-ci-requirements.txt | 4 + .../universal/online_store/mysql.py | 33 +++ setup.py | 2 + 16 files changed, 473 insertions(+) create mode 100644 docs/reference/online-stores/mysql.md create mode 100644 sdk/python/docs/source/feast.infra.online_stores.contrib.mysql_online_store.rst create mode 100644 sdk/python/feast/infra/online_stores/contrib/mysql_online_store/README.md create mode 100644 sdk/python/feast/infra/online_stores/contrib/mysql_online_store/__init__.py create mode 100644 sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py create mode 100644 sdk/python/feast/infra/online_stores/contrib/mysql_repo_configuration.py create mode 100644 sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py diff --git a/Makefile b/Makefile index 0bf0c82669f..f21c028dd7f 100644 --- a/Makefile +++ b/Makefile @@ -231,6 +231,27 @@ test-python-universal-postgres-online: not test_snowflake" \ sdk/python/tests + test-python-universal-mysql-online: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.mysql_repo_configuration \ + PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.mysql \ + FEAST_USAGE=False \ + IS_TEST=True \ + python -m pytest -n 8 --integration \ + -k "not test_universal_cli and \ + not test_go_feature_server and \ + not test_feature_logging and \ + not test_reorder_columns and \ + not test_logged_features_validation and \ + not test_lambda_materialization_consistency and \ + not test_offline_write and \ + not test_push_features_to_offline_store and \ + not gcs_registry and \ + not s3_registry and \ + not test_universal_types and \ + not test_snowflake" \ + sdk/python/tests + test-python-universal-cassandra: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \ diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 430f9a745d7..2b58fb277e3 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -94,6 +94,7 @@ * [DynamoDB](reference/online-stores/dynamodb.md) * [PostgreSQL (contrib)](reference/online-stores/postgres.md) * [Cassandra + Astra DB (contrib)](reference/online-stores/cassandra.md) + * [MySQL (contrib)](reference/online-stores/mysql.md) * [Providers](reference/providers/README.md) * [Local](reference/providers/local.md) * [Google Cloud Platform](reference/providers/google-cloud-platform.md) diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index 58c8705e9ee..2ade4f0de9b 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -34,3 +34,7 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli [cassandra.md](cassandra.md) {% endcontent-ref %} +{% content-ref url="mysql.md" %} +[mysql.md](mysql.md) +{% endcontent-ref %} + diff --git a/docs/reference/online-stores/mysql.md b/docs/reference/online-stores/mysql.md new file mode 100644 index 00000000000..cb90c7561c5 --- /dev/null +++ b/docs/reference/online-stores/mysql.md @@ -0,0 +1,53 @@ +# MySQL online store (contrib) + +## Description + +The MySQL online store provides support for materializing feature values into a MySQL database for serving online features. + +* Only the latest feature values are persisted + + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: mysql + host: DB_HOST + port: DB_PORT + database: DB_NAME + user: DB_USERNAME + password: DB_PASSWORD +``` +{% endcode %} + +The full set of configuration options is available in [MySQLOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.contrib.mysql.MySQLOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Mys online store. + +| | Mys | +| :-------------------------------------------------------- | :-- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.mysql_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.mysql_online_store.rst new file mode 100644 index 00000000000..036922d658f --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.mysql_online_store.rst @@ -0,0 +1,21 @@ +feast.infra.online\_stores.contrib.mysql\_online\_store package +=============================================================== + +Submodules +---------- + +feast.infra.online\_stores.contrib.mysql\_online\_store.mysql module +-------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.mysql_online_store.mysql + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.contrib.mysql_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst index 6afe9071ace..6b175f4584f 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst @@ -9,6 +9,7 @@ Subpackages feast.infra.online_stores.contrib.cassandra_online_store feast.infra.online_stores.contrib.hbase_online_store + feast.infra.online_stores.contrib.mysql_online_store Submodules ---------- @@ -29,6 +30,14 @@ feast.infra.online\_stores.contrib.hbase\_repo\_configuration module :undoc-members: :show-inheritance: +feast.infra.online\_stores.contrib.mysql\_repo\_configuration module +-------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.mysql_repo_configuration + :members: + :undoc-members: + :show-inheritance: + feast.infra.online\_stores.contrib.postgres module -------------------------------------------------- diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/README.md b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/README.md new file mode 100644 index 00000000000..ac38237cd11 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/README.md @@ -0,0 +1,81 @@ +# Mysql Online Store +Mysql is not included in current [Feast](https://github.com/feast-dev/feast) roadmap, this project intends to add Mysql support for Online Store. +We create a table _ which gets updated with data on every materialize call + + +#### Create a feature repository + +```shell +feast init feature_repo +cd feature_repo +``` + +#### Edit `feature_store.yaml` + +set `online_store` type to be `mysql` + +```yaml +project: feature_repo +registry: data/registry.db +provider: local +online_store: + type: mysql + host: 127.0.0.1 # mysql endpoint, default to 127.0.0.1 + port: 3306 # mysql port, default to 3306 + user: test # mysql user, default to test + password: test # mysql password, default to test + database: feast # mysql database, default to feast +``` + +#### Apply the feature definitions in `example.py` + +```shell +feast -c feature_repo apply +``` +##### Output +``` +Registered entity driver_id +Registered feature view driver_hourly_stats_view +Deploying infrastructure for driver_hourly_stats_view +``` + +### Materialize Latest Data to Online Feature Store (Mysql) +``` +$ CURRENT_TIME=$(date -u +"%Y-%m-%dT%H:%M:%S") +$ feast -c feature_repo materialize-incremental $CURRENT_TIME +``` +#### Output +``` +Materializing 1 feature views from 2022-04-16 15:30:39+05:30 to 2022-04-19 15:31:04+05:30 into the mysql online store. + +driver_hourly_stats_view from 2022-04-16 15:30:39+05:30 to 2022-04-19 15:31:04+05:30: +100%|████████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 120.59it/s] +``` + +### Fetch the latest features for some entity id +```python +from pprint import pprint +from feast import FeatureStore + +store = FeatureStore(repo_path=".") +feature_vector = store.get_online_features( + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + ], + entity_rows=[ + {"driver_id": 1004}, + {"driver_id": 1005}, + ], +).to_dict() +pprint(feature_vector) + +``` +#### Output +``` +{'acc_rate': [0.01390857808291912, 0.4063614010810852], + 'avg_daily_trips': [69, 706], + 'conv_rate': [0.6624961495399475, 0.7595928311347961], + 'driver_id': [1004, 1005]} +``` diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/__init__.py b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py new file mode 100644 index 00000000000..fa7dd2c2a49 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py @@ -0,0 +1,225 @@ +from __future__ import absolute_import + +from datetime import datetime +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +import pymysql +import pytz +from pydantic import StrictStr +from pymysql.connections import Connection +from pymysql.cursors import Cursor + +from feast import Entity, FeatureView, RepoConfig +from feast.infra.key_encoding_utils import serialize_entity_key +from feast.infra.online_stores.online_store import OnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel + + +class MySQLOnlineStoreConfig(FeastConfigBaseModel): + """ + Configuration for the MySQL online store. + NOTE: The class *must* end with the `OnlineStoreConfig` suffix. + """ + + type = "mysql" + + host: Optional[StrictStr] = None + user: Optional[StrictStr] = None + password: Optional[StrictStr] = None + database: Optional[StrictStr] = None + port: Optional[int] = None + + +class MySQLOnlineStore(OnlineStore): + """ + An online store implementation that uses MySQL. + NOTE: The class *must* end with the `OnlineStore` suffix. + """ + + _conn: Optional[Connection] = None + + def _get_conn(self, config: RepoConfig) -> Connection: + + online_store_config = config.online_store + assert isinstance(online_store_config, MySQLOnlineStoreConfig) + + if not self._conn: + self._conn = pymysql.connect( + host=online_store_config.host or "127.0.0.1", + user=online_store_config.user or "test", + password=online_store_config.password or "test", + database=online_store_config.database or "feast", + port=online_store_config.port or 3306, + autocommit=True, + ) + return self._conn + + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + + conn = self._get_conn(config) + cur = conn.cursor() + + project = config.project + + for entity_key, values, timestamp, created_ts in data: + entity_key_bin = serialize_entity_key( + entity_key, + entity_key_serialization_version=2, + ).hex() + timestamp = _to_naive_utc(timestamp) + if created_ts is not None: + created_ts = _to_naive_utc(created_ts) + + for feature_name, val in values.items(): + self.write_to_table( + created_ts, + cur, + entity_key_bin, + feature_name, + project, + table, + timestamp, + val, + ) + conn.commit() + if progress: + progress(1) + + @staticmethod + def write_to_table( + created_ts, cur, entity_key_bin, feature_name, project, table, timestamp, val + ) -> None: + cur.execute( + f""" + INSERT INTO {_table_id(project, table)} + (entity_key, feature_name, value, event_ts, created_ts) + values (%s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + value = %s, + event_ts = %s, + created_ts = %s; + """, + ( + # Insert + entity_key_bin, + feature_name, + val.SerializeToString(), + timestamp, + created_ts, + # Update on duplicate key + val.SerializeToString(), + timestamp, + created_ts, + ), + ) + + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + conn = self._get_conn(config) + cur = conn.cursor() + + result: List[Tuple[Optional[datetime], Optional[Dict[str, Any]]]] = [] + + project = config.project + for entity_key in entity_keys: + entity_key_bin = serialize_entity_key( + entity_key, + entity_key_serialization_version=2, + ).hex() + + cur.execute( + f"SELECT feature_name, value, event_ts FROM {_table_id(project, table)} WHERE entity_key = %s", + (entity_key_bin,), + ) + + res = {} + res_ts: Optional[datetime] = None + records = cur.fetchall() + if records: + for feature_name, val_bin, ts in records: + val = ValueProto() + val.ParseFromString(val_bin) + res[feature_name] = val + res_ts = ts + + if not res: + result.append((None, None)) + else: + result.append((res_ts, res)) + return result + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ) -> None: + conn = self._get_conn(config) + cur = conn.cursor() + project = config.project + + # We don't create any special state for the entities in this implementation. + for table in tables_to_keep: + cur.execute( + f"""CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key VARCHAR(512), + feature_name VARCHAR(256), + value BLOB, + event_ts timestamp NULL DEFAULT NULL, + created_ts timestamp NULL DEFAULT NULL, + PRIMARY KEY(entity_key, feature_name))""" + ) + + cur.execute( + f"ALTER TABLE {_table_id(project, table)} ADD INDEX {_table_id(project, table)}_ek (entity_key);" + ) + + for table in tables_to_delete: + _drop_table_and_index(cur, project, table) + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ) -> None: + conn = self._get_conn(config) + cur = conn.cursor() + project = config.project + + for table in tables: + _drop_table_and_index(cur, project, table) + + +def _drop_table_and_index(cur: Cursor, project: str, table: FeatureView) -> None: + table_name = _table_id(project, table) + cur.execute(f"DROP INDEX {table_name}_ek ON {table_name};") + cur.execute(f"DROP TABLE IF EXISTS {table_name}") + + +def _table_id(project: str, table: FeatureView) -> str: + return f"{project}_{table.name}" + + +def _to_naive_utc(ts: datetime) -> datetime: + if ts.tzinfo is None: + return ts + else: + return ts.astimezone(pytz.utc).replace(tzinfo=None) diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_repo_configuration.py b/sdk/python/feast/infra/online_stores/contrib/mysql_repo_configuration.py new file mode 100644 index 00000000000..3e92ead2d0b --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_repo_configuration.py @@ -0,0 +1,10 @@ +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) +from tests.integration.feature_repos.universal.online_store.mysql import ( + MySQLOnlineStoreCreator, +) + +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig(online_store_creator=MySQLOnlineStoreCreator), +] diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index fab0f25b410..d77707e5b14 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -51,6 +51,7 @@ "postgres": "feast.infra.online_stores.contrib.postgres.PostgreSQLOnlineStore", "hbase": "feast.infra.online_stores.contrib.hbase_online_store.hbase.HbaseOnlineStore", "cassandra": "feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store.CassandraOnlineStore", + "mysql": "feast.infra.online_stores.contrib.mysql_online_store.mysql.MySQLOnlineStore", } OFFLINE_STORE_CLASS_FOR_TYPE = { diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 07507ad0a3c..20ad4ba6ce4 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -557,6 +557,8 @@ pyjwt[crypto]==2.4.0 # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) +pymysql==1.0.2 + # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 @@ -761,6 +763,8 @@ types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf +types-pymysql==1.0.19 + # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 3e4bdefd4f8..0a37d1e784d 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -565,6 +565,8 @@ pyjwt[crypto]==2.4.0 # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) +pymysql==1.0.2 + # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 @@ -771,6 +773,8 @@ types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf +types-pymysql==1.0.19 + # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index b9441fb461b..1177da35d4b 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -557,6 +557,8 @@ pyjwt[crypto]==2.4.0 # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) +pymysql==1.0.2 + # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 @@ -763,6 +765,8 @@ types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf +types-pymysql==1.0.19 + # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py b/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py new file mode 100644 index 00000000000..093295c86ba --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py @@ -0,0 +1,33 @@ +from typing import Dict + +from testcontainers.mysql import MySqlContainer + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class MySQLOnlineStoreCreator(OnlineStoreCreator): + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.container = ( + MySqlContainer("mysql:latest", platform="linux/amd64") + .with_exposed_ports(3306) + .with_env("MYSQL_USER", "root") + .with_env("MYSQL_PASSWORD", "test") + .with_env("MYSQL_DATABASE", "test") + ) + + def create_online_store(self) -> Dict[str, str]: + self.container.start() + exposed_port = self.container.get_exposed_port(3306) + return { + "type": "mysql", + "user": "root", + "password": "test", + "database": "test", + "port": exposed_port, + } + + def teardown(self): + self.container.stop() diff --git a/setup.py b/setup.py index cddfb63b414..59c362ff9fc 100644 --- a/setup.py +++ b/setup.py @@ -113,6 +113,8 @@ MYSQL_REQUIRED = [ "mysqlclient", + "pymysql", + "types-PyMySQL" ] HBASE_REQUIRED = [ From b48d36bee074788626ed5c2d5cf130bd2fc8bda5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nhu=E1=BA=ADn=20Tr=E1=BA=A7n=20=28Steve=29?= Date: Fri, 23 Sep 2022 03:38:38 +0700 Subject: [PATCH 07/33] feat: Filter subset features in postgres #3174 (#3203) * Filter subset features Signed-off-by: NHUAN.TRAN Signed-off-by: NHUAN.TRAN * Fix python lint Signed-off-by: NHUAN.TRAN Signed-off-by: NHUAN.TRAN Signed-off-by: NHUAN.TRAN Co-authored-by: NHUAN.TRAN --- .../infra/online_stores/contrib/postgres.py | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/postgres.py b/sdk/python/feast/infra/online_stores/contrib/postgres.py index 18f3b189e7d..144b242a1d6 100644 --- a/sdk/python/feast/infra/online_stores/contrib/postgres.py +++ b/sdk/python/feast/infra/online_stores/contrib/postgres.py @@ -114,17 +114,30 @@ def online_read( ) ) - cur.execute( - sql.SQL( - """ - SELECT entity_key, feature_name, value, event_ts - FROM {} WHERE entity_key = ANY(%s); - """ - ).format( - sql.Identifier(_table_id(project, table)), - ), - (keys,), - ) + if not requested_features: + cur.execute( + sql.SQL( + """ + SELECT entity_key, feature_name, value, event_ts + FROM {} WHERE entity_key = ANY(%s); + """ + ).format( + sql.Identifier(_table_id(project, table)), + ), + (keys,), + ) + else: + cur.execute( + sql.SQL( + """ + SELECT entity_key, feature_name, value, event_ts + FROM {} WHERE entity_key = ANY(%s) and feature_name = ANY(%s); + """ + ).format( + sql.Identifier(_table_id(project, table)), + ), + (keys, requested_features), + ) rows = cur.fetchall() From 1b312fbb96a0629c0bd465ee09a01a43130c99c7 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Fri, 23 Sep 2022 21:14:05 -0400 Subject: [PATCH 08/33] fix: Enable users to upgrade a batch source into a push source (#3213) Signed-off-by: Danny Chiao Signed-off-by: Danny Chiao --- Makefile | 8 +++--- sdk/python/feast/data_source.py | 2 +- .../tests/unit/diff/test_registry_diff.py | 26 ++++++++++++++++++- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index f21c028dd7f..f29ec3c39cd 100644 --- a/Makefile +++ b/Makefile @@ -63,11 +63,9 @@ benchmark-python-local: FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests test-python: - @(docker info > /dev/null 2>&1 && \ - FEAST_USAGE=False \ - IS_TEST=True \ - python -m pytest -n 8 sdk/python/tests \ - ) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!"; + FEAST_USAGE=False \ + IS_TEST=True \ + python -m pytest -n 8 sdk/python/tests \ test-python-integration: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 19a780b32ca..54a68ed0480 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -760,7 +760,7 @@ def __init__( def __eq__(self, other): if not isinstance(other, PushSource): - raise TypeError("Comparisons should only involve PushSource class objects.") + return False if not super().__eq__(other): return False diff --git a/sdk/python/tests/unit/diff/test_registry_diff.py b/sdk/python/tests/unit/diff/test_registry_diff.py index 8af6c50a13d..ce40295f8b6 100644 --- a/sdk/python/tests/unit/diff/test_registry_diff.py +++ b/sdk/python/tests/unit/diff/test_registry_diff.py @@ -1,6 +1,6 @@ import pandas as pd -from feast import Field +from feast import Field, PushSource from feast.diff.registry_diff import ( diff_registry_objects, tag_objects_for_keep_delete_update_add, @@ -145,3 +145,27 @@ def post_changed(inputs: pd.DataFrame) -> pd.DataFrame: feast_object_diffs.feast_object_property_diffs[2].property_name == "user_defined_function.body_text" ) + + +def test_diff_registry_objects_batch_to_push_source(simple_dataset_1): + with prep_file_source(df=simple_dataset_1, timestamp_field="ts_1") as file_source: + entity = Entity(name="id", join_keys=["id"]) + pre_changed = FeatureView( + name="fv2", + entities=[entity], + source=file_source, + ) + post_changed = FeatureView( + name="fv2", + entities=[entity], + source=PushSource(name="push_source", batch_source=file_source), + ) + + feast_object_diffs = diff_registry_objects( + pre_changed, post_changed, "feature view" + ) + assert len(feast_object_diffs.feast_object_property_diffs) == 1 + assert ( + feast_object_diffs.feast_object_property_diffs[0].property_name + == "stream_source" + ) From 16de243ff18bf2b013cca55e71d54cd26840211e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Sep 2022 18:14:49 -0700 Subject: [PATCH 09/33] chore: Bump protobuf from 3.20.1 to 3.20.2 in /sdk/python/requirements (#3247) Bumps [protobuf](https://github.com/protocolbuffers/protobuf) from 3.20.1 to 3.20.2. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/main/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.20.1...v3.20.2) --- updated-dependencies: - dependency-name: protobuf dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.10-requirements.txt | 2 +- sdk/python/requirements/py3.8-ci-requirements.txt | 2 +- sdk/python/requirements/py3.8-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-requirements.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 20ad4ba6ce4..c1656ee7e6f 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -487,7 +487,7 @@ proto-plus==1.22.0 # google-cloud-bigquery-storage # google-cloud-datastore # google-cloud-firestore -protobuf==3.20.1 +protobuf==3.20.2 # via # feast (setup.py) # google-api-core diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index ac12befb87c..91369309175 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -107,7 +107,7 @@ partd==1.3.0 # via dask proto-plus==1.22.0 # via feast (setup.py) -protobuf==3.20.1 +protobuf==3.20.2 # via # feast (setup.py) # google-api-core diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 0a37d1e784d..84a8c7408e1 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -495,7 +495,7 @@ proto-plus==1.22.0 # google-cloud-bigquery-storage # google-cloud-datastore # google-cloud-firestore -protobuf==3.20.1 +protobuf==3.20.2 # via # feast (setup.py) # google-api-core diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index c2aef636733..b992ad3fc0a 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -111,7 +111,7 @@ pkgutil-resolve-name==1.3.10 # via jsonschema proto-plus==1.22.0 # via feast (setup.py) -protobuf==3.20.1 +protobuf==3.20.2 # via # feast (setup.py) # google-api-core diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 1177da35d4b..4893035a7e7 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -487,7 +487,7 @@ proto-plus==1.22.0 # google-cloud-bigquery-storage # google-cloud-datastore # google-cloud-firestore -protobuf==3.20.1 +protobuf==3.20.2 # via # feast (setup.py) # google-api-core diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 0d3cb22bbca..395302a9f9f 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -107,7 +107,7 @@ partd==1.3.0 # via dask proto-plus==1.22.0 # via feast (setup.py) -protobuf==3.20.1 +protobuf==3.20.2 # via # feast (setup.py) # google-api-core From 164e666ee8c425150903819a0035d6f6e48bcdd0 Mon Sep 17 00:00:00 2001 From: Ammar Alrashed Date: Fri, 23 Sep 2022 19:54:46 -0700 Subject: [PATCH 10/33] fix: Add `X-Trino-Extra-Credential` header and remove user override (#3246) * do changes with signing Signed-off-by: ammarar * lint fix Signed-off-by: ammarar Signed-off-by: ammarar --- .../contrib/trino_offline_store/trino_queries.py | 11 +++++++++++ .../contrib/trino_offline_store/trino_source.py | 1 - 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py index 1d4b5881240..97c61f78a60 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py @@ -36,6 +36,8 @@ def __init__( catalog: Optional[str] = None, auth: Optional[Any] = None, http_scheme: Optional[str] = None, + source: Optional[str] = None, + extra_credential: Optional[str] = None, ): self.host = host or os.getenv("TRINO_HOST") self.port = port or os.getenv("TRINO_PORT") @@ -43,6 +45,8 @@ def __init__( self.catalog = catalog or os.getenv("TRINO_CATALOG") self.auth = auth or os.getenv("TRINO_AUTH") self.http_scheme = http_scheme or os.getenv("TRINO_HTTP_SCHEME") + self.source = source or os.getenv("TRINO_SOURCE") + self.extra_credential = extra_credential or os.getenv("TRINO_EXTRA_CREDENTIAL") self._cursor: Optional[Cursor] = None if self.host is None: @@ -56,6 +60,11 @@ def __init__( def _get_cursor(self) -> Cursor: if self._cursor is None: + headers = ( + {trino.constants.HEADER_EXTRA_CREDENTIAL: self.extra_credential} + if self.extra_credential + else {} + ) self._cursor = trino.dbapi.connect( host=self.host, port=self.port, @@ -63,6 +72,8 @@ def _get_cursor(self) -> Cursor: catalog=self.catalog, auth=self.auth, http_scheme=self.http_scheme, + source=self.source, + http_headers=headers, ).cursor() return self._cursor diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py index 6e989bd40ce..f09b79069ca 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py @@ -228,7 +228,6 @@ def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: client = Trino( - user="user", catalog=config.offline_store.catalog, host=config.offline_store.host, port=config.offline_store.port, From 025f00c7a895241d4bf57d782cd0027897b62a62 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 26 Sep 2022 09:00:54 -0700 Subject: [PATCH 11/33] chore: Add other maintainers to the OWNERS file for /lgtm commands (#3243) * chore: Add other maintainers to the OWNERS file for /lgtm commands Signed-off-by: Achal Shah * add codeowners Signed-off-by: Danny Chiao * lint Signed-off-by: Danny Chiao Signed-off-by: Achal Shah Signed-off-by: Danny Chiao Co-authored-by: Danny Chiao --- OWNERS | 19 +++++++++++++++++-- README.md | 3 ++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/OWNERS b/OWNERS index bc7342c3d61..c34fd6baafe 100644 --- a/OWNERS +++ b/OWNERS @@ -3,16 +3,22 @@ # More info at https://www.kubernetes.dev/docs/guide/owners/ approvers: - woop - - tsotnet - achals - adchia - felixwang9817 - mavysavydav - MattDelac - kevjumba + - chhabrakadabra + - sfc-gh-madkins + - zhilingc + - whoahbot + - niklasvm + - toping4445 + - DvirDukhan + - hemidactylus reviewers: - woop - - tsotnet - achals - tedhtchang - adchia @@ -20,3 +26,12 @@ reviewers: - mavysavydav - MattDelac - kevjumba + - chhabrakadabra + - sfc-gh-madkins + - zhilingc + - whoahbot + - niklasvm + - toping4445 + - DvirDukhan + - hemidactylus + \ No newline at end of file diff --git a/README.md b/README.md index c8adfa5f22c..182637018f1 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,8 @@ The list below contains the functionality that contributors are planning to deve * **Feature Serving** * [x] Python Client * [x] [Python feature server](https://docs.feast.dev/reference/feature-servers/python-feature-server) - * [x] [Go feature server](https://docs.feast.dev/reference/feature-servers/go-feature-server) + * [x] [Java feature server (alpha)](https://github.com/feast-dev/feast/blob/master/infra/charts/feast/README.md) + * [x] [Go feature server (alpha)](https://docs.feast.dev/reference/feature-servers/go-feature-server) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** * [x] Data profiling and validation (Great Expectations) * **Feature Discovery and Governance** From d7d33c90ee09c5dd09db1e2a7d84e5fbbacd3cf6 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Mon, 26 Sep 2022 12:00:57 -0500 Subject: [PATCH 12/33] docs: Add references to the extras users need to install and quickstarts (#3249) docs: Add references to the extras users need to install and quickstart templates Signed-off-by: Danny Chiao Signed-off-by: Danny Chiao --- docs/reference/offline-stores/bigquery.md | 3 ++ docs/reference/offline-stores/mssql.md | 39 ++++++++------- docs/reference/offline-stores/postgres.md | 45 +++++++++-------- docs/reference/offline-stores/redshift.md | 45 +++++++++-------- docs/reference/offline-stores/snowflake.md | 49 +++++++++++-------- docs/reference/offline-stores/spark.md | 45 +++++++++-------- docs/reference/offline-stores/trino.md | 45 +++++++++-------- docs/reference/online-stores/cassandra.md | 39 ++++++++------- docs/reference/online-stores/datastore.md | 39 ++++++++------- docs/reference/online-stores/dynamodb.md | 41 +++++++++------- docs/reference/online-stores/mysql.md | 38 +++++++------- docs/reference/online-stores/postgres.md | 39 ++++++++------- docs/reference/online-stores/redis.md | 45 ++++++++++------- docs/reference/online-stores/snowflake.md | 39 ++++++++------- .../providers/amazon-web-services.md | 5 ++ docs/reference/providers/azure.md | 3 ++ .../providers/google-cloud-platform.md | 3 ++ 17 files changed, 311 insertions(+), 251 deletions(-) diff --git a/docs/reference/offline-stores/bigquery.md b/docs/reference/offline-stores/bigquery.md index 0e286d78c49..b7607abf595 100644 --- a/docs/reference/offline-stores/bigquery.md +++ b/docs/reference/offline-stores/bigquery.md @@ -7,6 +7,9 @@ The BigQuery offline store provides support for reading [BigQuerySources](../dat * All joins happen within BigQuery. * Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to BigQuery as a table (marked for expiration) in order to complete join operations. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[gcp]'`. You can get started by then running `feast init -t gcp`. + ## Example {% code title="feature_store.yaml" %} diff --git a/docs/reference/offline-stores/mssql.md b/docs/reference/offline-stores/mssql.md index bec0c8deb82..e352b3dd2aa 100644 --- a/docs/reference/offline-stores/mssql.md +++ b/docs/reference/offline-stores/mssql.md @@ -6,6 +6,9 @@ The MsSQL offline store provides support for reading [MsSQL Sources](../data-sou * Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[azure]'`. You can get started by then following this [tutorial](https://github.com/feast-dev/feast/blob/master/docs/tutorials/azure/README.md). + ## Disclaimer The MsSQL offline store does not achieve full test coverage. @@ -34,26 +37,26 @@ offline_store: The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Spark offline store. -| | MsSql | -| :-------------------------------- | :-- | -| `get_historical_features` (point-in-time correct join) | yes | -| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | -| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | -| `offline_write_batch` (persist dataframes to offline store) | no | -| `write_logged_features` (persist logged features to offline store) | no | +| | MsSql | +| :----------------------------------------------------------------- | :---- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | Below is a matrix indicating which functionality is supported by `MsSqlServerRetrievalJob`. -| | MsSql | -| --------------------------------- | --- | -| export to dataframe | yes | -| export to arrow table | yes | -| export to arrow batches | no | -| export to SQL | no | -| export to data lake (S3, GCS, etc.) | no | -| export to data warehouse | no | -| local execution of Python-based on-demand transforms | no | -| remote execution of Python-based on-demand transforms | no | -| persist results in the offline store | yes | +| | MsSql | +| ----------------------------------------------------- | ----- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | no | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | no | +| local execution of Python-based on-demand transforms | no | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/postgres.md b/docs/reference/offline-stores/postgres.md index 506666fc378..094ab4885f4 100644 --- a/docs/reference/offline-stores/postgres.md +++ b/docs/reference/offline-stores/postgres.md @@ -10,6 +10,9 @@ The PostgreSQL offline store provides support for reading [PostgreSQLSources](.. The PostgreSQL offline store does not achieve full test coverage. Please do not assume complete stability. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[postgres]'`. You can get started by then running `feast init -t postgres`. + ## Example {% code title="feature_store.yaml" %} @@ -42,29 +45,29 @@ The full set of configuration options is available in [PostgreSQLOfflineStoreCon The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the PostgreSQL offline store. -| | Postgres | -| :-------------------------------- | :-- | -| `get_historical_features` (point-in-time correct join) | yes | -| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | -| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | -| `offline_write_batch` (persist dataframes to offline store) | no | -| `write_logged_features` (persist logged features to offline store) | no | +| | Postgres | +| :----------------------------------------------------------------- | :------- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | Below is a matrix indicating which functionality is supported by `PostgreSQLRetrievalJob`. -| | Postgres | -| --------------------------------- | --- | -| export to dataframe | yes | -| export to arrow table | yes | -| export to arrow batches | no | -| export to SQL | yes | -| export to data lake (S3, GCS, etc.) | yes | -| export to data warehouse | yes | -| export as Spark dataframe | no | -| local execution of Python-based on-demand transforms | yes | -| remote execution of Python-based on-demand transforms | no | -| persist results in the offline store | yes | -| preview the query plan before execution | yes | -| read partitioned data | yes | +| | Postgres | +| ----------------------------------------------------- | -------- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | yes | +| export to data warehouse | yes | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/redshift.md b/docs/reference/offline-stores/redshift.md index 2cdf49bdb9f..98092c44bec 100644 --- a/docs/reference/offline-stores/redshift.md +++ b/docs/reference/offline-stores/redshift.md @@ -7,6 +7,9 @@ The Redshift offline store provides support for reading [RedshiftSources](../dat * All joins happen within Redshift. * Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Redshift temporarily in order to complete join operations. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[aws]'`. You can get started by then running `feast init -t aws`. + ## Example {% code title="feature_store.yaml" %} @@ -32,30 +35,30 @@ The full set of configuration options is available in [RedshiftOfflineStoreConfi The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Redshift offline store. -| | Redshift | -| :-------------------------------- | :-- | -| `get_historical_features` (point-in-time correct join) | yes | -| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | -| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | -| `offline_write_batch` (persist dataframes to offline store) | yes | -| `write_logged_features` (persist logged features to offline store) | yes | +| | Redshift | +| :----------------------------------------------------------------- | :------- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | yes | +| `write_logged_features` (persist logged features to offline store) | yes | Below is a matrix indicating which functionality is supported by `RedshiftRetrievalJob`. -| | Redshift | -| --------------------------------- | --- | -| export to dataframe | yes | -| export to arrow table | yes | -| export to arrow batches | yes | -| export to SQL | yes | -| export to data lake (S3, GCS, etc.) | no | -| export to data warehouse | yes | -| export as Spark dataframe | no | -| local execution of Python-based on-demand transforms | yes | -| remote execution of Python-based on-demand transforms | no | -| persist results in the offline store | yes | -| preview the query plan before execution | yes | -| read partitioned data | yes | +| | Redshift | +| ----------------------------------------------------- | -------- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | yes | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | yes | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/snowflake.md b/docs/reference/offline-stores/snowflake.md index e40ad7cd7a2..4ac7f164363 100644 --- a/docs/reference/offline-stores/snowflake.md +++ b/docs/reference/offline-stores/snowflake.md @@ -6,6 +6,13 @@ The [Snowflake](https://trial.snowflake.com) offline store provides support for * All joins happen within Snowflake. * Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Snowflake as a temporary table in order to complete join operations. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[snowflake]'`. + +If you're using a file based registry, then you'll also need to install the relevant cloud extra (`pip install 'feast[snowflake, CLOUD]'` where `CLOUD` is one of `aws`, `gcp`, `azure`) + +You can get started by then running `feast init -t snowflake`. + ## Example {% code title="feature_store.yaml" %} @@ -31,29 +38,29 @@ The full set of configuration options is available in [SnowflakeOfflineStoreConf The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Snowflake offline store. -| | Snowflake | -| :-------------------------------- | :-- | -| `get_historical_features` (point-in-time correct join) | yes | -| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | -| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | -| `offline_write_batch` (persist dataframes to offline store) | yes | -| `write_logged_features` (persist logged features to offline store) | yes | +| | Snowflake | +| :----------------------------------------------------------------- | :-------- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | yes | +| `write_logged_features` (persist logged features to offline store) | yes | Below is a matrix indicating which functionality is supported by `SnowflakeRetrievalJob`. -| | Snowflake | -| --------------------------------- | --- | -| export to dataframe | yes | -| export to arrow table | yes | -| export to arrow batches | no | -| export to SQL | yes | -| export to data lake (S3, GCS, etc.) | yes | -| export to data warehouse | yes | -| export as Spark dataframe | no | -| local execution of Python-based on-demand transforms | yes | -| remote execution of Python-based on-demand transforms | no | -| persist results in the offline store | yes | -| preview the query plan before execution | yes | -| read partitioned data | yes | +| | Snowflake | +| ----------------------------------------------------- | --------- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | yes | +| export to data warehouse | yes | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/spark.md b/docs/reference/offline-stores/spark.md index f1ef1300bd7..ae5ea78071e 100644 --- a/docs/reference/offline-stores/spark.md +++ b/docs/reference/offline-stores/spark.md @@ -11,6 +11,9 @@ The Spark offline store provides support for reading [SparkSources](../data-sour The Spark offline store does not achieve full test coverage. Please do not assume complete stability. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[spark]'`. You can get started by then running `feast init -t spark`. + ## Example {% code title="feature_store.yaml" %} @@ -39,29 +42,29 @@ The full set of configuration options is available in [SparkOfflineStoreConfig]( The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Spark offline store. -| | Spark | -| :-------------------------------- | :-- | -| `get_historical_features` (point-in-time correct join) | yes | -| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | -| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | -| `offline_write_batch` (persist dataframes to offline store) | no | -| `write_logged_features` (persist logged features to offline store) | no | +| | Spark | +| :----------------------------------------------------------------- | :---- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | Below is a matrix indicating which functionality is supported by `SparkRetrievalJob`. -| | Spark | -| --------------------------------- | --- | -| export to dataframe | yes | -| export to arrow table | yes | -| export to arrow batches | no | -| export to SQL | no | -| export to data lake (S3, GCS, etc.) | no | -| export to data warehouse | no | -| export as Spark dataframe | yes | -| local execution of Python-based on-demand transforms | no | -| remote execution of Python-based on-demand transforms | no | -| persist results in the offline store | yes | -| preview the query plan before execution | yes | -| read partitioned data | yes | +| | Spark | +| ----------------------------------------------------- | ----- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | no | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | no | +| export as Spark dataframe | yes | +| local execution of Python-based on-demand transforms | no | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/trino.md b/docs/reference/offline-stores/trino.md index 8cc604248ff..446db620e32 100644 --- a/docs/reference/offline-stores/trino.md +++ b/docs/reference/offline-stores/trino.md @@ -10,6 +10,9 @@ The Trino offline store provides support for reading [TrinoSources](../data-sour The Trino offline store does not achieve full test coverage. Please do not assume complete stability. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[trino]'`. You can then run `feast init`, then swap out `feature_store.yaml` with the below example to connect to Trino. + ## Example {% code title="feature_store.yaml" %} @@ -36,29 +39,29 @@ The full set of configuration options is available in [TrinoOfflineStoreConfig]( The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Trino offline store. -| | Trino | -| :-------------------------------- | :-- | -| `get_historical_features` (point-in-time correct join) | yes | -| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | -| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | -| `offline_write_batch` (persist dataframes to offline store) | no | -| `write_logged_features` (persist logged features to offline store) | no | +| | Trino | +| :----------------------------------------------------------------- | :---- | +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | Below is a matrix indicating which functionality is supported by `TrinoRetrievalJob`. -| | Trino | -| --------------------------------- | --- | -| export to dataframe | yes | -| export to arrow table | yes | -| export to arrow batches | no | -| export to SQL | yes | -| export to data lake (S3, GCS, etc.) | no | -| export to data warehouse | no | -| export as Spark dataframe | no | -| local execution of Python-based on-demand transforms | yes | -| remote execution of Python-based on-demand transforms | no | -| persist results in the offline store | no | -| preview the query plan before execution | yes | -| read partitioned data | yes | +| | Trino | +| ----------------------------------------------------- | ----- | +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | no | +| export to data warehouse | no | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | no | +| preview the query plan before execution | yes | +| read partitioned data | yes | To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/cassandra.md b/docs/reference/online-stores/cassandra.md index 48b7b73f439..e012ad250ae 100644 --- a/docs/reference/online-stores/cassandra.md +++ b/docs/reference/online-stores/cassandra.md @@ -8,7 +8,8 @@ The [Cassandra / Astra DB] online store provides support for materializing featu * Each feature view is mapped one-to-one to a specific Cassandra table * This implementation inherits all strengths of Cassandra such as high availability, fault-tolerance, and data distribution -An easy way to get started is the command `feast init REPO_NAME -t cassandra`. +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[cassandra]'`. You can then get started with the command `feast init REPO_NAME -t cassandra`. ### Example (Cassandra) @@ -66,23 +67,23 @@ Storage specifications can be found at `docs/specs/online_store_format.md`. The set of functionality supported by online stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Cassandra online store. -| | Cassandra | -| :-------------------------------------------------------- | :-- | -| write feature values to the online store | yes | -| read feature values from the online store | yes | -| update infrastructure (e.g. tables) in the online store | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | yes | -| support for on-demand transforms | yes | -| readable by Python SDK | yes | -| readable by Java | no | -| readable by Go | no | -| support for entityless feature views | yes | -| support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | -| collocated by feature view | yes | -| collocated by feature service | no | -| collocated by entity key | no | +| | Cassandra | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | yes | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/datastore.md b/docs/reference/online-stores/datastore.md index 0867853f15d..761d246ba7e 100644 --- a/docs/reference/online-stores/datastore.md +++ b/docs/reference/online-stores/datastore.md @@ -4,6 +4,9 @@ The [Datastore](https://cloud.google.com/datastore) online store provides support for materializing feature values into Cloud Datastore. The data model used to store feature values in Datastore is described in more detail [here](../../specs/online_store_format.md#google-datastore-online-store-format). +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[gcp]'`. You can then get started with the command `feast init REPO_NAME -t gcp`. + ## Example {% code title="feature_store.yaml" %} @@ -25,23 +28,23 @@ The full set of configuration options is available in [DatastoreOnlineStoreConfi The set of functionality supported by online stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Datastore online store. -| | Datastore | -| :-------------------------------------------------------- | :-- | -| write feature values to the online store | yes | -| read feature values from the online store | yes | -| update infrastructure (e.g. tables) in the online store | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | no | -| support for on-demand transforms | yes | -| readable by Python SDK | yes | -| readable by Java | no | -| readable by Go | no | -| support for entityless feature views | yes | -| support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | -| collocated by feature view | yes | -| collocated by feature service | no | -| collocated by entity key | no | +| | Datastore | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/dynamodb.md b/docs/reference/online-stores/dynamodb.md index 2f94c768199..344caccac1d 100644 --- a/docs/reference/online-stores/dynamodb.md +++ b/docs/reference/online-stores/dynamodb.md @@ -4,6 +4,9 @@ The [DynamoDB](https://aws.amazon.com/dynamodb/) online store provides support for materializing feature values into AWS DynamoDB. +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[aws]'`. You can then get started with the command `feast init REPO_NAME -t aws`. + ## Example {% code title="feature_store.yaml" %} @@ -27,7 +30,7 @@ Feast requires the following permissions in order to execute commands for Dynamo | ----------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------- | | **Apply** |

dynamodb:CreateTable

dynamodb:DescribeTable

dynamodb:DeleteTable

| arn:aws:dynamodb:\:\:table/\* | | **Materialize** | dynamodb.BatchWriteItem | arn:aws:dynamodb:\:\:table/\* | -| **Get Online Features** | dynamodb.BatchGetItem | arn:aws:dynamodb:\:\:table/\* | +| **Get Online Features** | dynamodb.BatchGetItem | arn:aws:dynamodb:\:\:table/\* | The following inline policy can be used to grant Feast the necessary permissions: @@ -59,23 +62,23 @@ Lastly, this IAM role needs to be associated with the desired Redshift cluster. The set of functionality supported by online stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the DynamoDB online store. -| | DynamoDB | -| :-------------------------------------------------------- | :-- | -| write feature values to the online store | yes | -| read feature values from the online store | yes | -| update infrastructure (e.g. tables) in the online store | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | no | -| support for on-demand transforms | yes | -| readable by Python SDK | yes | -| readable by Java | no | -| readable by Go | no | -| support for entityless feature views | yes | -| support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | -| collocated by feature view | yes | -| collocated by feature service | no | -| collocated by entity key | no | +| | DynamoDB | +| :-------------------------------------------------------- | :------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/mysql.md b/docs/reference/online-stores/mysql.md index cb90c7561c5..cbc48457e13 100644 --- a/docs/reference/online-stores/mysql.md +++ b/docs/reference/online-stores/mysql.md @@ -6,6 +6,8 @@ The MySQL online store provides support for materializing feature values into a * Only the latest feature values are persisted +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[mysql]'`. You can get started by then running `feast init` and then setting the `feature_store.yaml` as described below. ## Example @@ -31,23 +33,23 @@ The full set of configuration options is available in [MySQLOnlineStoreConfig](h The set of functionality supported by online stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Mys online store. -| | Mys | -| :-------------------------------------------------------- | :-- | -| write feature values to the online store | yes | -| read feature values from the online store | yes | -| update infrastructure (e.g. tables) in the online store | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | no | -| support for on-demand transforms | yes | -| readable by Python SDK | yes | -| readable by Java | no | -| readable by Go | no | -| support for entityless feature views | yes | -| support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | -| collocated by feature view | yes | -| collocated by feature service | no | -| collocated by entity key | no | +| | Mys | +| :-------------------------------------------------------- | :--- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/postgres.md b/docs/reference/online-stores/postgres.md index 083c0006359..3885867dd26 100644 --- a/docs/reference/online-stores/postgres.md +++ b/docs/reference/online-stores/postgres.md @@ -8,6 +8,9 @@ The PostgreSQL online store provides support for materializing feature values in * sslmode, sslkey_path, sslcert_path, and sslrootcert_path are optional +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[postgres]'`. You can get started by then running `feast init -t postgres`. + ## Example {% code title="feature_store.yaml" %} @@ -37,23 +40,23 @@ The full set of configuration options is available in [PostgreSQLOnlineStoreConf The set of functionality supported by online stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Postgres online store. -| | Postgres | -| :-------------------------------------------------------- | :-- | -| write feature values to the online store | yes | -| read feature values from the online store | yes | -| update infrastructure (e.g. tables) in the online store | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | no | -| support for on-demand transforms | yes | -| readable by Python SDK | yes | -| readable by Java | no | -| readable by Go | no | -| support for entityless feature views | yes | -| support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | -| collocated by feature view | yes | -| collocated by feature service | no | -| collocated by entity key | no | +| | Postgres | +| :-------------------------------------------------------- | :------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/redis.md b/docs/reference/online-stores/redis.md index 80e90348c55..2078ee16b95 100644 --- a/docs/reference/online-stores/redis.md +++ b/docs/reference/online-stores/redis.md @@ -7,6 +7,15 @@ The [Redis](https://redis.io) online store provides support for materializing fe * Both Redis and Redis Cluster are supported. * The data model used to store feature values in Redis is described in more detail [here](../../specs/online\_store\_format.md). +## Getting started +In order to use this online store, you'll need to install the redis extra (along with the dependency needed for the offline store of choice). E.g. +- `pip install 'feast[gcp, redis]'` +- `pip install 'feast[snowflake, redis]'` +- `pip install 'feast[aws, redis]'` +- `pip install 'feast[azure, redis]'` + +You can get started by using any of the other templates (e.g. `feast init -t gcp` or `feast init -t snowflake` or `feast init -t aws`), and then swapping in Redis as the online store as seen below in the examples. + ## Examples Connecting to a single Redis instance: @@ -43,23 +52,23 @@ The full set of configuration options is available in [RedisOnlineStoreConfig](h The set of functionality supported by online stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Redis online store. -| | Redis | -| :-------------------------------------------------------- | :-- | -| write feature values to the online store | yes | -| read feature values from the online store | yes | -| update infrastructure (e.g. tables) in the online store | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | no | -| support for on-demand transforms | yes | -| readable by Python SDK | yes | -| readable by Java | yes | -| readable by Go | yes | -| support for entityless feature views | yes | -| support for concurrent writing to the same key | yes | -| support for ttl (time to live) at retrieval | yes | -| support for deleting expired data | yes | -| collocated by feature view | no | -| collocated by feature service | no | -| collocated by entity key | yes | +| | Redis | +| :-------------------------------------------------------- | :---- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | yes | +| readable by Go | yes | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/snowflake.md b/docs/reference/online-stores/snowflake.md index d114c87144a..6b6d107285c 100644 --- a/docs/reference/online-stores/snowflake.md +++ b/docs/reference/online-stores/snowflake.md @@ -16,6 +16,9 @@ The data model for using a Snowflake Transient Table as an online store follows (This model may be subject to change when Snowflake Hybrid Tables are released) +## Getting started +In order to use this online store, you'll need to run `pip install 'feast[snowflake]'`. You can then get started with the command `feast init REPO_NAME -t snowflake`. + ## Example {% code title="feature_store.yaml" %} ```yaml @@ -53,23 +56,23 @@ The full set of configuration options is available in [SnowflakeOnlineStoreConfi The set of functionality supported by online stores is described in detail [here](overview.md#functionality). Below is a matrix indicating which functionality is supported by the Snowflake online store. -| | Snowflake | -| :-------------------------------------------------------- | :-- | -| write feature values to the online store | yes | -| read feature values from the online store | yes | -| update infrastructure (e.g. tables) in the online store | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | no | -| support for on-demand transforms | yes | -| readable by Python SDK | yes | -| readable by Java | no | -| readable by Go | no | -| support for entityless feature views | yes | -| support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | -| collocated by feature view | yes | -| collocated by feature service | no | -| collocated by entity key | no | +| | Snowflake | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/providers/amazon-web-services.md b/docs/reference/providers/amazon-web-services.md index 3135fedb740..68956a1be93 100644 --- a/docs/reference/providers/amazon-web-services.md +++ b/docs/reference/providers/amazon-web-services.md @@ -5,6 +5,11 @@ * Offline Store: Uses the **Redshift** offline store by default. Also supports File as the offline store. * Online Store: Uses the **DynamoDB** online store by default. Also supports Sqlite as an online store. +## Getting started +In order to use this offline store, you'll need to run (Snowflake) `pip install 'feast[aws, snowflake]'` or (Redshift) `pip install 'feast[aws]'`. + +You can get started by then running `feast init -t snowflake` or `feast init -t aws`. + ## Example {% code title="feature_store.yaml" %} diff --git a/docs/reference/providers/azure.md b/docs/reference/providers/azure.md index 123bf087635..0e7206f076e 100644 --- a/docs/reference/providers/azure.md +++ b/docs/reference/providers/azure.md @@ -10,6 +10,9 @@ The Azure provider does not achieve full test coverage. Please do not assume complete stability. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[azure]'`. You can get started by then following this [tutorial](https://github.com/feast-dev/feast/blob/master/docs/tutorials/azure/README.md). + ## Example {% code title="feature_store.yaml" %} diff --git a/docs/reference/providers/google-cloud-platform.md b/docs/reference/providers/google-cloud-platform.md index 713313d16be..96af3b6b2ff 100644 --- a/docs/reference/providers/google-cloud-platform.md +++ b/docs/reference/providers/google-cloud-platform.md @@ -5,6 +5,9 @@ * Offline Store: Uses the **BigQuery** offline store by default. Also supports File as the offline store. * Online Store: Uses the **Datastore** online store by default. Also supports Sqlite as an online store. +## Getting started +In order to use this offline store, you'll need to run `pip install 'feast[gcp]'`. You can get started by then running `feast init -t gcp`. + ## Example {% code title="feature_store.yaml" %} From 95fdb19ed3f4b4af7fe1fb732a1fd63182cf1072 Mon Sep 17 00:00:00 2001 From: arcabucero Date: Mon, 26 Sep 2022 22:32:57 +0300 Subject: [PATCH 13/33] feat: Add possibility to define feature_store.yaml path with env variable (#3231) * Add possibility to define feature_store.yaml path with environment variable Signed-off-by: arcabucero * Add possibility to define feature_store.yaml path with environment variable for aws and lambda_engine Signed-off-by: arcabucero * Add test for overriding feature_store.yaml path with environment variable Signed-off-by: arcabucero Signed-off-by: arcabucero --- sdk/python/feast/cli.py | 2 +- sdk/python/feast/constants.py | 3 +++ sdk/python/feast/feature_store.py | 2 +- sdk/python/feast/infra/aws.py | 6 +++++- .../materialization/aws_lambda/lambda_engine.py | 3 ++- sdk/python/feast/utils.py | 11 +++++++++++ sdk/python/tests/unit/cli/test_cli.py | 17 +++++++++++++++++ 7 files changed, 40 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index af9aa511918..f677584fe5f 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -99,7 +99,7 @@ def cli( ctx.obj["FS_YAML_FILE"] = ( Path(feature_store_yaml).absolute() if feature_store_yaml - else ctx.obj["CHDIR"] / "feature_store.yaml" + else utils.get_default_yaml_file_path(ctx.obj["CHDIR"]) ) try: level = getattr(logging, log_level.upper()) diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index a2fe6f15c58..574d79f4167 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -23,6 +23,9 @@ # feature_store.yaml environment variable name for remote feature server FEATURE_STORE_YAML_ENV_NAME: str = "FEATURE_STORE_YAML_BASE64" +# feature_store.yaml path environment variable name +FEAST_FS_YAML_FILE_PATH_ENV_NAME: str = "FEAST_FS_YAML_FILE_PATH" + # Environment variable for registry REGISTRY_ENV_NAME: str = "REGISTRY_BASE64" diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 9350220c21c..e534a5ffc93 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -160,7 +160,7 @@ def __init__( self.config = load_repo_config(self.repo_path, fs_yaml_file) else: self.config = load_repo_config( - self.repo_path, Path(self.repo_path) / "feature_store.yaml" + self.repo_path, utils.get_default_yaml_file_path(self.repo_path) ) registry_config = self.config.get_registry_config() diff --git a/sdk/python/feast/infra/aws.py b/sdk/python/feast/infra/aws.py index f334998e6bc..5a045de4016 100644 --- a/sdk/python/feast/infra/aws.py +++ b/sdk/python/feast/infra/aws.py @@ -8,6 +8,7 @@ from colorama import Fore, Style +from feast import utils from feast.constants import ( AWS_LAMBDA_FEATURE_SERVER_IMAGE, AWS_LAMBDA_FEATURE_SERVER_REPOSITORY, @@ -113,7 +114,10 @@ def _deploy_feature_server(self, project: str, image_uri: str): if not self.repo_config.repo_path: raise RepoConfigPathDoesNotExist() - with open(self.repo_config.repo_path / "feature_store.yaml", "rb") as f: + + with open( + utils.get_default_yaml_file_path(self.repo_config.repo_path), "rb" + ) as f: config_bytes = f.read() config_base64 = base64.b64encode(config_bytes).decode() diff --git a/sdk/python/feast/infra/materialization/aws_lambda/lambda_engine.py b/sdk/python/feast/infra/materialization/aws_lambda/lambda_engine.py index 53a845140ec..93e33d5949d 100644 --- a/sdk/python/feast/infra/materialization/aws_lambda/lambda_engine.py +++ b/sdk/python/feast/infra/materialization/aws_lambda/lambda_engine.py @@ -10,6 +10,7 @@ from pydantic import StrictStr from tqdm import tqdm +from feast import utils from feast.batch_feature_view import BatchFeatureView from feast.constants import FEATURE_STORE_YAML_ENV_NAME from feast.entity import Entity @@ -137,7 +138,7 @@ def __init__( ) repo_path = self.repo_config.repo_path assert repo_path - feature_store_path = repo_path / "feature_store.yaml" + feature_store_path = utils.get_default_yaml_file_path(repo_path) self.feature_store_base64 = str( base64.b64encode(bytes(feature_store_path.read_text(), "UTF-8")), "UTF-8" ) diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 1b999341597..50b1e73c86c 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -1,6 +1,8 @@ +import os import typing from collections import defaultdict from datetime import datetime +from pathlib import Path from typing import Dict, List, Optional, Tuple, Union import pandas as pd @@ -9,6 +11,7 @@ from dateutil.tz import tzlocal from pytz import utc +from feast.constants import FEAST_FS_YAML_FILE_PATH_ENV_NAME from feast.entity import Entity from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto @@ -51,6 +54,14 @@ def maybe_local_tz(t: datetime) -> datetime: return t +def get_default_yaml_file_path(repo_path: Path) -> Path: + if FEAST_FS_YAML_FILE_PATH_ENV_NAME in os.environ: + yaml_path = os.environ[FEAST_FS_YAML_FILE_PATH_ENV_NAME] + return Path(yaml_path) + else: + return repo_path / "feature_store.yaml" + + def _get_requested_feature_views_to_features_dict( feature_refs: List[str], feature_views: List["FeatureView"], diff --git a/sdk/python/tests/unit/cli/test_cli.py b/sdk/python/tests/unit/cli/test_cli.py index f55e5ffc068..25a1dfed349 100644 --- a/sdk/python/tests/unit/cli/test_cli.py +++ b/sdk/python/tests/unit/cli/test_cli.py @@ -1,7 +1,9 @@ +import os import tempfile from contextlib import contextmanager from pathlib import Path from textwrap import dedent +from unittest import mock from assertpy import assertpy @@ -85,6 +87,21 @@ def test_3rd_party_registry_store_with_fs_yaml_override() -> None: assertpy.assert_that(return_code).is_equal_to(0) +def test_3rd_party_registry_store_with_fs_yaml_override_by_env_var() -> None: + runner = CliRunner() + + fs_yaml_file = "test_fs.yaml" + with setup_third_party_registry_store_repo( + "foo.registry_store.FooRegistryStore", fs_yaml_file_name=fs_yaml_file + ) as repo_path: + custom_yaml_path = os.path.join(repo_path, fs_yaml_file) + with mock.patch.dict( + "os.environ", {"FEAST_FS_YAML_FILE_PATH": custom_yaml_path}, clear=True + ): + return_code, output = runner.run_with_output(["apply"], cwd=repo_path) + assertpy.assert_that(return_code).is_equal_to(0) + + @contextmanager def setup_third_party_provider_repo(provider_name: str): with tempfile.TemporaryDirectory() as repo_dir_name: From 0ad7fe3096d1733cdc0a0473f79525fae8dfe950 Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Mon, 26 Sep 2022 12:39:53 -0700 Subject: [PATCH 14/33] chore: Build docker images in `build_wheels.yml` (#3244) * Build docker images Signed-off-by: Felix Wang * Update docs Signed-off-by: Felix Wang * Also build Java docker image Signed-off-by: Felix Wang * Clean up Signed-off-by: Felix Wang Signed-off-by: Felix Wang --- .github/workflows/build_wheels.yml | 21 +++++++++++++++++++++ docs/project/release-process.md | 5 +++-- java/infra/docker/feature-server/Dockerfile | 7 ------- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 841f5da87b3..aef486d601a 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -151,6 +151,27 @@ jobs: name: wheels path: dist/* + # We add this step so the docker images can be built as part of the pre-release verification steps. + build-docker-images: + runs-on: ubuntu-latest + needs: get-version + strategy: + matrix: + component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server] + env: + REGISTRY: feastdev + steps: + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Build image + run: | + make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + env: + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + verify-python-wheels: runs-on: ${{ matrix.os }} needs: [build-python-wheel, build-source-distribution, get-version] diff --git a/docs/project/release-process.md b/docs/project/release-process.md index 2ddc697730e..0aa7d3fb5ba 100644 --- a/docs/project/release-process.md +++ b/docs/project/release-process.md @@ -19,8 +19,9 @@ After this step, you will have all the changes you need in the branch. ### 2. Pre-release verification A lot of things can go wrong. One of the most common is getting the wheels to build correctly (and not accidentally building dev wheels from improper tagging or local code changes during the release process). +Another possible failure is that the Docker images might not build correctly. -We verify the wheels building in **your fork** of Feast, not the main feast-dev/feast repo. +We verify the building the wheels and Docker images in **your fork** of Feast, not the main feast-dev/feast repo. #### For minor releases (e.g. v0.22.0) 1. Merge upstream master changes into your **fork**. Make sure you are running the workflow off of your fork! @@ -30,7 +31,7 @@ We verify the wheels building in **your fork** of Feast, not the main feast-dev/ > This is important. If you don't have a tag, then the wheels you build will be **dev wheels**, which we can't > push. The release process will automatically produce a tag for you via Semantic Release. 3. Access the `Actions` tab on your GitHub UI on your fork and click the `build_wheels` action. This workflow will - build the python sdk wheels for Python 3.8-3.10 on MacOS 10.15 and Linux and verify that these wheels are correct. + build the python sdk wheels for Python 3.8-3.10 on MacOS 10.15 and Linux and verify that these wheels are correct. It will also build the Docker images. The publish workflow uses this action to publish the python wheels for a new release to PyPI. 4. Look for the header `This workflow has a workflow_dispatch event trigger` and click `Run Workflow` on the right. 5. Run the workflow off of the tag you just created(`v0.22.0` in this case, **not** the master branch) and verify that diff --git a/java/infra/docker/feature-server/Dockerfile b/java/infra/docker/feature-server/Dockerfile index bf4e172f763..8d246ed86aa 100644 --- a/java/infra/docker/feature-server/Dockerfile +++ b/java/infra/docker/feature-server/Dockerfile @@ -12,13 +12,6 @@ COPY java/serving/pom.xml serving/pom.xml COPY java/serving-client/pom.xml serving-client/pom.xml COPY java/coverage/pom.xml coverage/pom.xml -# Setting Maven repository .m2 directory relative to /build folder gives the -# user to optionally use cached repository when building the image by copying -# the existing .m2 directory to $FEAST_REPO_ROOT/.m2 -ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3" -COPY java/pom.xml .m2/* .m2/ -RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true - COPY java/ . COPY protos/feast datatypes/src/main/proto/feast From e713dda7c5a4433f5d13b4685d36978645e0cd02 Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Tue, 27 Sep 2022 07:26:34 -0700 Subject: [PATCH 15/33] fix: Stream feature view not shown in the UI (#3251) fix stream feature view not shown in the UI Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- sdk/python/feast/infra/registry/base_registry.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdk/python/feast/infra/registry/base_registry.py b/sdk/python/feast/infra/registry/base_registry.py index 5edfae3472d..e1e9ba99e18 100644 --- a/sdk/python/feast/infra/registry/base_registry.py +++ b/sdk/python/feast/infra/registry/base_registry.py @@ -634,6 +634,13 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]: registry_dict["requestFeatureViews"].append( self._message_to_sorted_dict(request_feature_view.to_proto()) ) + for stream_feature_view in sorted( + self.list_stream_feature_views(project=project), + key=lambda stream_feature_view: stream_feature_view.name, + ): + registry_dict["streamFeatureViews"].append( + self._message_to_sorted_dict(stream_feature_view.to_proto()) + ) for saved_dataset in sorted( self.list_saved_datasets(project=project), key=lambda item: item.name ): From da20757aa1d5c5f898651295781ef61b52d7d712 Mon Sep 17 00:00:00 2001 From: arcabucero Date: Wed, 28 Sep 2022 17:35:34 +0300 Subject: [PATCH 16/33] feat: Add request_timeout setting for cassandra online store adapter (#3256) Add request_timeout setting for cassandra online store adapter Signed-off-by: arcabucero Signed-off-by: arcabucero --- .../cassandra_online_store/cassandra_online_store.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py b/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py index f89517c41eb..71d3167c46f 100644 --- a/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py +++ b/sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/cassandra_online_store.py @@ -32,7 +32,7 @@ ) from cassandra.policies import DCAwareRoundRobinPolicy, TokenAwarePolicy from cassandra.query import PreparedStatement -from pydantic import StrictInt, StrictStr +from pydantic import StrictFloat, StrictInt, StrictStr from pydantic.typing import Literal from feast import Entity, FeatureView, RepoConfig @@ -141,6 +141,9 @@ class CassandraOnlineStoreConfig(FeastConfigBaseModel): protocol_version: Optional[StrictInt] = None """Explicit specification of the CQL protocol version used.""" + request_timeout: Optional[StrictFloat] = None + """Request timeout in seconds.""" + class CassandraLoadBalancingPolicy(FeastConfigBaseModel): """ Configuration block related to the Cluster's load-balancing policy. @@ -240,7 +243,10 @@ def _get_session(self, config: RepoConfig): raise CassandraInvalidConfig(E_CASSANDRA_UNKNOWN_LB_POLICY) # wrap it up in a map of ex.profiles with a default - exe_profile = ExecutionProfile(load_balancing_policy=lb_policy) + exe_profile = ExecutionProfile( + request_timeout=online_store_config.request_timeout, + load_balancing_policy=lb_policy, + ) execution_profiles = {EXEC_PROFILE_DEFAULT: exe_profile} else: execution_profiles = None From 086f2790236b776f5c2397365266d9d4f4959142 Mon Sep 17 00:00:00 2001 From: Benjamin Tan Wei Hao Date: Wed, 28 Sep 2022 22:36:34 +0800 Subject: [PATCH 17/33] feat: Add tag description in Field in the Feast UI (#3258) Signed-off-by: Benjamin Tan Signed-off-by: Benjamin Tan Co-authored-by: Benjamin Tan --- ui/src/pages/features/FeatureOverviewTab.tsx | 21 ++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ui/src/pages/features/FeatureOverviewTab.tsx b/ui/src/pages/features/FeatureOverviewTab.tsx index 0a1c48509c2..11a2a0ab6a5 100644 --- a/ui/src/pages/features/FeatureOverviewTab.tsx +++ b/ui/src/pages/features/FeatureOverviewTab.tsx @@ -3,13 +3,16 @@ import { EuiHorizontalRule, EuiLoadingSpinner, EuiTitle, + EuiSpacer, EuiPanel, EuiFlexItem, + EuiText, EuiDescriptionList, EuiDescriptionListTitle, EuiDescriptionListDescription, } from "@elastic/eui"; import EuiCustomLink from "../../components/EuiCustomLink"; +import TagsDisplay from "../../components/TagsDisplay"; import React from "react"; import { useParams } from "react-router-dom"; import useLoadFeature from "./useLoadFeature"; @@ -53,14 +56,28 @@ const FeatureOverviewTab = () => { FeatureView - - {FeatureViewName} + {FeatureViewName} + + + +

Tags

+ + + {featureData?.tags ? ( + + ) : ( + No Tags specified on this field. + )} + From 97b7ab906081d1afe034fc017a2ec61f20818771 Mon Sep 17 00:00:00 2001 From: Ammar Alrashed Date: Wed, 28 Sep 2022 15:19:48 -0700 Subject: [PATCH 18/33] fix: Configuration to stop coercion of tz for entity_df (#3255) add option to skip coercion Signed-off-by: ammarar Signed-off-by: ammarar --- sdk/python/feast/feature_store.py | 3 ++- sdk/python/feast/repo_config.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index e534a5ffc93..9c722ff1a1c 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1111,7 +1111,8 @@ def get_historical_features( # Check that the right request data is present in the entity_df if type(entity_df) == pd.DataFrame: - entity_df = utils.make_df_tzaware(cast(pd.DataFrame, entity_df)) + if self.config.coerce_tz_aware: + entity_df = utils.make_df_tzaware(cast(pd.DataFrame, entity_df)) for fv in request_feature_views: for feature in fv.features: if feature.name not in entity_df.columns: diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index d77707e5b14..c11b25849e3 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -167,6 +167,9 @@ class RepoConfig(FeastBaseModel): feature values for entities that have already been written into the online store. """ + coerce_tz_aware: Optional[bool] = True + """ If True, coerces entity_df timestamp columns to be timezone aware (to UTC by default). """ + def __init__(self, **data: Any): super().__init__(**data) From b851e01ab4ae78e6714793c7efddb86b39d2a36e Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 28 Sep 2022 21:02:48 -0400 Subject: [PATCH 19/33] fix: Return 422 on bad push source name (#3214) * return 422 on bad push source name Signed-off-by: Rob Howley * fix: cant send empty df Signed-off-by: Rob Howley * return 422 on bad push source name Signed-off-by: Rob Howley * fix: cant send empty df Signed-off-by: Rob Howley * fix: json.dumps the body in the post :facepalm: Signed-off-by: Rob Howley Signed-off-by: Rob Howley Co-authored-by: Rob Howley --- sdk/python/feast/errors.py | 5 ++++ sdk/python/feast/feature_server.py | 6 +++++ sdk/python/feast/feature_store.py | 4 ++++ .../e2e/test_python_feature_server.py | 23 +++++++++++++++++++ 4 files changed, 38 insertions(+) diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 834df0e5c48..15ba86781df 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -398,3 +398,8 @@ def __init__(self): super().__init__( "The entity dataframe specified does not have the timestamp field as a datetime." ) + + +class PushSourceNotFoundException(Exception): + def __init__(self, push_source_name: str): + super().__init__(f"Unable to find push source '{push_source_name}'.") diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index c2596d411c6..7b0cfc4bed8 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -13,6 +13,7 @@ import feast from feast import proto_json from feast.data_source import PushMode +from feast.errors import PushSourceNotFoundException from feast.protos.feast.serving.ServingService_pb2 import GetOnlineFeaturesRequest @@ -98,6 +99,11 @@ def push(body=Depends(get_body)): allow_registry_cache=request.allow_registry_cache, to=to, ) + except PushSourceNotFoundException as e: + # Print the original exception on the server side + logger.exception(traceback.format_exc()) + # Raise HTTPException to return the error message to the client + raise HTTPException(status_code=422, detail=str(e)) except Exception as e: # Print the original exception on the server side logger.exception(traceback.format_exc()) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 9c722ff1a1c..c47b06c0471 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -59,6 +59,7 @@ EntityNotFoundException, FeatureNameCollisionError, FeatureViewNotFoundException, + PushSourceNotFoundException, RequestDataNotFoundInEntityDfException, RequestDataNotFoundInEntityRowsException, ) @@ -1445,6 +1446,9 @@ def push( ) } + if not fvs_with_push_sources: + raise PushSourceNotFoundException(push_source_name) + for fv in fvs_with_push_sources: if to == PushMode.ONLINE or to == PushMode.ONLINE_AND_OFFLINE: self.write_to_online_store( diff --git a/sdk/python/tests/integration/e2e/test_python_feature_server.py b/sdk/python/tests/integration/e2e/test_python_feature_server.py index 9c61f6fa198..089efd7a562 100644 --- a/sdk/python/tests/integration/e2e/test_python_feature_server.py +++ b/sdk/python/tests/integration/e2e/test_python_feature_server.py @@ -84,6 +84,29 @@ def test_push(python_fs_client): ) == [initial_temp * 100] +@pytest.mark.integration +@pytest.mark.universal_online_stores +def test_push_source_does_not_exist(python_fs_client): + initial_temp = _get_temperatures_from_feature_server( + python_fs_client, location_ids=[1] + )[0] + response = python_fs_client.post( + "/push", + data=json.dumps( + { + "push_source_name": "push_source_does_not_exist", + "df": { + "location_id": [1], + "temperature": [initial_temp * 100], + "event_timestamp": [str(datetime.utcnow())], + "created": [str(datetime.utcnow())], + }, + } + ), + ) + assert response.status_code == 422 + + def _get_temperatures_from_feature_server(client, location_ids: List[int]): get_request_data = { "features": ["pushable_location_stats:temperature"], From 532d8a1ec5dd50a959b512427fdbc3e5bb986eb4 Mon Sep 17 00:00:00 2001 From: Ammar Alrashed Date: Wed, 28 Sep 2022 18:33:48 -0700 Subject: [PATCH 20/33] fix: Use configured user in env var instead of "user" for Trino (#3254) * use env var user Signed-off-by: ammarar * lint fix Signed-off-by: ammarar Signed-off-by: ammarar --- .../contrib/trino_offline_store/trino.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py index 6c25b5768f4..a5a51311eb9 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py @@ -157,7 +157,7 @@ def pull_latest_from_table_or_query( created_timestamp_column: Optional[str], start_date: datetime, end_date: datetime, - user: str = "user", + user: Optional[str] = None, auth: Optional[Authentication] = None, http_scheme: Optional[str] = None, ) -> TrinoRetrievalJob: @@ -176,7 +176,6 @@ def pull_latest_from_table_or_query( timestamps.append(created_timestamp_column) timestamp_desc_string = " DESC, ".join(timestamps) + " DESC" field_string = ", ".join(join_key_columns + feature_name_columns + timestamps) - client = _get_trino_client( config=config, user=user, auth=auth, http_scheme=http_scheme ) @@ -212,7 +211,7 @@ def get_historical_features( registry: Registry, project: str, full_feature_names: bool = False, - user: str = "user", + user: Optional[str] = None, auth: Optional[Authentication] = None, http_scheme: Optional[str] = None, ) -> TrinoRetrievalJob: @@ -303,7 +302,7 @@ def pull_all_from_table_or_query( timestamp_field: str, start_date: datetime, end_date: datetime, - user: str = "user", + user: Optional[str] = None, auth: Optional[Authentication] = None, http_scheme: Optional[str] = None, ) -> RetrievalJob: @@ -375,7 +374,10 @@ def _upload_entity_df_and_get_entity_schema( def _get_trino_client( - config: RepoConfig, user: str, auth: Optional[Any], http_scheme: Optional[str] + config: RepoConfig, + user: Optional[str], + auth: Optional[Any], + http_scheme: Optional[str], ) -> Trino: client = Trino( user=user, From 1f70b3a9be738ab54f0cb59046ed1e299f92e3a2 Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Fri, 30 Sep 2022 07:36:28 -0700 Subject: [PATCH 21/33] fix: Add stream feature view in the Web UI (#3257) * add stream feature view to ui Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * update source Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * update example Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * add registry Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * fix lint Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * fix bug Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * add batch source Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * fix warning Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- ui/public/registry.json | 72 ++++++++++ ui/src/custom-tabs/TabsRegistryContext.tsx | 25 ++++ .../stream-fv-demo-tab/DemoCustomTab.tsx | 85 +++++++++++ .../stream-fv-demo-tab/useDemoQuery.tsx | 44 ++++++ ui/src/custom-tabs/types.ts | 21 +++ ui/src/index.tsx | 10 +- .../feature-views/FeatureViewInstance.tsx | 8 ++ .../feature-views/FeatureViewListingTable.tsx | 2 +- .../StreamFeatureViewInstance.tsx | 69 +++++++++ .../StreamFeatureViewOverviewTab.tsx | 135 ++++++++++++++++++ .../pages/feature-views/useLoadFeatureView.ts | 19 ++- ui/src/parsers/feastRegistry.ts | 2 + ui/src/parsers/feastSFVS.ts | 41 ++++++ ui/src/parsers/mergedFVTypes.ts | 25 +++- ui/src/parsers/parseEntityRelationships.ts | 26 ++++ ...reamFeatureViewCustomTabLoadingWrapper.tsx | 46 ++++++ 16 files changed, 625 insertions(+), 5 deletions(-) create mode 100644 ui/src/custom-tabs/stream-fv-demo-tab/DemoCustomTab.tsx create mode 100644 ui/src/custom-tabs/stream-fv-demo-tab/useDemoQuery.tsx create mode 100644 ui/src/pages/feature-views/StreamFeatureViewInstance.tsx create mode 100644 ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx create mode 100644 ui/src/parsers/feastSFVS.ts create mode 100644 ui/src/utils/custom-tabs/StreamFeatureViewCustomTabLoadingWrapper.tsx diff --git a/ui/public/registry.json b/ui/public/registry.json index 2d5c93c9620..279c9d08327 100644 --- a/ui/public/registry.json +++ b/ui/public/registry.json @@ -29,6 +29,24 @@ "name": "zipcode", "timestampField": "event_timestamp", "type": "BATCH_FILE" + }, + { + "batchSource": { + "fileOptions": { + "uri": "data/zipcode_table.parquet" + }, + "name": "user_stats", + "timestampField": "timestamp", + "type": "BATCH_FILE" + }, + "dataSourceClassType": "feast.data_source.KafkaSource", + "description": "The Kafka stream example", + "kafkaOptions": {"messageFormat": {"jsonFormat": {"schemaJson": "id string, timestamp timestamp"}}, + "watermarkDelayThreshold": "300s"}, + "name": "driver_stats_stream", + "owner": "test@gmail.com", + "timestampField": "timestamp", + "type": "STREAM_KAFKA" } ], "entities": [ @@ -630,5 +648,59 @@ } } ], + "streamFeatureViews": [ + { + "meta": { + "createdTimestamp": "2022-05-11T19:27:03.171556Z", + "lastUpdatedTimestamp": "2022-05-11T19:27:03.171556Z" + }, + "spec": { + "batchSource": { + "createdTimestampColumn": "created_timestamp", + "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", + "fileOptions": { + "uri": "data/zipcode_table.parquet" + }, + "name": "zipcode", + "timestampField": "event_timestamp", + "type": "BATCH_FILE" + }, + "features": [ + { + "name": "conv_percentage", + "valueType": "FLOAT" + }, + { + "name": "acc_percentage", + "valueType": "FLOAT" + } + ], + "name": "transaction_stream_example", + "streamSource": { + "batchSource": { + "fileOptions": { + "uri": "data/zipcode_table.parquet" + }, + "name": "user_stats", + "timestampField": "timestamp", + "type": "BATCH_FILE" + }, + "dataSourceClassType": "feast.data_source.KafkaSource", + "description": "The Kafka stream example", + "kafkaOptions": {"messageFormat": {"jsonFormat": {"schemaJson": "id string, timestamp timestamp"}}, + "watermarkDelayThreshold": "300s"}, + "name": "driver_stats_stream", + "owner": "test@gmail.com", + "timestampField": "timestamp", + "type": "STREAM_KAFKA" + }, + "ttl": "86400s", + "userDefinedFunction": { + "body": "@stream_feature_view(\n sources=[driver_stats_stream_source],\n mode=\"spark\",\n schema=[\n Field(name=\"conv_percentage\", dtype=Float32),\n Field(name=\"acc_percentage\", dtype=Float32),\n ],\n timestamp_field=\"event_timestamp\",\n online=True,\n source=driver_stats_stream_source,\n tags={},\n)\ndef driver_hourly_stats_stream(df: DataFrame) -> DataFrame:\n from pyspark.sql.functions import col\n return (\n df.withColumn(\"conv_percentage\", col(\"conv_rate\") * 100.0)\n .withColumn(\"acc_percentage\", col(\"acc_rate\") * 100.0)\n .drop(\"conv_rate\", \"acc_rate\")\n )\n", + "name": "driver_hourly_stats_stream" + } + } + } + ], "project": "credit_scoring_aws" } diff --git a/ui/src/custom-tabs/TabsRegistryContext.tsx b/ui/src/custom-tabs/TabsRegistryContext.tsx index 9f493e6d11b..83820de1535 100644 --- a/ui/src/custom-tabs/TabsRegistryContext.tsx +++ b/ui/src/custom-tabs/TabsRegistryContext.tsx @@ -10,6 +10,7 @@ import { import RegularFeatureViewCustomTabLoadingWrapper from "../utils/custom-tabs/RegularFeatureViewCustomTabLoadingWrapper"; import OnDemandFeatureViewCustomTabLoadingWrapper from "../utils/custom-tabs/OnDemandFeatureViewCustomTabLoadingWrapper"; +import StreamFeatureViewCustomTabLoadingWrapper from "../utils/custom-tabs/StreamFeatureViewCustomTabLoadingWrapper"; import FeatureServiceCustomTabLoadingWrapper from "../utils/custom-tabs/FeatureServiceCustomTabLoadingWrapper"; import FeatureCustomTabLoadingWrapper from "../utils/custom-tabs/FeatureCustomTabLoadingWrapper"; import DataSourceCustomTabLoadingWrapper from "../utils/custom-tabs/DataSourceCustomTabLoadingWrapper"; @@ -19,6 +20,7 @@ import DatasetCustomTabLoadingWrapper from "../utils/custom-tabs/DatasetCustomTa import { RegularFeatureViewCustomTabRegistrationInterface, OnDemandFeatureViewCustomTabRegistrationInterface, + StreamFeatureViewCustomTabRegistrationInterface, FeatureServiceCustomTabRegistrationInterface, FeatureCustomTabRegistrationInterface, DataSourceCustomTabRegistrationInterface, @@ -30,6 +32,7 @@ import { interface FeastTabsRegistryInterface { RegularFeatureViewCustomTabs?: RegularFeatureViewCustomTabRegistrationInterface[]; OnDemandFeatureViewCustomTabs?: OnDemandFeatureViewCustomTabRegistrationInterface[]; + StreamFeatureViewCustomTabs?: StreamFeatureViewCustomTabRegistrationInterface[]; FeatureServiceCustomTabs?: FeatureServiceCustomTabRegistrationInterface[]; FeatureCustomTabs?: FeatureCustomTabRegistrationInterface[]; DataSourceCustomTabs?: DataSourceCustomTabRegistrationInterface[]; @@ -148,6 +151,16 @@ const useOnDemandFeatureViewCustomTabs = (navigate: NavigateFunction) => { ); }; +const useStreamFeatureViewCustomTabs = (navigate: NavigateFunction) => { + const { StreamFeatureViewCustomTabs } = + React.useContext(TabsRegistryContext); + + return useGenericCustomTabsNavigation( + StreamFeatureViewCustomTabs || [], + navigate + ); +}; + const useFeatureServiceCustomTabs = (navigate: NavigateFunction) => { const { FeatureServiceCustomTabs } = React.useContext(TabsRegistryContext); @@ -214,6 +227,16 @@ const useOnDemandFeatureViewCustomTabRoutes = () => { ); }; +const useStreamFeatureViewCustomTabRoutes = () => { + const { StreamFeatureViewCustomTabs } = + React.useContext(TabsRegistryContext); + + return genericCustomTabRoutes( + StreamFeatureViewCustomTabs || [], + StreamFeatureViewCustomTabLoadingWrapper + ); +}; + const useFeatureServiceCustomTabRoutes = () => { const { FeatureServiceCustomTabs } = React.useContext(TabsRegistryContext); @@ -264,6 +287,7 @@ export { // Navigation useRegularFeatureViewCustomTabs, useOnDemandFeatureViewCustomTabs, + useStreamFeatureViewCustomTabs, useFeatureServiceCustomTabs, useFeatureCustomTabs, useDataSourceCustomTabs, @@ -272,6 +296,7 @@ export { // Routes useRegularFeatureViewCustomTabRoutes, useOnDemandFeatureViewCustomTabRoutes, + useStreamFeatureViewCustomTabRoutes, useFeatureServiceCustomTabRoutes, useFeatureCustomTabRoutes, useDataSourceCustomTabRoutes, diff --git a/ui/src/custom-tabs/stream-fv-demo-tab/DemoCustomTab.tsx b/ui/src/custom-tabs/stream-fv-demo-tab/DemoCustomTab.tsx new file mode 100644 index 00000000000..86e59d10c71 --- /dev/null +++ b/ui/src/custom-tabs/stream-fv-demo-tab/DemoCustomTab.tsx @@ -0,0 +1,85 @@ +import React from "react"; + +import { + // Feature View Custom Tabs will get these props + StreamFeatureViewCustomTabProps, +} from "../types"; + +import { + EuiLoadingContent, + EuiEmptyPrompt, + EuiFlexGroup, + EuiFlexItem, + EuiCode, + EuiSpacer, +} from "@elastic/eui"; + +// Separating out the query is not required, +// but encouraged for code readability +import useDemoQuery from "./useDemoQuery"; + +const DemoCustomTab = ({ + id, + feastObjectQuery, +}: StreamFeatureViewCustomTabProps) => { + // Use React Query to fetch data + // that is custom to this tab. + // See: https://react-query.tanstack.com/guides/queries + const { isLoading, isError, isSuccess, data } = useDemoQuery({ + featureView: id, + }); + + if (isLoading) { + // Handle Loading State + // https://elastic.github.io/eui/#/display/loading + return ; + } + + if (isError) { + // Handle Data Fetching Error + // https://elastic.github.io/eui/#/display/empty-prompt + return ( + Unable to load your demo page} + body={ +

+ There was an error loading the Dashboard application. Contact your + administrator for help. +

+ } + /> + ); + } + + // Feast UI uses the Elastic UI component system. + // and are particularly + // useful for layouts. + return ( + + + +

Hello World. The following is fetched data.

+ + {isSuccess && data && ( + + 
{JSON.stringify(data, null, 2)}
+ + )} + + +

... and this is data from Feast UI’s own query.

+ + {feastObjectQuery.isSuccess && feastObjectQuery.data && ( + + 
{JSON.stringify(feastObjectQuery.data, null, 2)}
+ + )} + + + + ); +}; + +export default DemoCustomTab; diff --git a/ui/src/custom-tabs/stream-fv-demo-tab/useDemoQuery.tsx b/ui/src/custom-tabs/stream-fv-demo-tab/useDemoQuery.tsx new file mode 100644 index 00000000000..b93602dbe3b --- /dev/null +++ b/ui/src/custom-tabs/stream-fv-demo-tab/useDemoQuery.tsx @@ -0,0 +1,44 @@ +import { useQuery } from "react-query"; +import { z } from "zod"; + +// Use Zod to check the shape of the +// json object being loaded +const demoSchema = z.object({ + hello: z.string(), + name: z.string().optional(), +}); + +// Make the type of the object available +type DemoDataType = z.infer; + +interface DemoQueryInterface { + featureView: string | undefined; +} + +const useDemoQuery = ({ featureView }: DemoQueryInterface) => { + // React Query manages caching for you based on query keys + // See: https://react-query.tanstack.com/guides/query-keys + const queryKey = `demo-tab-namespace:${featureView}`; + + // Pass the type to useQuery + // so that components consuming the + // result gets nice type hints + // on the other side. + return useQuery( + queryKey, + () => { + // Customizing the URL based on your needs + const url = `/demo-custom-tabs/demo.json`; + + return fetch(url) + .then((res) => res.json()) + .then((data) => demoSchema.parse(data)); // Use zod to parse results + }, + { + enabled: !!featureView, // Only start the query when the variable is not undefined + } + ); +}; + +export default useDemoQuery; +export type { DemoDataType }; diff --git a/ui/src/custom-tabs/types.ts b/ui/src/custom-tabs/types.ts index 1e555d6185c..ea1dbc8757b 100644 --- a/ui/src/custom-tabs/types.ts +++ b/ui/src/custom-tabs/types.ts @@ -1,5 +1,6 @@ import { useLoadOnDemandFeatureView, + useLoadStreamFeatureView, useLoadRegularFeatureView, } from "../pages/feature-views/useLoadFeatureView"; import useLoadFeature from "../pages/features/useLoadFeature"; @@ -48,6 +49,23 @@ interface OnDemandFeatureViewCustomTabRegistrationInterface }: OnDemandFeatureViewCustomTabProps) => JSX.Element; } +// Type for Stream Feature View Custom Tabs +type StreamFeatureViewQueryReturnType = ReturnType< + typeof useLoadStreamFeatureView +>; +interface StreamFeatureViewCustomTabProps { + id: string | undefined; + feastObjectQuery: StreamFeatureViewQueryReturnType; +} +interface StreamFeatureViewCustomTabRegistrationInterface + extends CustomTabRegistrationInterface { + Component: ({ + id, + feastObjectQuery, + ...args + }: StreamFeatureViewCustomTabProps) => JSX.Element; +} + // Type for Entity Custom Tabs interface EntityCustomTabProps { id: string | undefined; @@ -127,6 +145,9 @@ export type { OnDemandFeatureViewQueryReturnType, OnDemandFeatureViewCustomTabProps, OnDemandFeatureViewCustomTabRegistrationInterface, + StreamFeatureViewQueryReturnType, + StreamFeatureViewCustomTabProps, + StreamFeatureViewCustomTabRegistrationInterface, FeatureServiceCustomTabRegistrationInterface, FeatureServiceCustomTabProps, DataSourceCustomTabRegistrationInterface, diff --git a/ui/src/index.tsx b/ui/src/index.tsx index 2233b90c9e6..e38570929d4 100644 --- a/ui/src/index.tsx +++ b/ui/src/index.tsx @@ -18,6 +18,7 @@ import FeastUI from "./FeastUI"; import DataTab from "./custom-tabs/data-tab/DataTab"; import RFVDemoCustomTab from "./custom-tabs/reguar-fv-demo-tab/DemoCustomTab"; import ODFVDemoCustomTab from "./custom-tabs/ondemand-fv-demo-tab/DemoCustomTab"; +import SFVDemoCustomTab from "./custom-tabs/stream-fv-demo-tab/DemoCustomTab"; import FSDemoCustomTab from "./custom-tabs/feature-service-demo-tab/DemoCustomTab"; import DSDemoCustomTab from "./custom-tabs/data-source-demo-tab/DemoCustomTab"; import EntDemoCustomTab from "./custom-tabs/entity-demo-tab/DemoCustomTab"; @@ -46,6 +47,13 @@ const tabsRegistry = { Component: ODFVDemoCustomTab, }, ], + StreamFeatureViewCustomTabs: [ + { + label: "Custom Tab Demo", + path: "demo-tab", + Component: SFVDemoCustomTab, + }, + ], FeatureServiceCustomTabs: [ { label: "Custom Tab Demo", @@ -93,4 +101,4 @@ ReactDOM.render( /> , document.getElementById("root") -); \ No newline at end of file +); diff --git a/ui/src/pages/feature-views/FeatureViewInstance.tsx b/ui/src/pages/feature-views/FeatureViewInstance.tsx index b0fa7c32b03..5352507573f 100644 --- a/ui/src/pages/feature-views/FeatureViewInstance.tsx +++ b/ui/src/pages/feature-views/FeatureViewInstance.tsx @@ -7,8 +7,11 @@ import { FeastFeatureViewType } from "../../parsers/feastFeatureViews"; import RegularFeatureInstance from "./RegularFeatureViewInstance"; import { FEAST_FV_TYPES } from "../../parsers/mergedFVTypes"; import { FeastODFVType } from "../../parsers/feastODFVS"; +import { FeastSFVType } from "../../parsers/feastSFVS"; import useLoadFeatureView from "./useLoadFeatureView"; import OnDemandFeatureInstance from "./OnDemandFeatureViewInstance"; +import StreamFeatureInstance from "./StreamFeatureViewInstance"; + const FeatureViewInstance = () => { const { featureViewName } = useParams(); @@ -45,6 +48,11 @@ const FeatureViewInstance = () => { return ; } + if (data.type === FEAST_FV_TYPES.stream) { + const sfv: FeastSFVType = data.object; + + return ; + } } return

No Data So Sad

; diff --git a/ui/src/pages/feature-views/FeatureViewListingTable.tsx b/ui/src/pages/feature-views/FeatureViewListingTable.tsx index 59f8b1ed7aa..ceb756db804 100644 --- a/ui/src/pages/feature-views/FeatureViewListingTable.tsx +++ b/ui/src/pages/feature-views/FeatureViewListingTable.tsx @@ -35,7 +35,7 @@ const FeatureViewListingTable = ({ href={`/p/${projectName}/feature-view/${name}`} to={`/p/${projectName}/feature-view/${name}`} > - {name} {item.type === "ondemand" && ondemand} + {name} {(item.type === "ondemand" && ondemand) || (item.type === "stream" && stream)} ); }, diff --git a/ui/src/pages/feature-views/StreamFeatureViewInstance.tsx b/ui/src/pages/feature-views/StreamFeatureViewInstance.tsx new file mode 100644 index 00000000000..ba4c0087278 --- /dev/null +++ b/ui/src/pages/feature-views/StreamFeatureViewInstance.tsx @@ -0,0 +1,69 @@ +import React from "react"; +import { Route, Routes, useNavigate } from "react-router-dom"; +import { useParams } from "react-router-dom"; +import { + EuiPageHeader, + EuiPageContent, + EuiPageContentBody, +} from "@elastic/eui"; + +import { FeatureViewIcon32 } from "../../graphics/FeatureViewIcon"; +import { useMatchExact } from "../../hooks/useMatchSubpath"; +import { FeastSFVType } from "../../parsers/feastSFVS"; +import StreamFeatureViewOverviewTab from "./StreamFeatureViewOverviewTab"; + +import { + useStreamFeatureViewCustomTabs, + useStreamFeatureViewCustomTabRoutes, +} from "../../custom-tabs/TabsRegistryContext"; + +interface StreamFeatureInstanceProps { + data: FeastSFVType; +} + +const StreamFeatureInstance = ({ data }: StreamFeatureInstanceProps) => { + const navigate = useNavigate(); + let { featureViewName } = useParams(); + + const { customNavigationTabs } = useStreamFeatureViewCustomTabs(navigate); + const CustomTabRoutes = useStreamFeatureViewCustomTabRoutes(); + + return ( + + { + navigate(""); + }, + }, + ...customNavigationTabs, + ]} + /> + + + + } + /> + {CustomTabRoutes} + + + + + ); +}; + +export default StreamFeatureInstance; diff --git a/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx b/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx new file mode 100644 index 00000000000..56efc428453 --- /dev/null +++ b/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx @@ -0,0 +1,135 @@ +import { + EuiFlexGroup, + EuiFlexItem, + EuiHorizontalRule, + EuiText, + EuiTitle, + EuiPanel, + EuiCodeBlock, + EuiSpacer, +} from "@elastic/eui"; +import React from "react"; +import FeaturesListDisplay from "../../components/FeaturesListDisplay"; +import { + FeastSFVType, +} from "../../parsers/feastSFVS"; +import { useParams } from "react-router-dom"; +import { EntityRelation } from "../../parsers/parseEntityRelationships"; +import { FEAST_FCO_TYPES } from "../../parsers/types"; +import useLoadRelationshipData from "../../queries/useLoadRelationshipsData"; +import ConsumingFeatureServicesList from "./ConsumingFeatureServicesList"; +import EuiCustomLink from "../../components/EuiCustomLink"; + +interface StreamFeatureViewOverviewTabProps { + data: FeastSFVType; +} + +const whereFSconsumesThisFv = (fvName: string) => { + return (r: EntityRelation) => { + return ( + r.source.name === fvName && + r.target.type === FEAST_FCO_TYPES.featureService + ); + }; +}; + +const StreamFeatureViewOverviewTab = ({ + data, +}: StreamFeatureViewOverviewTabProps) => { + const inputs = Object.entries([data.spec.streamSource]); + const { projectName } = useParams(); + + const relationshipQuery = useLoadRelationshipData(); + const fsNames = relationshipQuery.data + ? relationshipQuery.data + .filter(whereFSconsumesThisFv(data.spec.name)) + .map((fs) => { + return fs.target.name; + }) + : []; + + return ( + + + + + +

Transformation

+ + + + {data.spec.userDefinedFunction.body} + + + + + + + + +

Features ({data.spec.features.length})

+ + + {projectName && data.spec.features ? ( + + ) : ( + No Tags sepcified on this feature view. + )} + + + + + +

Inputs ({inputs.length})

+ + + + {inputs.map(([key, inputGroup]) => { + + return ( + + + Stream Source + + + + {inputGroup.name} + + + + + {JSON.stringify(inputGroup, null, 2)} + + + + ); + })} + + + + + +

Consuming Feature Services

+ + + {fsNames.length > 0 ? ( + + ) : ( + No services consume this feature view + )} + + + + + ); +}; + +export default StreamFeatureViewOverviewTab; diff --git a/ui/src/pages/feature-views/useLoadFeatureView.ts b/ui/src/pages/feature-views/useLoadFeatureView.ts index ded7900ea94..7685171b72b 100644 --- a/ui/src/pages/feature-views/useLoadFeatureView.ts +++ b/ui/src/pages/feature-views/useLoadFeatureView.ts @@ -51,5 +51,22 @@ const useLoadOnDemandFeatureView = (featureViewName: string) => { }; }; +const useLoadStreamFeatureView = (featureViewName: string) => { + const registryUrl = useContext(RegistryPathContext); + const registryQuery = useLoadRegistry(registryUrl); + + const data = + registryQuery.data === undefined + ? undefined + : registryQuery.data.objects.streamFeatureViews?.find((fv) => { + return fv.spec.name === featureViewName; + }); + + return { + ...registryQuery, + data, + }; +}; + export default useLoadFeatureView; -export { useLoadRegularFeatureView, useLoadOnDemandFeatureView }; +export { useLoadRegularFeatureView, useLoadOnDemandFeatureView, useLoadStreamFeatureView }; diff --git a/ui/src/parsers/feastRegistry.ts b/ui/src/parsers/feastRegistry.ts index 98e4fccca2a..f84187046a8 100644 --- a/ui/src/parsers/feastRegistry.ts +++ b/ui/src/parsers/feastRegistry.ts @@ -5,6 +5,7 @@ import { FeastFeatureServiceSchema } from "./feastFeatureServices"; import { FeastFeatureViewSchema } from "./feastFeatureViews"; import { FeastSavedDatasetSchema } from "./feastSavedDataset"; import { FeastODFVSchema } from "./feastODFVS"; +import { FeastSFVSchema } from "./feastSFVS"; const FeastRegistrySchema = z.object({ project: z.string(), @@ -12,6 +13,7 @@ const FeastRegistrySchema = z.object({ entities: z.array(FeastEntitySchema).optional(), featureViews: z.array(FeastFeatureViewSchema).optional(), onDemandFeatureViews: z.array(FeastODFVSchema).optional(), + streamFeatureViews: z.array(FeastSFVSchema).optional(), featureServices: z.array(FeastFeatureServiceSchema).optional(), savedDatasets: z.array(FeastSavedDatasetSchema).optional(), }); diff --git a/ui/src/parsers/feastSFVS.ts b/ui/src/parsers/feastSFVS.ts new file mode 100644 index 00000000000..f21b3d1cdac --- /dev/null +++ b/ui/src/parsers/feastSFVS.ts @@ -0,0 +1,41 @@ +import { z } from "zod"; +import { FeastFeatureColumnSchema } from "./feastFeatureViews"; +import {FeastDatasourceSchema} from "./feastDatasources"; + +const FeatureViewProjectionSchema = z.object({ + featureViewProjection: z.object({ + featureViewName: z.string(), + featureColumns: z.array(FeastFeatureColumnSchema), + }), +}); + +const StreamSourceSchema = z.object({ + type: z.string(), + name: z.string(), + owner: z.string().optional(), + description: z.string().optional(), +}); + +const FeastSFVSchema = z.object({ + spec: z.object({ + name: z.string(), + features: z.array(FeastFeatureColumnSchema), + batchSource: FeastDatasourceSchema, + streamSource: StreamSourceSchema, + userDefinedFunction: z.object({ + name: z.string(), + body: z.string(), + }), + }), + meta: z.object({ + createdTimestamp: z.string().transform((val) => new Date(val)), + lastUpdatedTimestamp: z.string().transform((val) => new Date(val)), + }), +}); + +type FeastSFVType = z.infer; +type StreamSourceType = z.infer; +type FeatureViewProjectionType = z.infer; + +export { FeastSFVSchema }; +export type { FeastSFVType, StreamSourceType, FeatureViewProjectionType}; diff --git a/ui/src/parsers/mergedFVTypes.ts b/ui/src/parsers/mergedFVTypes.ts index 6a53b18e94d..edf1adee9e5 100644 --- a/ui/src/parsers/mergedFVTypes.ts +++ b/ui/src/parsers/mergedFVTypes.ts @@ -3,11 +3,13 @@ import { FeastFeatureViewType, } from "./feastFeatureViews"; import { FeastODFVType } from "./feastODFVS"; +import { FeastSFVType } from "./feastSFVS"; import { FeastRegistryType } from "./feastRegistry"; enum FEAST_FV_TYPES { regular = "regular", ondemand = "ondemand", + stream = "stream" } interface regularFVInterface { @@ -24,7 +26,14 @@ interface ODFVInterface { object: FeastODFVType; } -type genericFVType = regularFVInterface | ODFVInterface; +interface SFVInterface { + name: string; + type: FEAST_FV_TYPES.stream; + features: FeastFeatureColumnType[]; + object: FeastSFVType; +} + +type genericFVType = regularFVInterface | ODFVInterface | SFVInterface; const mergedFVTypes = (objects: FeastRegistryType) => { const mergedFVMap: Record = {}; @@ -55,9 +64,21 @@ const mergedFVTypes = (objects: FeastRegistryType) => { mergedFVList.push(obj); }); + objects.streamFeatureViews?.forEach((sfv) => { + const obj: genericFVType = { + name: sfv.spec.name, + type: FEAST_FV_TYPES.stream, + features: sfv.spec.features, + object: sfv, + }; + + mergedFVMap[sfv.spec.name] = obj; + mergedFVList.push(obj); + }); + return { mergedFVMap, mergedFVList }; }; export default mergedFVTypes; export { FEAST_FV_TYPES }; -export type { genericFVType, regularFVInterface, ODFVInterface }; +export type { genericFVType, regularFVInterface, ODFVInterface, SFVInterface }; diff --git a/ui/src/parsers/parseEntityRelationships.ts b/ui/src/parsers/parseEntityRelationships.ts index f54bff63a1c..8424bb7a44f 100644 --- a/ui/src/parsers/parseEntityRelationships.ts +++ b/ui/src/parsers/parseEntityRelationships.ts @@ -88,6 +88,32 @@ const parseEntityRelationships = (objects: FeastRegistryType) => { }); }); + objects.streamFeatureViews?.forEach((fv) => { + // stream source + links.push({ + source: { + type: FEAST_FCO_TYPES["dataSource"], + name: fv.spec.streamSource.name, + }, + target: { + type: FEAST_FCO_TYPES["featureView"], + name: fv.spec.name, + }, + }); + + // batch source + links.push({ + source: { + type: FEAST_FCO_TYPES["dataSource"], + name: fv.spec.batchSource.name, + }, + target: { + type: FEAST_FCO_TYPES["featureView"], + name: fv.spec.name, + }, + }); + }); + return links; }; diff --git a/ui/src/utils/custom-tabs/StreamFeatureViewCustomTabLoadingWrapper.tsx b/ui/src/utils/custom-tabs/StreamFeatureViewCustomTabLoadingWrapper.tsx new file mode 100644 index 00000000000..098ab848a55 --- /dev/null +++ b/ui/src/utils/custom-tabs/StreamFeatureViewCustomTabLoadingWrapper.tsx @@ -0,0 +1,46 @@ +import React from "react"; + +import { useParams } from "react-router-dom"; +import useLoadFeatureView from "../../pages/feature-views/useLoadFeatureView"; +import { + StreamFeatureViewCustomTabProps, + StreamFeatureViewQueryReturnType, +} from "../../custom-tabs/types"; +import { FEAST_FV_TYPES } from "../../parsers/mergedFVTypes"; + +interface StreamFeatureViewCustomTabLoadingWrapperProps { + Component: (props: StreamFeatureViewCustomTabProps) => JSX.Element; +} + +const StreamFeatureViewCustomTabLoadingWrapper = ({ + Component, +}: StreamFeatureViewCustomTabLoadingWrapperProps) => { + const { featureViewName } = useParams(); + + if (!featureViewName) { + throw new Error( + `This route has no 'featureViewName' part. This route is likely not supposed to render this component.` + ); + } + + const feastObjectQuery = useLoadFeatureView(featureViewName); + + if ( + feastObjectQuery.isSuccess && + feastObjectQuery.data && + feastObjectQuery.data.type !== FEAST_FV_TYPES.stream + ) { + throw new Error( + `This should not happen. Somehow a custom tab on a SFV page received data that does not have the shape?` + ); + } + + return ( + + ); +}; + +export default StreamFeatureViewCustomTabLoadingWrapper; From 6f6cbb76a216965666719e492c5ba854ada32605 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Fri, 30 Sep 2022 16:53:28 -0500 Subject: [PATCH 22/33] fix: Add postgres to the feature server Dockerfile to fix helm chart flow (#3261) fix: Add postgres to the feature server docker file Signed-off-by: Danny Chiao Signed-off-by: Danny Chiao --- sdk/python/feast/infra/feature_servers/multicloud/Dockerfile | 2 +- .../feast/infra/feature_servers/multicloud/Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index 7dbe5fd0c49..990c1fd8f03 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -9,7 +9,7 @@ RUN apt update && \ RUN pip install pip --upgrade COPY . . -RUN pip install ".[aws,gcp,snowflake,redis,go,mysql]" +RUN pip install ".[aws,gcp,snowflake,redis,go,mysql,postgres]" RUN apt update RUN apt install -y -V ca-certificates lsb-release wget diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev index 7dbe5fd0c49..990c1fd8f03 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -9,7 +9,7 @@ RUN apt update && \ RUN pip install pip --upgrade COPY . . -RUN pip install ".[aws,gcp,snowflake,redis,go,mysql]" +RUN pip install ".[aws,gcp,snowflake,redis,go,mysql,postgres]" RUN apt update RUN apt install -y -V ca-certificates lsb-release wget From 53dc8114b01f63caf3850d8eb1c9287c35702f9c Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Fri, 30 Sep 2022 17:32:26 -0500 Subject: [PATCH 23/33] ci: Add homebrew in CI Signed-off-by: Danny Chiao --- .github/workflows/release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ec56d60e4c5..0c9ac1e752c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -92,6 +92,9 @@ jobs: uses: actions/setup-node@v2 with: node-version: '16' + - name: Set up Homebrew + id: set-up-homebrew + uses: Homebrew/actions/setup-homebrew@master - name: Setup Helm-docs run: | brew install norwoodj/tap/helm-docs From f80f05f5d2042c6fff533b0dc85612acda5a7417 Mon Sep 17 00:00:00 2001 From: vmallya-123 Date: Sat, 1 Oct 2022 22:45:56 +0800 Subject: [PATCH 24/33] feat: Adding billing_project_id in BigQueryOfflineStoreConfig (#3253) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * adding_billing_project_in_config Signed-off-by: “Varun * Fix lint Signed-off-by: Danny Chiao Signed-off-by: “Varun Signed-off-by: Danny Chiao Co-authored-by: Danny Chiao --- .../feast/infra/offline_stores/bigquery.py | 60 ++++++++++++++----- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index c570b8d38ab..bf010c82aaf 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -19,7 +19,7 @@ import pandas as pd import pyarrow import pyarrow.parquet -from pydantic import StrictStr +from pydantic import StrictStr, validator from pydantic.typing import Literal from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed @@ -83,7 +83,8 @@ class BigQueryOfflineStoreConfig(FeastConfigBaseModel): project_id: Optional[StrictStr] = None """ (optional) GCP project name used for the BigQuery offline store """ - + billing_project_id: Optional[StrictStr] = None + """ (optional) GCP project name used to run the bigquery jobs at """ location: Optional[StrictStr] = None """ (optional) GCP location name used for the BigQuery offline store. Examples of location names include ``US``, ``EU``, ``us-central1``, ``us-west4``. @@ -94,6 +95,14 @@ class BigQueryOfflineStoreConfig(FeastConfigBaseModel): gcs_staging_location: Optional[str] = None """ (optional) GCS location used for offloading BigQuery results as parquet files.""" + @validator("billing_project_id") + def project_id_exists(cls, v, values, **kwargs): + if v and not values["project_id"]: + raise ValueError( + "please specify project_id if billing_project_id is specified" + ) + return v + class BigQueryOfflineStore(OfflineStore): @staticmethod @@ -122,9 +131,11 @@ def pull_latest_from_table_or_query( timestamps.append(created_timestamp_column) timestamp_desc_string = " DESC, ".join(timestamps) + " DESC" field_string = ", ".join(join_key_columns + feature_name_columns + timestamps) - + project_id = ( + config.offline_store.billing_project_id or config.offline_store.project_id + ) client = _get_bigquery_client( - project=config.offline_store.project_id, + project=project_id, location=config.offline_store.location, ) query = f""" @@ -162,9 +173,11 @@ def pull_all_from_table_or_query( assert isinstance(config.offline_store, BigQueryOfflineStoreConfig) assert isinstance(data_source, BigQuerySource) from_expression = data_source.get_table_query_string() - + project_id = ( + config.offline_store.billing_project_id or config.offline_store.project_id + ) client = _get_bigquery_client( - project=config.offline_store.project_id, + project=project_id, location=config.offline_store.location, ) field_string = ", ".join( @@ -197,17 +210,22 @@ def get_historical_features( assert isinstance(config.offline_store, BigQueryOfflineStoreConfig) for fv in feature_views: assert isinstance(fv.batch_source, BigQuerySource) - + project_id = ( + config.offline_store.billing_project_id or config.offline_store.project_id + ) client = _get_bigquery_client( - project=config.offline_store.project_id, + project=project_id, location=config.offline_store.location, ) assert isinstance(config.offline_store, BigQueryOfflineStoreConfig) - + if config.offline_store.billing_project_id: + dataset_project = str(config.offline_store.project_id) + else: + dataset_project = client.project table_reference = _get_table_reference_for_new_entity( client, - client.project, + dataset_project, config.offline_store.dataset, config.offline_store.location, ) @@ -295,9 +313,11 @@ def write_logged_features( ): destination = logging_config.destination assert isinstance(destination, BigQueryLoggingDestination) - + project_id = ( + config.offline_store.billing_project_id or config.offline_store.project_id + ) client = _get_bigquery_client( - project=config.offline_store.project_id, + project=project_id, location=config.offline_store.location, ) @@ -353,9 +373,11 @@ def offline_write_batch( if table.schema != pa_schema: table = table.cast(pa_schema) - + project_id = ( + config.offline_store.billing_project_id or config.offline_store.project_id + ) client = _get_bigquery_client( - project=config.offline_store.project_id, + project=project_id, location=config.offline_store.location, ) @@ -451,7 +473,10 @@ def to_bigquery( if not job_config: today = date.today().strftime("%Y%m%d") rand_id = str(uuid.uuid4())[:7] - path = f"{self.client.project}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" + if self.config.offline_store.billing_project_id: + path = f"{self.config.offline_store.project_id}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" + else: + path = f"{self.client.project}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" job_config = bigquery.QueryJobConfig(destination=path) if not job_config.dry_run and self.on_demand_feature_views: @@ -525,7 +550,10 @@ def to_remote_storage(self) -> List[str]: bucket: str prefix: str - storage_client = StorageClient(project=self.client.project) + if self.config.offline_store.billing_project_id: + storage_client = StorageClient(project=self.config.offline_store.project_id) + else: + storage_client = StorageClient(project=self.client.project) bucket, prefix = self._gcs_path[len("gs://") :].split("/", 1) prefix = prefix.rsplit("/", 1)[0] if prefix.startswith("/"): From 12e1a8f0e9f098cfc3ed66db6665cb5fa26240f4 Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Tue, 4 Oct 2022 10:36:13 -0700 Subject: [PATCH 25/33] fix: Stream feature view meta undefined created_timestamp issue (#3266) fix stream feature view meta undefied created_timestap issue Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- ui/src/parsers/feastSFVS.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/parsers/feastSFVS.ts b/ui/src/parsers/feastSFVS.ts index f21b3d1cdac..f65a0872220 100644 --- a/ui/src/parsers/feastSFVS.ts +++ b/ui/src/parsers/feastSFVS.ts @@ -28,8 +28,8 @@ const FeastSFVSchema = z.object({ }), }), meta: z.object({ - createdTimestamp: z.string().transform((val) => new Date(val)), - lastUpdatedTimestamp: z.string().transform((val) => new Date(val)), + createdTimestamp: z.string().transform((val) => new Date(val)).optional(), + lastUpdatedTimestamp: z.string().transform((val) => new Date(val)).optional(), }), }); From 07281178f8be78178b739366c90fd2074db0981b Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Tue, 4 Oct 2022 16:55:12 -0700 Subject: [PATCH 26/33] fix: Udf in stream feature view UI shows pickled data (#3268) * fix udf in stream feature view UI shows pickeled data Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * fix lint Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * fix lint Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * fix test Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> * fix lint Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- sdk/python/feast/infra/registry/base_registry.py | 9 ++++++--- sdk/python/feast/stream_feature_view.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/infra/registry/base_registry.py b/sdk/python/feast/infra/registry/base_registry.py index e1e9ba99e18..fc65932ea16 100644 --- a/sdk/python/feast/infra/registry/base_registry.py +++ b/sdk/python/feast/infra/registry/base_registry.py @@ -638,9 +638,12 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]: self.list_stream_feature_views(project=project), key=lambda stream_feature_view: stream_feature_view.name, ): - registry_dict["streamFeatureViews"].append( - self._message_to_sorted_dict(stream_feature_view.to_proto()) - ) + sfv_dict = self._message_to_sorted_dict(stream_feature_view.to_proto()) + + sfv_dict["spec"]["userDefinedFunction"][ + "body" + ] = stream_feature_view.udf_string + registry_dict["streamFeatureViews"].append(sfv_dict) for saved_dataset in sorted( self.list_saved_datasets(project=project), key=lambda item: item.name ): diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 176f38d093f..d3a2164788f 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -71,6 +71,7 @@ class StreamFeatureView(FeatureView): timestamp_field: str materialization_intervals: List[Tuple[datetime, datetime]] udf: Optional[FunctionType] + udf_string: Optional[str] def __init__( self, @@ -88,6 +89,7 @@ def __init__( mode: Optional[str] = "spark", timestamp_field: Optional[str] = "", udf: Optional[FunctionType] = None, + udf_string: Optional[str] = "", ): if not flags_helper.is_test(): warnings.warn( @@ -114,6 +116,7 @@ def __init__( self.mode = mode or "" self.timestamp_field = timestamp_field or "" self.udf = udf + self.udf_string = udf_string super().__init__( name=name, @@ -143,6 +146,7 @@ def __eq__(self, other): self.mode != other.mode or self.timestamp_field != other.timestamp_field or self.udf.__code__.co_code != other.udf.__code__.co_code + or self.udf_string != other.udf_string or self.aggregations != other.aggregations ): return False @@ -171,6 +175,7 @@ def to_proto(self): udf_proto = UserDefinedFunctionProto( name=self.udf.__name__, body=dill.dumps(self.udf, recurse=True), + body_text=self.udf_string, ) spec = StreamFeatureViewSpecProto( name=self.name, @@ -209,6 +214,11 @@ def from_proto(cls, sfv_proto): if sfv_proto.spec.HasField("user_defined_function") else None ) + udf_string = ( + sfv_proto.spec.user_defined_function.body_text + if sfv_proto.spec.HasField("user_defined_function") + else None + ) stream_feature_view = cls( name=sfv_proto.spec.name, description=sfv_proto.spec.description, @@ -226,6 +236,7 @@ def from_proto(cls, sfv_proto): source=stream_source, mode=sfv_proto.spec.mode, udf=udf, + udf_string=udf_string, aggregations=[ Aggregation.from_proto(agg_proto) for agg_proto in sfv_proto.spec.aggregations @@ -315,6 +326,7 @@ def mainify(obj): obj.__module__ = "__main__" def decorator(user_function): + udf_string = dill.source.getsource(user_function) mainify(user_function) stream_feature_view_obj = StreamFeatureView( name=user_function.__name__, @@ -323,6 +335,7 @@ def decorator(user_function): source=source, schema=schema, udf=user_function, + udf_string=udf_string, description=description, tags=tags, online=online, From b9b9c542bc0860cfb351be7d84d6461b70ad2c81 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Tue, 4 Oct 2022 17:59:10 -0600 Subject: [PATCH 27/33] fix: Updated quickstart notebook to patch an incorrect reference to an outdated featureview name (#3271) Signed-off-by: Francisco Javier Arceo Signed-off-by: Francisco Javier Arceo --- examples/quickstart/quickstart.ipynb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/quickstart/quickstart.ipynb b/examples/quickstart/quickstart.ipynb index 68b9d639110..cec4df91b11 100644 --- a/examples/quickstart/quickstart.ipynb +++ b/examples/quickstart/quickstart.ipynb @@ -512,7 +512,7 @@ "\n", "# This groups features into a model version\n", "driver_stats_fs = FeatureService(\n", - " name=\"driver_activity\", features=[driver_hourly_stats_view, transformed_conv_rate]\n", + " name=\"driver_activity_v1\", features=[driver_hourly_stats_view, transformed_conv_rate]\n", ")\n", "```" ] @@ -547,7 +547,7 @@ "Created entity \u001b[1m\u001b[32mdriver\u001b[0m\n", "Created feature view \u001b[1m\u001b[32mdriver_hourly_stats\u001b[0m\n", "Created on demand feature view \u001b[1m\u001b[32mtransformed_conv_rate\u001b[0m\n", - "Created feature service \u001b[1m\u001b[32mdriver_activity\u001b[0m\n", + "Created feature service \u001b[1m\u001b[32mdriver_activity_v1\u001b[0m\n", "\n", "Created sqlite table \u001b[1m\u001b[32mfeature_repo_driver_hourly_stats\u001b[0m\n", "\n" @@ -942,11 +942,11 @@ "### Fetching features using feature services\n", "You can also use feature services to manage multiple features, and decouple feature view definitions and the features needed by end applications. The feature store can also be used to fetch either online or historical features using the same api below. More information can be found [here](https://docs.feast.dev/getting-started/concepts/feature-retrieval).\n", "\n", - " The `driver_activity` feature service pulls all features from the `driver_hourly_stats` feature view:\n", + " The `driver_activity_v1` feature service pulls all features from the `driver_hourly_stats` feature view:\n", "\n", "```python\n", "driver_stats_fs = FeatureService(\n", - " name=\"driver_activity\", features=[driver_hourly_stats_view]\n", + " name=\"driver_activity_v1\", features=[driver_hourly_stats_view]\n", ")\n", "```" ] @@ -979,7 +979,7 @@ "from feast import FeatureStore\n", "feature_store = FeatureStore('.') # Initialize the feature store\n", "\n", - "feature_service = feature_store.get_feature_service(\"driver_activity\")\n", + "feature_service = feature_store.get_feature_service(\"driver_activity_v1\")\n", "feature_vector = feature_store.get_online_features(\n", " features=feature_service,\n", " entity_rows=[\n", @@ -1101,4 +1101,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} From 6bc91c23545a6cf765c0e70887afb22eada2c8e1 Mon Sep 17 00:00:00 2001 From: Abhin Chhabra Date: Wed, 5 Oct 2022 14:10:11 -0400 Subject: [PATCH 28/33] feat: BigTable online store (#3140) * Initial implementation of BigTable online store. Signed-off-by: Abhin Chhabra * Attempt to run bigtable integration tests. Currently focusing on just getting the tests running locally. I've only build python3.8 requirements. Signed-off-by: Abhin Chhabra * Got the BigTable tests running in local containers Signed-off-by: Abhin Chhabra * Set serialization version when computing entity ID Signed-off-by: Abhin Chhabra * Switch to the recommended layout in bigtable. This was recommended by the BigTable dev team. Details of this layout will be added to the documentation in a future commit. Signed-off-by: Abhin Chhabra * Minor bugfixes. - If a row is empty when fetching data, don't process it more. - If a task in the threadpool fails, bubble up that failure. - If a `created_ts` is not available, use an empty string. `None` does not automatically serialize to bytes. Signed-off-by: Abhin Chhabra * Move BigTable online store out of contrib As per feedback on the PR. Signed-off-by: Abhin Chhabra * Attempt to run integration tests in CI. Provide the GCP project and the bigtable instance ID for the tests to connect to. Signed-off-by: Abhin Chhabra * Delete tables for entity-less feature views. Signed-off-by: Abhin Chhabra * Table names should be smaller than 50 characters This is BigTable's table length limit and it's causing test failures. Signed-off-by: Abhin Chhabra * Optimize bigtable reads. - Fetch all the rows in one bigtable fetch. - Get only the columns that are necessary (using a column regex filter). Signed-off-by: Abhin Chhabra * dynamodb: switch to `mock_dynamodb` The latest rebuilding of requirements has upgraded the `moto` library past the `4.0.0` release, which has a couple of breaking changes. Specifically, the `mock_dynamodb2` decorator has been deprecated. See https://github.com/spulec/moto/blob/master/CHANGELOG.md#400 for more details. The actual PR (https://github.com/spulec/moto/pull/4919) mentions that it's because the `mock_dynamodb` decorator is now equivalent to the `mock_dynamodb2` decorator. Signed-off-by: Abhin Chhabra * minor: rename `BigTable` to `Bigtable` This matches the GCP docs. Signed-off-by: Abhin Chhabra * Wrote some Bigtable documentation. Closely mirrors the docs for the other online stores. Signed-off-by: Abhin Chhabra * Bugfix: Deal with missing row keys. It looks like the bigtable client will just skip over non-existent row keys. Signed-off-by: Abhin Chhabra * Fix linting issues. Signed-off-by: Abhin Chhabra * Generate requirements files. - As of version `1.49`, the various python packages in the [grpc repo](https://github.com/grpc/grpc/tree/master/src/python) require `protobuf>=4.21.3`. Unfortunately, this is incompatible with all versions of `tensorflow-metadata` (see [this issue](https://github.com/tensorflow/metadata/issues/37)). And since `piptools` doesn't backtrack during dependency resolution, the requirement files cannot be regenerated without adding an upper limit on these grpc libraries directly in `setup.py`. - The previous attempt to upgrade usages of the `mock_dynamodb2` decorator to the newest version failed. Since I'm not an expert in dynamodb, it made sense to just cap the test tool to the version already being used in CI. Signed-off-by: Abhin Chhabra * Don't bother materializing created timestamp. Had a discussion with Danny about whether it's useful to copy this column. He agreed that there's no value to storing this in the online store. Signed-off-by: Abhin Chhabra * Remove `tensorflow-metadata`. Turns out that this dependency is not required. We removed all references to it in [this PR](https://github.com/feast-dev/feast/pull/2063), but did not remove it from `setup.py`. Removing it has caused many of the restrictions imposed in previous commits to be unnecessary. Signed-off-by: Abhin Chhabra * Minor fix to Bigtable documentation. Feedback from Danny mentioned that Bigtable should be able to store multiple versions of the same key and fetch the latest at read time. This makes sense and means that concurrent writes should work just fine. Signed-off-by: Abhin Chhabra * update roadmap docs Signed-off-by: Danny Chiao * Fix roadmap doc Signed-off-by: Danny Chiao * Change link to point to roadmap page Signed-off-by: Danny Chiao * change order in roadmap Signed-off-by: Danny Chiao Signed-off-by: Abhin Chhabra Signed-off-by: Abhin Chhabra Signed-off-by: Danny Chiao Co-authored-by: Danny Chiao --- README.md | 4 +- docs/SUMMARY.md | 1 + .../third-party-integrations.md | 2 +- docs/reference/online-stores/README.md | 4 + docs/reference/online-stores/bigtable.md | 56 +++ docs/roadmap.md | 4 +- docs/specs/online_store_format.md | 25 +- .../feast/infra/online_stores/bigtable.py | 338 ++++++++++++++++++ sdk/python/feast/repo_config.py | 1 + .../gcp/feature_repo/feature_store.yaml | 5 + .../requirements/py3.10-ci-requirements.txt | 187 +++++----- .../requirements/py3.10-requirements.txt | 70 ++-- .../requirements/py3.8-ci-requirements.txt | 187 +++++----- .../requirements/py3.8-requirements.txt | 70 ++-- .../requirements/py3.9-ci-requirements.txt | 193 +++++----- .../requirements/py3.9-requirements.txt | 70 ++-- .../feature_repos/repo_configuration.py | 11 + .../universal/online_store/bigtable.py | 47 +++ setup.py | 32 +- 19 files changed, 900 insertions(+), 407 deletions(-) create mode 100644 docs/reference/online-stores/bigtable.md create mode 100644 sdk/python/feast/infra/online_stores/bigtable.py create mode 100644 sdk/python/tests/integration/feature_repos/universal/online_store/bigtable.py diff --git a/README.md b/README.md index 182637018f1..d57247c60b5 100644 --- a/README.md +++ b/README.md @@ -173,12 +173,12 @@ The list below contains the functionality that contributors are planning to deve * [x] [DynamoDB](https://docs.feast.dev/reference/online-stores/dynamodb) * [x] [Redis](https://docs.feast.dev/reference/online-stores/redis) * [x] [Datastore](https://docs.feast.dev/reference/online-stores/datastore) + * [x] [Bigtable](https://docs.feast.dev/reference/online-stores/bigtable) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) + * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/adding-support-for-a-new-online-store) - * [x] [Cassandra / AstraDB](https://docs.feast.dev/reference/online-stores/cassandra) - * [ ] Bigtable (in progress) * **Feature Engineering** * [x] On-demand Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) * [x] Streaming Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1UzEyETHUaGpn0ap4G82DHluiCj7zEbrQLkJJkKSv4e8/edit)) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 2b58fb277e3..cb4a2664cd6 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -92,6 +92,7 @@ * [Redis](reference/online-stores/redis.md) * [Datastore](reference/online-stores/datastore.md) * [DynamoDB](reference/online-stores/dynamodb.md) + * [Bigtable](reference/online-stores/bigtable.md) * [PostgreSQL (contrib)](reference/online-stores/postgres.md) * [Cassandra + Astra DB (contrib)](reference/online-stores/cassandra.md) * [MySQL (contrib)](reference/online-stores/mysql.md) diff --git a/docs/getting-started/third-party-integrations.md b/docs/getting-started/third-party-integrations.md index 8e6a600aa05..ca974880ed3 100644 --- a/docs/getting-started/third-party-integrations.md +++ b/docs/getting-started/third-party-integrations.md @@ -11,7 +11,7 @@ Don't see your offline store or online store of choice here? Check out our guide ## Integrations -See [Functionality and Roadmap](../../#-functionality-and-roadmap) +See [Functionality and Roadmap](../roadmap.md) ## Standards diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index 2ade4f0de9b..e46fc28d162 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -26,6 +26,10 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli [dynamodb.md](dynamodb.md) {% endcontent-ref %} +{% content-ref url="bigtable.md" %} +[bigtable.md](mysql.md) +{% endcontent-ref %} + {% content-ref url="postgres.md" %} [postgres.md](postgres.md) {% endcontent-ref %} diff --git a/docs/reference/online-stores/bigtable.md b/docs/reference/online-stores/bigtable.md new file mode 100644 index 00000000000..0d6e7cfb13b --- /dev/null +++ b/docs/reference/online-stores/bigtable.md @@ -0,0 +1,56 @@ +# Bigtable online store + +## Description + +The [Bigtable](https://cloud.google.com/bigtable) online store provides support for +materializing feature values into Cloud Bigtable. The data model used to store feature +values in Bigtable is described in more detail +[here](../../specs/online_store_format.md#google-bigtable-online-store-format). + +## Getting started + +In order to use this online store, you'll need to run `pip install 'feast[gcp]'`. You +can then get started with the command `feast init REPO_NAME -t gcp`. + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: gcp +online_store: + type: bigtable + project_id: my_gcp_project + instance: my_bigtable_instance +``` +{% endcode %} + +The full set of configuration options is available in +[BigtableOnlineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.bigtable.BigtableOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Bigtable online store. + +| | Bigtable | +|-----------------------------------------------------------|----------| +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/roadmap.md b/docs/roadmap.md index 30f4317054b..cea646a8f5f 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -31,12 +31,12 @@ The list below contains the functionality that contributors are planning to deve * [x] [DynamoDB](https://docs.feast.dev/reference/online-stores/dynamodb) * [x] [Redis](https://docs.feast.dev/reference/online-stores/redis) * [x] [Datastore](https://docs.feast.dev/reference/online-stores/datastore) + * [x] [Bigtable](https://docs.feast.dev/reference/online-stores/bigtable) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) + * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/adding-support-for-a-new-online-store) - * [x] [Cassandra / AstraDB](https://docs.feast.dev/reference/online-stores/cassandra) - * [ ] Bigtable (in progress) * **Feature Engineering** * [x] On-demand Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) * [x] Streaming Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1UzEyETHUaGpn0ap4G82DHluiCj7zEbrQLkJJkKSv4e8/edit)) diff --git a/docs/specs/online_store_format.md b/docs/specs/online_store_format.md index 5c3c545c8d6..4022138e319 100644 --- a/docs/specs/online_store_format.md +++ b/docs/specs/online_store_format.md @@ -92,6 +92,29 @@ Other types of entity keys are not supported in this version of the specificatio ![Datastore Online Example](datastore_online_example.png) +## Google Bigtable Online Store Format + +[Bigtable storage model](https://cloud.google.com/bigtable/docs/overview#storage-model) +consists of massively scalable tables, with each row keyed by a "row key". The rows in a +table are stored lexicographically sorted by this row key. + +We use the following structure to store feature data in Bigtable: + +* All feature data for an entity or a specific group of entities is stored in the same + table. The table name is derived by concatenating the lexicographically sorted names + of entities. +* This implementation only uses one column family per table, named `features`. +* Each row key is created by concatenating a hash derived from the specific entity keys + and the name of the feature view. Each row only stores feature values for a specific + feature view. This arrangement also means that feature values for a given group of + entities are colocated. +* The columns used in each row are named after the features in the feature view. + Bigtable is perfectly content being sparsely populated. +* By default, we store 1 historical value of each feature value. This can be configured + using the `max_versions` setting in `BigtableOnlineStoreConfig`. This implementation + of the online store does not have the ability to revert any given value to its old + self. To use the historical version, you'll have to use custom code. + ## Cassandra/Astra DB Online Store Format ### Overview @@ -250,4 +273,4 @@ message BoolList { repeated bool val = 1; } -``` \ No newline at end of file +``` diff --git a/sdk/python/feast/infra/online_stores/bigtable.py b/sdk/python/feast/infra/online_stores/bigtable.py new file mode 100644 index 00000000000..e08bc44bdbe --- /dev/null +++ b/sdk/python/feast/infra/online_stores/bigtable.py @@ -0,0 +1,338 @@ +import hashlib +import logging +from concurrent import futures +from datetime import datetime +from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple + +import google +from google.cloud import bigtable +from google.cloud.bigtable import row_filters +from pydantic import StrictStr +from pydantic.typing import Literal + +from feast import Entity, FeatureView, utils +from feast.feature_view import DUMMY_ENTITY_NAME +from feast.infra.online_stores.helpers import compute_entity_id +from feast.infra.online_stores.online_store import OnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel, RepoConfig +from feast.usage import log_exceptions_and_usage + +logger = logging.getLogger(__name__) + +# Number of mutations per Bigtable write operation we're aiming for. The official max is +# 100K; we're being conservative. +MUTATIONS_PER_OP = 50_000 +# The Bigtable client library limits the connection pool size to 10. This imposes a +# limitation to the concurrency we can get using a thread pool in each worker. +BIGTABLE_CLIENT_CONNECTION_POOL_SIZE = 10 + + +class BigtableOnlineStoreConfig(FeastConfigBaseModel): + """Online store config for GCP Bigtable""" + + type: Literal["bigtable"] = "bigtable" + """Online store typee selector""" + + project_id: Optional[StrictStr] = None + """(optional) GCP Project ID""" + + instance: StrictStr + """The Bigtable instance's ID""" + + max_versions: int = 2 + """The number of historical versions of data that will be kept around.""" + + +class BigtableOnlineStore(OnlineStore): + _client: Optional[bigtable.Client] = None + + feature_column_family: str = "features" + + @log_exceptions_and_usage(online_store="bigtable") + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + # Potential performance improvement opportunity described in + # https://github.com/feast-dev/feast/issues/3259 + feature_view = table + bt_table_name = self._get_table_name(config=config, feature_view=feature_view) + + client = self._get_client(online_config=config.online_store) + bt_instance = client.instance(instance_id=config.online_store.instance) + bt_table = bt_instance.table(bt_table_name) + row_keys = [ + self._compute_row_key( + entity_key=entity_key, + feature_view_name=feature_view.name, + config=config, + ) + for entity_key in entity_keys + ] + + row_set = bigtable.row_set.RowSet() + for row_key in row_keys: + row_set.add_row_key(row_key) + rows = bt_table.read_rows( + row_set=row_set, + filter_=( + row_filters.ColumnQualifierRegexFilter( + f"^({'|'.join(requested_features)}|event_ts)$".encode() + ) + if requested_features + else None + ), + ) + + # The BigTable client library only returns rows for keys that are found. This + # means that it's our responsibility to match the returned rows to the original + # `row_keys` and make sure that we're returning a list of the same length as + # `entity_keys`. + bt_rows_dict: Dict[bytes, bigtable.row.PartialRowData] = { + row.row_key: row for row in rows + } + return [self._process_bt_row(bt_rows_dict.get(row_key)) for row_key in row_keys] + + def _process_bt_row( + self, row: Optional[bigtable.row.PartialRowData] + ) -> Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]: + res = {} + + if row is None: + return (None, None) + + row_values = row.cells[self.feature_column_family] + event_ts = datetime.fromisoformat(row_values.pop(b"event_ts")[0].value.decode()) + for feature_name, feature_values in row_values.items(): + # We only want to retrieve the latest value for each feature + feature_value = feature_values[0] + val = ValueProto() + val.ParseFromString(feature_value.value) + res[feature_name.decode()] = val + + return (event_ts, res) + + @log_exceptions_and_usage(online_store="bigtable") + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + feature_view = table + bt_table_name = self._get_table_name(config=config, feature_view=feature_view) + + client = self._get_client(online_config=config.online_store) + bt_instance = client.instance(instance_id=config.online_store.instance) + bt_table = bt_instance.table(bt_table_name) + + # `columns_per_row` is used to calculate the number of rows we are allowed to + # mutate in one request. + columns_per_row = len(feature_view.features) + 1 # extra for event timestamp + rows_per_write = MUTATIONS_PER_OP // columns_per_row + + with futures.ThreadPoolExecutor( + max_workers=BIGTABLE_CLIENT_CONNECTION_POOL_SIZE + ) as executor: + fs = [] + while data: + rows_to_write, data = data[:rows_per_write], data[rows_per_write:] + fs.append( + executor.submit( + self._write_rows_to_bt, + rows_to_write=rows_to_write, + bt_table=bt_table, + feature_view_name=feature_view.name, + config=config, + progress=progress, + ) + ) + done_tasks, not_done_tasks = futures.wait(fs) + for task in done_tasks: + # If a task raised an exception, this will raise it here as well + task.result() + if not_done_tasks: + raise RuntimeError( + f"Not all batches were written to Bigtable: {not_done_tasks}" + ) + + def _write_rows_to_bt( + self, + rows_to_write: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + bt_table: bigtable.table.Table, + feature_view_name: str, + config: RepoConfig, + progress: Optional[Callable[[int], Any]], + ): + rows = [] + for entity_key, features, timestamp, created_ts in rows_to_write: + bt_row = bt_table.direct_row( + self._compute_row_key( + entity_key=entity_key, + feature_view_name=feature_view_name, + config=config, + ) + ) + + for feature_name, feature_value in features.items(): + bt_row.set_cell( + self.feature_column_family, + feature_name.encode(), + feature_value.SerializeToString(), + ) + bt_row.set_cell( + self.feature_column_family, + b"event_ts", + utils.make_tzaware(timestamp).isoformat().encode(), + ) + rows.append(bt_row) + bt_table.mutate_rows(rows) + + if progress: + progress(len(rows)) + + def _compute_row_key( + self, entity_key: EntityKeyProto, feature_view_name: str, config: RepoConfig + ) -> bytes: + entity_id = compute_entity_id( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + # Even though `entity_id` uniquely identifies an entity, we use the same table + # for multiple feature_views with the same set of entities. To uniquely identify + # the row for a feature_view, we suffix the name of the feature_view itself. + # This also ensures that features for entities from various feature_views are + # colocated. + return f"{entity_id}#{feature_view_name}".encode() + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + """Creates the appropriate tables and column family in Bigtable. + + We use a dedicated table for each entity combination. For example, if a + FeatureView uses the entities `shop` and `customer`, the resulting table would + be called `customer-shop` (entities are sorted lexicographically first). + """ + online_config = config.online_store + assert isinstance(online_config, BigtableOnlineStoreConfig) + client = self._get_client(online_config, admin=True) + bt_instance = client.instance(instance_id=online_config.instance) + max_versions_gc_rule = bigtable.column_family.MaxVersionsGCRule( + online_config.max_versions + ) + # The word "table" in the arguments refers to feature_views (this is for legacy + # reasons). To reduce confusion with bigtable tables, we use alternate variable + # names + feature_views_to_keep = tables_to_keep + feature_views_to_delete = tables_to_delete + # Multiple feature views can share the same tables. So just because a feature + # view has been deleted does not mean that we can just delete the table. We map + # feature views to Bigtable table names and figure out which ones to create + # and/or delete. + bt_tables_to_keep: Set[str] = { + self._get_table_name(config=config, feature_view=feature_view) + for feature_view in feature_views_to_keep + } + bt_tables_to_delete: Set[str] = { + self._get_table_name(config=config, feature_view=feature_view) + for feature_view in feature_views_to_delete + } - bt_tables_to_keep # we don't delete a table if it's in `bt_tables_to_keep` + + for bt_table_name in bt_tables_to_keep: + bt_table = bt_instance.table(bt_table_name) + if not bt_table.exists(): + logger.info(f"Creating table `{bt_table_name}` in Bigtable") + bt_table.create() + else: + logger.info(f"Table {bt_table_name} already exists in Bigtable") + + if self.feature_column_family not in bt_table.list_column_families(): + bt_table.column_family( + self.feature_column_family, gc_rule=max_versions_gc_rule + ).create() + + for bt_table_name in bt_tables_to_delete: + bt_table = bt_instance.table(bt_table_name) + logger.info(f"Deleting table {bt_table_name} in Bigtable") + bt_table.delete() + + @staticmethod + def _get_table_name(config: RepoConfig, feature_view: FeatureView) -> str: + entities_part = ( + "-".join(sorted(feature_view.entities)) + if feature_view.entities + else DUMMY_ENTITY_NAME + ) + BIGTABLE_TABLE_MAX_LENGTH = 50 + ENTITIES_PART_MAX_LENGTH = 24 + # Bigtable limits table names to 50 characters. We'll limit the max size of of + # the `entities_part` and if that's not enough, we'll just hash the + # entities_part. The remaining length is dedicated to the project name. This + # allows multiple projects to coexist in the same bigtable instance. This also + # allows multiple integration test executions to run simultaneously without + # conflicts. + if len(entities_part) > ENTITIES_PART_MAX_LENGTH: + entities_part = hashlib.md5(entities_part.encode()).hexdigest()[ + :ENTITIES_PART_MAX_LENGTH + ] + remaining_length = BIGTABLE_TABLE_MAX_LENGTH - len(entities_part) - 1 + if len(config.project) > remaining_length: + HUMAN_READABLE_PART_LENGTH = 10 + HASH_PART_LENGTH = remaining_length - HUMAN_READABLE_PART_LENGTH - 1 + project_part = ( + config.project[:HUMAN_READABLE_PART_LENGTH] + + "_" + + hashlib.md5(config.project.encode()).hexdigest()[:HASH_PART_LENGTH] + ) + else: + project_part = config.project + return f"{project_part}.{entities_part}" + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ): + # Because of historical reasons, Feast calls them tables. We use this alias for + # readability. + feature_views = tables + + bt_tables = { + self._get_table_name(config=config, feature_view=fv) for fv in feature_views + } + + online_config = config.online_store + assert isinstance(online_config, BigtableOnlineStoreConfig) + client = self._get_client(online_config, admin=True) + bt_instance = client.instance(instance_id=online_config.instance) + for table_name in bt_tables: + try: + logger.info(f"Deleting Bigtable table `{table_name}`") + bt_instance.table(table_name).delete() + except google.api_core.exceptions.NotFound: + logger.warning( + f"Table `{table_name}` was not found. Skipping deletion." + ) + + def _get_client( + self, online_config: BigtableOnlineStoreConfig, admin: bool = False + ): + return bigtable.Client(project=online_config.project_id, admin=admin) diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index c11b25849e3..673d039ff0b 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -48,6 +48,7 @@ "redis": "feast.infra.online_stores.redis.RedisOnlineStore", "dynamodb": "feast.infra.online_stores.dynamodb.DynamoDBOnlineStore", "snowflake.online": "feast.infra.online_stores.snowflake.SnowflakeOnlineStore", + "bigtable": "feast.infra.online_stores.bigtable.BigtableOnlineStore", "postgres": "feast.infra.online_stores.contrib.postgres.PostgreSQLOnlineStore", "hbase": "feast.infra.online_stores.contrib.hbase_online_store.hbase.HbaseOnlineStore", "cassandra": "feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store.CassandraOnlineStore", diff --git a/sdk/python/feast/templates/gcp/feature_repo/feature_store.yaml b/sdk/python/feast/templates/gcp/feature_repo/feature_store.yaml index 7d4096615ac..e3d9d1d1e61 100644 --- a/sdk/python/feast/templates/gcp/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/gcp/feature_repo/feature_store.yaml @@ -13,6 +13,11 @@ online_store: # type: datastore # project_id: my_gcp_project # namespace: my_datastore_namespace +# See https://docs.feast.dev/reference/online-stores/bigtable +#online_store: +# type: bigtable +# project_id: my_gcp_project +# instance: my_bigtable_instance # See https://docs.feast.dev/reference/online-stores/redis #online_store: # type: redis diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index c1656ee7e6f..93729efc518 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt # -absl-py==1.2.0 - # via tensorflow-metadata adal==1.2.7 # via # azure-datalake-store @@ -14,13 +12,13 @@ adlfs==0.5.9 # via feast (setup.py) aiobotocore==2.1.2 # via s3fs -aiohttp==3.8.1 +aiohttp==3.8.3 # via # adlfs # aiobotocore # gcsfs # s3fs -aioitertools==0.10.0 +aioitertools==0.11.0 # via aiobotocore aiosignal==1.2.0 # via aiohttp @@ -56,7 +54,7 @@ attrs==22.1.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.25.0 +azure-core==1.25.1 # via # adlfs # azure-identity @@ -64,7 +62,7 @@ azure-core==1.25.0 # msrest azure-datalake-store==0.0.52 # via adlfs -azure-identity==1.10.0 +azure-identity==1.11.0 # via # adlfs # feast (setup.py) @@ -76,7 +74,7 @@ babel==2.10.3 # via sphinx backcall==0.2.0 # via ipython -black==22.6.0 +black==22.8.0 # via feast (setup.py) boto3==1.20.23 # via @@ -102,7 +100,7 @@ cachetools==5.2.0 # via google-auth cassandra-driver==3.25.0 # via feast (setup.py) -certifi==2022.6.15 +certifi==2022.9.24 # via # kubernetes # minio @@ -116,7 +114,7 @@ cffi==1.15.1 # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via # aiohttp # requests @@ -131,13 +129,13 @@ click==8.1.3 # moreorless # pip-tools # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via # feast (setup.py) # great-expectations -coverage[toml]==6.4.4 +coverage[toml]==6.5.0 # via pytest-cov cryptography==35.0.0 # via @@ -155,7 +153,7 @@ dask==2022.1.1 # via feast (setup.py) dataclasses==0.6 # via great-expectations -db-dtypes==1.0.3 +db-dtypes==1.0.4 # via google-cloud-bigquery decorator==5.1.1 # via @@ -169,7 +167,7 @@ dill==0.3.5.1 # via # feast (setup.py) # multiprocess -distlib==0.3.5 +distlib==0.3.6 # via virtualenv docker==6.0.0 # via @@ -183,19 +181,21 @@ entrypoints==0.4 # via altair execnet==1.9.0 # via pytest-xdist -executing==0.10.0 +executing==1.1.0 # via stack-data -fastapi==0.79.1 +fastapi==0.85.0 # via feast (setup.py) -fastavro==1.6.0 +fastavro==1.6.1 # via # feast (setup.py) # pandavro -fastjsonschema==2.16.1 +fastjsonschema==2.16.2 # via nbformat filelock==3.8.0 - # via virtualenv -firebase-admin==5.2.0 + # via + # snowflake-connector-python + # virtualenv +firebase-admin==5.4.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -215,20 +215,21 @@ gcsfs==2022.1.0 # via feast (setup.py) geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.8.2 +google-api-core[grpc]==2.10.1 # via # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-core # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.57.0 +google-api-python-client==2.63.0 # via firebase-admin -google-auth==2.10.0 +google-auth==2.12.0 # via # gcsfs # google-api-core @@ -240,63 +241,68 @@ google-auth==2.10.0 # kubernetes google-auth-httplib2==0.1.0 # via google-api-python-client -google-auth-oauthlib==0.5.2 +google-auth-oauthlib==0.5.3 # via gcsfs -google-cloud-bigquery[pandas]==3.3.2 +google-cloud-bigquery[pandas]==3.3.3 # via feast (setup.py) -google-cloud-bigquery-storage==2.14.2 +google-cloud-bigquery-storage==2.16.1 # via # feast (setup.py) # google-cloud-bigquery +google-cloud-bigtable==2.12.0 + # via feast (setup.py) google-cloud-core==2.3.2 # via # google-cloud-bigquery + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.8.1 # via feast (setup.py) -google-cloud-firestore==2.6.1 +google-cloud-firestore==2.7.0 # via firebase-admin google-cloud-storage==2.5.0 # via # feast (setup.py) # firebase-admin # gcsfs -google-crc32c==1.3.0 +google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.3.3 +google-resumable-media==2.4.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos==1.56.4 +googleapis-common-protos[grpc]==1.56.4 # via # feast (setup.py) # google-api-core + # grpc-google-iam-v1 # grpcio-status - # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.47.0 +grpc-google-iam-v1==0.12.4 + # via google-cloud-bigtable +grpcio==1.49.1 # via # feast (setup.py) # google-api-core # google-cloud-bigquery + # googleapis-common-protos + # grpc-google-iam-v1 # grpcio-reflection # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.47.0 +grpcio-reflection==1.49.1 # via feast (setup.py) -grpcio-status==1.47.0 +grpcio-status==1.49.1 # via google-api-core -grpcio-testing==1.47.0 +grpcio-testing==1.49.1 # via feast (setup.py) -grpcio-tools==1.47.0 +grpcio-tools==1.49.1 # via feast (setup.py) -h11==0.13.0 +h11==0.14.0 # via uvicorn happybase==1.2.0 # via feast (setup.py) @@ -306,11 +312,11 @@ httplib2==0.20.4 # via # google-api-python-client # google-auth-httplib2 -httptools==0.4.0 +httptools==0.5.0 # via uvicorn -identify==2.5.3 +identify==2.5.5 # via pre-commit -idna==3.3 +idna==3.4 # via # anyio # requests @@ -322,7 +328,7 @@ importlib-metadata==4.12.0 # via great-expectations iniconfig==1.1.1 # via pytest -ipython==8.4.0 +ipython==8.5.0 # via great-expectations isodate==0.6.1 # via msrest @@ -345,7 +351,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.13.0 +jsonschema==4.16.0 # via # altair # feast (setup.py) @@ -377,7 +383,7 @@ moreorless==0.4.0 # via bowler moto==3.1.18 # via feast (setup.py) -msal==1.18.0 +msal==1.19.0 # via # azure-identity # msal-extensions @@ -397,7 +403,7 @@ multidict==6.0.2 # yarl multiprocess==0.70.13 # via bytewax -mypy==0.971 +mypy==0.981 # via # feast (setup.py) # sqlalchemy @@ -409,11 +415,11 @@ mypy-protobuf==3.1 # via feast (setup.py) mysqlclient==2.1.1 # via feast (setup.py) -nbformat==5.4.0 +nbformat==5.6.1 # via great-expectations nodeenv==1.7.0 # via pre-commit -numpy==1.23.2 +numpy==1.23.3 # via # altair # db-dtypes @@ -439,7 +445,7 @@ packaging==21.3 # pytest # redis # sphinx -pandas==1.4.3 +pandas==1.4.4 # via # altair # db-dtypes @@ -454,7 +460,7 @@ parso==0.8.3 # via jedi partd==1.3.0 # via dask -pathspec==0.9.0 +pathspec==0.10.1 # via black pbr==5.10.0 # via mock @@ -478,21 +484,23 @@ portalocker==2.5.1 # via msal-extensions pre-commit==2.20.0 # via feast (setup.py) -prompt-toolkit==3.0.30 +prompt-toolkit==3.0.31 # via ipython -proto-plus==1.22.0 +proto-plus==1.22.1 # via # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==3.20.2 +protobuf==4.21.7 # via # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # googleapis-common-protos @@ -502,7 +510,6 @@ protobuf==3.20.2 # grpcio-tools # mypy-protobuf # proto-plus - # tensorflow-metadata psutil==5.9.0 # via feast (setup.py) psycopg2-binary==2.9.3 @@ -519,7 +526,7 @@ py-cpuinfo==8.0.0 # via pytest-benchmark py4j==0.10.9.5 # via pyspark -pyarrow==6.0.1 +pyarrow==8.0.0 # via # db-dtypes # feast (setup.py) @@ -539,7 +546,7 @@ pycparser==2.21 # via cffi pycryptodomex==3.15.0 # via snowflake-connector-python -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -550,7 +557,7 @@ pygments==2.13.0 # feast (setup.py) # ipython # sphinx -pyjwt[crypto]==2.4.0 +pyjwt[crypto]==2.5.0 # via # adal # msal @@ -562,7 +569,9 @@ pymysql==1.0.2 pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 - # via snowflake-connector-python + # via + # feast (setup.py) + # snowflake-connector-python pyparsing==2.4.7 # via # great-expectations @@ -572,7 +581,7 @@ pyrsistent==0.18.1 # via jsonschema pyspark==3.3.0 # via feast (setup.py) -pytest==7.1.2 +pytest==7.1.3 # via # feast (setup.py) # pytest-benchmark @@ -585,7 +594,7 @@ pytest==7.1.2 # pytest-xdist pytest-benchmark==3.4.1 # via feast (setup.py) -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via feast (setup.py) pytest-forked==1.4.0 # via pytest-xdist @@ -608,7 +617,7 @@ python-dateutil==2.8.2 # kubernetes # moto # pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via @@ -666,7 +675,7 @@ s3fs==2022.1.0 # via feast (setup.py) s3transfer==0.5.2 # via boto3 -scipy==1.9.0 +scipy==1.9.1 # via great-expectations six==1.16.0 # via @@ -684,11 +693,11 @@ six==1.16.0 # msrestazure # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==2.7.8 +snowflake-connector-python[pandas]==2.8.0 # via feast (setup.py) sphinx==4.3.2 # via @@ -708,23 +717,21 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -stack-data==0.4.0 +stack-data==0.5.1 # via ipython -starlette==0.19.1 +starlette==0.20.4 # via fastapi tabulate==0.8.10 # via feast (setup.py) -tenacity==8.0.1 +tenacity==8.1.0 # via feast (setup.py) -tensorflow-metadata==1.9.0 - # via feast (setup.py) -termcolor==1.1.0 +termcolor==2.0.1 # via great-expectations -testcontainers==3.6.1 +testcontainers==3.7.0 # via feast (setup.py) thriftpy2==0.4.14 # via happybase @@ -745,21 +752,23 @@ toolz==0.12.0 # altair # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via # feast (setup.py) # great-expectations -traitlets==5.3.0 +traitlets==5.4.0 # via # ipython # jupyter-core # matplotlib-inline # nbformat -trino==0.315.0 +trino==0.316.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==3.19.22 +types-cryptography==3.3.23 + # via pyjwt +types-protobuf==3.20.4 # via # feast (setup.py) # mypy-protobuf @@ -769,17 +778,17 @@ types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 # via feast (setup.py) -types-pyyaml==6.0.11 +types-pyyaml==6.0.12 # via feast (setup.py) -types-redis==4.3.14 +types-redis==4.3.21 # via feast (setup.py) -types-requests==2.28.9 +types-requests==2.28.11 # via feast (setup.py) -types-setuptools==65.1.0 +types-setuptools==65.4.0.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.23 +types-urllib3==1.26.25 # via types-requests typing-extensions==4.3.0 # via @@ -787,14 +796,15 @@ typing-extensions==4.3.0 # great-expectations # mypy # pydantic + # snowflake-connector-python # sqlalchemy2-stubs -tzdata==2022.2 +tzdata==2022.4 # via pytz-deprecation-shim tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.11 +urllib3==1.26.12 # via # botocore # docker @@ -804,19 +814,20 @@ urllib3==1.26.11 # minio # requests # responses -uvicorn[standard]==0.18.2 + # snowflake-connector-python +uvicorn[standard]==0.18.3 # via feast (setup.py) -uvloop==0.16.0 +uvloop==0.17.0 # via uvicorn -virtualenv==20.16.3 +virtualenv==20.16.5 # via pre-commit volatile==2.1.0 # via bowler -watchfiles==0.16.1 +watchfiles==0.17.0 # via uvicorn wcwidth==0.2.5 # via prompt-toolkit -websocket-client==1.3.3 +websocket-client==1.4.1 # via # docker # kubernetes diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 91369309175..4770b765e50 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt # -absl-py==1.2.0 - # via tensorflow-metadata anyio==3.6.1 # via # starlette @@ -20,9 +18,9 @@ bowler==0.9.0 # via feast (setup.py) cachetools==5.2.0 # via google-auth -certifi==2022.6.15 +certifi==2022.9.24 # via requests -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via requests click==8.1.3 # via @@ -30,7 +28,7 @@ click==8.1.3 # feast (setup.py) # moreorless # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via feast (setup.py) @@ -38,44 +36,41 @@ dask==2022.1.1 # via feast (setup.py) dill==0.3.5.1 # via feast (setup.py) -fastapi==0.79.1 +fastapi==0.85.0 # via feast (setup.py) -fastavro==1.6.0 +fastavro==1.6.1 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2022.7.1 +fsspec==2022.8.2 # via dask -google-api-core==2.8.2 +google-api-core==2.10.1 # via feast (setup.py) -google-auth==2.10.0 +google-auth==2.12.0 # via google-api-core googleapis-common-protos==1.56.4 # via # feast (setup.py) # google-api-core - # tensorflow-metadata -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.47.0 +grpcio==1.49.1 # via # feast (setup.py) # grpcio-reflection -grpcio-reflection==1.47.0 +grpcio-reflection==1.49.1 # via feast (setup.py) -h11==0.13.0 +h11==0.14.0 # via uvicorn -httptools==0.4.0 +httptools==0.5.0 # via uvicorn -idna==3.3 +idna==3.4 # via # anyio # requests jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.13.0 +jsonschema==4.16.0 # via feast (setup.py) locket==1.0.0 # via partd @@ -85,11 +80,11 @@ mmh3==3.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==0.971 +mypy==0.981 # via sqlalchemy mypy-extensions==0.4.3 # via mypy -numpy==1.23.2 +numpy==1.23.3 # via # feast (setup.py) # pandas @@ -97,7 +92,7 @@ numpy==1.23.2 # pyarrow packaging==21.3 # via dask -pandas==1.4.3 +pandas==1.5.0 # via # feast (setup.py) # pandavro @@ -105,16 +100,15 @@ pandavro==1.5.2 # via feast (setup.py) partd==1.3.0 # via dask -proto-plus==1.22.0 +proto-plus==1.22.1 # via feast (setup.py) -protobuf==3.20.2 +protobuf==4.21.7 # via # feast (setup.py) # google-api-core # googleapis-common-protos # grpcio-reflection # proto-plus - # tensorflow-metadata pyarrow==8.0.0 # via feast (setup.py) pyasn1==0.4.8 @@ -123,7 +117,7 @@ pyasn1==0.4.8 # rsa pyasn1-modules==0.2.8 # via google-auth -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -135,7 +129,7 @@ pyrsistent==0.18.1 # via jsonschema python-dateutil==2.8.2 # via pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via pandas @@ -154,19 +148,17 @@ six==1.16.0 # grpcio # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -starlette==0.19.1 +starlette==0.20.4 # via fastapi tabulate==0.8.10 # via feast (setup.py) -tenacity==8.0.1 - # via feast (setup.py) -tensorflow-metadata==1.9.0 +tenacity==8.1.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) @@ -176,7 +168,7 @@ toolz==0.12.0 # via # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) @@ -185,15 +177,15 @@ typing-extensions==4.3.0 # mypy # pydantic # sqlalchemy2-stubs -urllib3==1.26.11 +urllib3==1.26.12 # via requests -uvicorn[standard]==0.18.2 +uvicorn[standard]==0.18.3 # via feast (setup.py) -uvloop==0.16.0 +uvloop==0.17.0 # via uvicorn volatile==2.1.0 # via bowler -watchfiles==0.16.1 +watchfiles==0.17.0 # via uvicorn websockets==10.3 # via uvicorn diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 84a8c7408e1..e1b99210aa5 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.8-ci-requirements.txt # -absl-py==1.2.0 - # via tensorflow-metadata adal==1.2.7 # via # azure-datalake-store @@ -14,13 +12,13 @@ adlfs==0.5.9 # via feast (setup.py) aiobotocore==2.1.2 # via s3fs -aiohttp==3.8.1 +aiohttp==3.8.3 # via # adlfs # aiobotocore # gcsfs # s3fs -aioitertools==0.10.0 +aioitertools==0.11.0 # via aiobotocore aiosignal==1.2.0 # via aiohttp @@ -56,7 +54,7 @@ attrs==22.1.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.25.0 +azure-core==1.25.1 # via # adlfs # azure-identity @@ -64,7 +62,7 @@ azure-core==1.25.0 # msrest azure-datalake-store==0.0.52 # via adlfs -azure-identity==1.10.0 +azure-identity==1.11.0 # via # adlfs # feast (setup.py) @@ -80,7 +78,7 @@ backports-zoneinfo==0.2.1 # via # pytz-deprecation-shim # tzlocal -black==22.6.0 +black==22.8.0 # via feast (setup.py) boto3==1.20.23 # via @@ -106,7 +104,7 @@ cachetools==5.2.0 # via google-auth cassandra-driver==3.25.0 # via feast (setup.py) -certifi==2022.6.15 +certifi==2022.9.24 # via # kubernetes # minio @@ -120,7 +118,7 @@ cffi==1.15.1 # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via # aiohttp # requests @@ -135,13 +133,13 @@ click==8.1.3 # moreorless # pip-tools # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via # feast (setup.py) # great-expectations -coverage[toml]==6.4.4 +coverage[toml]==6.5.0 # via pytest-cov cryptography==35.0.0 # via @@ -159,7 +157,7 @@ dask==2022.1.1 # via feast (setup.py) dataclasses==0.6 # via great-expectations -db-dtypes==1.0.3 +db-dtypes==1.0.4 # via google-cloud-bigquery decorator==5.1.1 # via @@ -173,7 +171,7 @@ dill==0.3.5.1 # via # feast (setup.py) # multiprocess -distlib==0.3.5 +distlib==0.3.6 # via virtualenv docker==6.0.0 # via @@ -187,19 +185,21 @@ entrypoints==0.4 # via altair execnet==1.9.0 # via pytest-xdist -executing==0.10.0 +executing==1.1.0 # via stack-data -fastapi==0.79.1 +fastapi==0.85.0 # via feast (setup.py) -fastavro==1.6.0 +fastavro==1.6.1 # via # feast (setup.py) # pandavro -fastjsonschema==2.16.1 +fastjsonschema==2.16.2 # via nbformat filelock==3.8.0 - # via virtualenv -firebase-admin==5.2.0 + # via + # snowflake-connector-python + # virtualenv +firebase-admin==5.4.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -219,20 +219,21 @@ gcsfs==2022.1.0 # via feast (setup.py) geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.8.2 +google-api-core[grpc]==2.10.1 # via # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-core # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.57.0 +google-api-python-client==2.63.0 # via firebase-admin -google-auth==2.10.0 +google-auth==2.12.0 # via # gcsfs # google-api-core @@ -244,63 +245,68 @@ google-auth==2.10.0 # kubernetes google-auth-httplib2==0.1.0 # via google-api-python-client -google-auth-oauthlib==0.5.2 +google-auth-oauthlib==0.5.3 # via gcsfs -google-cloud-bigquery[pandas]==3.3.2 +google-cloud-bigquery[pandas]==3.3.3 # via feast (setup.py) -google-cloud-bigquery-storage==2.14.2 +google-cloud-bigquery-storage==2.16.1 # via # feast (setup.py) # google-cloud-bigquery +google-cloud-bigtable==2.12.0 + # via feast (setup.py) google-cloud-core==2.3.2 # via # google-cloud-bigquery + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.8.1 # via feast (setup.py) -google-cloud-firestore==2.6.1 +google-cloud-firestore==2.7.0 # via firebase-admin google-cloud-storage==2.5.0 # via # feast (setup.py) # firebase-admin # gcsfs -google-crc32c==1.3.0 +google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.3.3 +google-resumable-media==2.4.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos==1.56.4 +googleapis-common-protos[grpc]==1.56.4 # via # feast (setup.py) # google-api-core + # grpc-google-iam-v1 # grpcio-status - # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.47.0 +grpc-google-iam-v1==0.12.4 + # via google-cloud-bigtable +grpcio==1.49.1 # via # feast (setup.py) # google-api-core # google-cloud-bigquery + # googleapis-common-protos + # grpc-google-iam-v1 # grpcio-reflection # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.47.0 +grpcio-reflection==1.49.1 # via feast (setup.py) -grpcio-status==1.47.0 +grpcio-status==1.49.1 # via google-api-core -grpcio-testing==1.47.0 +grpcio-testing==1.49.1 # via feast (setup.py) -grpcio-tools==1.47.0 +grpcio-tools==1.49.1 # via feast (setup.py) -h11==0.13.0 +h11==0.14.0 # via uvicorn happybase==1.2.0 # via feast (setup.py) @@ -310,11 +316,11 @@ httplib2==0.20.4 # via # google-api-python-client # google-auth-httplib2 -httptools==0.4.0 +httptools==0.5.0 # via uvicorn -identify==2.5.3 +identify==2.5.5 # via pre-commit -idna==3.3 +idna==3.4 # via # anyio # requests @@ -328,7 +334,7 @@ importlib-resources==5.9.0 # via jsonschema iniconfig==1.1.1 # via pytest -ipython==8.4.0 +ipython==8.5.0 # via great-expectations isodate==0.6.1 # via msrest @@ -351,7 +357,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.13.0 +jsonschema==4.16.0 # via # altair # feast (setup.py) @@ -383,7 +389,7 @@ moreorless==0.4.0 # via bowler moto==3.1.18 # via feast (setup.py) -msal==1.18.0 +msal==1.19.0 # via # azure-identity # msal-extensions @@ -403,7 +409,7 @@ multidict==6.0.2 # yarl multiprocess==0.70.13 # via bytewax -mypy==0.971 +mypy==0.981 # via # feast (setup.py) # sqlalchemy @@ -415,11 +421,11 @@ mypy-protobuf==3.1 # via feast (setup.py) mysqlclient==2.1.1 # via feast (setup.py) -nbformat==5.4.0 +nbformat==5.6.1 # via great-expectations nodeenv==1.7.0 # via pre-commit -numpy==1.23.2 +numpy==1.23.3 # via # altair # db-dtypes @@ -445,7 +451,7 @@ packaging==21.3 # pytest # redis # sphinx -pandas==1.4.3 +pandas==1.4.4 # via # altair # db-dtypes @@ -460,7 +466,7 @@ parso==0.8.3 # via jedi partd==1.3.0 # via dask -pathspec==0.9.0 +pathspec==0.10.1 # via black pbr==5.10.0 # via mock @@ -486,21 +492,23 @@ portalocker==2.5.1 # via msal-extensions pre-commit==2.20.0 # via feast (setup.py) -prompt-toolkit==3.0.30 +prompt-toolkit==3.0.31 # via ipython -proto-plus==1.22.0 +proto-plus==1.22.1 # via # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==3.20.2 +protobuf==4.21.7 # via # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # googleapis-common-protos @@ -510,7 +518,6 @@ protobuf==3.20.2 # grpcio-tools # mypy-protobuf # proto-plus - # tensorflow-metadata psutil==5.9.0 # via feast (setup.py) psycopg2-binary==2.9.3 @@ -527,7 +534,7 @@ py-cpuinfo==8.0.0 # via pytest-benchmark py4j==0.10.9.5 # via pyspark -pyarrow==6.0.1 +pyarrow==8.0.0 # via # db-dtypes # feast (setup.py) @@ -547,7 +554,7 @@ pycparser==2.21 # via cffi pycryptodomex==3.15.0 # via snowflake-connector-python -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -558,7 +565,7 @@ pygments==2.13.0 # feast (setup.py) # ipython # sphinx -pyjwt[crypto]==2.4.0 +pyjwt[crypto]==2.5.0 # via # adal # msal @@ -570,7 +577,9 @@ pymysql==1.0.2 pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 - # via snowflake-connector-python + # via + # feast (setup.py) + # snowflake-connector-python pyparsing==2.4.7 # via # great-expectations @@ -580,7 +589,7 @@ pyrsistent==0.18.1 # via jsonschema pyspark==3.3.0 # via feast (setup.py) -pytest==7.1.2 +pytest==7.1.3 # via # feast (setup.py) # pytest-benchmark @@ -593,7 +602,7 @@ pytest==7.1.2 # pytest-xdist pytest-benchmark==3.4.1 # via feast (setup.py) -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via feast (setup.py) pytest-forked==1.4.0 # via pytest-xdist @@ -616,7 +625,7 @@ python-dateutil==2.8.2 # kubernetes # moto # pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via @@ -676,7 +685,7 @@ s3fs==2022.1.0 # via feast (setup.py) s3transfer==0.5.2 # via boto3 -scipy==1.9.0 +scipy==1.9.1 # via great-expectations six==1.16.0 # via @@ -694,11 +703,11 @@ six==1.16.0 # msrestazure # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==2.7.8 +snowflake-connector-python[pandas]==2.8.0 # via feast (setup.py) sphinx==4.3.2 # via @@ -718,23 +727,21 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -stack-data==0.4.0 +stack-data==0.5.1 # via ipython -starlette==0.19.1 +starlette==0.20.4 # via fastapi tabulate==0.8.10 # via feast (setup.py) -tenacity==8.0.1 +tenacity==8.1.0 # via feast (setup.py) -tensorflow-metadata==1.9.0 - # via feast (setup.py) -termcolor==1.1.0 +termcolor==2.0.1 # via great-expectations -testcontainers==3.6.1 +testcontainers==3.7.0 # via feast (setup.py) thriftpy2==0.4.14 # via happybase @@ -755,21 +762,23 @@ toolz==0.12.0 # altair # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via # feast (setup.py) # great-expectations -traitlets==5.3.0 +traitlets==5.4.0 # via # ipython # jupyter-core # matplotlib-inline # nbformat -trino==0.315.0 +trino==0.316.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==3.19.22 +types-cryptography==3.3.23 + # via pyjwt +types-protobuf==3.20.4 # via # feast (setup.py) # mypy-protobuf @@ -779,17 +788,17 @@ types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 # via feast (setup.py) -types-pyyaml==6.0.11 +types-pyyaml==6.0.12 # via feast (setup.py) -types-redis==4.3.14 +types-redis==4.3.21 # via feast (setup.py) -types-requests==2.28.9 +types-requests==2.28.11 # via feast (setup.py) -types-setuptools==65.1.0 +types-setuptools==65.4.0.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.23 +types-urllib3==1.26.25 # via types-requests typing-extensions==4.3.0 # via @@ -799,15 +808,16 @@ typing-extensions==4.3.0 # great-expectations # mypy # pydantic + # snowflake-connector-python # sqlalchemy2-stubs # starlette -tzdata==2022.2 +tzdata==2022.4 # via pytz-deprecation-shim tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.11 +urllib3==1.26.12 # via # botocore # docker @@ -817,19 +827,20 @@ urllib3==1.26.11 # minio # requests # responses -uvicorn[standard]==0.18.2 + # snowflake-connector-python +uvicorn[standard]==0.18.3 # via feast (setup.py) -uvloop==0.16.0 +uvloop==0.17.0 # via uvicorn -virtualenv==20.16.3 +virtualenv==20.16.5 # via pre-commit volatile==2.1.0 # via bowler -watchfiles==0.16.1 +watchfiles==0.17.0 # via uvicorn wcwidth==0.2.5 # via prompt-toolkit -websocket-client==1.3.3 +websocket-client==1.4.1 # via # docker # kubernetes diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index b992ad3fc0a..9a321bf9a50 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --output-file=sdk/python/requirements/py3.8-requirements.txt # -absl-py==1.2.0 - # via tensorflow-metadata anyio==3.6.1 # via # starlette @@ -20,9 +18,9 @@ bowler==0.9.0 # via feast (setup.py) cachetools==5.2.0 # via google-auth -certifi==2022.6.15 +certifi==2022.9.24 # via requests -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via requests click==8.1.3 # via @@ -30,7 +28,7 @@ click==8.1.3 # feast (setup.py) # moreorless # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via feast (setup.py) @@ -38,38 +36,35 @@ dask==2022.1.1 # via feast (setup.py) dill==0.3.5.1 # via feast (setup.py) -fastapi==0.79.1 +fastapi==0.85.0 # via feast (setup.py) -fastavro==1.6.0 +fastavro==1.6.1 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2022.7.1 +fsspec==2022.8.2 # via dask -google-api-core==2.8.2 +google-api-core==2.10.1 # via feast (setup.py) -google-auth==2.10.0 +google-auth==2.12.0 # via google-api-core googleapis-common-protos==1.56.4 # via # feast (setup.py) # google-api-core - # tensorflow-metadata -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.47.0 +grpcio==1.49.1 # via # feast (setup.py) # grpcio-reflection -grpcio-reflection==1.47.0 +grpcio-reflection==1.49.1 # via feast (setup.py) -h11==0.13.0 +h11==0.14.0 # via uvicorn -httptools==0.4.0 +httptools==0.5.0 # via uvicorn -idna==3.3 +idna==3.4 # via # anyio # requests @@ -77,7 +72,7 @@ importlib-resources==5.9.0 # via jsonschema jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.13.0 +jsonschema==4.16.0 # via feast (setup.py) locket==1.0.0 # via partd @@ -87,11 +82,11 @@ mmh3==3.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==0.971 +mypy==0.981 # via sqlalchemy mypy-extensions==0.4.3 # via mypy -numpy==1.23.2 +numpy==1.23.3 # via # feast (setup.py) # pandas @@ -99,7 +94,7 @@ numpy==1.23.2 # pyarrow packaging==21.3 # via dask -pandas==1.4.3 +pandas==1.5.0 # via # feast (setup.py) # pandavro @@ -109,16 +104,15 @@ partd==1.3.0 # via dask pkgutil-resolve-name==1.3.10 # via jsonschema -proto-plus==1.22.0 +proto-plus==1.22.1 # via feast (setup.py) -protobuf==3.20.2 +protobuf==4.21.7 # via # feast (setup.py) # google-api-core # googleapis-common-protos # grpcio-reflection # proto-plus - # tensorflow-metadata pyarrow==8.0.0 # via feast (setup.py) pyasn1==0.4.8 @@ -127,7 +121,7 @@ pyasn1==0.4.8 # rsa pyasn1-modules==0.2.8 # via google-auth -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -139,7 +133,7 @@ pyrsistent==0.18.1 # via jsonschema python-dateutil==2.8.2 # via pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via pandas @@ -158,19 +152,17 @@ six==1.16.0 # grpcio # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -starlette==0.19.1 +starlette==0.20.4 # via fastapi tabulate==0.8.10 # via feast (setup.py) -tenacity==8.0.1 - # via feast (setup.py) -tensorflow-metadata==1.9.0 +tenacity==8.1.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) @@ -180,7 +172,7 @@ toolz==0.12.0 # via # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) @@ -190,15 +182,15 @@ typing-extensions==4.3.0 # pydantic # sqlalchemy2-stubs # starlette -urllib3==1.26.11 +urllib3==1.26.12 # via requests -uvicorn[standard]==0.18.2 +uvicorn[standard]==0.18.3 # via feast (setup.py) -uvloop==0.16.0 +uvloop==0.17.0 # via uvicorn volatile==2.1.0 # via bowler -watchfiles==0.16.1 +watchfiles==0.17.0 # via uvicorn websockets==10.3 # via uvicorn diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 4893035a7e7..af7b79ea229 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt # -absl-py==1.2.0 - # via tensorflow-metadata adal==1.2.7 # via # azure-datalake-store @@ -14,13 +12,13 @@ adlfs==0.5.9 # via feast (setup.py) aiobotocore==2.1.2 # via s3fs -aiohttp==3.8.1 +aiohttp==3.8.3 # via # adlfs # aiobotocore # gcsfs # s3fs -aioitertools==0.10.0 +aioitertools==0.11.0 # via aiobotocore aiosignal==1.2.0 # via aiohttp @@ -56,7 +54,7 @@ attrs==22.1.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.25.0 +azure-core==1.25.1 # via # adlfs # azure-identity @@ -64,7 +62,7 @@ azure-core==1.25.0 # msrest azure-datalake-store==0.0.52 # via adlfs -azure-identity==1.10.0 +azure-identity==1.11.0 # via # adlfs # feast (setup.py) @@ -76,7 +74,7 @@ babel==2.10.3 # via sphinx backcall==0.2.0 # via ipython -black==22.6.0 +black==22.8.0 # via feast (setup.py) boto3==1.20.23 # via @@ -102,7 +100,7 @@ cachetools==5.2.0 # via google-auth cassandra-driver==3.25.0 # via feast (setup.py) -certifi==2022.6.15 +certifi==2022.9.24 # via # kubernetes # minio @@ -116,7 +114,7 @@ cffi==1.15.1 # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via # aiohttp # requests @@ -131,13 +129,13 @@ click==8.1.3 # moreorless # pip-tools # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via # feast (setup.py) # great-expectations -coverage[toml]==6.4.4 +coverage[toml]==6.5.0 # via pytest-cov cryptography==35.0.0 # via @@ -155,7 +153,7 @@ dask==2022.1.1 # via feast (setup.py) dataclasses==0.6 # via great-expectations -db-dtypes==1.0.3 +db-dtypes==1.0.4 # via google-cloud-bigquery decorator==5.1.1 # via @@ -169,7 +167,7 @@ dill==0.3.5.1 # via # feast (setup.py) # multiprocess -distlib==0.3.5 +distlib==0.3.6 # via virtualenv docker==6.0.0 # via @@ -183,19 +181,21 @@ entrypoints==0.4 # via altair execnet==1.9.0 # via pytest-xdist -executing==0.10.0 +executing==1.1.0 # via stack-data -fastapi==0.79.1 +fastapi==0.85.0 # via feast (setup.py) -fastavro==1.6.0 +fastavro==1.6.1 # via # feast (setup.py) # pandavro -fastjsonschema==2.16.1 +fastjsonschema==2.16.2 # via nbformat filelock==3.8.0 - # via virtualenv -firebase-admin==5.2.0 + # via + # snowflake-connector-python + # virtualenv +firebase-admin==5.4.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -215,20 +215,21 @@ gcsfs==2022.1.0 # via feast (setup.py) geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.8.2 +google-api-core[grpc]==2.10.1 # via # feast (setup.py) # firebase-admin # google-api-python-client # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-core # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.57.0 +google-api-python-client==2.63.0 # via firebase-admin -google-auth==2.10.0 +google-auth==2.12.0 # via # gcsfs # google-api-core @@ -240,63 +241,68 @@ google-auth==2.10.0 # kubernetes google-auth-httplib2==0.1.0 # via google-api-python-client -google-auth-oauthlib==0.5.2 +google-auth-oauthlib==0.5.3 # via gcsfs -google-cloud-bigquery[pandas]==3.3.2 +google-cloud-bigquery[pandas]==3.3.3 # via feast (setup.py) -google-cloud-bigquery-storage==2.14.2 +google-cloud-bigquery-storage==2.16.1 # via # feast (setup.py) # google-cloud-bigquery +google-cloud-bigtable==2.12.0 + # via feast (setup.py) google-cloud-core==2.3.2 # via # google-cloud-bigquery + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage google-cloud-datastore==2.8.1 # via feast (setup.py) -google-cloud-firestore==2.6.1 +google-cloud-firestore==2.7.0 # via firebase-admin google-cloud-storage==2.5.0 # via # feast (setup.py) # firebase-admin # gcsfs -google-crc32c==1.3.0 +google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.3.3 +google-resumable-media==2.4.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos==1.56.4 +googleapis-common-protos[grpc]==1.56.4 # via # feast (setup.py) # google-api-core + # grpc-google-iam-v1 # grpcio-status - # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.47.0 +grpc-google-iam-v1==0.12.4 + # via google-cloud-bigtable +grpcio==1.49.1 # via # feast (setup.py) # google-api-core # google-cloud-bigquery + # googleapis-common-protos + # grpc-google-iam-v1 # grpcio-reflection # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.47.0 +grpcio-reflection==1.49.1 # via feast (setup.py) -grpcio-status==1.47.0 +grpcio-status==1.49.1 # via google-api-core -grpcio-testing==1.47.0 +grpcio-testing==1.49.1 # via feast (setup.py) -grpcio-tools==1.47.0 +grpcio-tools==1.49.1 # via feast (setup.py) -h11==0.13.0 +h11==0.14.0 # via uvicorn happybase==1.2.0 # via feast (setup.py) @@ -306,11 +312,11 @@ httplib2==0.20.4 # via # google-api-python-client # google-auth-httplib2 -httptools==0.4.0 +httptools==0.5.0 # via uvicorn -identify==2.5.3 +identify==2.5.5 # via pre-commit -idna==3.3 +idna==3.4 # via # anyio # requests @@ -322,7 +328,7 @@ importlib-metadata==4.12.0 # via great-expectations iniconfig==1.1.1 # via pytest -ipython==8.4.0 +ipython==8.5.0 # via great-expectations isodate==0.6.1 # via msrest @@ -345,7 +351,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.13.0 +jsonschema==4.16.0 # via # altair # feast (setup.py) @@ -377,7 +383,7 @@ moreorless==0.4.0 # via bowler moto==3.1.18 # via feast (setup.py) -msal==1.18.0 +msal==1.19.0 # via # azure-identity # msal-extensions @@ -397,7 +403,7 @@ multidict==6.0.2 # yarl multiprocess==0.70.13 # via bytewax -mypy==0.971 +mypy==0.981 # via # feast (setup.py) # sqlalchemy @@ -409,11 +415,11 @@ mypy-protobuf==3.1 # via feast (setup.py) mysqlclient==2.1.1 # via feast (setup.py) -nbformat==5.4.0 +nbformat==5.6.1 # via great-expectations nodeenv==1.7.0 # via pre-commit -numpy==1.23.2 +numpy==1.23.3 # via # altair # db-dtypes @@ -439,7 +445,7 @@ packaging==21.3 # pytest # redis # sphinx -pandas==1.4.3 +pandas==1.4.4 # via # altair # db-dtypes @@ -454,7 +460,7 @@ parso==0.8.3 # via jedi partd==1.3.0 # via dask -pathspec==0.9.0 +pathspec==0.10.1 # via black pbr==5.10.0 # via mock @@ -478,21 +484,23 @@ portalocker==2.5.1 # via msal-extensions pre-commit==2.20.0 # via feast (setup.py) -prompt-toolkit==3.0.30 +prompt-toolkit==3.0.31 # via ipython -proto-plus==1.22.0 +proto-plus==1.22.1 # via # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==3.20.2 +protobuf==4.21.7 # via # feast (setup.py) # google-api-core # google-cloud-bigquery # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # googleapis-common-protos @@ -502,7 +510,6 @@ protobuf==3.20.2 # grpcio-tools # mypy-protobuf # proto-plus - # tensorflow-metadata psutil==5.9.0 # via feast (setup.py) psycopg2-binary==2.9.3 @@ -519,7 +526,7 @@ py-cpuinfo==8.0.0 # via pytest-benchmark py4j==0.10.9.5 # via pyspark -pyarrow==6.0.1 +pyarrow==8.0.0 # via # db-dtypes # feast (setup.py) @@ -539,7 +546,7 @@ pycparser==2.21 # via cffi pycryptodomex==3.15.0 # via snowflake-connector-python -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -550,7 +557,7 @@ pygments==2.13.0 # feast (setup.py) # ipython # sphinx -pyjwt[crypto]==2.4.0 +pyjwt[crypto]==2.5.0 # via # adal # msal @@ -562,7 +569,9 @@ pymysql==1.0.2 pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 - # via snowflake-connector-python + # via + # feast (setup.py) + # snowflake-connector-python pyparsing==2.4.7 # via # great-expectations @@ -572,7 +581,7 @@ pyrsistent==0.18.1 # via jsonschema pyspark==3.3.0 # via feast (setup.py) -pytest==7.1.2 +pytest==7.1.3 # via # feast (setup.py) # pytest-benchmark @@ -585,7 +594,7 @@ pytest==7.1.2 # pytest-xdist pytest-benchmark==3.4.1 # via feast (setup.py) -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via feast (setup.py) pytest-forked==1.4.0 # via pytest-xdist @@ -608,7 +617,7 @@ python-dateutil==2.8.2 # kubernetes # moto # pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via @@ -660,15 +669,15 @@ responses==0.21.0 # via moto rsa==4.9 # via google-auth -ruamel.yaml==0.17.17 +ruamel-yaml==0.17.17 # via great-expectations -ruamel.yaml.clib==0.2.6 - # via ruamel.yaml +ruamel-yaml-clib==0.2.6 + # via ruamel-yaml s3fs==2022.1.0 # via feast (setup.py) s3transfer==0.5.2 # via boto3 -scipy==1.9.0 +scipy==1.9.1 # via great-expectations six==1.16.0 # via @@ -686,11 +695,11 @@ six==1.16.0 # msrestazure # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==2.7.8 +snowflake-connector-python[pandas]==2.8.0 # via feast (setup.py) sphinx==4.3.2 # via @@ -710,23 +719,21 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -stack-data==0.4.0 +stack-data==0.5.1 # via ipython -starlette==0.19.1 +starlette==0.20.4 # via fastapi tabulate==0.8.10 # via feast (setup.py) -tenacity==8.0.1 +tenacity==8.1.0 # via feast (setup.py) -tensorflow-metadata==1.9.0 - # via feast (setup.py) -termcolor==1.1.0 +termcolor==2.0.1 # via great-expectations -testcontainers==3.6.1 +testcontainers==3.7.0 # via feast (setup.py) thriftpy2==0.4.14 # via happybase @@ -747,21 +754,23 @@ toolz==0.12.0 # altair # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via # feast (setup.py) # great-expectations -traitlets==5.3.0 +traitlets==5.4.0 # via # ipython # jupyter-core # matplotlib-inline # nbformat -trino==0.315.0 +trino==0.316.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==3.19.22 +types-cryptography==3.3.23 + # via pyjwt +types-protobuf==3.20.4 # via # feast (setup.py) # mypy-protobuf @@ -771,17 +780,17 @@ types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 # via feast (setup.py) -types-pyyaml==6.0.11 +types-pyyaml==6.0.12 # via feast (setup.py) -types-redis==4.3.14 +types-redis==4.3.21 # via feast (setup.py) -types-requests==2.28.9 +types-requests==2.28.11 # via feast (setup.py) -types-setuptools==65.1.0 +types-setuptools==65.4.0.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.23 +types-urllib3==1.26.25 # via types-requests typing-extensions==4.3.0 # via @@ -791,15 +800,16 @@ typing-extensions==4.3.0 # great-expectations # mypy # pydantic + # snowflake-connector-python # sqlalchemy2-stubs # starlette -tzdata==2022.2 +tzdata==2022.4 # via pytz-deprecation-shim tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.11 +urllib3==1.26.12 # via # botocore # docker @@ -809,19 +819,20 @@ urllib3==1.26.11 # minio # requests # responses -uvicorn[standard]==0.18.2 + # snowflake-connector-python +uvicorn[standard]==0.18.3 # via feast (setup.py) -uvloop==0.16.0 +uvloop==0.17.0 # via uvicorn -virtualenv==20.16.3 +virtualenv==20.16.5 # via pre-commit volatile==2.1.0 # via bowler -watchfiles==0.16.1 +watchfiles==0.17.0 # via uvicorn wcwidth==0.2.5 # via prompt-toolkit -websocket-client==1.3.3 +websocket-client==1.4.1 # via # docker # kubernetes diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 395302a9f9f..3b5135b602d 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --output-file=sdk/python/requirements/py3.9-requirements.txt # -absl-py==1.2.0 - # via tensorflow-metadata anyio==3.6.1 # via # starlette @@ -20,9 +18,9 @@ bowler==0.9.0 # via feast (setup.py) cachetools==5.2.0 # via google-auth -certifi==2022.6.15 +certifi==2022.9.24 # via requests -charset-normalizer==2.1.0 +charset-normalizer==2.1.1 # via requests click==8.1.3 # via @@ -30,7 +28,7 @@ click==8.1.3 # feast (setup.py) # moreorless # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via feast (setup.py) @@ -38,44 +36,41 @@ dask==2022.1.1 # via feast (setup.py) dill==0.3.5.1 # via feast (setup.py) -fastapi==0.79.1 +fastapi==0.85.0 # via feast (setup.py) -fastavro==1.6.0 +fastavro==1.6.1 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2022.7.1 +fsspec==2022.8.2 # via dask -google-api-core==2.8.2 +google-api-core==2.10.1 # via feast (setup.py) -google-auth==2.10.0 +google-auth==2.12.0 # via google-api-core googleapis-common-protos==1.56.4 # via # feast (setup.py) # google-api-core - # tensorflow-metadata -greenlet==1.1.2 - # via sqlalchemy -grpcio==1.47.0 +grpcio==1.49.1 # via # feast (setup.py) # grpcio-reflection -grpcio-reflection==1.47.0 +grpcio-reflection==1.49.1 # via feast (setup.py) -h11==0.13.0 +h11==0.14.0 # via uvicorn -httptools==0.4.0 +httptools==0.5.0 # via uvicorn -idna==3.3 +idna==3.4 # via # anyio # requests jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.13.0 +jsonschema==4.16.0 # via feast (setup.py) locket==1.0.0 # via partd @@ -85,11 +80,11 @@ mmh3==3.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==0.971 +mypy==0.981 # via sqlalchemy mypy-extensions==0.4.3 # via mypy -numpy==1.23.2 +numpy==1.23.3 # via # feast (setup.py) # pandas @@ -97,7 +92,7 @@ numpy==1.23.2 # pyarrow packaging==21.3 # via dask -pandas==1.4.3 +pandas==1.5.0 # via # feast (setup.py) # pandavro @@ -105,16 +100,15 @@ pandavro==1.5.2 # via feast (setup.py) partd==1.3.0 # via dask -proto-plus==1.22.0 +proto-plus==1.22.1 # via feast (setup.py) -protobuf==3.20.2 +protobuf==4.21.7 # via # feast (setup.py) # google-api-core # googleapis-common-protos # grpcio-reflection # proto-plus - # tensorflow-metadata pyarrow==8.0.0 # via feast (setup.py) pyasn1==0.4.8 @@ -123,7 +117,7 @@ pyasn1==0.4.8 # rsa pyasn1-modules==0.2.8 # via google-auth -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -135,7 +129,7 @@ pyrsistent==0.18.1 # via jsonschema python-dateutil==2.8.2 # via pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via pandas @@ -154,19 +148,17 @@ six==1.16.0 # grpcio # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -starlette==0.19.1 +starlette==0.20.4 # via fastapi tabulate==0.8.10 # via feast (setup.py) -tenacity==8.0.1 - # via feast (setup.py) -tensorflow-metadata==1.9.0 +tenacity==8.1.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) @@ -176,7 +168,7 @@ toolz==0.12.0 # via # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) @@ -186,15 +178,15 @@ typing-extensions==4.3.0 # pydantic # sqlalchemy2-stubs # starlette -urllib3==1.26.11 +urllib3==1.26.12 # via requests -uvicorn[standard]==0.18.2 +uvicorn[standard]==0.18.3 # via feast (setup.py) -uvloop==0.16.0 +uvloop==0.17.0 # via uvicorn volatile==2.1.0 # via bowler -watchfiles==0.16.1 +watchfiles==0.17.0 # via uvicorn websockets==10.3 # via uvicorn diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 708d9c0a142..174b0b91ad1 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -51,6 +51,9 @@ create_order_feature_view, create_pushable_feature_view, ) +from tests.integration.feature_repos.universal.online_store.bigtable import ( + BigtableOnlineStoreCreator, +) from tests.integration.feature_repos.universal.online_store.datastore import ( DatastoreOnlineStoreCreator, ) @@ -84,6 +87,12 @@ "schema": "ONLINE", } +BIGTABLE_CONFIG = { + "type": "bigtable", + "project_id": os.getenv("GCLOUD_PROJECT", "kf-feast"), + "instance": os.getenv("BIGTABLE_INSTANCE_ID", "feast-integration-tests"), +} + OFFLINE_STORE_TO_PROVIDER_CONFIG: Dict[str, DataSourceCreator] = { "file": ("local", FileDataSourceCreator), "bigquery": ("gcp", BigQueryDataSourceCreator), @@ -115,6 +124,7 @@ AVAILABLE_ONLINE_STORES["dynamodb"] = (DYNAMO_CONFIG, None) AVAILABLE_ONLINE_STORES["datastore"] = ("datastore", None) AVAILABLE_ONLINE_STORES["snowflake"] = (SNOWFLAKE_CONFIG, None) + AVAILABLE_ONLINE_STORES["bigtable"] = (BIGTABLE_CONFIG, None) full_repo_configs_module = os.environ.get(FULL_REPO_CONFIGS_MODULE_ENV_NAME) @@ -161,6 +171,7 @@ "redis": (REDIS_CONFIG, RedisOnlineStoreCreator), "dynamodb": (DYNAMO_CONFIG, DynamoDBOnlineStoreCreator), "datastore": ("datastore", DatastoreOnlineStoreCreator), + "bigtable": ("bigtable", BigtableOnlineStoreCreator), } for key, replacement in replacements.items(): diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/bigtable.py b/sdk/python/tests/integration/feature_repos/universal/online_store/bigtable.py new file mode 100644 index 00000000000..c06143e245b --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/bigtable.py @@ -0,0 +1,47 @@ +import os +from typing import Dict + +from google.cloud import bigtable +from testcontainers.core.container import DockerContainer +from testcontainers.core.waiting_utils import wait_for_logs + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class BigtableOnlineStoreCreator(OnlineStoreCreator): + gcp_project = "test-project" + host = "0.0.0.0" + port = "8086" + bt_instance = "test-instance" + + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.container = ( + DockerContainer( + "gcr.io/google.com/cloudsdktool/cloud-sdk:380.0.0-emulators" + ) + .with_command( + f"gcloud beta emulators bigtable start --project {self.gcp_project} --host-port {self.host}:{self.port}" + ) + .with_exposed_ports(self.port) + ) + + def create_online_store(self) -> Dict[str, str]: + self.container.start() + log_string_to_wait_for = r"\[bigtable\] Cloud Bigtable emulator running" + wait_for_logs( + container=self.container, predicate=log_string_to_wait_for, timeout=10 + ) + exposed_port = self.container.get_exposed_port(self.port) + os.environ[bigtable.client.BIGTABLE_EMULATOR] = f"{self.host}:{exposed_port}" + return { + "type": "bigtable", + "project_id": self.gcp_project, + "instance": self.bt_instance, + } + + def teardown(self): + del os.environ[bigtable.client.BIGTABLE_EMULATOR] + self.container.stop() diff --git a/setup.py b/setup.py index 59c362ff9fc..2b09bcbe946 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ "mmh3", "numpy>=1.22,<3", "pandas>=1.4.3,<2", - "pandavro==1.5.*", # For some reason pandavro higher than 1.5.* only support pandas less than 1.3. + "pandavro==1.5.*", # For some reason pandavro higher than 1.5.* only support pandas less than 1.3. "protobuf<5,>3", "proto-plus>=1.20.0,<2", "pyarrow>=4,<9", @@ -74,7 +74,6 @@ "typeguard", "fastapi>=0.68.0,<1", "uvicorn[standard]>=0.14.0,<1", - "tensorflow-metadata>=1.0.0,<2.0.0", "dask>=2021.*,<2022.02.0", "bowler", # Needed for automatic repo upgrades ] @@ -84,6 +83,7 @@ "google-cloud-bigquery-storage >= 2.0.0,<3", "google-cloud-datastore>=2.1.*,<3", "google-cloud-storage>=1.34.*,<3", + "google-cloud-bigtable>=2.11.*,<3", ] REDIS_REQUIRED = [ @@ -97,6 +97,10 @@ SNOWFLAKE_REQUIRED = [ "snowflake-connector-python[pandas]>=2.7.3,<3", + # `pyOpenSSL==22.1.0` requires `cryptography<39,>=38.0.0`, which is incompatible + # with `snowflake-connector-python[pandas]==2.8.0`, which depends on + # `cryptography<37.0.0,>=3.1.0`. + "pyOpenSSL<22.1.0", ] SPARK_REQUIRED = [ @@ -111,11 +115,7 @@ "psycopg2-binary>=2.8.3,<3", ] -MYSQL_REQUIRED = [ - "mysqlclient", - "pymysql", - "types-PyMySQL" -] +MYSQL_REQUIRED = ["mysqlclient", "pymysql", "types-PyMySQL"] HBASE_REQUIRED = [ "happybase>=1.2.0,<3", @@ -131,15 +131,13 @@ "cffi==1.15.*,<2", ] -AZURE_REQUIRED = ( - [ - "azure-storage-blob>=0.37.0", - "azure-identity>=1.6.1", - "SQLAlchemy>=1.4.19", - "pyodbc>=4.0.30", - "pymssql", - ] -) +AZURE_REQUIRED = [ + "azure-storage-blob>=0.37.0", + "azure-identity>=1.6.1", + "SQLAlchemy>=1.4.19", + "pyodbc>=4.0.30", + "pymssql", +] CI_REQUIRED = ( [ @@ -152,7 +150,7 @@ "grpcio-testing>=1.47.0", "minio==7.1.0", "mock==2.0.0", - "moto", + "moto<4", "mypy>=0.931", "mypy-protobuf==3.1", "avro==1.10.0", From c9820ca9f5f22ab24d73dd43a75526906b9ddb94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Viveret?= Date: Thu, 6 Oct 2022 01:32:17 +0200 Subject: [PATCH 29/33] chore: Replace zod by protobufjs (#3208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Replace zod by protobufjs Signed-off-by: Jérôme Viveret Signed-off-by: Danny Chiao * Remove unneeded pages and fix bugs Signed-off-by: Danny Chiao * Update the ui_server Signed-off-by: Danny Chiao * Replace manually generated json registry by protobuf version Signed-off-by: Danny Chiao * Correctly display protobufjs enums Signed-off-by: Danny Chiao Signed-off-by: Jérôme Viveret Signed-off-by: Danny Chiao Co-authored-by: Jérôme Viveret Co-authored-by: Danny Chiao --- protos/feast/core/DataSource.proto | 7 + sdk/python/feast/ui/README.md | 5 +- sdk/python/feast/ui/package.json | 12 +- sdk/python/feast/ui/yarn.lock | 80 +- sdk/python/feast/ui_server.py | 11 +- ui/.gitignore | 2 + ui/feature_repo/test_get_features.py | 86 --- ui/package.json | 21 +- ui/public/projects-list.json | 2 +- ui/public/registry.db | Bin 0 -> 5527 bytes ui/public/registry.json | 706 ------------------ .../components/FeaturesInServiceDisplay.tsx | 43 +- ui/src/components/FeaturesListDisplay.tsx | 80 +- ui/src/components/NumericFeaturesTable.tsx | 60 -- ui/src/components/SparklineHistogram.tsx | 58 -- ui/src/hooks/useFCOExploreSuggestions.ts | 6 +- ui/src/hooks/useTagsAggregation.ts | 27 +- .../BatchSourcePropertiesView.tsx | 63 +- ui/src/pages/data-sources/DataSourceDbt.tsx | 35 - .../pages/data-sources/DataSourceInstance.tsx | 18 +- .../data-sources/DataSourceOverviewTab.tsx | 11 +- .../data-sources/DataSourcesListingTable.tsx | 9 +- .../RequestDataSourceSchemaTable.tsx | 8 +- .../pages/entities/EntitiesListingTable.tsx | 16 +- ui/src/pages/entities/EntityOverviewTab.tsx | 20 +- ui/src/pages/entities/useLoadEntity.ts | 4 +- .../FeatureServiceListingTable.tsx | 28 +- .../FeatureServiceOverviewTab.tsx | 23 +- ui/src/pages/feature-services/Index.tsx | 10 +- .../feature-services/useLoadFeatureService.ts | 24 +- .../feature-views/FeatureViewInstance.tsx | 12 +- .../feature-views/FeatureViewListingTable.tsx | 2 +- .../FeatureViewSummaryStatisticsTab.tsx | 84 --- ui/src/pages/feature-views/Index.tsx | 2 +- .../OnDemandFeatureViewInstance.tsx | 4 +- .../OnDemandFeatureViewOverviewTab.tsx | 33 +- .../RegularFeatureViewInstance.tsx | 11 +- .../RegularFeatureViewOverviewTab.tsx | 31 +- .../StreamFeatureViewInstance.tsx | 4 +- .../StreamFeatureViewOverviewTab.tsx | 24 +- .../FeatureViewProjectionDisplayPanel.tsx | 17 +- .../components/RequestDataDisplayPanel.tsx | 12 +- .../pages/feature-views/useLoadFeatureView.ts | 10 +- ui/src/pages/features/FeatureOverviewTab.tsx | 3 +- ui/src/pages/features/useLoadFeature.ts | 12 +- .../DatasetExpectationsTab.tsx | 12 +- .../saved-data-sets/DatasetOverviewTab.tsx | 13 +- .../saved-data-sets/DatasetsListingTable.tsx | 13 +- .../pages/saved-data-sets/useLoadDataset.ts | 4 +- ui/src/parsers/feastDatasources.ts | 30 - ui/src/parsers/feastEntities.ts | 21 - ui/src/parsers/feastFeatureServices.ts | 31 - ui/src/parsers/feastFeatureViews.ts | 64 -- ui/src/parsers/feastFeatures.ts | 11 - ui/src/parsers/feastODFVS.ts | 47 -- ui/src/parsers/feastRegistry.ts | 24 - ui/src/parsers/feastSFVS.ts | 41 - ui/src/parsers/feastSavedDataset.ts | 37 - .../parsers/featureViewSummaryStatistics.ts | 56 -- ui/src/parsers/mergedFVTypes.ts | 40 +- ui/src/parsers/parseEntityRelationships.ts | 86 +-- ui/src/parsers/parseIndirectRelationships.ts | 8 +- ui/src/parsers/types.ts | 23 +- .../useLoadFeatureViewSummaryStatistics.ts | 37 - ui/src/queries/useLoadRegistry.ts | 17 +- ui/src/utils/timestamp.ts | 13 + ui/yarn.lock | 309 +++++++- 67 files changed, 693 insertions(+), 1980 deletions(-) create mode 100644 ui/.gitignore delete mode 100644 ui/feature_repo/test_get_features.py create mode 100644 ui/public/registry.db delete mode 100644 ui/public/registry.json delete mode 100644 ui/src/components/NumericFeaturesTable.tsx delete mode 100644 ui/src/components/SparklineHistogram.tsx delete mode 100644 ui/src/pages/data-sources/DataSourceDbt.tsx delete mode 100644 ui/src/pages/feature-views/FeatureViewSummaryStatisticsTab.tsx delete mode 100644 ui/src/parsers/feastDatasources.ts delete mode 100644 ui/src/parsers/feastEntities.ts delete mode 100644 ui/src/parsers/feastFeatureServices.ts delete mode 100644 ui/src/parsers/feastFeatureViews.ts delete mode 100644 ui/src/parsers/feastFeatures.ts delete mode 100644 ui/src/parsers/feastODFVS.ts delete mode 100644 ui/src/parsers/feastRegistry.ts delete mode 100644 ui/src/parsers/feastSFVS.ts delete mode 100644 ui/src/parsers/feastSavedDataset.ts delete mode 100644 ui/src/parsers/featureViewSummaryStatistics.ts delete mode 100644 ui/src/queries/useLoadFeatureViewSummaryStatistics.ts create mode 100644 ui/src/utils/timestamp.ts diff --git a/protos/feast/core/DataSource.proto b/protos/feast/core/DataSource.proto index 5258618f3bd..3992d2c247d 100644 --- a/protos/feast/core/DataSource.proto +++ b/protos/feast/core/DataSource.proto @@ -23,6 +23,7 @@ option java_outer_classname = "DataSourceProto"; option java_package = "feast.proto.core"; import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; import "feast/core/DataFormat.proto"; import "feast/types/Value.proto"; import "feast/core/Feature.proto"; @@ -89,6 +90,12 @@ message DataSource { // Optional batch source for streaming sources for historical features and materialization. DataSource batch_source = 26; + SourceMeta meta = 50; + + message SourceMeta { + google.protobuf.Timestamp earliestEventTimestamp = 1; + google.protobuf.Timestamp latestEventTimestamp = 2; + } // Defines options for DataSource that sources features from a file message FileOptions { diff --git a/sdk/python/feast/ui/README.md b/sdk/python/feast/ui/README.md index 0c11dcf134c..220f40a2251 100644 --- a/sdk/python/feast/ui/README.md +++ b/sdk/python/feast/ui/README.md @@ -22,4 +22,7 @@ It is used by the `feast ui` command to scaffold a local UI server. The feast py The `feast ui` command will generate the necessary `projects-list.json` file and initialize it for the UI to read. -**Note**: yarn start will not work on this because of the above dependency. +**Note**: `yarn start` will not work on this because of the above dependency. + +## Dev +To test, do `yarn link` in ui/ and then come here to do `yarn link @feast-dev/feast` \ No newline at end of file diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 358aa2cdd24..6ffebcc834d 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -4,7 +4,7 @@ "private": true, "dependencies": { "@elastic/datemath": "^5.0.3", - "@elastic/eui": "^57.0.0", + "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", "@feast-dev/feast-ui": "latest", "@testing-library/jest-dom": "^5.16.4", @@ -16,11 +16,11 @@ "moment": "^2.29.4", "prop-types": "^15.8.1", "query-string": "^7.1.1", - "react": "^18.1.0", - "react-dom": "^18.1.0", - "react-query": "^3.39.0", - "react-router-dom": "^6.3.0", - "react-scripts": "5.0.1", + "react": "^17.0.2", + "react-dom": "^17.0.2", + "react-query": "^3.34.12", + "react-router-dom": "6", + "react-scripts": "^5.0.0", "typescript": "^4.6.4", "use-query-params": "^1.2.3", "web-vitals": "^2.1.4", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index df2bfe45ff2..1b9cf5f067f 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1202,51 +1202,6 @@ uuid "^8.3.0" vfile "^4.2.0" -"@elastic/eui@^57.0.0": - version "57.0.0" - resolved "https://registry.yarnpkg.com/@elastic/eui/-/eui-57.0.0.tgz#86d43e27196f9997ef44d2a4c701d092ce99e132" - integrity sha512-VBgW6Pr0JJB3JhJ59MV8guxb2v4Gd3SJEmsMGKGyIY+KcvSMWbVEGa44Ep12VAJYynIA05Z3OXXc/ge5dMycpA== - dependencies: - "@types/chroma-js" "^2.0.0" - "@types/lodash" "^4.14.160" - "@types/numeral" "^0.0.28" - "@types/react-beautiful-dnd" "^13.1.2" - "@types/react-input-autosize" "^2.2.1" - "@types/react-virtualized-auto-sizer" "^1.0.1" - "@types/react-window" "^1.8.5" - "@types/refractor" "^3.0.0" - "@types/resize-observer-browser" "^0.1.5" - "@types/vfile-message" "^2.0.0" - chroma-js "^2.1.0" - classnames "^2.2.6" - lodash "^4.17.21" - mdast-util-to-hast "^10.0.0" - numeral "^2.0.6" - prop-types "^15.6.0" - react-beautiful-dnd "^13.1.0" - react-dropzone "^11.5.3" - react-element-to-jsx-string "^14.3.4" - react-focus-on "^3.5.4" - react-input-autosize "^3.0.0" - react-is "^17.0.2" - react-virtualized-auto-sizer "^1.0.6" - react-window "^1.8.6" - refractor "^3.5.0" - rehype-raw "^5.0.0" - rehype-react "^6.0.0" - rehype-stringify "^8.0.0" - remark-breaks "^2.0.2" - remark-emoji "^2.1.0" - remark-parse "^8.0.3" - remark-rehype "^8.0.0" - tabbable "^5.2.1" - text-diff "^1.0.1" - unified "^9.2.0" - unist-util-visit "^2.0.3" - url-parse "^1.5.10" - uuid "^8.3.0" - vfile "^4.2.0" - "@emotion/babel-plugin@^11.7.1": version "11.9.2" resolved "https://registry.yarnpkg.com/@emotion/babel-plugin/-/babel-plugin-11.9.2.tgz#723b6d394c89fb2ef782229d92ba95a740576e95" @@ -8403,13 +8358,14 @@ react-dev-utils@^12.0.1: strip-ansi "^6.0.1" text-table "^0.2.0" -react-dom@^18.1.0: - version "18.1.0" - resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-18.1.0.tgz#7f6dd84b706408adde05e1df575b3a024d7e8a2f" - integrity sha512-fU1Txz7Budmvamp7bshe4Zi32d0ll7ect+ccxNu9FlObT605GOEB8BfO4tmRJ39R5Zj831VCpvQ05QPBW5yb+w== +react-dom@^17.0.2: + version "17.0.2" + resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-17.0.2.tgz#ecffb6845e3ad8dbfcdc498f0d0a939736502c23" + integrity sha512-s4h96KtLDUQlsENhMn1ar8t2bEa+q/YAtj8pPPdIjPDGBDIVNsrD9aXNWqspUe6AzKCIG0C1HZZLqLV7qpOBGA== dependencies: loose-envify "^1.1.0" - scheduler "^0.22.0" + object-assign "^4.1.1" + scheduler "^0.20.2" react-dropzone@^11.5.3: version "11.7.1" @@ -8481,7 +8437,7 @@ react-is@^18.0.0: resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.1.0.tgz#61aaed3096d30eacf2a2127118b5b41387d32a67" integrity sha512-Fl7FuabXsJnV5Q1qIOQwx/sagGF18kogb4gpfcG4gjLBWO0WDiiz1ko/ExayuxE7InyQkBLkxRFG5oxY6Uu3Kg== -react-query@^3.34.12, react-query@^3.39.0: +react-query@^3.34.12: version "3.39.0" resolved "https://registry.yarnpkg.com/react-query/-/react-query-3.39.0.tgz#0caca7b0da98e65008bbcd4df0d25618c2100050" integrity sha512-Od0IkSuS79WJOhzWBx/ys0x13+7wFqgnn64vBqqAAnZ9whocVhl/y1padD5uuZ6EIkXbFbInax0qvY7zGM0thA== @@ -8526,7 +8482,7 @@ react-remove-scroll@^2.5.2: use-callback-ref "^1.3.0" use-sidecar "^1.1.2" -react-router-dom@6, react-router-dom@^6.3.0: +react-router-dom@6: version "6.3.0" resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.3.0.tgz#a0216da813454e521905b5fa55e0e5176123f43d" integrity sha512-uaJj7LKytRxZNQV8+RbzJWnJ8K2nPsOOEuX7aQstlMZKQT0164C+X2w6bnkqU3sjtLvpd5ojrezAyfZ1+0sStw== @@ -8541,7 +8497,7 @@ react-router@6.3.0: dependencies: history "^5.2.0" -react-scripts@5.0.1, react-scripts@^5.0.0: +react-scripts@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/react-scripts/-/react-scripts-5.0.1.tgz#6285dbd65a8ba6e49ca8d651ce30645a6d980003" integrity sha512-8VAmEm/ZAwQzJ+GOMLbBsTdDKOpuZh7RPs0UymvBR2vRk4iZWCskjbFnxqjrzoIvlNNRZ3QJFx6/qDSi6zSnaQ== @@ -8618,12 +8574,13 @@ react-window@^1.8.6: "@babel/runtime" "^7.0.0" memoize-one ">=3.1.1 <6" -react@^18.1.0: - version "18.1.0" - resolved "https://registry.yarnpkg.com/react/-/react-18.1.0.tgz#6f8620382decb17fdc5cc223a115e2adbf104890" - integrity sha512-4oL8ivCz5ZEPyclFQXaNksK3adutVS8l2xzZU0cqEFrE9Sb7fC0EFK5uEk74wIreL1DERyjvsU915j1pcT2uEQ== +react@^17.0.2: + version "17.0.2" + resolved "https://registry.yarnpkg.com/react/-/react-17.0.2.tgz#d0b5cc516d29eb3eee383f75b62864cfb6800037" + integrity sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA== dependencies: loose-envify "^1.1.0" + object-assign "^4.1.1" readable-stream@^2.0.1: version "2.3.7" @@ -9001,12 +8958,13 @@ saxes@^5.0.1: dependencies: xmlchars "^2.2.0" -scheduler@^0.22.0: - version "0.22.0" - resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.22.0.tgz#83a5d63594edf074add9a7198b1bae76c3db01b8" - integrity sha512-6QAm1BgQI88NPYymgGQLCZgvep4FyePDWFpXVK+zNSUgHwlqpJy8VEh8Et0KxTACS4VWwMousBElAZOH9nkkoQ== +scheduler@^0.20.2: + version "0.20.2" + resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.20.2.tgz#4baee39436e34aa93b4874bddcbf0fe8b8b50e91" + integrity sha512-2eWfGgAqqWFGqtdMmcL5zCMK1U8KlXv8SQFGglL3CEtd0aDVDWgeF/YoCmvln55m5zSk3J/20hTaSBeSObsQDQ== dependencies: loose-envify "^1.1.0" + object-assign "^4.1.1" schema-utils@2.7.0: version "2.7.0" diff --git a/sdk/python/feast/ui_server.py b/sdk/python/feast/ui_server.py index 4d1fd67dc1e..f79030e8d35 100644 --- a/sdk/python/feast/ui_server.py +++ b/sdk/python/feast/ui_server.py @@ -30,14 +30,14 @@ def get_app( ) # Asynchronously refresh registry, notifying shutdown and canceling the active timer if the app is shutting down - registry_json = "" + registry_proto = None shutting_down = False active_timer: Optional[threading.Timer] = None def async_refresh(): store.refresh_registry() - nonlocal registry_json - registry_json = get_registry_dump(store.config, store.repo_path) + nonlocal registry_proto + registry_proto = store.registry.proto() if shutting_down: return nonlocal active_timer @@ -70,7 +70,10 @@ def shutdown_event(): @app.get("/registry") def read_registry(): - return json.loads(registry_json) + return Response( + content=registry_proto.SerializeToString(), + media_type="application/octet-stream", + ) # For all other paths (such as paths that would otherwise be handled by react router), pass to React @app.api_route("/p/{path_name:path}", methods=["GET"]) diff --git a/ui/.gitignore b/ui/.gitignore new file mode 100644 index 00000000000..728f2aab717 --- /dev/null +++ b/ui/.gitignore @@ -0,0 +1,2 @@ +src/protos.d.ts +src/protos.js diff --git a/ui/feature_repo/test_get_features.py b/ui/feature_repo/test_get_features.py deleted file mode 100644 index 722c5a3dd46..00000000000 --- a/ui/feature_repo/test_get_features.py +++ /dev/null @@ -1,86 +0,0 @@ -import pandas as pd -from great_expectations.core.expectation_suite import ExpectationSuite -from great_expectations.dataset import PandasDataset - -from feast import FeatureStore -from feast.dqm.profilers.ge_profiler import ge_profiler -from feast.infra.offline_stores.file import SavedDatasetFileStorage - -DELTA = 0.1 # controlling allowed window in fraction of the value on scale [0, 1] -# Note: the GE integration allows asserting differences between datasets. The "ds" below is the reference dataset to check and this generates the expectation suite which can be used against future datasets. -# It's used via ge.validate(new_dataset, ExpectationSuite) -# For this demo though, we ignore this and - - -@ge_profiler -def credit_profiler(ds: PandasDataset) -> ExpectationSuite: - # simple checks on data consistency - ds.expect_column_values_to_be_between( - "credit_card_due", min_value=0, mostly=0.99, # allow some outliers - ) - - ds.expect_column_values_to_be_between( - "missed_payments_1y", - min_value=0, - max_value=5, - mostly=0.99, # allow some outliers - ) - - return ds.get_expectation_suite() - - -def generate_saved_dataset(): - store = FeatureStore(repo_path=".") - entity_df = pd.read_parquet(path="data/loan_table.parquet") - - fs = store.get_feature_service("credit_score_v1") - job = store.get_historical_features(entity_df=entity_df, features=fs,) - store.create_saved_dataset( - from_=job, - name="my_training_ds", - storage=SavedDatasetFileStorage(path="my_training_ds.parquet"), - feature_service=fs, - profiler=credit_profiler, - ) - - -def get_latest_timestamps(): - store = FeatureStore(repo_path=".") - feature_views = store.list_feature_views() - for fv in feature_views: - print( - f"Data source latest event for {fv.name} is {fv.batch_source._meta.latest_event_timestamp}" - ) - - -def test_ge(): - store = FeatureStore(repo_path=".") - - print("--- Historical features (from saved dataset) ---") - ds = store.get_saved_dataset("my_training_ds") - print(ds._profile) - - -def run_demo(): - store = FeatureStore(repo_path=".") - - print("--- Historical features (from saved dataset) ---") - ds = store.get_saved_dataset("my_training_ds") - print(ds.to_df()) - - print("\n--- Online features ---") - features = store.get_online_features( - features=store.get_feature_service("credit_score_v3"), - entity_rows=[ - {"zipcode": 30721, "dob_ssn": "19530219_5179", "transaction_amt": 1023} - ], - ).to_dict() - for key, value in sorted(features.items()): - print(key, " : ", value) - - -if __name__ == "__main__": - generate_saved_dataset() - get_latest_timestamps() - # test_ge() - run_demo() diff --git a/ui/package.json b/ui/package.json index 73165523252..7d50b3e086b 100644 --- a/ui/package.json +++ b/ui/package.json @@ -25,8 +25,7 @@ "react-router-dom": "6", "react-scripts": "^5.0.0", "typescript": "^4.4.2", - "use-query-params": "^1.2.3", - "zod": "^3.11.6" + "use-query-params": "^1.2.3" }, "dependencies": { "@elastic/datemath": "^5.0.3", @@ -41,20 +40,21 @@ "inter-ui": "^3.19.3", "moment": "^2.29.1", "prop-types": "^15.8.1", + "protobufjs": "^7.1.1", "query-string": "^7.1.1", "react-query": "^3.34.12", "react-router-dom": "6", "react-scripts": "^5.0.0", - "use-query-params": "^1.2.3", - "zod": "^3.11.6" + "use-query-params": "^1.2.3" }, "scripts": { - "start": "react-scripts start", - "build": "react-scripts build", - "build:lib": "rimraf ./dist && tsc && rollup -c", - "build:lib-dev": "rimraf ./dist && tsc && rollup -c && yalc publish -f", - "test": "react-scripts test", - "eject": "react-scripts eject" + "start": "npm run generate-protos && react-scripts start", + "build": "npm run generate-protos && react-scripts build", + "build:lib": "npm run generate-protos && rimraf ./dist && tsc && rollup -c", + "build:lib-dev": "npm run generate-protos && rimraf ./dist && tsc && rollup -c && yalc publish -f", + "test": "npm run generate-protos && react-scripts test", + "eject": "react-scripts eject", + "generate-protos": "pbjs --no-encode -o src/protos.js -w commonjs -t static-module `find ../protos/feast/ -iname *.proto` && pbts -n protos -o src/protos.d.ts src/protos.js" }, "eslintConfig": { "extends": [ @@ -92,6 +92,7 @@ "@testing-library/react": "^12.0.0", "@testing-library/user-event": "^13.2.1", "msw": "^0.36.8", + "protobufjs-cli": "^1.0.2", "react": "^17.0.2", "react-dom": "^17.0.2", "rimraf": "^3.0.2", diff --git a/ui/public/projects-list.json b/ui/public/projects-list.json index d3d7c3b7d91..8e2bc3c616a 100644 --- a/ui/public/projects-list.json +++ b/ui/public/projects-list.json @@ -4,7 +4,7 @@ "name": "Credit Score Project", "description": "Project for credit scoring team and associated models.", "id": "credit_score_project", - "registryPath": "/registry.json" + "registryPath": "/registry.db" }, { "name": "Empty Registry", diff --git a/ui/public/registry.db b/ui/public/registry.db new file mode 100644 index 0000000000000000000000000000000000000000..617771999c7e43ef6bdb1e6acf9eed18b06b1dae GIT binary patch literal 5527 zcmds5TZkJ~7|tZ=ZuabUH#_!XQ!nGR=x*3#cc*&|uGC#A=t_mwr`9^0%sJVNom*#4 zcAL^lu~?->5gls_Vs9@+P%QXRq%Rg96cp>5FG5j4QADd?`yl8!lgwtanQhYw-X@Sa z|37EW`Op7<|82lI4!q2Pc4X#d$1zg(KLiOD%z}_tZBi7VfdmaYL@1agTR`fZN*wmo zF%=bRf`gTkt&*}}l=M8dMSq#;wP1z0U|^f?CZ?o$K8gnvSaQHhwSK{ppSC6+PzdQZHNse!zz{8*)w97Io>3UfNwSlan z8!UD0v!j8fJPJGwo&@dtR7)`tPIWve)P|zZ3lVtwcK>ia2pYlD-C+NpUpx`m-=k98 zu(%OT;E94XI+~fBz+;(_5ea7|M#pewa&khEM$zPuG&-L4HE{NW%V#+0g*f;=2D)pS zE2<7L?J^&t$&_yr@ix#?o320`QRU%eTug#4-L%OZoWnKFDv)$Y31NfCnhA|McODm2 zMZ-aEHvmPpmQ~U;hpx5;@Ve?am~voQr%O1pR1W74FT)?xt5W5mvD+?LL{TwaIlUGD zM1d(ogfQYeq#;Sl3`v=>^hVGH6$LwvJZGCFi(gg16s{SDu{!F~>p>^pYhhdUgyOsE zMCL)MEphPLH@`%4N4Nx+>cR8u=!DL62-K}~FI5`d8P!=k?&Q}~Vc=k+_OlkW_mnU> z!VP;eWL2YJ!>m~-XsUr_W+Cik3o2EiL!C*%*)4Ru-5Z|fmY8xgm+xG3%49U)59hx) z8vv00HB+nRtj=pa*mklR!;g zpvYhb7;V(xy+I8Q1c>P%G1H=Xma*=5uQ;)Ac_gKx$MAC3=x2|oCxPM+EdHV_*c4pcm}&CwJhE zZSeQCG%eXKc6lhC!+Z?kd2zX#AiZkFD*%LA0U+o!7i$6kI(F#2)AbU-qjFa`aESve zegCSy5|;n|0X>TA`~o2fby`8edM0XANxJ*dnrAni`Rs>Z1&EK3tibmrbq9INDOjYP9-w;Bg&5JpaR+ zXX;MiQMvb{;CK@thC=}{)fAup?a^pid()zoB}W=o+S!e^3^$A*m#@4NP@YFkzXZUK z9W6z&P=&S}1i$8*RDX#}ecH?oHAr(72`$ih%55&tS;P+(=wZo*73g7nmW0CitU%|f zRE-6?I7th}dg|cQd|#sd6>fEk9xA|>p-z(V(_C#tb?hE?D{ZP_ zm54*1p#r0&!J?_-!KbjzZZvHUddGoGJV3yT4lGGm_0O;h^$CNyZ>m5 zx&8gb-R2%}D+yUP=r*$K4Rxp*bm$ZlTensx&?!B#Oj}r5>9d6I(8mX;-d0w6Eta;N zEUS33w2Ps#7CT?1(pLKfw|G~{Iqc?(tGpvoMR&`pZqY`P_8zFDF;!REs%yBi=7RoLfuwTeL(7?!f(xHMd z)xhrSukm-bB;_vpdQ~Y*G1ZM5-=uCN?h+nYe72-Q6L)$0Rd=nLOh5$?D5=i>`o^hx zHu0!!=VIUp2L?R6`OT{l>|LEbno4kXyLWq>byngIFxnzuy|&Es*2|(ie@{*1z(*gy z&a8dS0F|2+=80PsWh$T*bX2%dz00_gd7b^rhX literal 0 HcmV?d00001 diff --git a/ui/public/registry.json b/ui/public/registry.json deleted file mode 100644 index 279c9d08327..00000000000 --- a/ui/public/registry.json +++ /dev/null @@ -1,706 +0,0 @@ -{ - "dataSources": [ - { - "createdTimestampColumn": "created_timestamp", - "fileOptions": { - "uri": "data/credit_history.parquet" - }, - "name": "credit_history", - "timestampField": "event_timestamp", - "type": "BATCH_FILE" - }, - { - "name": "transaction", - "requestDataOptions": { - "schema": [ - { - "name": "transaction_amt", - "valueType": "INT64" - } - ] - }, - "type": "REQUEST_SOURCE" - }, - { - "createdTimestampColumn": "created_timestamp", - "fileOptions": { - "uri": "data/zipcode_table.parquet" - }, - "name": "zipcode", - "timestampField": "event_timestamp", - "type": "BATCH_FILE" - }, - { - "batchSource": { - "fileOptions": { - "uri": "data/zipcode_table.parquet" - }, - "name": "user_stats", - "timestampField": "timestamp", - "type": "BATCH_FILE" - }, - "dataSourceClassType": "feast.data_source.KafkaSource", - "description": "The Kafka stream example", - "kafkaOptions": {"messageFormat": {"jsonFormat": {"schemaJson": "id string, timestamp timestamp"}}, - "watermarkDelayThreshold": "300s"}, - "name": "driver_stats_stream", - "owner": "test@gmail.com", - "timestampField": "timestamp", - "type": "STREAM_KAFKA" - } - ], - "entities": [ - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171062Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.171062Z" - }, - "spec": { - "joinKey": "__dummy_id", - "name": "__dummy", - "valueType": "STRING" - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171112Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.171112Z" - }, - "spec": { - "description": "Date of birth and last four digits of social security number", - "joinKey": "dob_ssn", - "name": "dob_ssn", - "tags": { - "owner": "tony@tecton.ai", - "team": "hack week" - }, - "valueType": "STRING" - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171153Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.171153Z" - }, - "spec": { - "description": "A zipcode", - "joinKey": "zipcode", - "name": "zipcode", - "tags": { - "owner": "danny@tecton.ai", - "team": "hack week" - }, - "valueType": "INT64" - } - } - ], - "featureServices": [ - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.172623Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.172623Z" - }, - "spec": { - "description": "Credit scoring model", - "features": [ - { - "featureColumns": [ - { - "name": "credit_card_due", - "valueType": "INT64" - }, - { - "name": "missed_payments_1y", - "valueType": "INT64" - } - ], - "featureViewName": "credit_history" - }, - { - "featureColumns": [ - { - "name": "city", - "valueType": "STRING" - }, - { - "name": "state", - "valueType": "STRING" - }, - { - "name": "location_type", - "valueType": "STRING" - }, - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "population", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "featureViewName": "zipcode_features" - } - ], - "name": "credit_score_v1", - "tags": { - "owner": "tony@tecton.ai", - "stage": "staging" - } - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.172405Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.172405Z" - }, - "spec": { - "description": "Credit scoring model", - "features": [ - { - "featureColumns": [ - { - "name": "credit_card_due", - "valueType": "INT64" - }, - { - "name": "mortgage_due", - "valueType": "INT64" - }, - { - "name": "missed_payments_1y", - "valueType": "INT64" - } - ], - "featureViewName": "credit_history" - }, - { - "featureColumns": [ - { - "name": "city", - "valueType": "STRING" - }, - { - "name": "state", - "valueType": "STRING" - }, - { - "name": "location_type", - "valueType": "STRING" - }, - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "population", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "featureViewName": "zipcode_features" - } - ], - "name": "credit_score_v2", - "tags": { - "owner": "tony@tecton.ai", - "stage": "prod" - } - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.172264Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.172264Z" - }, - "spec": { - "description": "Credit scoring model", - "features": [ - { - "featureColumns": [ - { - "name": "credit_card_due", - "valueType": "INT64" - }, - { - "name": "mortgage_due", - "valueType": "INT64" - }, - { - "name": "missed_payments_1y", - "valueType": "INT64" - } - ], - "featureViewName": "credit_history" - }, - { - "featureColumns": [ - { - "name": "city", - "valueType": "STRING" - }, - { - "name": "state", - "valueType": "STRING" - }, - { - "name": "location_type", - "valueType": "STRING" - }, - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "population", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "featureViewName": "zipcode_features" - }, - { - "featureColumns": [ - { - "name": "transaction_gt_last_credit_card_due", - "valueType": "BOOL" - } - ], - "featureViewName": "transaction_gt_last_credit_card_due" - } - ], - "name": "credit_score_v3", - "tags": { - "owner": "tony@tecton.ai", - "stage": "dev" - } - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.172530Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.172530Z" - }, - "spec": { - "description": "Location model", - "features": [ - { - "featureColumns": [ - { - "name": "city", - "valueType": "STRING" - }, - { - "name": "state", - "valueType": "STRING" - }, - { - "name": "location_type", - "valueType": "STRING" - }, - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "population", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "featureViewName": "zipcode_features" - } - ], - "name": "zipcode_model", - "tags": { - "owner": "amanda@tecton.ai", - "stage": "dev" - } - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.172188Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.172188Z" - }, - "spec": { - "description": "Location model", - "features": [ - { - "featureColumns": [ - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "featureViewName": "zipcode_money_features" - } - ], - "name": "zipcode_model_v2", - "tags": { - "owner": "amanda@tecton.ai", - "stage": "dev" - } - } - } - ], - "featureViews": [ - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171421Z", - "lastUpdatedTimestamp": "2022-05-11T19:28:21.444739Z", - "materializationIntervals": [ - { - "endTime": "2019-05-11T19:27:05Z", - "startTime": "1997-09-19T19:27:13.273753Z" - }, - { - "endTime": "2022-05-11T19:27:43Z", - "startTime": "2019-05-11T19:27:05Z" - } - ] - }, - "spec": { - "batchSource": { - "createdTimestampColumn": "created_timestamp", - "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", - "fileOptions": { - "uri": "data/credit_history.parquet" - }, - "name": "credit_history", - "timestampField": "event_timestamp", - "type": "BATCH_FILE" - }, - "entities": [ - "dob_ssn" - ], - "features": [ - { - "name": "credit_card_due", - "valueType": "INT64" - }, - { - "name": "mortgage_due", - "valueType": "INT64" - }, - { - "name": "student_loan_due", - "valueType": "INT64" - }, - { - "name": "vehicle_loan_due", - "valueType": "INT64" - }, - { - "name": "hard_pulls", - "valueType": "INT64" - }, - { - "name": "missed_payments_2y", - "valueType": "INT64" - }, - { - "name": "missed_payments_1y", - "valueType": "INT64" - }, - { - "name": "missed_payments_6m", - "valueType": "INT64" - }, - { - "name": "bankruptcies", - "valueType": "INT64" - } - ], - "name": "credit_history", - "online": true, - "tags": { - "access_group": "feast-team@tecton.ai", - "date_added": "2022-02-6", - "experiments": "experiment-A" - }, - "ttl": "777600000s" - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171300Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:46.002348Z", - "materializationIntervals": [ - { - "endTime": "2019-05-11T19:27:05Z", - "startTime": "2012-05-13T19:27:08.483036Z" - }, - { - "endTime": "2022-05-11T19:27:43Z", - "startTime": "2019-05-11T19:27:05Z" - } - ] - }, - "spec": { - "batchSource": { - "createdTimestampColumn": "created_timestamp", - "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", - "fileOptions": { - "uri": "data/zipcode_table.parquet" - }, - "name": "zipcode", - "timestampField": "event_timestamp", - "type": "BATCH_FILE" - }, - "entities": [ - "zipcode" - ], - "features": [ - { - "name": "city", - "valueType": "STRING" - }, - { - "name": "state", - "valueType": "STRING" - }, - { - "name": "location_type", - "valueType": "STRING" - }, - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "population", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "name": "zipcode_features", - "online": true, - "tags": { - "access_group": "feast-team@tecton.ai", - "date_added": "2022-02-7", - "experiments": "experiment-A,experiment-B,experiment-C" - }, - "ttl": "315360000s" - } - }, - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171195Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:45.942549Z", - "materializationIntervals": [ - { - "endTime": "2019-05-11T19:27:05Z", - "startTime": "2012-05-13T19:27:06.493847Z" - }, - { - "endTime": "2022-05-11T19:27:43Z", - "startTime": "2019-05-11T19:27:05Z" - } - ] - }, - "spec": { - "batchSource": { - "createdTimestampColumn": "created_timestamp", - "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", - "fileOptions": { - "uri": "data/zipcode_table.parquet" - }, - "name": "zipcode", - "timestampField": "event_timestamp", - "type": "BATCH_FILE" - }, - "entities": [ - "zipcode" - ], - "features": [ - { - "name": "tax_returns_filed", - "valueType": "INT64" - }, - { - "name": "total_wages", - "valueType": "INT64" - } - ], - "name": "zipcode_money_features", - "online": true, - "tags": { - "access_group": "feast-team@tecton.ai", - "date_added": "2022-02-7", - "experiments": "experiment-A,experiment-B,experiment-C" - }, - "ttl": "315360000s" - } - } - ], - "infra": [ - { - "name": "credit_scoring_aws_credit_history", - "path": "/Users/dannychiao/GitHub/feast/ui/feature_repo/data/online.db" - }, - { - "name": "credit_scoring_aws_zipcode_features", - "path": "/Users/dannychiao/GitHub/feast/ui/feature_repo/data/online.db" - }, - { - "name": "credit_scoring_aws_zipcode_money_features", - "path": "/Users/dannychiao/GitHub/feast/ui/feature_repo/data/online.db" - } - ], - "onDemandFeatureViews": [ - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171556Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.171556Z" - }, - "spec": { - "features": [ - { - "name": "transaction_gt_last_credit_card_due", - "valueType": "BOOL" - } - ], - "name": "transaction_gt_last_credit_card_due", - "sources": { - "credit_history": { - "featureViewProjection": { - "featureColumns": [ - { - "name": "credit_card_due", - "valueType": "INT64" - }, - { - "name": "mortgage_due", - "valueType": "INT64" - }, - { - "name": "student_loan_due", - "valueType": "INT64" - }, - { - "name": "vehicle_loan_due", - "valueType": "INT64" - }, - { - "name": "hard_pulls", - "valueType": "INT64" - }, - { - "name": "missed_payments_2y", - "valueType": "INT64" - }, - { - "name": "missed_payments_1y", - "valueType": "INT64" - }, - { - "name": "missed_payments_6m", - "valueType": "INT64" - }, - { - "name": "bankruptcies", - "valueType": "INT64" - } - ], - "featureViewName": "credit_history" - } - }, - "transaction": { - "requestDataSource": { - "name": "transaction", - "requestDataOptions": { - "schema": [ - { - "name": "transaction_amt", - "valueType": "INT64" - } - ] - }, - "type": "REQUEST_SOURCE" - } - } - }, - "userDefinedFunction": { - "body": "@on_demand_feature_view(\n sources=[credit_history, input_request],\n schema=[\n Field(name=\"transaction_gt_last_credit_card_due\", dtype=Bool),\n ],\n)\ndef transaction_gt_last_credit_card_due(inputs: pd.DataFrame) -> pd.DataFrame:\n df = pd.DataFrame()\n df[\"transaction_gt_last_credit_card_due\"] = (\n inputs[\"transaction_amt\"] > inputs[\"credit_card_due\"]\n )\n return df\n", - "name": "transaction_gt_last_credit_card_due" - } - } - } - ], - "streamFeatureViews": [ - { - "meta": { - "createdTimestamp": "2022-05-11T19:27:03.171556Z", - "lastUpdatedTimestamp": "2022-05-11T19:27:03.171556Z" - }, - "spec": { - "batchSource": { - "createdTimestampColumn": "created_timestamp", - "dataSourceClassType": "feast.infra.offline_stores.file_source.FileSource", - "fileOptions": { - "uri": "data/zipcode_table.parquet" - }, - "name": "zipcode", - "timestampField": "event_timestamp", - "type": "BATCH_FILE" - }, - "features": [ - { - "name": "conv_percentage", - "valueType": "FLOAT" - }, - { - "name": "acc_percentage", - "valueType": "FLOAT" - } - ], - "name": "transaction_stream_example", - "streamSource": { - "batchSource": { - "fileOptions": { - "uri": "data/zipcode_table.parquet" - }, - "name": "user_stats", - "timestampField": "timestamp", - "type": "BATCH_FILE" - }, - "dataSourceClassType": "feast.data_source.KafkaSource", - "description": "The Kafka stream example", - "kafkaOptions": {"messageFormat": {"jsonFormat": {"schemaJson": "id string, timestamp timestamp"}}, - "watermarkDelayThreshold": "300s"}, - "name": "driver_stats_stream", - "owner": "test@gmail.com", - "timestampField": "timestamp", - "type": "STREAM_KAFKA" - }, - "ttl": "86400s", - "userDefinedFunction": { - "body": "@stream_feature_view(\n sources=[driver_stats_stream_source],\n mode=\"spark\",\n schema=[\n Field(name=\"conv_percentage\", dtype=Float32),\n Field(name=\"acc_percentage\", dtype=Float32),\n ],\n timestamp_field=\"event_timestamp\",\n online=True,\n source=driver_stats_stream_source,\n tags={},\n)\ndef driver_hourly_stats_stream(df: DataFrame) -> DataFrame:\n from pyspark.sql.functions import col\n return (\n df.withColumn(\"conv_percentage\", col(\"conv_rate\") * 100.0)\n .withColumn(\"acc_percentage\", col(\"acc_rate\") * 100.0)\n .drop(\"conv_rate\", \"acc_rate\")\n )\n", - "name": "driver_hourly_stats_stream" - } - } - } - ], - "project": "credit_scoring_aws" -} diff --git a/ui/src/components/FeaturesInServiceDisplay.tsx b/ui/src/components/FeaturesInServiceDisplay.tsx index 39091b81dec..6ca2fcc8b09 100644 --- a/ui/src/components/FeaturesInServiceDisplay.tsx +++ b/ui/src/components/FeaturesInServiceDisplay.tsx @@ -1,36 +1,26 @@ import React from "react"; -import { z } from "zod"; import { EuiBasicTable } from "@elastic/eui"; -import { FeastFeatureInServiceType } from "../parsers/feastFeatureServices"; import EuiCustomLink from "./EuiCustomLink"; -import { FEAST_FEATURE_VALUE_TYPES } from "../parsers/types"; import { useParams } from "react-router-dom"; +import { feast } from "../protos"; interface FeatureViewsListInterace { - featureViews: FeastFeatureInServiceType[]; + featureViews: feast.core.IFeatureViewProjection[]; +} + +interface IFeatureColumnInService { + featureViewName: string; + name: string; + valueType: feast.types.ValueType.Enum; } const FeaturesInServiceList = ({ featureViews }: FeatureViewsListInterace) => { const { projectName } = useParams(); - - const FeatureInService = z.object({ - featureViewName: z.string(), - featureColumnName: z.string(), - valueType: z.nativeEnum(FEAST_FEATURE_VALUE_TYPES), - }); - type FeatureInServiceType = z.infer; - - var items: FeatureInServiceType[] = []; - featureViews.forEach((featureView) => { - featureView.featureColumns.forEach((featureColumn) => { - const row: FeatureInServiceType = { - featureViewName: featureView.featureViewName, - featureColumnName: featureColumn.name, - valueType: featureColumn.valueType, - }; - items.push(row); - }); - }); + const items: IFeatureColumnInService[] = featureViews.flatMap(featureView => featureView.featureColumns!.map(featureColumn => ({ + featureViewName: featureView.featureViewName!, + name: featureColumn.name!, + valueType: featureColumn.valueType!, + }))); const columns = [ { @@ -49,15 +39,18 @@ const FeaturesInServiceList = ({ featureViews }: FeatureViewsListInterace) => { }, { name: "Feature Column", - field: "featureColumnName", + field: "name", }, { name: "Value Type", field: "valueType", + render: (valueType: feast.types.ValueType.Enum) => { + return feast.types.ValueType.Enum[valueType]; + }, }, ]; - const getRowProps = (item: FeatureInServiceType) => { + const getRowProps = (item: IFeatureColumnInService) => { return { "data-test-subj": `row-${item.featureViewName}`, }; diff --git a/ui/src/components/FeaturesListDisplay.tsx b/ui/src/components/FeaturesListDisplay.tsx index dcb6ba81eb1..a40730c6873 100644 --- a/ui/src/components/FeaturesListDisplay.tsx +++ b/ui/src/components/FeaturesListDisplay.tsx @@ -1,38 +1,39 @@ -import React, { useContext } from "react"; -import { EuiBasicTable, EuiLoadingSpinner, EuiBadge } from "@elastic/eui"; -import { FeastFeatureColumnType } from "../parsers/feastFeatureViews"; -import useLoadFeatureViewSummaryStatistics from "../queries/useLoadFeatureViewSummaryStatistics"; -import SparklineHistogram from "./SparklineHistogram"; -import FeatureFlagsContext from "../contexts/FeatureFlagsContext"; +import { EuiBasicTable } from "@elastic/eui"; import EuiCustomLink from "./EuiCustomLink"; +import { feast } from "../protos"; interface FeaturesListProps { projectName: string; featureViewName: string; - features: FeastFeatureColumnType[]; + features: feast.core.IFeatureSpecV2[]; link: boolean; } -const FeaturesList = ({ projectName, featureViewName, features, link }: FeaturesListProps) => { - const { enabledFeatureStatistics } = useContext(FeatureFlagsContext); - const { isLoading, isError, isSuccess, data } = - useLoadFeatureViewSummaryStatistics(featureViewName); - +const FeaturesList = ({ + projectName, + featureViewName, + features, + link, +}: FeaturesListProps) => { let columns: { name: string; render?: any; field: any }[] = [ - { + { name: "Name", field: "name", - render: (item: string) => ( - ( + + to={`/p/${projectName}/feature-view/${featureViewName}/feature/${item}`} + > {item} - ) + ), }, { name: "Value Type", field: "valueType", + render: (valueType: feast.types.ValueType.Enum) => { + return feast.types.ValueType.Enum[valueType]; + }, }, ]; @@ -40,50 +41,7 @@ const FeaturesList = ({ projectName, featureViewName, features, link }: Features columns[0].render = undefined; } - if (enabledFeatureStatistics) { - columns.push( - ...[ - { - name: "Sample", - field: "", - render: (item: FeastFeatureColumnType) => { - const statistics = - isSuccess && data && data.columnsSummaryStatistics[item.name]; - - return ( - - {isLoading && } - {isError && ( - error loading samples - )} - {statistics && statistics.sampleValues.join(",")} - - ); - }, - }, - { - name: "Sparklines", - field: "", - render: (item: FeastFeatureColumnType) => { - const statistics = - isSuccess && data && data.columnsSummaryStatistics[item.name]; - - if ( - statistics && - statistics.valueType === "INT64" && - statistics.histogram - ) { - return ; - } else { - return ""; - } - }, - }, - ] - ); - } - - const getRowProps = (item: FeastFeatureColumnType) => { + const getRowProps = (item: feast.core.IFeatureSpecV2) => { return { "data-test-subj": `row-${item.name}`, }; diff --git a/ui/src/components/NumericFeaturesTable.tsx b/ui/src/components/NumericFeaturesTable.tsx deleted file mode 100644 index 7c55f5ddbae..00000000000 --- a/ui/src/components/NumericFeaturesTable.tsx +++ /dev/null @@ -1,60 +0,0 @@ -import { EuiBasicTable } from "@elastic/eui"; -import React from "react"; -import { NumericColumnSummaryStatisticType } from "../parsers/featureViewSummaryStatistics"; -import SparklineHistogram from "./SparklineHistogram"; - -interface NumericFeaturesTableProps { - data: NumericColumnSummaryStatisticType[]; -} - -const NumericFeaturesTable = ({ data }: NumericFeaturesTableProps) => { - const columns = [ - { name: "Name", field: "name" }, - { - name: "Value Type", - field: "valueType", - }, - { - name: "Sample", - render: (statistics: NumericColumnSummaryStatisticType) => { - return ( - - {statistics && statistics.sampleValues.join(",")} - - ); - }, - }, - { - name: "Min/Max", - render: (statistics: NumericColumnSummaryStatisticType) => { - return statistics.min !== undefined && statistics.max !== undefined - ? `${statistics.min}/${statistics.max}` - : undefined; - }, - }, - { name: "zeros", field: "proportionOfZeros" }, - { name: "missing", field: "proportionMissing" }, - { - name: "Sparklines", - render: (statistics: NumericColumnSummaryStatisticType) => { - if (statistics && statistics.histogram) { - return ; - } else { - return ""; - } - }, - }, - ]; - - const getRowProps = (item: NumericColumnSummaryStatisticType) => { - return { - "data-test-subj": `row-${item.name}`, - }; - }; - - return ( - - ); -}; - -export default NumericFeaturesTable; diff --git a/ui/src/components/SparklineHistogram.tsx b/ui/src/components/SparklineHistogram.tsx deleted file mode 100644 index bd632ec20d1..00000000000 --- a/ui/src/components/SparklineHistogram.tsx +++ /dev/null @@ -1,58 +0,0 @@ -import React from "react"; -import { HistogramDataType } from "../parsers/featureViewSummaryStatistics"; -import { extent } from "d3-array"; -import { scaleLinear } from "d3"; -import { EuiBadge, useEuiTheme } from "@elastic/eui"; - -interface SparklineHistogramProps { - data: HistogramDataType; -} - -const SparklineHistogram = ({ data }: SparklineHistogramProps) => { - const width = 100; - const height = 24; - - const yMax = height - 2; - - const { euiTheme } = useEuiTheme(); - - if (data.length > 0) { - const x0Extent = extent(data, (d) => d.x0) as [number, number]; - const xScale = scaleLinear() - .domain(x0Extent) - .range([0, width - width / data.length]); - - const yExtent = extent(data, (d) => d.count) as [number, number]; - const yScale = scaleLinear().domain(yExtent).range([0, yMax]); - - return ( - - - {data.map((d) => { - const barHeight = yScale(d.count); - - return ( - - ); - })} - - ); - } else { - return histogram n/a; - } -}; - -export default SparklineHistogram; diff --git a/ui/src/hooks/useFCOExploreSuggestions.ts b/ui/src/hooks/useFCOExploreSuggestions.ts index 5767f73beda..822642daf4b 100644 --- a/ui/src/hooks/useFCOExploreSuggestions.ts +++ b/ui/src/hooks/useFCOExploreSuggestions.ts @@ -1,9 +1,9 @@ import { encodeSearchQueryString } from "./encodeSearchQueryString"; import { FEAST_FCO_TYPES } from "../parsers/types"; -import { FeastFeatureViewType } from "../parsers/feastFeatureViews"; import { useParams } from "react-router-dom"; import { useFeatureViewTagsAggregation } from "./useTagsAggregation"; +import { feast } from "../protos"; interface ExplorationSuggestionItem { name: string; @@ -66,14 +66,14 @@ const sortTagsByTotalUsage = ( }; const generateExplorationSuggestions = ( - tagAggregation: Record>, + tagAggregation: Record>, projectName: string ) => { const suggestions: ExplorationSuggestion[] = []; if (tagAggregation) { const SortedCandidates = - sortTagByUniqueValues(tagAggregation); + sortTagByUniqueValues(tagAggregation); SortedCandidates.slice(0, NUMBER_OF_SUGGESTION_GROUPS).forEach( ([selectedTag, selectedTagValuesMap]) => { diff --git a/ui/src/hooks/useTagsAggregation.ts b/ui/src/hooks/useTagsAggregation.ts index 21480fa8b0c..13d0c2f9669 100644 --- a/ui/src/hooks/useTagsAggregation.ts +++ b/ui/src/hooks/useTagsAggregation.ts @@ -1,8 +1,7 @@ import { useContext, useMemo } from "react"; import RegistryPathContext from "../contexts/RegistryPathContext"; -import { FeastFeatureServiceType } from "../parsers/feastFeatureServices"; -import { FeastFeatureViewType } from "../parsers/feastFeatureViews"; import useLoadRegistry from "../queries/useLoadRegistry"; +import { feast } from "../protos"; // Usage of generic type parameter T // https://stackoverflow.com/questions/53203409/how-to-tell-typescript-that-im-returning-an-array-of-arrays-of-the-input-type @@ -44,12 +43,12 @@ const useFeatureViewTagsAggregation = () => { const data = useMemo(() => { return query.data && query.data.objects && query.data.objects.featureViews - ? buildTagCollection( - query.data.objects.featureViews, - (fv) => { - return fv.spec.tags; - } - ) + ? buildTagCollection( + query.data.objects.featureViews!, + (fv) => { + return fv.spec?.tags!; + } + ) : undefined; }, [query.data]); @@ -67,12 +66,12 @@ const useFeatureServiceTagsAggregation = () => { return query.data && query.data.objects && query.data.objects.featureServices - ? buildTagCollection( - query.data.objects.featureServices, - (fs) => { - return fs.spec.tags; - } - ) + ? buildTagCollection( + query.data.objects.featureServices, + (fs) => { + return fs.spec?.tags!; + } + ) : undefined; }, [query.data]); diff --git a/ui/src/pages/data-sources/BatchSourcePropertiesView.tsx b/ui/src/pages/data-sources/BatchSourcePropertiesView.tsx index 97f9f58d274..c19e4ff50f8 100644 --- a/ui/src/pages/data-sources/BatchSourcePropertiesView.tsx +++ b/ui/src/pages/data-sources/BatchSourcePropertiesView.tsx @@ -1,38 +1,16 @@ import React from "react"; import { - EuiCodeBlock, EuiDescriptionList, EuiDescriptionListDescription, EuiDescriptionListTitle, EuiFlexGroup, EuiFlexItem, - EuiSpacer, - EuiTitle, } from "@elastic/eui"; +import { feast } from "../../protos"; +import { toDate } from "../../utils/timestamp"; interface BatchSourcePropertiesViewProps { - batchSource: { - type?: string | undefined; - owner?: string | undefined; - description?: string | undefined; - dataSourceClassType?: string | undefined; - fileOptions?: - | { - uri?: string | undefined; - } - | undefined; - meta?: - | { - latestEventTimestamp?: Date | undefined; - earliestEventTimestamp?: Date | undefined; - } - | undefined; - bigqueryOptions?: - | { - dbtModelSerialized?: string | undefined; - } - | undefined; - }; + batchSource: feast.core.IDataSource; } const BatchSourcePropertiesView = (props: BatchSourcePropertiesViewProps) => { @@ -49,7 +27,7 @@ const BatchSourcePropertiesView = (props: BatchSourcePropertiesViewProps) => { {batchSource.dataSourceClassType.split(".").at(-1)} - ) : batchSource.type ? ( + ) : feast.core.DataSource.SourceType[batchSource.type!] ? ( {batchSource.type} @@ -87,9 +65,9 @@ const BatchSourcePropertiesView = (props: BatchSourcePropertiesViewProps) => { Latest Event - {batchSource.meta.latestEventTimestamp.toLocaleDateString( - "en-CA" - )} + {toDate( + batchSource.meta.latestEventTimestamp + ).toLocaleDateString("en-CA")} )} @@ -99,35 +77,14 @@ const BatchSourcePropertiesView = (props: BatchSourcePropertiesViewProps) => { Earliest Event - {batchSource.meta.earliestEventTimestamp.toLocaleDateString( - "en-CA" - )} + {toDate( + batchSource.meta?.earliestEventTimestamp + ).toLocaleDateString("en-CA")} )} - - {batchSource.bigqueryOptions?.dbtModelSerialized && ( - - - - )} - {batchSource.bigqueryOptions?.dbtModelSerialized && ( - - -

Dbt Transformation

- - - {batchSource.bigqueryOptions.dbtModelSerialized} - - - )} ); diff --git a/ui/src/pages/data-sources/DataSourceDbt.tsx b/ui/src/pages/data-sources/DataSourceDbt.tsx deleted file mode 100644 index 4e61ba80266..00000000000 --- a/ui/src/pages/data-sources/DataSourceDbt.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import React from "react"; -import { - EuiCodeBlock, - EuiPanel, - EuiHorizontalRule, - EuiTitle, -} from "@elastic/eui"; -import { useParams } from "react-router-dom"; -import useLoadDataSource from "./useLoadDataSource"; - -const DataSourceDbt = () => { - let { dataSourceName } = useParams(); - - const dsName = dataSourceName === undefined ? "" : dataSourceName; - - const { isSuccess, data } = useLoadDataSource(dsName); - - return isSuccess && data && data.bigqueryOptions ? ( - - -

Dbt Transformation

- - - - {data.bigqueryOptions.dbtModelSerialized} - - - ) : ( - - No data so sad - - ); -}; - -export default DataSourceDbt; diff --git a/ui/src/pages/data-sources/DataSourceInstance.tsx b/ui/src/pages/data-sources/DataSourceInstance.tsx index 332f51e0235..d6db4c87472 100644 --- a/ui/src/pages/data-sources/DataSourceInstance.tsx +++ b/ui/src/pages/data-sources/DataSourceInstance.tsx @@ -7,12 +7,10 @@ import { } from "@elastic/eui"; import { DataSourceIcon32 } from "../../graphics/DataSourceIcon"; -import { useMatchExact, useMatchSubpath } from "../../hooks/useMatchSubpath"; +import { useMatchExact } from "../../hooks/useMatchSubpath"; import { useDocumentTitle } from "../../hooks/useDocumentTitle"; import DataSourceRawData from "./DataSourceRawData"; import DataSourceOverviewTab from "./DataSourceOverviewTab"; -import DataSourceDbt from "./DataSourceDbt"; -import useLoadDataSource from "./useLoadDataSource"; import { useDataSourceCustomTabs, @@ -24,8 +22,6 @@ const DataSourceInstance = () => { let { dataSourceName } = useParams(); useDocumentTitle(`${dataSourceName} | Data Source | Feast`); - const dsName = dataSourceName === undefined ? "" : dataSourceName; - const { isSuccess, data } = useLoadDataSource(dsName); let tabs = [ { @@ -37,17 +33,6 @@ const DataSourceInstance = () => { }, ]; - const dbtTab = { - label: "Dbt Definition", - isSelected: useMatchSubpath("dbt"), - onClick: () => { - navigate("dbt"); - }, - }; - if (isSuccess && data?.bigqueryOptions?.dbtModelSerialized) { - tabs.push(dbtTab); - } - const { customNavigationTabs } = useDataSourceCustomTabs(navigate); tabs = tabs.concat(customNavigationTabs); @@ -72,7 +57,6 @@ const DataSourceInstance = () => { } /> } /> - } /> {CustomTabRoutes} diff --git a/ui/src/pages/data-sources/DataSourceOverviewTab.tsx b/ui/src/pages/data-sources/DataSourceOverviewTab.tsx index 124a0e6ab92..6eadfc42d0a 100644 --- a/ui/src/pages/data-sources/DataSourceOverviewTab.tsx +++ b/ui/src/pages/data-sources/DataSourceOverviewTab.tsx @@ -19,6 +19,7 @@ import BatchSourcePropertiesView from "./BatchSourcePropertiesView"; import FeatureViewEdgesList from "../entities/FeatureViewEdgesList"; import RequestDataSourceSchemaTable from "./RequestDataSourceSchemaTable"; import useLoadDataSource from "./useLoadDataSource"; +import { feast } from "../../protos"; const DataSourceOverviewTab = () => { let { dataSourceName } = useParams(); @@ -57,7 +58,7 @@ const DataSourceOverviewTab = () => { Source Type - {data.type} + {feast.core.DataSource.SourceType[data.type]} @@ -77,12 +78,12 @@ const DataSourceOverviewTab = () => { { + fields={data?.requestDataOptions?.schema!.map((obj) => { return { - fieldName: obj.name, - valueType: obj.valueType, + fieldName: obj.name!, + valueType: obj.valueType!, }; - })} + })!} /> ) : ( diff --git a/ui/src/pages/data-sources/DataSourcesListingTable.tsx b/ui/src/pages/data-sources/DataSourcesListingTable.tsx index 661d18c7e9d..50c1f933a9c 100644 --- a/ui/src/pages/data-sources/DataSourcesListingTable.tsx +++ b/ui/src/pages/data-sources/DataSourcesListingTable.tsx @@ -1,11 +1,11 @@ import React from "react"; import { EuiBasicTable } from "@elastic/eui"; import EuiCustomLink from "../../components/EuiCustomLink"; -import { FeastDatasourceType } from "../../parsers/feastDatasources"; import { useParams } from "react-router-dom"; +import { feast } from "../../protos"; interface DatasourcesListingTableProps { - dataSources: FeastDatasourceType[]; + dataSources: feast.core.IDataSource[]; } const DatasourcesListingTable = ({ @@ -33,10 +33,13 @@ const DatasourcesListingTable = ({ name: "Type", field: "type", sortable: true, + render: (valueType: feast.types.ValueType.Enum) => { + return feast.types.ValueType.Enum[valueType]; + }, }, ]; - const getRowProps = (item: FeastDatasourceType) => { + const getRowProps = (item: feast.core.IDataSource) => { return { "data-test-subj": `row-${item.name}`, }; diff --git a/ui/src/pages/data-sources/RequestDataSourceSchemaTable.tsx b/ui/src/pages/data-sources/RequestDataSourceSchemaTable.tsx index 60ef4c406a4..a55aeac0280 100644 --- a/ui/src/pages/data-sources/RequestDataSourceSchemaTable.tsx +++ b/ui/src/pages/data-sources/RequestDataSourceSchemaTable.tsx @@ -1,10 +1,10 @@ import React from "react"; import { EuiBasicTable } from "@elastic/eui"; -import { FEAST_FEATURE_VALUE_TYPES } from "../../parsers/types"; +import { feast } from "../../protos"; interface RequestDataSourceSchemaField { fieldName: string; - valueType: FEAST_FEATURE_VALUE_TYPES; + valueType: feast.types.ValueType.Enum; } interface RequestDataSourceSchema { @@ -12,7 +12,6 @@ interface RequestDataSourceSchema { } const RequestDataSourceSchemaTable = ({ fields }: RequestDataSourceSchema) => { - console.log(fields); const columns = [ { name: "Field", @@ -21,6 +20,9 @@ const RequestDataSourceSchemaTable = ({ fields }: RequestDataSourceSchema) => { { name: "Value Type", field: "valueType", + render: (valueType: feast.types.ValueType.Enum) => { + return feast.types.ValueType.Enum[valueType]; + }, }, ]; diff --git a/ui/src/pages/entities/EntitiesListingTable.tsx b/ui/src/pages/entities/EntitiesListingTable.tsx index 2e608e37692..2a017b18aac 100644 --- a/ui/src/pages/entities/EntitiesListingTable.tsx +++ b/ui/src/pages/entities/EntitiesListingTable.tsx @@ -1,12 +1,12 @@ import React from "react"; import { EuiBasicTable } from "@elastic/eui"; import EuiCustomLink from "../../components/EuiCustomLink"; -import { FeastEntityType } from "../../parsers/feastEntities"; import useFeatureViewEdgesByEntity from "./useFeatureViewEdgesByEntity"; import { useParams } from "react-router-dom"; +import { feast } from "../../protos"; interface EntitiesListingTableProps { - entities: FeastEntityType[]; + entities: feast.core.IEntity[]; } const EntitiesListingTable = ({ entities }: EntitiesListingTableProps) => { @@ -33,15 +33,15 @@ const EntitiesListingTable = ({ entities }: EntitiesListingTableProps) => { name: "Type", field: "spec.valueType", sortable: true, - render: (valueType: string) => { - return valueType; + render: (valueType: feast.types.ValueType.Enum) => { + return feast.types.ValueType.Enum[valueType]; }, }, { name: "# of FVs", - render: (item: FeastEntityType) => { + render: (item: feast.core.IEntity) => { if (isSuccess && data) { - return data[item.spec.name] ? data[item.spec.name].length : "0"; + return data[item?.spec?.name!] ? data[item?.spec?.name!].length : "0"; } else { return "."; } @@ -49,9 +49,9 @@ const EntitiesListingTable = ({ entities }: EntitiesListingTableProps) => { }, ]; - const getRowProps = (item: FeastEntityType) => { + const getRowProps = (item: feast.core.IEntity) => { return { - "data-test-subj": `row-${item.spec.name}`, + "data-test-subj": `row-${item?.spec?.name}`, }; }; diff --git a/ui/src/pages/entities/EntityOverviewTab.tsx b/ui/src/pages/entities/EntityOverviewTab.tsx index dc649c2a9fb..b4df67bba49 100644 --- a/ui/src/pages/entities/EntityOverviewTab.tsx +++ b/ui/src/pages/entities/EntityOverviewTab.tsx @@ -19,6 +19,8 @@ import TagsDisplay from "../../components/TagsDisplay"; import FeatureViewEdgesList from "./FeatureViewEdgesList"; import useFeatureViewEdgesByEntity from "./useFeatureViewEdgesByEntity"; import useLoadEntity from "./useLoadEntity"; +import { toDate } from "../../utils/timestamp"; +import { feast } from "../../protos"; const EntityOverviewTab = () => { let { entityName } = useParams(); @@ -52,17 +54,17 @@ const EntityOverviewTab = () => { Join Key - {data.spec.joinKey} + {data?.spec?.joinKey} Description - {data.spec.description} + {data?.spec?.description} Value Type - {data.spec.valueType} + {feast.types.ValueType.Enum[data?.spec?.valueType!]} @@ -71,8 +73,8 @@ const EntityOverviewTab = () => { Created - {data.meta.createdTimestamp ? ( - data.meta.createdTimestamp.toLocaleDateString("en-CA") + {data?.meta?.createdTimestamp ? ( + toDate(data.meta.createdTimestamp).toLocaleDateString("en-CA") ) : ( No createdTimestamp specified on this entity. )} @@ -80,8 +82,8 @@ const EntityOverviewTab = () => { Updated - {data.meta.lastUpdatedTimestamp ? ( - data.meta.lastUpdatedTimestamp.toLocaleDateString("en-CA") + {data?.meta?.lastUpdatedTimestamp ? ( + toDate(data.meta.lastUpdatedTimestamp).toLocaleDateString("en-CA") ) : ( No lastUpdatedTimestamp specified on this entity. )} @@ -117,8 +119,8 @@ const EntityOverviewTab = () => {

Labels

- {data.spec.labels ? ( - + {data?.spec?.tags ? ( + ) : ( No labels specified on this entity. )} diff --git a/ui/src/pages/entities/useLoadEntity.ts b/ui/src/pages/entities/useLoadEntity.ts index a1ca6d55c16..6f7bb1f59cf 100644 --- a/ui/src/pages/entities/useLoadEntity.ts +++ b/ui/src/pages/entities/useLoadEntity.ts @@ -10,8 +10,8 @@ const useLoadEntity = (entityName: string) => { registryQuery.data === undefined ? undefined : registryQuery.data.objects.entities?.find( - (fv) => fv.spec.name === entityName - ); + (fv) => fv?.spec?.name === entityName + ); return { ...registryQuery, diff --git a/ui/src/pages/feature-services/FeatureServiceListingTable.tsx b/ui/src/pages/feature-services/FeatureServiceListingTable.tsx index b865da6e23c..c81edeaeb58 100644 --- a/ui/src/pages/feature-services/FeatureServiceListingTable.tsx +++ b/ui/src/pages/feature-services/FeatureServiceListingTable.tsx @@ -5,20 +5,18 @@ import { EuiTableFieldDataColumnType, } from "@elastic/eui"; import EuiCustomLink from "../../components/EuiCustomLink"; -import { - FeastFeatureInServiceType, - FeastFeatureServiceType, -} from "../../parsers/feastFeatureServices"; import { useParams } from "react-router-dom"; +import { feast } from "../../protos"; +import { toDate } from "../../utils/timestamp"; interface FeatureServiceListingTableProps { tagKeysSet: Set; - featureServices: FeastFeatureServiceType[]; + featureServices: feast.core.IFeatureService[]; } type FeatureServiceTypeColumn = - | EuiTableFieldDataColumnType - | EuiTableComputedColumnType; + | EuiTableFieldDataColumnType + | EuiTableComputedColumnType; const FeatureServiceListingTable = ({ tagKeysSet, @@ -44,10 +42,10 @@ const FeatureServiceListingTable = ({ { name: "# of Features", field: "spec.features", - render: (featureViews: FeastFeatureInServiceType[]) => { + render: (featureViews: feast.core.IFeatureViewProjection[]) => { var numFeatures = 0; featureViews.forEach((featureView) => { - numFeatures += featureView.featureColumns.length; + numFeatures += featureView.featureColumns!.length; }); return numFeatures; }, @@ -55,8 +53,8 @@ const FeatureServiceListingTable = ({ { name: "Last updated", field: "meta.lastUpdatedTimestamp", - render: (date: Date) => { - return date ? date.toLocaleDateString("en-CA") : "n/a"; + render: (date: any) => { + return date ? toDate(date).toLocaleDateString("en-CA") : "n/a"; }, }, ]; @@ -64,10 +62,10 @@ const FeatureServiceListingTable = ({ tagKeysSet.forEach((key) => { columns.push({ name: key, - render: (item: FeastFeatureServiceType) => { + render: (item: feast.core.IFeatureService) => { let tag = n/a; - const value = item.spec.tags ? item.spec.tags[key] : undefined; + const value = item?.spec?.tags ? item.spec.tags[key] : undefined; if (value) { tag = {value}; @@ -78,9 +76,9 @@ const FeatureServiceListingTable = ({ }); }); - const getRowProps = (item: FeastFeatureServiceType) => { + const getRowProps = (item: feast.core.IFeatureService) => { return { - "data-test-subj": `row-${item.spec.name}`, + "data-test-subj": `row-${item?.spec?.name}`, }; }; diff --git a/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx b/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx index ea62b3b3a70..387320778ff 100644 --- a/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx +++ b/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx @@ -19,6 +19,7 @@ import TagsDisplay from "../../components/TagsDisplay"; import { encodeSearchQueryString } from "../../hooks/encodeSearchQueryString"; import FeatureViewEdgesList from "../entities/FeatureViewEdgesList"; import useLoadFeatureService from "./useLoadFeatureService"; +import { toDate } from "../../utils/timestamp"; const FeatureServiceOverviewTab = () => { let { featureServiceName, projectName } = useParams(); @@ -32,9 +33,9 @@ const FeatureServiceOverviewTab = () => { let numFeatures = 0; let numFeatureViews = 0; if (data) { - data.spec.features.forEach((featureView) => { + data?.spec?.features?.forEach((featureView) => { numFeatureViews += 1; - numFeatures += featureView.featureColumns.length; + numFeatures += featureView?.featureColumns!.length; }); } @@ -66,10 +67,10 @@ const FeatureServiceOverviewTab = () => { description="Feature Views" /> - {data.meta.lastUpdatedTimestamp ? ( + {data?.meta?.lastUpdatedTimestamp ? ( {

Features

- {data.spec.features ? ( - + {data?.spec?.features ? ( + ) : ( No features specified for this feature service. @@ -103,7 +104,7 @@ const FeatureServiceOverviewTab = () => {

Tags

- {data.spec.tags ? ( + {data?.spec?.tags ? ( { @@ -154,11 +155,11 @@ const FeatureServiceOverviewTab = () => {

All Feature Views

- {data.spec.features.length > 0 ? ( + {data?.spec?.features?.length! > 0 ? ( { - return f.featureViewName; - })} + fvNames={data?.spec?.features?.map((f) => { + return f.featureViewName!; + })!} /> ) : ( No feature views in this feature service diff --git a/ui/src/pages/feature-services/Index.tsx b/ui/src/pages/feature-services/Index.tsx index 441f3cf82c9..d034a24bd7c 100644 --- a/ui/src/pages/feature-services/Index.tsx +++ b/ui/src/pages/feature-services/Index.tsx @@ -22,12 +22,12 @@ import { filterInputInterface, tagTokenGroupsType, } from "../../hooks/useSearchInputWithTags"; -import { FeastFeatureServiceType } from "../../parsers/feastFeatureServices"; import { useDocumentTitle } from "../../hooks/useDocumentTitle"; import RegistryPathContext from "../../contexts/RegistryPathContext"; import FeatureServiceIndexEmptyState from "./FeatureServiceIndexEmptyState"; import TagSearch from "../../components/TagSearch"; import { useFeatureServiceTagsAggregation } from "../../hooks/useTagsAggregation"; +import { feast } from "../../protos"; const useLoadFeatureServices = () => { const registryUrl = useContext(RegistryPathContext); @@ -45,11 +45,11 @@ const useLoadFeatureServices = () => { }; const shouldIncludeFSsGivenTokenGroups = ( - entry: FeastFeatureServiceType, + entry: feast.core.IFeatureService, tagTokenGroups: tagTokenGroupsType ) => { return Object.entries(tagTokenGroups).every(([key, values]) => { - const entryTagValue = entry.spec.tags ? entry.spec.tags[key] : undefined; + const entryTagValue = entry?.spec?.tags ? entry.spec.tags[key] : undefined; if (entryTagValue) { return values.every((value) => { @@ -62,7 +62,7 @@ const shouldIncludeFSsGivenTokenGroups = ( }; const filterFn = ( - data: FeastFeatureServiceType[], + data: feast.core.IFeatureService[], filterInput: filterInputInterface ) => { let filteredByTags = data; @@ -79,7 +79,7 @@ const filterFn = ( if (filterInput.searchTokens.length) { return filteredByTags.filter((entry) => { return filterInput.searchTokens.find((token) => { - return token.length >= 3 && entry.spec.name.indexOf(token) >= 0; + return token.length >= 3 && entry?.spec?.name?.indexOf(token)! >= 0; }); }); } diff --git a/ui/src/pages/feature-services/useLoadFeatureService.ts b/ui/src/pages/feature-services/useLoadFeatureService.ts index be2242eae04..7d991533a1d 100644 --- a/ui/src/pages/feature-services/useLoadFeatureService.ts +++ b/ui/src/pages/feature-services/useLoadFeatureService.ts @@ -13,23 +13,23 @@ const useLoadFeatureService = (featureServiceName: string) => { registryQuery.data === undefined ? undefined : registryQuery.data.objects.featureServices?.find( - (fs) => fs.spec.name === featureServiceName - ); + (fs) => fs?.spec?.name === featureServiceName + ); let entities = data === undefined ? undefined : registryQuery.data?.indirectRelationships - .filter((relationship) => { - return ( - relationship.target.type === FEAST_FCO_TYPES.featureService && - relationship.target.name === data.spec.name && - relationship.source.type === FEAST_FCO_TYPES.entity - ); - }) - .map((relationship) => { - return relationship.source; - }); + .filter((relationship) => { + return ( + relationship.target.type === FEAST_FCO_TYPES.featureService && + relationship.target.name === data?.spec?.name && + relationship.source.type === FEAST_FCO_TYPES.entity + ); + }) + .map((relationship) => { + return relationship.source; + }); // Deduplicate on name of entity if (entities) { let entityToName: { [key: string]: EntityReference } = {}; diff --git a/ui/src/pages/feature-views/FeatureViewInstance.tsx b/ui/src/pages/feature-views/FeatureViewInstance.tsx index 5352507573f..dbe4dad6ec1 100644 --- a/ui/src/pages/feature-views/FeatureViewInstance.tsx +++ b/ui/src/pages/feature-views/FeatureViewInstance.tsx @@ -3,15 +3,13 @@ import React from "react"; import { useParams } from "react-router-dom"; import { EuiLoadingSpinner } from "@elastic/eui"; -import { FeastFeatureViewType } from "../../parsers/feastFeatureViews"; import RegularFeatureInstance from "./RegularFeatureViewInstance"; import { FEAST_FV_TYPES } from "../../parsers/mergedFVTypes"; -import { FeastODFVType } from "../../parsers/feastODFVS"; -import { FeastSFVType } from "../../parsers/feastSFVS"; + import useLoadFeatureView from "./useLoadFeatureView"; import OnDemandFeatureInstance from "./OnDemandFeatureViewInstance"; import StreamFeatureInstance from "./StreamFeatureViewInstance"; - +import { feast } from "../../protos"; const FeatureViewInstance = () => { const { featureViewName } = useParams(); @@ -38,18 +36,18 @@ const FeatureViewInstance = () => { if (isSuccess && !isEmpty) { if (data.type === FEAST_FV_TYPES.regular) { - const fv: FeastFeatureViewType = data.object; + const fv: feast.core.IFeatureView = data.object; return ; } if (data.type === FEAST_FV_TYPES.ondemand) { - const odfv: FeastODFVType = data.object; + const odfv: feast.core.IOnDemandFeatureView = data.object; return ; } if (data.type === FEAST_FV_TYPES.stream) { - const sfv: FeastSFVType = data.object; + const sfv: feast.core.IStreamFeatureView = data.object; return ; } diff --git a/ui/src/pages/feature-views/FeatureViewListingTable.tsx b/ui/src/pages/feature-views/FeatureViewListingTable.tsx index ceb756db804..e4eccecc975 100644 --- a/ui/src/pages/feature-views/FeatureViewListingTable.tsx +++ b/ui/src/pages/feature-views/FeatureViewListingTable.tsx @@ -58,7 +58,7 @@ const FeatureViewListingTable = ({ let tag = n/a; if (item.type === "regular") { - const value = item.object.spec.tags + const value = item?.object?.spec!.tags ? item.object.spec.tags[key] : undefined; diff --git a/ui/src/pages/feature-views/FeatureViewSummaryStatisticsTab.tsx b/ui/src/pages/feature-views/FeatureViewSummaryStatisticsTab.tsx deleted file mode 100644 index 7371d4a73bf..00000000000 --- a/ui/src/pages/feature-views/FeatureViewSummaryStatisticsTab.tsx +++ /dev/null @@ -1,84 +0,0 @@ -import React from "react"; - -import { EuiEmptyPrompt, EuiLoadingContent, EuiTitle } from "@elastic/eui"; -import { useParams } from "react-router-dom"; -import useLoadFeatureViewSummaryStatistics from "../../queries/useLoadFeatureViewSummaryStatistics"; -import { - NumericColumnSummaryStatisticType, - StringColumnSummaryStatisticType, -} from "../../parsers/featureViewSummaryStatistics"; -import NumericFeaturesTable from "../../components/NumericFeaturesTable"; - -interface ColumnsByGroup { - INT64?: NumericColumnSummaryStatisticType[]; - STRING?: StringColumnSummaryStatisticType[]; -} - -const FeatureViewSummaryStatisticsTab = () => { - let { featureViewName } = useParams(); - - if (!featureViewName) { - throw new Error("Unable to get Feature View Name"); - } - - const { isError, data } = - useLoadFeatureViewSummaryStatistics(featureViewName); - - if (isError) { - return ( - Error loading Statistics} - body={ -

- There was an error loading statistics for{" "} - {featureViewName}. Please check that statistics - have been generated. -

- } - /> - ); - } - - if (data) { - const columnsByGroup = Object.entries( - data.columnsSummaryStatistics - ).reduce((memo, [key, columnStatistics]) => { - if (columnStatistics.valueType === "INT64") { - if (!memo["INT64"]) { - memo[columnStatistics.valueType] = [columnStatistics]; - } else { - memo["INT64"].push(columnStatistics); - } - } - - if (columnStatistics.valueType === "STRING") { - if (!memo["STRING"]) { - memo[columnStatistics.valueType] = [columnStatistics]; - } else { - memo["STRING"].push(columnStatistics); - } - } - - return memo; - }, {}); - - return ( - - {columnsByGroup["INT64"] && ( - <> - -

Numeric Columns

- - - - )} - - ); - } - - return ; -}; - -export default FeatureViewSummaryStatisticsTab; diff --git a/ui/src/pages/feature-views/Index.tsx b/ui/src/pages/feature-views/Index.tsx index 3abd42a22b2..b874a532362 100644 --- a/ui/src/pages/feature-views/Index.tsx +++ b/ui/src/pages/feature-views/Index.tsx @@ -48,7 +48,7 @@ const shouldIncludeFVsGivenTokenGroups = ( tagTokenGroups: Record ) => { return Object.entries(tagTokenGroups).every(([key, values]) => { - const entryTagValue = entry.object.spec.tags + const entryTagValue = entry?.object?.spec!.tags ? entry.object.spec.tags[key] : undefined; diff --git a/ui/src/pages/feature-views/OnDemandFeatureViewInstance.tsx b/ui/src/pages/feature-views/OnDemandFeatureViewInstance.tsx index 1db2f5194fd..166746df220 100644 --- a/ui/src/pages/feature-views/OnDemandFeatureViewInstance.tsx +++ b/ui/src/pages/feature-views/OnDemandFeatureViewInstance.tsx @@ -9,16 +9,16 @@ import { import { FeatureViewIcon32 } from "../../graphics/FeatureViewIcon"; import { useMatchExact } from "../../hooks/useMatchSubpath"; -import { FeastODFVType } from "../../parsers/feastODFVS"; import OnDemandFeatureViewOverviewTab from "./OnDemandFeatureViewOverviewTab"; import { useOnDemandFeatureViewCustomTabs, useOnDemandFeatureViewCustomTabRoutes, } from "../../custom-tabs/TabsRegistryContext"; +import { feast } from "../../protos"; interface OnDemandFeatureInstanceProps { - data: FeastODFVType; + data: feast.core.IOnDemandFeatureView; } const OnDemandFeatureInstance = ({ data }: OnDemandFeatureInstanceProps) => { diff --git a/ui/src/pages/feature-views/OnDemandFeatureViewOverviewTab.tsx b/ui/src/pages/feature-views/OnDemandFeatureViewOverviewTab.tsx index 0922f62102b..ee8e41bbf6c 100644 --- a/ui/src/pages/feature-views/OnDemandFeatureViewOverviewTab.tsx +++ b/ui/src/pages/feature-views/OnDemandFeatureViewOverviewTab.tsx @@ -10,11 +10,6 @@ import { } from "@elastic/eui"; import React from "react"; import FeaturesListDisplay from "../../components/FeaturesListDisplay"; -import { - FeastODFVType, - RequestDataSourceType, - FeatureViewProjectionType, -} from "../../parsers/feastODFVS"; import { useParams } from "react-router-dom"; import { EntityRelation } from "../../parsers/parseEntityRelationships"; import { FEAST_FCO_TYPES } from "../../parsers/types"; @@ -22,9 +17,10 @@ import useLoadRelationshipData from "../../queries/useLoadRelationshipsData"; import FeatureViewProjectionDisplayPanel from "./components/FeatureViewProjectionDisplayPanel"; import RequestDataDisplayPanel from "./components/RequestDataDisplayPanel"; import ConsumingFeatureServicesList from "./ConsumingFeatureServicesList"; +import { feast } from "../../protos"; interface OnDemandFeatureViewOverviewTabProps { - data: FeastODFVType; + data: feast.core.IOnDemandFeatureView; } const whereFSconsumesThisFv = (fvName: string) => { @@ -39,13 +35,13 @@ const whereFSconsumesThisFv = (fvName: string) => { const OnDemandFeatureViewOverviewTab = ({ data, }: OnDemandFeatureViewOverviewTabProps) => { - const inputs = Object.entries(data.spec.sources); + const inputs = Object.entries(data?.spec?.sources!); const { projectName } = useParams(); const relationshipQuery = useLoadRelationshipData(); const fsNames = relationshipQuery.data ? relationshipQuery.data - .filter(whereFSconsumesThisFv(data.spec.name)) + .filter(whereFSconsumesThisFv(data?.spec?.name!)) .map((fs) => { return fs.target.name; }) @@ -61,7 +57,7 @@ const OnDemandFeatureViewOverviewTab = ({ - {data.spec.userDefinedFunction.body} + {data?.spec?.userDefinedFunction?.bodyText} @@ -70,13 +66,13 @@ const OnDemandFeatureViewOverviewTab = ({ -

Features ({data.spec.features.length})

+

Features ({data?.spec?.features!.length})

 - {projectName && data.spec.features ? ( + {projectName && data?.spec?.features ? ( @@ -93,21 +89,26 @@ const OnDemandFeatureViewOverviewTab = ({ {inputs.map(([key, inputGroup]) => { - if ((inputGroup as RequestDataSourceType).requestDataSource) { + if ( + (inputGroup as feast.core.IOnDemandSource).requestDataSource + ) { return ( ); } - if (inputGroup as FeatureViewProjectionType) { + if ( + (inputGroup as feast.core.IOnDemandSource) + .featureViewProjection + ) { return ( ); diff --git a/ui/src/pages/feature-views/RegularFeatureViewInstance.tsx b/ui/src/pages/feature-views/RegularFeatureViewInstance.tsx index 7200163614b..6d34745d7cb 100644 --- a/ui/src/pages/feature-views/RegularFeatureViewInstance.tsx +++ b/ui/src/pages/feature-views/RegularFeatureViewInstance.tsx @@ -9,18 +9,17 @@ import { import { FeatureViewIcon32 } from "../../graphics/FeatureViewIcon"; import { useMatchExact, useMatchSubpath } from "../../hooks/useMatchSubpath"; -import { FeastFeatureViewType } from "../../parsers/feastFeatureViews"; import RegularFeatureViewOverviewTab from "./RegularFeatureViewOverviewTab"; -import FeatureViewSummaryStatisticsTab from "./FeatureViewSummaryStatisticsTab"; import { useRegularFeatureViewCustomTabs, useRegularFeatureViewCustomTabRoutes, } from "../../custom-tabs/TabsRegistryContext"; import FeatureFlagsContext from "../../contexts/FeatureFlagsContext"; +import { feast } from "../../protos"; interface RegularFeatureInstanceProps { - data: FeastFeatureViewType; + data: feast.core.IFeatureView; } const RegularFeatureInstance = ({ data }: RegularFeatureInstanceProps) => { @@ -58,7 +57,7 @@ const RegularFeatureInstance = ({ data }: RegularFeatureInstanceProps) => { { path="/" element={} /> - } - /> {TabRoutes} diff --git a/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx b/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx index 689bc6b9024..3bbb906e05b 100644 --- a/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx +++ b/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx @@ -15,12 +15,13 @@ import { useNavigate, useParams } from "react-router-dom"; import FeaturesListDisplay from "../../components/FeaturesListDisplay"; import TagsDisplay from "../../components/TagsDisplay"; import { encodeSearchQueryString } from "../../hooks/encodeSearchQueryString"; -import { FeastFeatureViewType } from "../../parsers/feastFeatureViews"; import { EntityRelation } from "../../parsers/parseEntityRelationships"; import { FEAST_FCO_TYPES } from "../../parsers/types"; import useLoadRelationshipData from "../../queries/useLoadRelationshipsData"; import BatchSourcePropertiesView from "../data-sources/BatchSourcePropertiesView"; import ConsumingFeatureServicesList from "./ConsumingFeatureServicesList"; +import { feast } from "../../protos"; +import { toDate } from "../../utils/timestamp"; const whereFSconsumesThisFv = (fvName: string) => { return (r: EntityRelation) => { @@ -32,7 +33,7 @@ const whereFSconsumesThisFv = (fvName: string) => { }; interface RegularFeatureViewOverviewTabProps { - data: FeastFeatureViewType; + data: feast.core.IFeatureView; } const RegularFeatureViewOverviewTab = ({ @@ -49,8 +50,8 @@ const RegularFeatureViewOverviewTab = ({ const fsNames = relationshipQuery.data ? relationshipQuery.data.filter(whereFSconsumesThisFv(fvName)).map((fs) => { - return fs.target.name; - }) + return fs.target.name; + }) : []; const numOfFs = fsNames.length; @@ -66,13 +67,13 @@ const RegularFeatureViewOverviewTab = ({ -

Features ({data.spec.features.length})

+

Features ({data?.spec?.features?.length})

 - {projectName && data.spec.features ? ( + {projectName && data?.spec?.features ? ( @@ -87,7 +88,7 @@ const RegularFeatureViewOverviewTab = ({

Entities

- {data.spec.entities ? ( + {data?.spec?.entities ? ( {data.spec.entities.map((entity) => { return ( @@ -128,7 +129,7 @@ const RegularFeatureViewOverviewTab = ({

Tags

- {data.spec.tags ? ( + {data?.spec?.tags ? ( { @@ -137,8 +138,8 @@ const RegularFeatureViewOverviewTab = ({ encodeSearchQueryString(`${key}:${value}`) ); }} - owner={data.spec.owner} - description={data.spec.description} + owner={data?.spec?.owner!} + description={data?.spec?.description!} /> ) : ( No Tags specified on this feature view. @@ -154,7 +155,7 @@ const RegularFeatureViewOverviewTab = ({

Batch Source

- + @@ -164,11 +165,11 @@ const RegularFeatureViewOverviewTab = ({

Materialization Intervals

- {data.meta.materializationIntervals?.map((interval, i) => { + {data?.meta?.materializationIntervals?.map((interval, i) => { return (

- {interval.startTime.toLocaleDateString("en-CA")} to{" "} - {interval.endTime.toLocaleDateString("en-CA")} + {toDate(interval.startTime!).toLocaleDateString("en-CA")} to{" "} + {toDate(interval.endTime!).toLocaleDateString("en-CA")}

); })} diff --git a/ui/src/pages/feature-views/StreamFeatureViewInstance.tsx b/ui/src/pages/feature-views/StreamFeatureViewInstance.tsx index ba4c0087278..52bc06bc5e1 100644 --- a/ui/src/pages/feature-views/StreamFeatureViewInstance.tsx +++ b/ui/src/pages/feature-views/StreamFeatureViewInstance.tsx @@ -9,16 +9,16 @@ import { import { FeatureViewIcon32 } from "../../graphics/FeatureViewIcon"; import { useMatchExact } from "../../hooks/useMatchSubpath"; -import { FeastSFVType } from "../../parsers/feastSFVS"; import StreamFeatureViewOverviewTab from "./StreamFeatureViewOverviewTab"; import { useStreamFeatureViewCustomTabs, useStreamFeatureViewCustomTabRoutes, } from "../../custom-tabs/TabsRegistryContext"; +import { feast } from "../../protos"; interface StreamFeatureInstanceProps { - data: FeastSFVType; + data: feast.core.IStreamFeatureView; } const StreamFeatureInstance = ({ data }: StreamFeatureInstanceProps) => { diff --git a/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx b/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx index 56efc428453..b6cf12d3725 100644 --- a/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx +++ b/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx @@ -10,18 +10,16 @@ import { } from "@elastic/eui"; import React from "react"; import FeaturesListDisplay from "../../components/FeaturesListDisplay"; -import { - FeastSFVType, -} from "../../parsers/feastSFVS"; import { useParams } from "react-router-dom"; import { EntityRelation } from "../../parsers/parseEntityRelationships"; import { FEAST_FCO_TYPES } from "../../parsers/types"; import useLoadRelationshipData from "../../queries/useLoadRelationshipsData"; import ConsumingFeatureServicesList from "./ConsumingFeatureServicesList"; import EuiCustomLink from "../../components/EuiCustomLink"; +import { feast } from "../../protos"; interface StreamFeatureViewOverviewTabProps { - data: FeastSFVType; + data: feast.core.IStreamFeatureView; } const whereFSconsumesThisFv = (fvName: string) => { @@ -36,13 +34,13 @@ const whereFSconsumesThisFv = (fvName: string) => { const StreamFeatureViewOverviewTab = ({ data, }: StreamFeatureViewOverviewTabProps) => { - const inputs = Object.entries([data.spec.streamSource]); + const inputs = Object.entries([data.spec?.streamSource]); const { projectName } = useParams(); const relationshipQuery = useLoadRelationshipData(); const fsNames = relationshipQuery.data ? relationshipQuery.data - .filter(whereFSconsumesThisFv(data.spec.name)) + .filter(whereFSconsumesThisFv(data.spec?.name!)) .map((fs) => { return fs.target.name; }) @@ -58,7 +56,7 @@ const StreamFeatureViewOverviewTab = ({ - {data.spec.userDefinedFunction.body} + {data.spec?.userDefinedFunction?.body} @@ -67,13 +65,13 @@ const StreamFeatureViewOverviewTab = ({ -

Features ({data.spec.features.length})

+

Features ({data.spec?.features?.length})

 - {projectName && data.spec.features ? ( + {projectName && data.spec?.features ? ( @@ -98,10 +96,10 @@ const StreamFeatureViewOverviewTab = ({ - {inputGroup.name} + {inputGroup?.name} diff --git a/ui/src/pages/feature-views/components/FeatureViewProjectionDisplayPanel.tsx b/ui/src/pages/feature-views/components/FeatureViewProjectionDisplayPanel.tsx index 7b110f326d7..156f6db1ec6 100644 --- a/ui/src/pages/feature-views/components/FeatureViewProjectionDisplayPanel.tsx +++ b/ui/src/pages/feature-views/components/FeatureViewProjectionDisplayPanel.tsx @@ -1,25 +1,26 @@ import React from "react"; import { EuiBasicTable, EuiPanel, EuiText, EuiTitle } from "@elastic/eui"; -import { FeatureViewProjectionType } from "../../../parsers/feastODFVS"; import { useParams } from "react-router-dom"; import EuiCustomLink from "../../../components/EuiCustomLink"; +import { feast } from "../../../protos"; -interface RequestDataDisplayPanelProps extends FeatureViewProjectionType {} +interface RequestDataDisplayPanelProps extends feast.core.IFeatureViewProjection { } -const FeatureViewProjectionDisplayPanel = ({ - featureViewProjection, -}: RequestDataDisplayPanelProps) => { +const FeatureViewProjectionDisplayPanel = (featureViewProjection: RequestDataDisplayPanelProps) => { const { projectName } = useParams(); const columns = [ { name: "Column Name", - field: "name", + field: "name" }, { name: "Type", field: "valueType", + render: (valueType: any) => { + return feast.types.ValueType.Enum[valueType]; + }, }, ]; @@ -33,12 +34,12 @@ const FeatureViewProjectionDisplayPanel = ({ href={`/p/${projectName}/feature-view/${featureViewProjection.featureViewName}`} to={`/p/${projectName}/feature-view/${featureViewProjection.featureViewName}`} > - {featureViewProjection.featureViewName} + {featureViewProjection?.featureViewName} ); diff --git a/ui/src/pages/feature-views/components/RequestDataDisplayPanel.tsx b/ui/src/pages/feature-views/components/RequestDataDisplayPanel.tsx index a6e546d9d89..e8e6854389a 100644 --- a/ui/src/pages/feature-views/components/RequestDataDisplayPanel.tsx +++ b/ui/src/pages/feature-views/components/RequestDataDisplayPanel.tsx @@ -1,17 +1,17 @@ import React from "react"; import { EuiBasicTable, EuiPanel, EuiText, EuiTitle } from "@elastic/eui"; import { useParams } from "react-router-dom"; -import { RequestDataSourceType } from "../../../parsers/feastODFVS"; import EuiCustomLink from "../../../components/EuiCustomLink"; +import { feast } from "../../../protos"; -interface RequestDataDisplayPanelProps extends RequestDataSourceType {} +interface RequestDataDisplayPanelProps extends feast.core.IOnDemandSource { } const RequestDataDisplayPanel = ({ requestDataSource, }: RequestDataDisplayPanelProps) => { const { projectName } = useParams(); - const items = Object.entries(requestDataSource.requestDataOptions.schema).map( + const items = Object.entries(requestDataSource?.requestDataOptions?.schema!).map( ([key, type]) => { return { key, @@ -38,10 +38,10 @@ const RequestDataDisplayPanel = ({ - {requestDataSource.name} + {requestDataSource?.name} diff --git a/ui/src/pages/feature-views/useLoadFeatureView.ts b/ui/src/pages/feature-views/useLoadFeatureView.ts index 7685171b72b..14970360f24 100644 --- a/ui/src/pages/feature-views/useLoadFeatureView.ts +++ b/ui/src/pages/feature-views/useLoadFeatureView.ts @@ -25,8 +25,8 @@ const useLoadRegularFeatureView = (featureViewName: string) => { registryQuery.data === undefined ? undefined : registryQuery.data.objects.featureViews?.find((fv) => { - return fv.spec.name === featureViewName; - }); + return fv?.spec?.name === featureViewName; + }); return { ...registryQuery, @@ -42,8 +42,8 @@ const useLoadOnDemandFeatureView = (featureViewName: string) => { registryQuery.data === undefined ? undefined : registryQuery.data.objects.onDemandFeatureViews?.find((fv) => { - return fv.spec.name === featureViewName; - }); + return fv?.spec?.name === featureViewName; + }); return { ...registryQuery, @@ -59,7 +59,7 @@ const useLoadStreamFeatureView = (featureViewName: string) => { registryQuery.data === undefined ? undefined : registryQuery.data.objects.streamFeatureViews?.find((fv) => { - return fv.spec.name === featureViewName; + return fv.spec?.name === featureViewName; }); return { diff --git a/ui/src/pages/features/FeatureOverviewTab.tsx b/ui/src/pages/features/FeatureOverviewTab.tsx index 11a2a0ab6a5..92adfe1501d 100644 --- a/ui/src/pages/features/FeatureOverviewTab.tsx +++ b/ui/src/pages/features/FeatureOverviewTab.tsx @@ -16,6 +16,7 @@ import TagsDisplay from "../../components/TagsDisplay"; import React from "react"; import { useParams } from "react-router-dom"; import useLoadFeature from "./useLoadFeature"; +import { feast } from "../../protos"; const FeatureOverviewTab = () => { let { projectName, FeatureViewName, FeatureName } = useParams(); @@ -51,7 +52,7 @@ const FeatureOverviewTab = () => { Value Type - {featureData?.valueType} + {feast.types.ValueType.Enum[featureData?.valueType!]} FeatureView diff --git a/ui/src/pages/features/useLoadFeature.ts b/ui/src/pages/features/useLoadFeature.ts index 5ddaf282043..f1918dd4d59 100644 --- a/ui/src/pages/features/useLoadFeature.ts +++ b/ui/src/pages/features/useLoadFeature.ts @@ -10,15 +10,15 @@ const useLoadFeature = (featureViewName: string, featureName: string) => { registryQuery.data === undefined ? undefined : registryQuery.data.objects.featureViews?.find((fv) => { - return fv.spec.name === featureViewName; - }); + return fv?.spec?.name === featureViewName; + }); - const featureData = + const featureData = data === undefined ? undefined - : data?.spec.features.find((f) => { - return f.name === featureName; - }); + : data?.spec?.features?.find((f) => { + return f.name === featureName; + }); return { ...registryQuery, diff --git a/ui/src/pages/saved-data-sets/DatasetExpectationsTab.tsx b/ui/src/pages/saved-data-sets/DatasetExpectationsTab.tsx index 10ebb87297b..dc49355c28f 100644 --- a/ui/src/pages/saved-data-sets/DatasetExpectationsTab.tsx +++ b/ui/src/pages/saved-data-sets/DatasetExpectationsTab.tsx @@ -11,7 +11,7 @@ const DatasetExpectationsTab = () => { } const { isSuccess, data } = useLoadDataset(datasetName); - if (!data || !data.spec.profile) { + if (!data || !data.spec?.name) { return ( No data so sad @@ -21,15 +21,9 @@ const DatasetExpectationsTab = () => { let expectationsData; - try { - expectationsData = JSON.parse(data.spec.profile); - } catch (e) { - throw new Error(`Unable to parse Expectations Profile: ${e}`); - } - - return isSuccess && expectationsData ? ( + return isSuccess ? ( - 
{JSON.stringify(expectationsData, null, 2)}
+ 
{JSON.stringify(data.spec, null, 2)}
 ) : ( diff --git a/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx b/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx index a20c83b1e21..5f6ffa2101c 100644 --- a/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx +++ b/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx @@ -15,6 +15,7 @@ import { useParams } from "react-router-dom"; import DatasetFeaturesTable from "./DatasetFeaturesTable"; import DatasetJoinKeysTable from "./DatasetJoinKeysTable"; import useLoadDataset from "./useLoadDataset"; +import { toDate } from "../../utils/timestamp"; const EntityOverviewTab = () => { let { datasetName } = useParams(); @@ -47,7 +48,7 @@ const EntityOverviewTab = () => { { + features={data.spec?.features!.map((joinedName: string) => { const [featureViewName, featureName] = joinedName.split(":"); @@ -55,7 +56,7 @@ const EntityOverviewTab = () => { featureViewName, featureName, }; - })} + })!} /> @@ -65,9 +66,9 @@ const EntityOverviewTab = () => { { + joinKeys={data?.spec?.joinKeys!.map((joinKey) => { return { name: joinKey }; - })} + })!} /> @@ -82,7 +83,7 @@ const EntityOverviewTab = () => { Source Feature Service - {data.spec.featureService} + {data?.spec?.featureServiceName!} @@ -91,7 +92,7 @@ const EntityOverviewTab = () => { Created - {data.meta.createdTimestamp.toLocaleDateString("en-CA")} + {toDate(data?.meta?.createdTimestamp!).toLocaleDateString("en-CA")} diff --git a/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx b/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx index 97d11b0b24c..a1a97084171 100644 --- a/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx +++ b/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx @@ -2,10 +2,11 @@ import React from "react"; import { EuiBasicTable } from "@elastic/eui"; import EuiCustomLink from "../../components/EuiCustomLink"; import { useParams } from "react-router-dom"; -import { FeastSavedDatasetType } from "../../parsers/feastSavedDataset"; +import { feast } from "../../protos"; +import { toDate } from "../../utils/timestamp"; interface DatasetsListingTableProps { - datasets: FeastSavedDatasetType[]; + datasets: feast.core.ISavedDataset[]; } const DatasetsListingTable = ({ datasets }: DatasetsListingTableProps) => { @@ -33,15 +34,15 @@ const DatasetsListingTable = ({ datasets }: DatasetsListingTableProps) => { }, { name: "Created", - render: (item: FeastSavedDatasetType) => { - return item.meta.createdTimestamp.toLocaleDateString("en-CA"); + render: (item: feast.core.ISavedDataset) => { + return toDate(item?.meta?.createdTimestamp!).toLocaleString("en-CA")!; }, }, ]; - const getRowProps = (item: FeastSavedDatasetType) => { + const getRowProps = (item: feast.core.ISavedDataset) => { return { - "data-test-subj": `row-${item.spec.name}`, + "data-test-subj": `row-${item.spec?.name}`, }; }; diff --git a/ui/src/pages/saved-data-sets/useLoadDataset.ts b/ui/src/pages/saved-data-sets/useLoadDataset.ts index a3dbd3225d8..17a77d97993 100644 --- a/ui/src/pages/saved-data-sets/useLoadDataset.ts +++ b/ui/src/pages/saved-data-sets/useLoadDataset.ts @@ -10,8 +10,8 @@ const useLoadEntity = (entityName: string) => { registryQuery.data === undefined ? undefined : registryQuery.data.objects.savedDatasets?.find( - (fv) => fv.spec.name === entityName - ); + (fv) => fv.spec?.name === entityName + ); return { ...registryQuery, diff --git a/ui/src/parsers/feastDatasources.ts b/ui/src/parsers/feastDatasources.ts deleted file mode 100644 index 3e1dca72d1f..00000000000 --- a/ui/src/parsers/feastDatasources.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { z } from "zod"; -import { FeastFeatureColumnSchema } from "./feastFeatureViews"; - -const FeastDatasourceSchema = z.object({ - type: z.string(), - eventTimestampColumn: z.string().optional(), - createdTimestampColumn: z.string().optional(), - fileOptions: z.object({ - uri: z.string().optional(), - }).optional(), - name: z.string(), - description: z.string().optional(), - owner: z.string().optional(), - meta: z.object({ - latestEventTimestamp: z.string().transform((val) => new Date(val)), - earliestEventTimestamp: z.string().transform((val) => new Date(val)), - }).optional(), - requestDataOptions: z.object({ - schema: z.array(FeastFeatureColumnSchema), - }).optional(), - bigqueryOptions: z.object({ - tableRef: z.string().optional(), - dbtModelSerialized: z.string().optional() - }).optional(), -}); - -type FeastDatasourceType = z.infer; - -export { FeastDatasourceSchema }; -export type { FeastDatasourceType }; diff --git a/ui/src/parsers/feastEntities.ts b/ui/src/parsers/feastEntities.ts deleted file mode 100644 index 09057c6fe92..00000000000 --- a/ui/src/parsers/feastEntities.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { z } from "zod"; -import { FEAST_FEATURE_VALUE_TYPES } from "./types"; - -const FeastEntitySchema = z.object({ - spec: z.object({ - name: z.string(), - valueType: z.nativeEnum(FEAST_FEATURE_VALUE_TYPES).optional(), - joinKey: z.string(), - description: z.string().optional(), - labels: z.record(z.string()).optional(), - }), - meta: z.object({ - createdTimestamp: z.string().transform((val) => new Date(val)).optional(), - lastUpdatedTimestamp: z.string().transform((val) => new Date(val)).optional(), - }), -}); - -type FeastEntityType = z.infer; - -export { FeastEntitySchema }; -export type { FeastEntityType }; diff --git a/ui/src/parsers/feastFeatureServices.ts b/ui/src/parsers/feastFeatureServices.ts deleted file mode 100644 index 6812b7e02cb..00000000000 --- a/ui/src/parsers/feastFeatureServices.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { z } from "zod"; -import { FEAST_FEATURE_VALUE_TYPES } from "./types"; - -const FeatureColumnInService = z.object({ - name: z.string(), - valueType: z.nativeEnum(FEAST_FEATURE_VALUE_TYPES), -}); - -const FeatureInServiceSchema = z.object({ - featureViewName: z.string(), - featureColumns: z.array(FeatureColumnInService), -}); - -const FeastFeatureServiceSchema = z.object({ - spec: z.object({ - name: z.string(), - features: z.array(FeatureInServiceSchema), - tags: z.record(z.string()).optional(), - description: z.string().optional(), - }), - meta: z.object({ - createdTimestamp: z.string().transform((val) => new Date(val)).optional(), - lastUpdatedTimestamp: z.string().transform((val) => new Date(val)).optional(), - }), -}); - -type FeastFeatureServiceType = z.infer; -type FeastFeatureInServiceType = z.infer; - -export { FeastFeatureServiceSchema }; -export type { FeastFeatureServiceType, FeastFeatureInServiceType }; diff --git a/ui/src/parsers/feastFeatureViews.ts b/ui/src/parsers/feastFeatureViews.ts deleted file mode 100644 index cbf15d280e6..00000000000 --- a/ui/src/parsers/feastFeatureViews.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { z } from "zod"; -import { FEAST_FEATURE_VALUE_TYPES } from "./types"; - -const FeastFeatureColumnSchema = z.object({ - name: z.string(), - valueType: z.nativeEnum(FEAST_FEATURE_VALUE_TYPES), - tags: z.record(z.string()).optional(), -}); - -const FeastBatchSourceSchema = z.object({ - type: z.string(), - eventTimestampColumn: z.string().optional(), - createdTimestampColumn: z.string().optional(), - fileOptions: z.object({ - uri: z.string().optional(), - }).optional(), - name: z.string().optional(), - description: z.string().optional(), - owner: z.string().optional(), - meta: z.object({ - earliestEventTimestamp: z.string().transform((val) => new Date(val)), - latestEventTimestamp: z.string().transform((val) => new Date(val)), - }).optional(), - requestDataOptions: z.object({ - schema: z.record(z.nativeEnum(FEAST_FEATURE_VALUE_TYPES)), - }).optional(), - bigqueryOptions: z.object({ - tableRef: z.string().optional(), - dbtModelSerialized: z.string().optional() - }).optional(), - dataSourceClassType: z.string(), -}); - -const FeastFeatureViewSchema = z.object({ - spec: z.object({ - description: z.string().optional(), - name: z.string(), - entities: z.array(z.string()), - features: z.array(FeastFeatureColumnSchema), - ttl: z.string().transform((val) => parseInt(val)), - batchSource: FeastBatchSourceSchema, - online: z.boolean().optional(), - owner: z.string().optional(), - tags: z.record(z.string()).optional(), - }), - meta: z.object({ - createdTimestamp: z.string().transform((val) => new Date(val)).optional(), - lastUpdatedTimestamp: z.string().transform((val) => new Date(val)).optional(), - materializationIntervals: z - .array( - z.object({ - startTime: z.string().transform((val) => new Date(val)), - endTime: z.string().transform((val) => new Date(val)), - }) - ) - .optional(), - }), -}); - -type FeastFeatureViewType = z.infer; -type FeastFeatureColumnType = z.infer; - -export { FeastFeatureViewSchema, FeastFeatureColumnSchema }; -export type { FeastFeatureViewType, FeastFeatureColumnType }; diff --git a/ui/src/parsers/feastFeatures.ts b/ui/src/parsers/feastFeatures.ts deleted file mode 100644 index 129120c168d..00000000000 --- a/ui/src/parsers/feastFeatures.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { z } from "zod"; -import { FEAST_FEATURE_VALUE_TYPES } from "./types"; -import { jsonSchema } from "./jsonType" - -const FeastFeatureSchema = z.object({ - name: z.string(), - valueType: z.nativeEnum(FEAST_FEATURE_VALUE_TYPES), - metadata: jsonSchema.optional(), -}); - -export { FeastFeatureSchema }; diff --git a/ui/src/parsers/feastODFVS.ts b/ui/src/parsers/feastODFVS.ts deleted file mode 100644 index 4d09cc72dfa..00000000000 --- a/ui/src/parsers/feastODFVS.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { z } from "zod"; -import { FeastFeatureColumnSchema } from "./feastFeatureViews"; - -const FeatureViewProjectionSchema = z.object({ - featureViewProjection: z.object({ - featureViewName: z.string(), - featureColumns: z.array(FeastFeatureColumnSchema), - }), -}); - -const RequestDataSourceSchema = z.object({ - requestDataSource: z.object({ - type: z.string(), - name: z.string(), - requestDataOptions: z.object({ - schema: z.array(FeastFeatureColumnSchema), - }), - }), -}); - -const ODFVInputsSchema = z.union([ - FeatureViewProjectionSchema, - RequestDataSourceSchema, -]); - -const FeastODFVSchema = z.object({ - spec: z.object({ - name: z.string(), - features: z.array(FeastFeatureColumnSchema), - sources: z.record(ODFVInputsSchema), - userDefinedFunction: z.object({ - name: z.string(), - body: z.string(), - }), - }), - meta: z.object({ - createdTimestamp: z.string().transform((val) => new Date(val)), - lastUpdatedTimestamp: z.string().transform((val) => new Date(val)), - }), -}); - -type FeastODFVType = z.infer; -type RequestDataSourceType = z.infer; -type FeatureViewProjectionType = z.infer; - -export { FeastODFVSchema }; -export type { FeastODFVType, RequestDataSourceType, FeatureViewProjectionType }; diff --git a/ui/src/parsers/feastRegistry.ts b/ui/src/parsers/feastRegistry.ts deleted file mode 100644 index f84187046a8..00000000000 --- a/ui/src/parsers/feastRegistry.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { z } from "zod"; -import { FeastDatasourceSchema } from "./feastDatasources"; -import { FeastEntitySchema } from "./feastEntities"; -import { FeastFeatureServiceSchema } from "./feastFeatureServices"; -import { FeastFeatureViewSchema } from "./feastFeatureViews"; -import { FeastSavedDatasetSchema } from "./feastSavedDataset"; -import { FeastODFVSchema } from "./feastODFVS"; -import { FeastSFVSchema } from "./feastSFVS"; - -const FeastRegistrySchema = z.object({ - project: z.string(), - dataSources: z.array(FeastDatasourceSchema).optional(), - entities: z.array(FeastEntitySchema).optional(), - featureViews: z.array(FeastFeatureViewSchema).optional(), - onDemandFeatureViews: z.array(FeastODFVSchema).optional(), - streamFeatureViews: z.array(FeastSFVSchema).optional(), - featureServices: z.array(FeastFeatureServiceSchema).optional(), - savedDatasets: z.array(FeastSavedDatasetSchema).optional(), -}); - -type FeastRegistryType = z.infer; - -export { FeastRegistrySchema }; -export type { FeastRegistryType }; diff --git a/ui/src/parsers/feastSFVS.ts b/ui/src/parsers/feastSFVS.ts deleted file mode 100644 index f65a0872220..00000000000 --- a/ui/src/parsers/feastSFVS.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { z } from "zod"; -import { FeastFeatureColumnSchema } from "./feastFeatureViews"; -import {FeastDatasourceSchema} from "./feastDatasources"; - -const FeatureViewProjectionSchema = z.object({ - featureViewProjection: z.object({ - featureViewName: z.string(), - featureColumns: z.array(FeastFeatureColumnSchema), - }), -}); - -const StreamSourceSchema = z.object({ - type: z.string(), - name: z.string(), - owner: z.string().optional(), - description: z.string().optional(), -}); - -const FeastSFVSchema = z.object({ - spec: z.object({ - name: z.string(), - features: z.array(FeastFeatureColumnSchema), - batchSource: FeastDatasourceSchema, - streamSource: StreamSourceSchema, - userDefinedFunction: z.object({ - name: z.string(), - body: z.string(), - }), - }), - meta: z.object({ - createdTimestamp: z.string().transform((val) => new Date(val)).optional(), - lastUpdatedTimestamp: z.string().transform((val) => new Date(val)).optional(), - }), -}); - -type FeastSFVType = z.infer; -type StreamSourceType = z.infer; -type FeatureViewProjectionType = z.infer; - -export { FeastSFVSchema }; -export type { FeastSFVType, StreamSourceType, FeatureViewProjectionType}; diff --git a/ui/src/parsers/feastSavedDataset.ts b/ui/src/parsers/feastSavedDataset.ts deleted file mode 100644 index ce1d39b4e73..00000000000 --- a/ui/src/parsers/feastSavedDataset.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { z } from "zod"; - -const FeastSavedDatasetSchema = z.object({ - spec: z.object({ - name: z.string(), - features: z.array(z.string()), - joinKeys: z.array(z.string()), - storage: z.object({ - fileStorage: z.object({ - fileFormat: z.object({ - parquetFormat: z.object({}).optional(), - }).optional(), - fileUrl: z.string(), - }).optional(), - }).optional(), - featureService: z - .object({ - spec: z.object({ - name: z.string(), - }), - }) - .transform((obj) => { - return obj.spec.name; - }).optional(), - profile: z.string().optional(), - }), - meta: z.object({ - createdTimestamp: z.string().transform((val) => new Date(val)), - minEventTimestamp: z.string().transform((val) => new Date(val)), - maxEventTimestamp: z.string().transform((val) => new Date(val)), - }), -}); - -type FeastSavedDatasetType = z.infer; - -export { FeastSavedDatasetSchema }; -export type { FeastSavedDatasetType }; diff --git a/ui/src/parsers/featureViewSummaryStatistics.ts b/ui/src/parsers/featureViewSummaryStatistics.ts deleted file mode 100644 index f8eca669d37..00000000000 --- a/ui/src/parsers/featureViewSummaryStatistics.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { z } from "zod"; - -const histogramSchema = z.array( - z.object({ - x0: z.number(), - x1: z.number(), - count: z.number(), - }) -); - -const numericColumnSummaryStaticsSchema = z.object({ - name: z.string(), - valueType: z.literal("INT64"), - sampleValues: z.array(z.number()), - histogram: histogramSchema.optional(), - proportionOfZeros: z.number().optional(), - proportionMissing: z.number().optional(), - min: z.number().optional(), - max: z.number().optional(), -}); - -const stringColumnSummaryStaticsSchema = z.object({ - name: z.string(), - valueType: z.literal("STRING"), - sampleValues: z.array(z.string()), -}); - -const columnsSummaryStatisticsSchema = z.union([ - numericColumnSummaryStaticsSchema, - stringColumnSummaryStaticsSchema, -]); - -const featureViewSummaryStatisticsSchema = z.object({ - columnsSummaryStatistics: z.record(columnsSummaryStatisticsSchema), -}); - -type FeatureViewSummaryStatisticsType = z.infer< - typeof featureViewSummaryStatisticsSchema ->; - -type NumericColumnSummaryStatisticType = z.infer< - typeof numericColumnSummaryStaticsSchema ->; -type StringColumnSummaryStatisticType = z.infer< - typeof stringColumnSummaryStaticsSchema ->; - -type HistogramDataType = z.infer; - -export { featureViewSummaryStatisticsSchema }; -export type { - FeatureViewSummaryStatisticsType, - HistogramDataType, - NumericColumnSummaryStatisticType, - StringColumnSummaryStatisticType, -}; diff --git a/ui/src/parsers/mergedFVTypes.ts b/ui/src/parsers/mergedFVTypes.ts index edf1adee9e5..7e23e095209 100644 --- a/ui/src/parsers/mergedFVTypes.ts +++ b/ui/src/parsers/mergedFVTypes.ts @@ -1,10 +1,4 @@ -import { - FeastFeatureColumnType, - FeastFeatureViewType, -} from "./feastFeatureViews"; -import { FeastODFVType } from "./feastODFVS"; -import { FeastSFVType } from "./feastSFVS"; -import { FeastRegistryType } from "./feastRegistry"; +import { feast } from "../protos"; enum FEAST_FV_TYPES { regular = "regular", @@ -15,64 +9,64 @@ enum FEAST_FV_TYPES { interface regularFVInterface { name: string; type: FEAST_FV_TYPES.regular; - features: FeastFeatureColumnType[]; - object: FeastFeatureViewType; + features: feast.core.IFeatureSpecV2[]; + object: feast.core.IFeatureView; } interface ODFVInterface { name: string; type: FEAST_FV_TYPES.ondemand; - features: FeastFeatureColumnType[]; - object: FeastODFVType; + features: feast.core.IOnDemandFeatureViewSpec[]; + object: feast.core.IOnDemandFeatureView; } interface SFVInterface { name: string; type: FEAST_FV_TYPES.stream; - features: FeastFeatureColumnType[]; - object: FeastSFVType; + features: feast.core.IFeatureSpecV2[]; + object: feast.core.IStreamFeatureView; } type genericFVType = regularFVInterface | ODFVInterface | SFVInterface; -const mergedFVTypes = (objects: FeastRegistryType) => { +const mergedFVTypes = (objects: feast.core.Registry) => { const mergedFVMap: Record = {}; const mergedFVList: genericFVType[] = []; objects.featureViews?.forEach((fv) => { const obj: genericFVType = { - name: fv.spec.name, + name: fv.spec?.name!, type: FEAST_FV_TYPES.regular, - features: fv.spec.features, + features: fv.spec?.features!, object: fv, }; - mergedFVMap[fv.spec.name] = obj; + mergedFVMap[fv.spec?.name!] = obj; mergedFVList.push(obj); }); objects.onDemandFeatureViews?.forEach((odfv) => { const obj: genericFVType = { - name: odfv.spec.name, + name: odfv.spec?.name!, type: FEAST_FV_TYPES.ondemand, - features: odfv.spec.features, + features: odfv.spec?.features!, object: odfv, }; - mergedFVMap[odfv.spec.name] = obj; + mergedFVMap[odfv.spec?.name!] = obj; mergedFVList.push(obj); }); objects.streamFeatureViews?.forEach((sfv) => { const obj: genericFVType = { - name: sfv.spec.name, + name: sfv.spec?.name!, type: FEAST_FV_TYPES.stream, - features: sfv.spec.features, + features: sfv.spec?.features!, object: sfv, }; - mergedFVMap[sfv.spec.name] = obj; + mergedFVMap[sfv.spec?.name!] = obj; mergedFVList.push(obj); }); diff --git a/ui/src/parsers/parseEntityRelationships.ts b/ui/src/parsers/parseEntityRelationships.ts index 8424bb7a44f..7a791fd0a15 100644 --- a/ui/src/parsers/parseEntityRelationships.ts +++ b/ui/src/parsers/parseEntityRelationships.ts @@ -1,5 +1,5 @@ -import { FeastRegistryType } from "./feastRegistry"; import { FEAST_FCO_TYPES } from "./types"; +import { feast } from "../protos"; interface EntityReference { type: FEAST_FCO_TYPES; @@ -11,26 +11,26 @@ interface EntityRelation { target: EntityReference; } -const parseEntityRelationships = (objects: FeastRegistryType) => { +const parseEntityRelationships = (objects: feast.core.Registry) => { const links: EntityRelation[] = []; objects.featureServices?.forEach((fs) => { - fs.spec.features.forEach((feature) => { + fs.spec?.features!.forEach((feature) => { links.push({ source: { type: FEAST_FCO_TYPES["featureView"], - name: feature.featureViewName, + name: feature?.featureViewName!, }, target: { type: FEAST_FCO_TYPES["featureService"], - name: fs.spec.name, + name: fs.spec?.name!, }, }); }); }); objects.featureViews?.forEach((fv) => { - fv.spec.entities.forEach((ent) => { + fv.spec?.entities?.forEach((ent) => { links.push({ source: { type: FEAST_FCO_TYPES["entity"], @@ -38,11 +38,11 @@ const parseEntityRelationships = (objects: FeastRegistryType) => { }, target: { type: FEAST_FCO_TYPES["featureView"], - name: fv.spec.name, + name: fv.spec?.name!, }, }); }); - if (fv.spec.batchSource) { + if (fv.spec?.batchSource) { links.push({ source: { type: FEAST_FCO_TYPES["dataSource"], @@ -50,54 +50,54 @@ const parseEntityRelationships = (objects: FeastRegistryType) => { }, target: { type: FEAST_FCO_TYPES["featureView"], - name: fv.spec.name, + name: fv.spec?.name!, } }) } }); objects.onDemandFeatureViews?.forEach((fv) => { - Object.values(fv.spec.sources).forEach((input: { [key: string]: any }) => { - if (input.requestDataSource) { - links.push({ - source: { - type: FEAST_FCO_TYPES["dataSource"], - name: input.requestDataSource.name, - }, - target: { - type: FEAST_FCO_TYPES["featureView"], - name: fv.spec.name, - }, - }); - } else if (input.featureViewProjection?.featureViewName) { - const source_fv = objects.featureViews?.find(el => el.spec.name === input.featureViewProjection.featureViewName); - if (!source_fv) { - return; - } - links.push({ - source: { - type: FEAST_FCO_TYPES["dataSource"], - name: source_fv?.spec.batchSource.name || '', - }, - target: { - type: FEAST_FCO_TYPES["featureView"], - name: fv.spec.name, - }, - }); - } - }); - }); + Object.values(fv.spec?.sources!).forEach((input: { [key: string]: any }) => { + if (input.requestDataSource) { + links.push({ + source: { + type: FEAST_FCO_TYPES["dataSource"], + name: input.requestDataSource.name, + }, + target: { + type: FEAST_FCO_TYPES["featureView"], + name: fv.spec?.name!, + }, + }); + } else if (input.featureViewProjection?.featureViewName) { + const source_fv = objects.featureViews?.find(el => el.spec?.name === input.featureViewProjection.featureViewName); + if (!source_fv) { + return; + } + links.push({ + source: { + type: FEAST_FCO_TYPES["dataSource"], + name: source_fv.spec?.batchSource?.name || '', + }, + target: { + type: FEAST_FCO_TYPES["featureView"], + name: fv.spec?.name!, + }, + }); + } + }); + }); objects.streamFeatureViews?.forEach((fv) => { // stream source links.push({ source: { type: FEAST_FCO_TYPES["dataSource"], - name: fv.spec.streamSource.name, + name: fv.spec?.streamSource?.name!, }, target: { type: FEAST_FCO_TYPES["featureView"], - name: fv.spec.name, + name: fv.spec?.name!, }, }); @@ -105,11 +105,11 @@ const parseEntityRelationships = (objects: FeastRegistryType) => { links.push({ source: { type: FEAST_FCO_TYPES["dataSource"], - name: fv.spec.batchSource.name, + name: fv.spec?.batchSource?.name!, }, target: { type: FEAST_FCO_TYPES["featureView"], - name: fv.spec.name, + name: fv.spec?.name!, }, }); }); diff --git a/ui/src/parsers/parseIndirectRelationships.ts b/ui/src/parsers/parseIndirectRelationships.ts index d7d532ad3e7..092176c9b62 100644 --- a/ui/src/parsers/parseIndirectRelationships.ts +++ b/ui/src/parsers/parseIndirectRelationships.ts @@ -1,16 +1,16 @@ -import { FeastRegistryType } from "./feastRegistry"; import { EntityRelation } from "./parseEntityRelationships"; import { FEAST_FCO_TYPES } from "./types"; +import { feast } from "../protos"; const parseIndirectRelationships = ( relationships: EntityRelation[], - objects: FeastRegistryType + objects: feast.core.Registry ) => { const indirectLinks: EntityRelation[] = []; // Only contains Entity -> FS or DS -> FS relationships objects.featureServices?.forEach((featureService) => { - featureService.spec.features.forEach((featureView) => { + featureService.spec?.features?.forEach((featureView) => { relationships .filter( (relationship) => @@ -21,7 +21,7 @@ const parseIndirectRelationships = ( source: relationship.source, target: { type: FEAST_FCO_TYPES["featureService"], - name: featureService.spec.name, + name: featureService.spec?.name!, }, }); }); diff --git a/ui/src/parsers/types.ts b/ui/src/parsers/types.ts index 2f88eea4f06..1e515f23f34 100644 --- a/ui/src/parsers/types.ts +++ b/ui/src/parsers/types.ts @@ -5,25 +5,4 @@ enum FEAST_FCO_TYPES { featureService = "featureService", } -enum FEAST_FEATURE_VALUE_TYPES { - FLOAT = "FLOAT", - INT64 = "INT64", - STRING = "STRING", - BOOL = "BOOL", - BYTES = "BYTES", - INT32 = "INT32", - DOUBLE = "DOUBLE", - UNIX_TIMESTAMP = "UNIX_TIMESTAMP", - INVALID = "INVALID", - BYTES_LIST = "BYTES_LIST", - STRING_LIST = "STRING_LIST", - INT32_LIST = "INT32_LIST", - INT64_LIST = "INT64_LIST", - DOUBLE_LIST = "DOUBLE_LIST", - FLOAT_LIST = "FLOAT_LIST", - BOOL_LIST = "BOOL_LIST", - UNIX_TIMESTAMP_LIST = "UNIX_TIMESTAMP_LIST", - NULL = "NULL" -} - -export { FEAST_FCO_TYPES, FEAST_FEATURE_VALUE_TYPES }; +export { FEAST_FCO_TYPES }; diff --git a/ui/src/queries/useLoadFeatureViewSummaryStatistics.ts b/ui/src/queries/useLoadFeatureViewSummaryStatistics.ts deleted file mode 100644 index fea0bd9d816..00000000000 --- a/ui/src/queries/useLoadFeatureViewSummaryStatistics.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { useQuery } from "react-query"; -import { useParams } from "react-router-dom"; -import { - featureViewSummaryStatisticsSchema, - FeatureViewSummaryStatisticsType, -} from "../parsers/featureViewSummaryStatistics"; - -const useLoadFeatureViewSummaryStatistics = (featureViewName: string) => { - const { projectName } = useParams(); - - const queryKey = `featureViewSummaryStatistics:${featureViewName}`; - const url = `/data/${projectName}/featureView/${featureViewName}.json`; - - return useQuery( - queryKey, - () => { - return fetch(url, { - headers: { - "Content-Type": "application/json", - }, - }) - .then((res) => { - return res.json(); - }) - .then((json) => { - const summary = featureViewSummaryStatisticsSchema.parse(json); - - return summary; - }); - }, - { - staleTime: 15 * 60 * 1000, // Given that we are reading from a registry dump, this seems reasonable for now. - } - ); -}; - -export default useLoadFeatureViewSummaryStatistics; diff --git a/ui/src/queries/useLoadRegistry.ts b/ui/src/queries/useLoadRegistry.ts index ffb06756437..be8ab65a8cd 100644 --- a/ui/src/queries/useLoadRegistry.ts +++ b/ui/src/queries/useLoadRegistry.ts @@ -1,18 +1,15 @@ import { useQuery } from "react-query"; -import { - FeastRegistrySchema, - FeastRegistryType, -} from "../parsers/feastRegistry"; import mergedFVTypes, { genericFVType } from "../parsers/mergedFVTypes"; import parseEntityRelationships, { EntityRelation, } from "../parsers/parseEntityRelationships"; import parseIndirectRelationships from "../parsers/parseIndirectRelationships"; +import { feast } from "../protos"; interface FeatureStoreAllData { project: string; description?: string; - objects: FeastRegistryType; + objects: feast.core.Registry; relationships: EntityRelation[]; mergedFVMap: Record; mergedFVList: genericFVType[]; @@ -29,10 +26,12 @@ const useLoadRegistry = (url: string) => { }, }) .then((res) => { - return res.json(); + return res.arrayBuffer(); }) - .then((json) => { - const objects = FeastRegistrySchema.parse(json); + .then((arrayBuffer) => { + + const objects = feast.core.Registry.decode(new Uint8Array(arrayBuffer)); + // const objects = FeastRegistrySchema.parse(json); const { mergedFVMap, mergedFVList } = mergedFVTypes(objects); @@ -53,7 +52,7 @@ const useLoadRegistry = (url: string) => { // }); return { - project: objects.project, + project: objects.projectMetadata[0].project!, objects, mergedFVMap, mergedFVList, diff --git a/ui/src/utils/timestamp.ts b/ui/src/utils/timestamp.ts new file mode 100644 index 00000000000..869d24870f0 --- /dev/null +++ b/ui/src/utils/timestamp.ts @@ -0,0 +1,13 @@ +import long from 'long'; +import { google } from '../protos'; + +export function toDate(ts: google.protobuf.ITimestamp) { + var seconds: number; + if (ts.seconds instanceof long) { + seconds = ts.seconds.low + } else { + seconds = ts.seconds!; + } + + return new Date(seconds * 1000); +} \ No newline at end of file diff --git a/ui/yarn.lock b/ui/yarn.lock index ad31cbeac51..e056cad6179 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -334,6 +334,11 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.17.3.tgz#b07702b982990bf6fdc1da5049a23fece4c5c3d0" integrity sha512-7yJPvPV+ESz2IUTPbOL+YkIGyCqOyNIzdguKQuJGnH7bg1WTIifuM21YqokFt/THWh1AkCRn9IgoykTRCBVpzA== +"@babel/parser@^7.9.4": + version "7.19.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.19.0.tgz#497fcafb1d5b61376959c1c338745ef0577aa02c" + integrity sha512-74bEXKX2h+8rrfQUfsBfuZZHzsEs6Eql4pqy/T4Nn6Y9wNPggQOqD6z6pn5Bl8ZfysKouFZT/UXEH94ummEeQw== + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.16.7": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.16.7.tgz#4eda6d6c2a0aa79c70fa7b6da67763dfe2141050" @@ -1592,6 +1597,59 @@ schema-utils "^3.0.0" source-map "^0.7.3" +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" + integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== + +"@protobufjs/eventemitter@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== + +"@protobufjs/fetch@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== + dependencies: + "@protobufjs/aspromise" "^1.1.1" + "@protobufjs/inquire" "^1.1.0" + +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== + +"@protobufjs/inquire@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" + integrity sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q== + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== + +"@protobufjs/utf8@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" + integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== + "@rollup/plugin-babel@^5.2.0": version "5.3.0" resolved "https://registry.yarnpkg.com/@rollup/plugin-babel/-/plugin-babel-5.3.0.tgz#9cb1c5146ddd6a4968ad96f209c50c62f92f9879" @@ -2316,11 +2374,24 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4= +"@types/linkify-it@*": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@types/linkify-it/-/linkify-it-3.0.2.tgz#fd2cd2edbaa7eaac7e7f3c1748b52a19143846c9" + integrity sha512-HZQYqbiFVWufzCwexrvh694SOim8z2d+xJl5UNamcvQFejLY/2YUtzXHYi3cHdI7PMlS8ejH2slRAOJQ32aNbA== + "@types/lodash@^4.14.160": version "4.14.178" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.178.tgz#341f6d2247db528d4a13ddbb374bcdc80406f4f8" integrity sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw== +"@types/markdown-it@^12.2.3": + version "12.2.3" + resolved "https://registry.yarnpkg.com/@types/markdown-it/-/markdown-it-12.2.3.tgz#0d6f6e5e413f8daaa26522904597be3d6cd93b51" + integrity sha512-GKMHFfv3458yYy+v/N8gjufHO6MSZKCOXpZc5GXIWWy8uldwfmPn98vp81gZ5f9SVw8YYBctgfJ22a2d7AOMeQ== + dependencies: + "@types/linkify-it" "*" + "@types/mdurl" "*" + "@types/mdast@^3.0.0": version "3.0.10" resolved "https://registry.yarnpkg.com/@types/mdast/-/mdast-3.0.10.tgz#4724244a82a4598884cbbe9bcfd73dff927ee8af" @@ -2328,6 +2399,11 @@ dependencies: "@types/unist" "*" +"@types/mdurl@*": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@types/mdurl/-/mdurl-1.0.2.tgz#e2ce9d83a613bacf284c7be7d491945e39e1f8e9" + integrity sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA== + "@types/mime@^1": version "1.3.2" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" @@ -2343,6 +2419,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.13.tgz#5ed7ed7c662948335fcad6c412bb42d99ea754e3" integrity sha512-Y86MAxASe25hNzlDbsviXl8jQHb0RDvKt4c40ZJQ1Don0AAL0STLZSs4N+6gLEO55pedy7r2cLwS+ZDxPm/2Bw== +"@types/node@>=13.7.0": + version "18.7.18" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.7.18.tgz#633184f55c322e4fb08612307c274ee6d5ed3154" + integrity sha512-m+6nTEOadJZuTPkKR/SYK3A2d7FZrgElol9UP1Kae90VVU4a6mxnPuLiIW1m4Cq4gZ/nWb9GrdVXJCoCazDAbg== + "@types/node@^16.7.13": version "16.11.21" resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.21.tgz#474d7589a30afcf5291f59bd49cca9ad171ffde4" @@ -2809,7 +2890,7 @@ acorn-import-assertions@^1.7.6: resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz#ba2b5939ce62c238db6d93d81c9b111b29b855e9" integrity sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw== -acorn-jsx@^5.3.1: +acorn-jsx@^5.3.1, acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== @@ -2838,6 +2919,11 @@ acorn@^8.2.4, acorn@^8.4.1, acorn@^8.5.0, acorn@^8.7.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30" integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A== +acorn@^8.8.0: + version "8.8.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.0.tgz#88c0187620435c7f6015803f5539dae05a9dbea8" + integrity sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w== + address@^1.0.1, address@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/address/-/address-1.1.2.tgz#bf1116c9c758c51b7a933d296b72c221ed9428b6" @@ -3296,7 +3382,7 @@ bl@^4.1.0: inherits "^2.0.4" readable-stream "^3.4.0" -bluebird@^3.5.5: +bluebird@^3.5.5, bluebird@^3.7.2: version "3.7.2" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== @@ -3482,6 +3568,13 @@ case-sensitive-paths-webpack-plugin@^2.4.0: resolved "https://registry.yarnpkg.com/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.4.0.tgz#db64066c6422eed2e08cc14b986ca43796dbc6d4" integrity sha512-roIFONhcxog0JSSWbvVAh3OocukmSgpqOH6YpMkCvav/ySIV3JKg4Dc8vYtQjYi/UxpNE36r/9v+VqTQqgkYmw== +catharsis@^0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/catharsis/-/catharsis-0.9.0.tgz#40382a168be0e6da308c277d3a2b3eb40c7d2121" + integrity sha512-prMTQVpcns/tzFgFVkVp6ak6RykZyWb3gu8ckUpd6YkTlacOd3DXGJjIpD4Q6zJirizvaiAjSSHlOsA+6sNh2A== + dependencies: + lodash "^4.17.15" + ccount@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/ccount/-/ccount-1.1.0.tgz#246687debb6014735131be8abab2d93898f8d043" @@ -4736,6 +4829,11 @@ entities@^2.0.0: resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== +entities@~2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-2.1.0.tgz#992d3129cf7df6870b96c57858c249a120f8b8b5" + integrity sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w== + error-ex@^1.3.1: version "1.3.2" resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" @@ -4815,6 +4913,18 @@ escape-string-regexp@^4.0.0: resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== +escodegen@^1.13.0: + version "1.14.3" + resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-1.14.3.tgz#4e7b81fba61581dc97582ed78cab7f0e8d63f503" + integrity sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw== + dependencies: + esprima "^4.0.1" + estraverse "^4.2.0" + esutils "^2.0.2" + optionator "^0.8.1" + optionalDependencies: + source-map "~0.6.1" + escodegen@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-2.0.0.tgz#5e32b12833e8aa8fa35e1bf0befa89380484c7dd" @@ -4980,6 +5090,11 @@ eslint-visitor-keys@^3.0.0, eslint-visitor-keys@^3.1.0, eslint-visitor-keys@^3.2 resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.2.0.tgz#6fbb166a6798ee5991358bc2daa1ba76cc1254a1" integrity sha512-IOzT0X126zn7ALX0dwFiUQEdsfzrm4+ISsQS8nukaJXwEyYKRSnEIIDULYg1mCtGp7UUXgfGl7BIolXREQK+XQ== +eslint-visitor-keys@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz#f6480fa6b1f30efe2d1968aa8ac745b862469826" + integrity sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA== + eslint-webpack-plugin@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/eslint-webpack-plugin/-/eslint-webpack-plugin-3.1.1.tgz#83dad2395e5f572d6f4d919eedaa9cf902890fcb" @@ -5032,6 +5147,15 @@ eslint@^8.3.0: text-table "^0.2.0" v8-compile-cache "^2.0.3" +espree@^9.0.0: + version "9.4.0" + resolved "https://registry.yarnpkg.com/espree/-/espree-9.4.0.tgz#cd4bc3d6e9336c433265fc0aa016fc1aaf182f8a" + integrity sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw== + dependencies: + acorn "^8.8.0" + acorn-jsx "^5.3.2" + eslint-visitor-keys "^3.3.0" + espree@^9.2.0, espree@^9.3.0: version "9.3.0" resolved "https://registry.yarnpkg.com/espree/-/espree-9.3.0.tgz#c1240d79183b72aaee6ccfa5a90bc9111df085a8" @@ -5060,7 +5184,7 @@ esrecurse@^4.3.0: dependencies: estraverse "^5.2.0" -estraverse@^4.1.1: +estraverse@^4.1.1, estraverse@^4.2.0: version "4.3.0" resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.3.0.tgz#398ad3f3c5a24948be7725e83d11a7de28cdbd1d" integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw== @@ -5536,6 +5660,17 @@ glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^8.0.0: + version "8.0.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-8.0.3.tgz#415c6eb2deed9e502c68fa44a272e6da6eeca42e" + integrity sha512-ull455NHSHI/Y1FqGaaYFaLGkNMMJbavMrEGFXG/PGrg6y7sutWHUHrz6gy6WEBH6akM1M414dWKCNs+IhKdiQ== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^5.0.1" + once "^1.3.0" + global-modules@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780" @@ -5595,6 +5730,11 @@ graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4, resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.9.tgz#041b05df45755e587a24942279b9d113146e1c96" integrity sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ== +graceful-fs@^4.1.9: + version "4.2.10" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" + integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== + graphql@^15.5.1: version "15.8.0" resolved "https://registry.yarnpkg.com/graphql/-/graphql-15.8.0.tgz#33410e96b012fa3bdb1091cc99a94769db212b38" @@ -6862,6 +7002,34 @@ js-yaml@^4.1.0: dependencies: argparse "^2.0.1" +js2xmlparser@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/js2xmlparser/-/js2xmlparser-4.0.2.tgz#2a1fdf01e90585ef2ae872a01bc169c6a8d5e60a" + integrity sha512-6n4D8gLlLf1n5mNLQPRfViYzu9RATblzPEtm1SthMX1Pjao0r9YI9nw7ZIfRxQMERS87mcswrg+r/OYrPRX6jA== + dependencies: + xmlcreate "^2.0.4" + +jsdoc@^3.6.3: + version "3.6.11" + resolved "https://registry.yarnpkg.com/jsdoc/-/jsdoc-3.6.11.tgz#8bbb5747e6f579f141a5238cbad4e95e004458ce" + integrity sha512-8UCU0TYeIYD9KeLzEcAu2q8N/mx9O3phAGl32nmHlE0LpaJL71mMkP4d+QE5zWfNt50qheHtOZ0qoxVrsX5TUg== + dependencies: + "@babel/parser" "^7.9.4" + "@types/markdown-it" "^12.2.3" + bluebird "^3.7.2" + catharsis "^0.9.0" + escape-string-regexp "^2.0.0" + js2xmlparser "^4.0.2" + klaw "^3.0.0" + markdown-it "^12.3.2" + markdown-it-anchor "^8.4.1" + marked "^4.0.10" + mkdirp "^1.0.4" + requizzle "^0.2.3" + strip-json-comments "^3.1.0" + taffydb "2.6.2" + underscore "~1.13.2" + jsdom@^16.6.0: version "16.7.0" resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-16.7.0.tgz#918ae71965424b197c819f8183a754e18977b710" @@ -6983,6 +7151,13 @@ kind-of@^6.0.2: resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== +klaw@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/klaw/-/klaw-3.0.0.tgz#b11bec9cf2492f06756d6e809ab73a2910259146" + integrity sha512-0Fo5oir+O9jnXu5EefYbVK+mHMBeEVEy2cmctR1O1NECcCkPRreJKrS6Qt/j3KC2C148Dfo9i3pCmCMsdqGr0g== + dependencies: + graceful-fs "^4.1.9" + kleur@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" @@ -7036,6 +7211,13 @@ lines-and-columns@^1.1.6: resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== +linkify-it@^3.0.1: + version "3.0.3" + resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-3.0.3.tgz#a98baf44ce45a550efb4d49c769d07524cc2fa2e" + integrity sha512-ynTsyrFSdE5oZ/O9GEf00kPngmOfVwazR5GKDq6EYfhlpFug3J2zybX56a2PRRpc9P+FuSoGNAwjlbDs9jJBPQ== + dependencies: + uc.micro "^1.0.1" + loader-runner@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.2.0.tgz#d7022380d66d14c5fb1d496b89864ebcfd478384" @@ -7132,6 +7314,11 @@ log-symbols@^4.1.0: chalk "^4.1.0" is-unicode-supported "^0.1.0" +long@^5.0.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/long/-/long-5.2.0.tgz#2696dadf4b4da2ce3f6f6b89186085d94d52fd61" + integrity sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w== + loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -7184,6 +7371,27 @@ markdown-escapes@^1.0.0: resolved "https://registry.yarnpkg.com/markdown-escapes/-/markdown-escapes-1.0.4.tgz#c95415ef451499d7602b91095f3c8e8975f78535" integrity sha512-8z4efJYk43E0upd0NbVXwgSTQs6cT3T06etieCMEg7dRbzCbxUCK/GHlX8mhHRDcp+OLlHkPKsvqQTCvsRl2cg== +markdown-it-anchor@^8.4.1: + version "8.6.5" + resolved "https://registry.yarnpkg.com/markdown-it-anchor/-/markdown-it-anchor-8.6.5.tgz#30c4bc5bbff327f15ce3c429010ec7ba75e7b5f8" + integrity sha512-PI1qEHHkTNWT+X6Ip9w+paonfIQ+QZP9sCeMYi47oqhH+EsW8CrJ8J7CzV19QVOj6il8ATGbK2nTECj22ZHGvQ== + +markdown-it@^12.3.2: + version "12.3.2" + resolved "https://registry.yarnpkg.com/markdown-it/-/markdown-it-12.3.2.tgz#bf92ac92283fe983fe4de8ff8abfb5ad72cd0c90" + integrity sha512-TchMembfxfNVpHkbtriWltGWc+m3xszaRD0CZup7GFFhzIgQqxIfn3eGj1yZpfuflzPvfkt611B2Q/Bsk1YnGg== + dependencies: + argparse "^2.0.1" + entities "~2.1.0" + linkify-it "^3.0.1" + mdurl "^1.0.1" + uc.micro "^1.0.5" + +marked@^4.0.10: + version "4.1.0" + resolved "https://registry.yarnpkg.com/marked/-/marked-4.1.0.tgz#3fc6e7485f21c1ca5d6ec4a39de820e146954796" + integrity sha512-+Z6KDjSPa6/723PQYyc1axYZpYYpDnECDaU6hkaf5gqBieBkMKYReL5hteF2QizhlMbgbo8umXl/clZ67+GlsA== + match-sorter@^6.0.2: version "6.3.1" resolved "https://registry.yarnpkg.com/match-sorter/-/match-sorter-6.3.1.tgz#98cc37fda756093424ddf3cbc62bfe9c75b92bda" @@ -7228,7 +7436,7 @@ mdn-data@~1.1.0: resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-1.1.4.tgz#50b5d4ffc4575276573c4eedb8780812a8419f01" integrity sha512-FSYbp3lyKjyj3E7fMl6rYvUdX0FBXaluGqlFoYESWQlyUTq8R+wp0rkFxoYFqZlHCvsUXGjyJmLQSnXToYhOSA== -mdurl@^1.0.0: +mdurl@^1.0.0, mdurl@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e" integrity sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4= @@ -7355,6 +7563,11 @@ mkdirp@^0.5.5, mkdirp@~0.5.1: dependencies: minimist "^1.2.5" +mkdirp@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== + moment@^2.29.1: version "2.29.4" resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108" @@ -8502,6 +8715,40 @@ property-information@^5.0.0, property-information@^5.3.0: dependencies: xtend "^4.0.0" +protobufjs-cli@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/protobufjs-cli/-/protobufjs-cli-1.0.2.tgz#905fc49007cf4aaf3c45d5f250eb294eedeea062" + integrity sha512-cz9Pq9p/Zs7okc6avH20W7QuyjTclwJPgqXG11jNaulfS3nbVisID8rC+prfgq0gbZE0w9LBFd1OKFF03kgFzg== + dependencies: + chalk "^4.0.0" + escodegen "^1.13.0" + espree "^9.0.0" + estraverse "^5.1.0" + glob "^8.0.0" + jsdoc "^3.6.3" + minimist "^1.2.0" + semver "^7.1.2" + tmp "^0.2.1" + uglify-js "^3.7.7" + +protobufjs@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.1.1.tgz#0117befb4b0f5a49d028e93f2ca62c3c1f5e7c65" + integrity sha512-d0nMQqS/aT3lfV8bKi9Gbg73vPd2LcDdTDOu6RE/M+h9DY8g1EmDzk3ADPccthEWfTBjkR2oxNdx9Gs8YubT+g== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/node" ">=13.7.0" + long "^5.0.0" + proxy-addr@~2.0.7: version "2.0.7" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" @@ -9104,6 +9351,13 @@ requires-port@^1.0.0: resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" integrity sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8= +requizzle@^0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/requizzle/-/requizzle-0.2.3.tgz#4675c90aacafb2c036bd39ba2daa4a1cb777fded" + integrity sha512-YanoyJjykPxGHii0fZP0uUPEXpvqfBDxWV7s6GKAiiOsiqhX6vHNyW3Qzdmqp/iq/ExbhaGbVrjB4ruEVSM4GQ== + dependencies: + lodash "^4.17.14" + resolve-cwd@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d" @@ -9379,6 +9633,13 @@ semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +semver@^7.1.2: + version "7.3.7" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f" + integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g== + dependencies: + lru-cache "^6.0.0" + semver@^7.3.2, semver@^7.3.5: version "7.3.5" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" @@ -9929,6 +10190,11 @@ tabbable@^5.2.1: resolved "https://registry.yarnpkg.com/tabbable/-/tabbable-5.3.2.tgz#66d6119ee8a533634c3f17deb0caa1c379e36ac7" integrity sha512-6G/8EWRFx8CiSe2++/xHhXkmCRq2rHtDtZbQFHx34cvDfZzIBfvwG9zGUNTWMXWLCYvDj3aQqOzdl3oCxKuBkQ== +taffydb@2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/taffydb/-/taffydb-2.6.2.tgz#7cbcb64b5a141b6a2efc2c5d2c67b4e150b2a268" + integrity sha512-y3JaeRSplks6NYQuCOj3ZFMO3j60rTwbuKCvZxsAraGYH2epusatvZ0baZYA01WsGqJBq/Dl6vOrMUJqyMj8kA== + tailwindcss@^3.0.2: version "3.0.18" resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-3.0.18.tgz#ea4825e6496d77dc21877b6b61c7cc56cda3add5" @@ -10060,6 +10326,13 @@ tmp@^0.0.33: dependencies: os-tmpdir "~1.0.2" +tmp@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.1.tgz#8457fc3037dcf4719c251367a1af6500ee1ccf14" + integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ== + dependencies: + rimraf "^3.0.0" + tmpl@1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc" @@ -10221,6 +10494,16 @@ typescript@^4.4.2: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3" integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA== +uc.micro@^1.0.1, uc.micro@^1.0.5: + version "1.0.6" + resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac" + integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA== + +uglify-js@^3.7.7: + version "3.17.0" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.17.0.tgz#55bd6e9d19ce5eef0d5ad17cd1f587d85b180a85" + integrity sha512-aTeNPVmgIMPpm1cxXr2Q/nEbvkmV8yq66F3om7X3P/cvOXQ0TMQ64Wk63iyT1gPlmdmGzjGpyLh1f3y8MZWXGg== + unbox-primitive@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.1.tgz#085e215625ec3162574dc8859abee78a59b14471" @@ -10231,6 +10514,11 @@ unbox-primitive@^1.0.1: has-symbols "^1.0.2" which-boxed-primitive "^1.0.2" +underscore@~1.13.2: + version "1.13.4" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.4.tgz#7886b46bbdf07f768e0052f1828e1dcab40c0dee" + integrity sha512-BQFnUDuAQ4Yf/cYY5LNrK9NCJFKriaRbD9uR1fTeXnBeoa97W0i41qkZfGO9pSo8I5KzjAcSY2XYtdf0oKd7KQ== + unherit@^1.0.4: version "1.1.3" resolved "https://registry.yarnpkg.com/unherit/-/unherit-1.1.3.tgz#6c9b503f2b41b262330c80e91c8614abdaa69c22" @@ -10988,6 +11276,11 @@ xmlchars@^2.2.0: resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw== +xmlcreate@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/xmlcreate/-/xmlcreate-2.0.4.tgz#0c5ab0f99cdd02a81065fa9cd8f8ae87624889be" + integrity sha512-nquOebG4sngPmGPICTS5EnxqhKbCmz5Ox5hsszI2T6U5qdrJizBc+0ilYSEjTSzU0yZcmvppztXe/5Al5fUwdg== + xtend@^4.0.0, xtend@^4.0.1, xtend@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" @@ -11049,10 +11342,10 @@ yocto-queue@^0.1.0: resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== -zod@^3.11.6: - version "3.11.6" - resolved "https://registry.yarnpkg.com/zod/-/zod-3.11.6.tgz#e43a5e0c213ae2e02aefe7cb2b1a6fa3d7f1f483" - integrity sha512-daZ80A81I3/9lIydI44motWe6n59kRBfNzTuS2bfzVh1nAXi667TOTWWtatxyG+fwgNUiagSj/CWZwRRbevJIg== +zod@^3.19.1: + version "3.19.1" + resolved "https://registry.yarnpkg.com/zod/-/zod-3.19.1.tgz#112f074a97b50bfc4772d4ad1576814bd8ac4473" + integrity sha512-LYjZsEDhCdYET9ikFu6dVPGp2YH9DegXjdJToSzD9rO6fy4qiRYFoyEYwps88OseJlPyl2NOe2iJuhEhL7IpEA== zwitch@^1.0.0: version "1.0.5" From a9d48d07f634deb0b992e36d2ae1577d9b627940 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Wed, 5 Oct 2022 20:14:59 -0400 Subject: [PATCH 30/33] fix: Fix Feast UI release process to update the feast-ui package (#3267) fix: Fix Feast UI release process to update the feast-ui package to the latest version Signed-off-by: Danny Chiao Signed-off-by: Danny Chiao --- .releaserc.js | 1 + Makefile | 2 +- sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 8 ++++---- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.releaserc.js b/.releaserc.js index 124ad8801c2..114d65d1a2a 100644 --- a/.releaserc.js +++ b/.releaserc.js @@ -67,6 +67,7 @@ module.exports = { "java/pom.xml", "infra/charts/**/*.*", "ui/package.json", + "sdk/python/feast/ui/package.json", "sdk/python/feast/ui/yarn.lock" ], message: "chore(release): release ${nextRelease.version}\n\n${nextRelease.notes}" diff --git a/Makefile b/Makefile index f29ec3c39cd..718df6af4ab 100644 --- a/Makefile +++ b/Makefile @@ -467,4 +467,4 @@ build-helm-docs: # Note: requires node and yarn to be installed build-ui: - cd $(ROOT_DIR)/sdk/python/feast/ui && yarn install && npm run build --omit=dev + cd $(ROOT_DIR)/sdk/python/feast/ui && yarn upgrade @feast-dev/feast-ui --latest && yarn install && npm run build --omit=dev diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 6ffebcc834d..5854f205709 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "latest", + "@feast-dev/feast-ui": "0.25.1", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 1b9cf5f067f..af820b085a1 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1300,10 +1300,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@latest": - version "0.24.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.24.0.tgz#a52037247563290f92d0d993fcaf0d88e9741f36" - integrity sha512-Te27bSVFp7gCE7+p9bbCkCEQ7+nsRCzBtwWivNPBFRn8HC2ewBzmRzzasXlCHok1cXHDbh7Xj7y+2Hshp91LTg== +"@feast-dev/feast-ui@0.25.1": + version "0.25.1" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.25.1.tgz#c15e04dfdd4c616fcce254fc22f1474ba4c7b3fe" + integrity sha512-WKJEF2KwzJ9ydoDHlkaYDboHEud+Q/ij8ShGs9CxygNFMSXR00cc63QDdXPLNG4dCE0afSjlgqLtV4R3EJTBfg== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^55.0.1" From eff01d1ed3a76ff539595455f69ff1c954ad46cd Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 5 Oct 2022 17:15:58 -0700 Subject: [PATCH 31/33] fix: Fix docker image for feature-server (#3272) * Fix Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang Signed-off-by: Kevin Zhang --- .github/workflows/publish.yml | 2 +- sdk/python/feast/infra/feature_servers/multicloud/Dockerfile | 4 ++-- .../feast/infra/feature_servers/multicloud/Dockerfile.dev | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9587044a84d..039ab6a24e7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -46,7 +46,7 @@ jobs: build-publish-docker-images: runs-on: ubuntu-latest - needs: get-version + needs: [get-version, publish-python-sdk] strategy: matrix: component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server] diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index 990c1fd8f03..c95c515fb4b 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -8,8 +8,8 @@ RUN apt update && \ build-essential RUN pip install pip --upgrade -COPY . . -RUN pip install ".[aws,gcp,snowflake,redis,go,mysql,postgres]" +RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres]" + RUN apt update RUN apt install -y -V ca-certificates lsb-release wget diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev index 990c1fd8f03..ecbc199a5b9 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -9,7 +9,8 @@ RUN apt update && \ RUN pip install pip --upgrade COPY . . -RUN pip install ".[aws,gcp,snowflake,redis,go,mysql,postgres]" + +RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres]" RUN apt update RUN apt install -y -V ca-certificates lsb-release wget From a84ace0eba438cb4eedab48b2f476ff6d9843492 Mon Sep 17 00:00:00 2001 From: Danny Chiao Date: Thu, 6 Oct 2022 14:21:30 -0400 Subject: [PATCH 32/33] ci: Fix Feast UI tests (#3273) * ci: Fix Feast UI tests Signed-off-by: Danny Chiao * add test to workflow Signed-off-by: Danny Chiao * add test to workflow Signed-off-by: Danny Chiao * fix test suite in workflow Signed-off-by: Danny Chiao * fix README Signed-off-by: Danny Chiao * fix build Signed-off-by: Danny Chiao * fix test name Signed-off-by: Danny Chiao * upgrade caniuse Signed-off-by: Danny Chiao Signed-off-by: Danny Chiao --- .github/workflows/unit_tests.yml | 20 +++++++++++++++++ sdk/python/feast/ui/README.md | 15 ++++++++++++- ui/package.json | 6 ++++-- ui/src/FeastUISansProviders.test.tsx | 32 +++++++++++++--------------- ui/src/mocks/handlers.ts | 14 ++++++++---- ui/src/parsers/jsonType.ts | 11 ---------- ui/yarn.lock | 8 +++---- 7 files changed, 67 insertions(+), 39 deletions(-) delete mode 100644 ui/src/parsers/jsonType.ts diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index de6d98d1400..ebc09f0080e 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -108,3 +108,23 @@ jobs: sudo apt install -y -V libarrow-dev - name: Test run: make test-go + + unit-test-ui: + runs-on: ubuntu-latest + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '17.x' + registry-url: 'https://registry.npmjs.org' + - name: Install yarn dependencies + working-directory: ./ui + run: yarn install + - name: Build yarn rollup + working-directory: ./ui + run: yarn build:lib + - name: Run yarn tests + working-directory: ./ui + run: yarn test --watchAll=false diff --git a/sdk/python/feast/ui/README.md b/sdk/python/feast/ui/README.md index 220f40a2251..28290f0326a 100644 --- a/sdk/python/feast/ui/README.md +++ b/sdk/python/feast/ui/README.md @@ -25,4 +25,17 @@ The `feast ui` command will generate the necessary `projects-list.json` file and **Note**: `yarn start` will not work on this because of the above dependency. ## Dev -To test, do `yarn link` in ui/ and then come here to do `yarn link @feast-dev/feast` \ No newline at end of file +To test with a locally built Feast UI package, do: +1. `yarn link` in ui/ +2. `yarn install` in ui/ +3. `yarn link` in ui/node_modules/react +4. `yarn link` in ui/node_modules/react-dom +5. and then come here to do: + ```bash + yarn link "@feast-dev/feast" + yarn link react + yarn link react-dom + yarn start + ``` + +See also https://github.com/facebook/react/issues/14257. \ No newline at end of file diff --git a/ui/package.json b/ui/package.json index 7d50b3e086b..64952897a29 100644 --- a/ui/package.json +++ b/ui/package.json @@ -25,7 +25,8 @@ "react-router-dom": "6", "react-scripts": "^5.0.0", "typescript": "^4.4.2", - "use-query-params": "^1.2.3" + "use-query-params": "^1.2.3", + "zod": "^3.11.6" }, "dependencies": { "@elastic/datemath": "^5.0.3", @@ -45,7 +46,8 @@ "react-query": "^3.34.12", "react-router-dom": "6", "react-scripts": "^5.0.0", - "use-query-params": "^1.2.3" + "use-query-params": "^1.2.3", + "zod": "^3.11.6" }, "scripts": { "start": "npm run generate-protos && react-scripts start", diff --git a/ui/src/FeastUISansProviders.test.tsx b/ui/src/FeastUISansProviders.test.tsx index 09985bc1338..46702328090 100644 --- a/ui/src/FeastUISansProviders.test.tsx +++ b/ui/src/FeastUISansProviders.test.tsx @@ -15,13 +15,17 @@ import { creditHistoryRegistry, } from "./mocks/handlers"; -import registry from "../public/registry.json"; +import { readFileSync } from "fs"; +import { feast } from "./protos"; +import path from "path"; // declare which API requests to mock const server = setupServer( projectsListWithDefaultProject, creditHistoryRegistry ); +const registry = readFileSync(path.resolve(__dirname, "../public/registry.db")); +const parsedRegistry = feast.core.Registry.decode(registry); // establish API mocking before all tests beforeAll(() => server.listen()); @@ -50,7 +54,10 @@ test("full app rendering", async () => { // Explore Panel Should Appear expect(screen.getByText(/Explore this Project/i)).toBeInTheDocument(); - const projectNameRegExp = new RegExp(registry.project, "i"); + const projectNameRegExp = new RegExp( + parsedRegistry.projectMetadata[0].project!, + "i" + ); // It should load the default project, which is credit_scoring_aws await waitFor(() => { @@ -95,9 +102,9 @@ test("routes are reachable", async () => { } }); - -const featureViewName = registry.featureViews[0].spec.name; -const featureName = registry.featureViews[0].spec.features[0].name; +const spec = parsedRegistry.featureViews[0].spec!; +const featureViewName = spec.name!; +const featureName = spec.features![0]!.name!; test("features are reachable", async () => { render(); @@ -106,10 +113,7 @@ test("features are reachable", async () => { await screen.findByText(/Explore this Project/i); const routeRegExp = new RegExp("Feature Views", "i"); - userEvent.click( - screen.getByRole("button", { name: routeRegExp }), - leftClick - ); + userEvent.click(screen.getByRole("button", { name: routeRegExp }), leftClick); screen.getByRole("heading", { name: "Feature Views", @@ -118,18 +122,12 @@ test("features are reachable", async () => { await screen.findAllByText(/Feature Views/i); const fvRegExp = new RegExp(featureViewName, "i"); - userEvent.click( - screen.getByRole("link", { name: fvRegExp }), - leftClick - ) + userEvent.click(screen.getByRole("link", { name: fvRegExp }), leftClick); await screen.findByText(featureName); const fRegExp = new RegExp(featureName, "i"); - userEvent.click( - screen.getByRole("link", { name: fRegExp }), - leftClick - ) + userEvent.click(screen.getByRole("link", { name: fRegExp }), leftClick); // Should land on a page with the heading // await screen.findByText("Feature: " + featureName); screen.getByRole("heading", { diff --git a/ui/src/mocks/handlers.ts b/ui/src/mocks/handlers.ts index e7b0040f0dc..39f30b62a6d 100644 --- a/ui/src/mocks/handlers.ts +++ b/ui/src/mocks/handlers.ts @@ -1,5 +1,8 @@ import { rest } from "msw"; -import registry from "../../public/registry.json"; +import {readFileSync} from 'fs'; +import path from "path"; + +const registry = readFileSync(path.resolve(__dirname, "../../public/registry.db")); const projectsListWithDefaultProject = rest.get( "/projects-list.json", @@ -14,7 +17,7 @@ const projectsListWithDefaultProject = rest.get( description: "Project for credit scoring team and associated models.", id: "credit_score_project", - registryPath: "/registry.json", + registryPath: "/registry.pb", }, ], }) @@ -22,8 +25,11 @@ const projectsListWithDefaultProject = rest.get( } ); -const creditHistoryRegistry = rest.get("/registry.json", (req, res, ctx) => { - return res(ctx.status(200), ctx.json(registry)); +const creditHistoryRegistry = rest.get("/registry.pb", (req, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/octet-stream'), + ctx.body(registry)); }); export { projectsListWithDefaultProject, creditHistoryRegistry }; diff --git a/ui/src/parsers/jsonType.ts b/ui/src/parsers/jsonType.ts deleted file mode 100644 index be484b5477f..00000000000 --- a/ui/src/parsers/jsonType.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { z } from "zod"; - -// Taken from the zod documentation code - accepts any JSON object. -const literalSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]); -type Literal = z.infer; -type Json = Literal | { [key: string]: Json } | Json[]; -const jsonSchema: z.ZodType = z.lazy(() => - z.union([literalSchema, z.array(jsonSchema), z.record(jsonSchema)]) -); - -export { jsonSchema }; diff --git a/ui/yarn.lock b/ui/yarn.lock index e056cad6179..948eb78796e 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -3559,9 +3559,9 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001286, caniuse-lite@^1.0.30001297, caniuse-lite@^1.0.30001299: - version "1.0.30001303" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001303.tgz#9b168e4f43ccfc372b86f4bc5a551d9b909c95c9" - integrity sha512-/Mqc1oESndUNszJP0kx0UaQU9kEv9nNtJ7Kn8AdA0mNnH8eR1cj0kG+NbNuC1Wq/b21eA8prhKRA3bbkjONegQ== + version "1.0.30001416" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001416.tgz" + integrity sha512-06wzzdAkCPZO+Qm4e/eNghZBDfVNDsCgw33T27OwBH9unE9S478OYw//Q2L7Npf/zBzs7rjZOszIFQkwQKAEqA== case-sensitive-paths-webpack-plugin@^2.4.0: version "2.4.0" @@ -11342,7 +11342,7 @@ yocto-queue@^0.1.0: resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== -zod@^3.19.1: +zod@^3.11.6: version "3.19.1" resolved "https://registry.yarnpkg.com/zod/-/zod-3.19.1.tgz#112f074a97b50bfc4772d4ad1576814bd8ac4473" integrity sha512-LYjZsEDhCdYET9ikFu6dVPGp2YH9DegXjdJToSzD9rO6fy4qiRYFoyEYwps88OseJlPyl2NOe2iJuhEhL7IpEA== From 7417dd3d8cba2030a4a3ad67628eb606148f1c95 Mon Sep 17 00:00:00 2001 From: feast-ci-bot Date: Thu, 6 Oct 2022 20:02:27 +0000 Subject: [PATCH 33/33] chore(release): release 0.26.0 # [0.26.0](https://github.com/feast-dev/feast/compare/v0.25.0...v0.26.0) (2022-10-06) ### Bug Fixes * Add `X-Trino-Extra-Credential` header and remove user override ([#3246](https://github.com/feast-dev/feast/issues/3246)) ([164e666](https://github.com/feast-dev/feast/commit/164e666ee8c425150903819a0035d6f6e48bcdd0)) * Add postgres to the feature server Dockerfile to fix helm chart flow ([#3261](https://github.com/feast-dev/feast/issues/3261)) ([6f6cbb7](https://github.com/feast-dev/feast/commit/6f6cbb76a216965666719e492c5ba854ada32605)) * Add stream feature view in the Web UI ([#3257](https://github.com/feast-dev/feast/issues/3257)) ([1f70b3a](https://github.com/feast-dev/feast/commit/1f70b3a9be738ab54f0cb59046ed1e299f92e3a2)) * Build dockerfile correctly ([#3239](https://github.com/feast-dev/feast/issues/3239)) ([a2dc0d0](https://github.com/feast-dev/feast/commit/a2dc0d0410eb297afddfb1dd4f1f899ab70fa14f)) * Configuration to stop coercion of tz for entity_df ([#3255](https://github.com/feast-dev/feast/issues/3255)) ([97b7ab9](https://github.com/feast-dev/feast/commit/97b7ab906081d1afe034fc017a2ec61f20818771)) * Enable users to upgrade a batch source into a push source ([#3213](https://github.com/feast-dev/feast/issues/3213)) ([1b312fb](https://github.com/feast-dev/feast/commit/1b312fbb96a0629c0bd465ee09a01a43130c99c7)) * Fix docker image for feature-server ([#3272](https://github.com/feast-dev/feast/issues/3272)) ([eff01d1](https://github.com/feast-dev/feast/commit/eff01d1ed3a76ff539595455f69ff1c954ad46cd)) * Fix Feast UI release process to update the feast-ui package ([#3267](https://github.com/feast-dev/feast/issues/3267)) ([a9d48d0](https://github.com/feast-dev/feast/commit/a9d48d07f634deb0b992e36d2ae1577d9b627940)) * Return 422 on bad push source name ([#3214](https://github.com/feast-dev/feast/issues/3214)) ([b851e01](https://github.com/feast-dev/feast/commit/b851e01ab4ae78e6714793c7efddb86b39d2a36e)) * Stream feature view meta undefined created_timestamp issue ([#3266](https://github.com/feast-dev/feast/issues/3266)) ([12e1a8f](https://github.com/feast-dev/feast/commit/12e1a8f0e9f098cfc3ed66db6665cb5fa26240f4)) * Stream feature view not shown in the UI ([#3251](https://github.com/feast-dev/feast/issues/3251)) ([e713dda](https://github.com/feast-dev/feast/commit/e713dda7c5a4433f5d13b4685d36978645e0cd02)) * Udf in stream feature view UI shows pickled data ([#3268](https://github.com/feast-dev/feast/issues/3268)) ([0728117](https://github.com/feast-dev/feast/commit/07281178f8be78178b739366c90fd2074db0981b)) * Update snowflake materialization messages ([#3230](https://github.com/feast-dev/feast/issues/3230)) ([a63d440](https://github.com/feast-dev/feast/commit/a63d440e4207c1e360630423bcda2c329673fddd)) * Updated quickstart notebook to patch an incorrect reference to an outdated featureview name ([#3271](https://github.com/feast-dev/feast/issues/3271)) ([b9b9c54](https://github.com/feast-dev/feast/commit/b9b9c542bc0860cfb351be7d84d6461b70ad2c81)) * Use configured user in env var instead of "user" for Trino ([#3254](https://github.com/feast-dev/feast/issues/3254)) ([532d8a1](https://github.com/feast-dev/feast/commit/532d8a1ec5dd50a959b512427fdbc3e5bb986eb4)) ### Features * Add mysql as online store ([#3190](https://github.com/feast-dev/feast/issues/3190)) ([cb8db84](https://github.com/feast-dev/feast/commit/cb8db84a1b200c7a716d23e17f150b013d5689ff)) * Add possibility to define feature_store.yaml path with env variable ([#3231](https://github.com/feast-dev/feast/issues/3231)) ([95fdb19](https://github.com/feast-dev/feast/commit/95fdb19ed3f4b4af7fe1fb732a1fd63182cf1072)) * Add request_timeout setting for cassandra online store adapter ([#3256](https://github.com/feast-dev/feast/issues/3256)) ([da20757](https://github.com/feast-dev/feast/commit/da20757aa1d5c5f898651295781ef61b52d7d712)) * Add tag description in Field in the Feast UI ([#3258](https://github.com/feast-dev/feast/issues/3258)) ([086f279](https://github.com/feast-dev/feast/commit/086f2790236b776f5c2397365266d9d4f4959142)) * Adding billing_project_id in BigQueryOfflineStoreConfig ([#3253](https://github.com/feast-dev/feast/issues/3253)) ([f80f05f](https://github.com/feast-dev/feast/commit/f80f05f5d2042c6fff533b0dc85612acda5a7417)) * BigTable online store ([#3140](https://github.com/feast-dev/feast/issues/3140)) ([6bc91c2](https://github.com/feast-dev/feast/commit/6bc91c23545a6cf765c0e70887afb22eada2c8e1)), closes [/github.com/spulec/moto/blob/master/CHANGELOG.md#400](https://github.com//github.com/spulec/moto/blob/master/CHANGELOG.md/issues/400) * Filter subset features in postgres [#3174](https://github.com/feast-dev/feast/issues/3174) ([#3203](https://github.com/feast-dev/feast/issues/3203)) ([b48d36b](https://github.com/feast-dev/feast/commit/b48d36bee074788626ed5c2d5cf130bd2fc8bda5)) --- CHANGELOG.md | 32 +++++++ infra/charts/feast-feature-server/Chart.yaml | 2 +- infra/charts/feast-feature-server/README.md | 4 +- infra/charts/feast-feature-server/values.yaml | 2 +- infra/charts/feast-python-server/Chart.yaml | 2 +- infra/charts/feast-python-server/README.md | 2 +- infra/charts/feast/Chart.yaml | 2 +- infra/charts/feast/README.md | 6 +- .../feast/charts/feature-server/Chart.yaml | 4 +- .../feast/charts/feature-server/README.md | 4 +- .../feast/charts/feature-server/values.yaml | 2 +- .../charts/transformation-service/Chart.yaml | 4 +- .../charts/transformation-service/README.md | 4 +- .../charts/transformation-service/values.yaml | 2 +- infra/charts/feast/requirements.yaml | 4 +- java/pom.xml | 2 +- sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 90 ++++++++++++++++++- ui/package.json | 2 +- 19 files changed, 143 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b657e9ddd1a..65aa693691e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,37 @@ # Changelog +# [0.26.0](https://github.com/feast-dev/feast/compare/v0.25.0...v0.26.0) (2022-10-06) + + +### Bug Fixes + +* Add `X-Trino-Extra-Credential` header and remove user override ([#3246](https://github.com/feast-dev/feast/issues/3246)) ([164e666](https://github.com/feast-dev/feast/commit/164e666ee8c425150903819a0035d6f6e48bcdd0)) +* Add postgres to the feature server Dockerfile to fix helm chart flow ([#3261](https://github.com/feast-dev/feast/issues/3261)) ([6f6cbb7](https://github.com/feast-dev/feast/commit/6f6cbb76a216965666719e492c5ba854ada32605)) +* Add stream feature view in the Web UI ([#3257](https://github.com/feast-dev/feast/issues/3257)) ([1f70b3a](https://github.com/feast-dev/feast/commit/1f70b3a9be738ab54f0cb59046ed1e299f92e3a2)) +* Build dockerfile correctly ([#3239](https://github.com/feast-dev/feast/issues/3239)) ([a2dc0d0](https://github.com/feast-dev/feast/commit/a2dc0d0410eb297afddfb1dd4f1f899ab70fa14f)) +* Configuration to stop coercion of tz for entity_df ([#3255](https://github.com/feast-dev/feast/issues/3255)) ([97b7ab9](https://github.com/feast-dev/feast/commit/97b7ab906081d1afe034fc017a2ec61f20818771)) +* Enable users to upgrade a batch source into a push source ([#3213](https://github.com/feast-dev/feast/issues/3213)) ([1b312fb](https://github.com/feast-dev/feast/commit/1b312fbb96a0629c0bd465ee09a01a43130c99c7)) +* Fix docker image for feature-server ([#3272](https://github.com/feast-dev/feast/issues/3272)) ([eff01d1](https://github.com/feast-dev/feast/commit/eff01d1ed3a76ff539595455f69ff1c954ad46cd)) +* Fix Feast UI release process to update the feast-ui package ([#3267](https://github.com/feast-dev/feast/issues/3267)) ([a9d48d0](https://github.com/feast-dev/feast/commit/a9d48d07f634deb0b992e36d2ae1577d9b627940)) +* Return 422 on bad push source name ([#3214](https://github.com/feast-dev/feast/issues/3214)) ([b851e01](https://github.com/feast-dev/feast/commit/b851e01ab4ae78e6714793c7efddb86b39d2a36e)) +* Stream feature view meta undefined created_timestamp issue ([#3266](https://github.com/feast-dev/feast/issues/3266)) ([12e1a8f](https://github.com/feast-dev/feast/commit/12e1a8f0e9f098cfc3ed66db6665cb5fa26240f4)) +* Stream feature view not shown in the UI ([#3251](https://github.com/feast-dev/feast/issues/3251)) ([e713dda](https://github.com/feast-dev/feast/commit/e713dda7c5a4433f5d13b4685d36978645e0cd02)) +* Udf in stream feature view UI shows pickled data ([#3268](https://github.com/feast-dev/feast/issues/3268)) ([0728117](https://github.com/feast-dev/feast/commit/07281178f8be78178b739366c90fd2074db0981b)) +* Update snowflake materialization messages ([#3230](https://github.com/feast-dev/feast/issues/3230)) ([a63d440](https://github.com/feast-dev/feast/commit/a63d440e4207c1e360630423bcda2c329673fddd)) +* Updated quickstart notebook to patch an incorrect reference to an outdated featureview name ([#3271](https://github.com/feast-dev/feast/issues/3271)) ([b9b9c54](https://github.com/feast-dev/feast/commit/b9b9c542bc0860cfb351be7d84d6461b70ad2c81)) +* Use configured user in env var instead of "user" for Trino ([#3254](https://github.com/feast-dev/feast/issues/3254)) ([532d8a1](https://github.com/feast-dev/feast/commit/532d8a1ec5dd50a959b512427fdbc3e5bb986eb4)) + + +### Features + +* Add mysql as online store ([#3190](https://github.com/feast-dev/feast/issues/3190)) ([cb8db84](https://github.com/feast-dev/feast/commit/cb8db84a1b200c7a716d23e17f150b013d5689ff)) +* Add possibility to define feature_store.yaml path with env variable ([#3231](https://github.com/feast-dev/feast/issues/3231)) ([95fdb19](https://github.com/feast-dev/feast/commit/95fdb19ed3f4b4af7fe1fb732a1fd63182cf1072)) +* Add request_timeout setting for cassandra online store adapter ([#3256](https://github.com/feast-dev/feast/issues/3256)) ([da20757](https://github.com/feast-dev/feast/commit/da20757aa1d5c5f898651295781ef61b52d7d712)) +* Add tag description in Field in the Feast UI ([#3258](https://github.com/feast-dev/feast/issues/3258)) ([086f279](https://github.com/feast-dev/feast/commit/086f2790236b776f5c2397365266d9d4f4959142)) +* Adding billing_project_id in BigQueryOfflineStoreConfig ([#3253](https://github.com/feast-dev/feast/issues/3253)) ([f80f05f](https://github.com/feast-dev/feast/commit/f80f05f5d2042c6fff533b0dc85612acda5a7417)) +* BigTable online store ([#3140](https://github.com/feast-dev/feast/issues/3140)) ([6bc91c2](https://github.com/feast-dev/feast/commit/6bc91c23545a6cf765c0e70887afb22eada2c8e1)), closes [/github.com/spulec/moto/blob/master/CHANGELOG.md#400](https://github.com//github.com/spulec/moto/blob/master/CHANGELOG.md/issues/400) +* Filter subset features in postgres [#3174](https://github.com/feast-dev/feast/issues/3174) ([#3203](https://github.com/feast-dev/feast/issues/3203)) ([b48d36b](https://github.com/feast-dev/feast/commit/b48d36bee074788626ed5c2d5cf130bd2fc8bda5)) + # [0.25.0](https://github.com/feast-dev/feast/compare/v0.24.0...v0.25.0) (2022-09-20) diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index 7095866e4f6..252dceb9f6f 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.25.0 +version: 0.26.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index 9cf5c31a631..0ae45328598 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.25.0` +Current chart version is `0.26.0` ## Installation @@ -30,7 +30,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.25.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.26.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 90954a23cb1..d897346efe4 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.25.0 + tag: 0.26.0 imagePullSecrets: [] nameOverride: "" diff --git a/infra/charts/feast-python-server/Chart.yaml b/infra/charts/feast-python-server/Chart.yaml index 30d90876182..0c7690deb27 100644 --- a/infra/charts/feast-python-server/Chart.yaml +++ b/infra/charts/feast-python-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-python-server description: Feast Feature Server in Python type: application -version: 0.25.0 +version: 0.26.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-python-server/README.md b/infra/charts/feast-python-server/README.md index c5d5393e29f..ab2de1d3a83 100644 --- a/infra/charts/feast-python-server/README.md +++ b/infra/charts/feast-python-server/README.md @@ -2,7 +2,7 @@ > Note: this helm chart is deprecated in favor of [feast-feature-server](../feast-feature-server/README.md) -Current chart version is `0.25.0` +Current chart version is `0.26.0` ## Installation Docker repository and tag are required. Helm install example: diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 02f689f5419..f0d6774a3cb 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.25.0 +version: 0.26.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 2afdaf645cc..b0dde94610b 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.25.0` +Feature store for machine learning Current chart version is `0.26.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.25.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.25.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.26.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.26.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index bdaa9ea1fcc..6e577e1fb83 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.25.0 -appVersion: v0.25.0 +version: 0.26.0 +appVersion: v0.26.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index aef8c0329a6..f13a6488c39 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.25.0](https://img.shields.io/badge/Version-0.25.0-informational?style=flat-square) ![AppVersion: v0.25.0](https://img.shields.io/badge/AppVersion-v0.25.0-informational?style=flat-square) +![Version: 0.26.0](https://img.shields.io/badge/Version-0.26.0-informational?style=flat-square) ![AppVersion: v0.26.0](https://img.shields.io/badge/AppVersion-v0.26.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.25.0"` | Image tag | +| image.tag | string | `"0.26.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index d3b8a33a645..66795aeba71 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.25.0 + tag: 0.26.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 104b2f24278..20321e687a3 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.25.0 -appVersion: v0.25.0 +version: 0.26.0 +appVersion: v0.26.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index 37be5b0f106..8b44c218df9 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.25.0](https://img.shields.io/badge/Version-0.25.0-informational?style=flat-square) ![AppVersion: v0.25.0](https://img.shields.io/badge/AppVersion-v0.25.0-informational?style=flat-square) +![Version: 0.26.0](https://img.shields.io/badge/Version-0.26.0-informational?style=flat-square) ![AppVersion: v0.26.0](https://img.shields.io/badge/AppVersion-v0.26.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.25.0"` | Image tag | +| image.tag | string | `"0.26.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index 5232ac68ca1..ecaa16e614f 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.25.0 + tag: 0.26.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index a599ac23a44..f9f23092f5d 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.25.0 + version: 0.26.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.25.0 + version: 0.26.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/java/pom.xml b/java/pom.xml index 874daa27984..3b9910a7cfa 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.25.0 + 0.26.0 https://github.com/feast-dev/feast UTF-8 diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 5854f205709..442cb2466e2 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.25.1", + "@feast-dev/feast-ui": "0.26.0", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index af820b085a1..66abae62440 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1300,10 +1300,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.25.1": - version "0.25.1" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.25.1.tgz#c15e04dfdd4c616fcce254fc22f1474ba4c7b3fe" - integrity sha512-WKJEF2KwzJ9ydoDHlkaYDboHEud+Q/ij8ShGs9CxygNFMSXR00cc63QDdXPLNG4dCE0afSjlgqLtV4R3EJTBfg== +"@feast-dev/feast-ui@0.26.0": + version "0.26.0" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.26.0.tgz#502e61652024a2db9bc5b9b5fcfe3f3dce04161b" + integrity sha512-7TBL9zetO/7Kx+FECIaDYG+fCr7P4R6esWyZW08xJTEFEvds5VXdXvUTWCibA/sKH24dlNXwpiLpyujwqwpQgA== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^55.0.1" @@ -1317,6 +1317,7 @@ inter-ui "^3.19.3" moment "^2.29.1" prop-types "^15.8.1" + protobufjs "^7.1.1" query-string "^7.1.1" react-query "^3.34.12" react-router-dom "6" @@ -1665,6 +1666,59 @@ schema-utils "^3.0.0" source-map "^0.7.3" +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" + integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== + +"@protobufjs/eventemitter@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== + +"@protobufjs/fetch@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== + dependencies: + "@protobufjs/aspromise" "^1.1.1" + "@protobufjs/inquire" "^1.1.0" + +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== + +"@protobufjs/inquire@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" + integrity sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q== + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== + +"@protobufjs/utf8@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" + integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== + "@rollup/plugin-babel@^5.2.0": version "5.3.1" resolved "https://registry.yarnpkg.com/@rollup/plugin-babel/-/plugin-babel-5.3.1.tgz#04bc0608f4aa4b2e4b1aebf284344d0f68fda283" @@ -2328,6 +2382,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.32.tgz#51d59d7a90ef2d0ae961791e0900cad2393a0149" integrity sha512-eAIcfAvhf/BkHcf4pkLJ7ECpBAhh9kcxRBpip9cTiO+hf+aJrsxYxBeS6OXvOd9WqNAJmavXVpZvY1rBjNsXmw== +"@types/node@>=13.7.0": + version "18.8.3" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.8.3.tgz#ce750ab4017effa51aed6a7230651778d54e327c" + integrity sha512-0os9vz6BpGwxGe9LOhgP/ncvYN5Tx1fNcd2TM3rD/aCGBkysb+ZWpXEocG24h6ZzOi13+VB8HndAQFezsSOw1w== + "@types/node@^16.7.13": version "16.11.34" resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.34.tgz#520224e4be4448c279ecad09639ab460cc441a50" @@ -6902,6 +6961,11 @@ lodash@^4.17.15, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7.0: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== +long@^5.0.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/long/-/long-5.2.0.tgz#2696dadf4b4da2ce3f6f6b89186085d94d52fd61" + integrity sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w== + loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -8207,6 +8271,24 @@ property-information@^5.0.0, property-information@^5.3.0: dependencies: xtend "^4.0.0" +protobufjs@^7.1.1: + version "7.1.2" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.1.2.tgz#a0cf6aeaf82f5625bffcf5a38b7cd2a7de05890c" + integrity sha512-4ZPTPkXCdel3+L81yw3dG6+Kq3umdWKh7Dc7GW/CpNk4SX3hK58iPCWeCyhVTDrbkNeKrYNZ7EojM5WDaEWTLQ== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/node" ">=13.7.0" + long "^5.0.0" + proxy-addr@~2.0.7: version "2.0.7" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" diff --git a/ui/package.json b/ui/package.json index 64952897a29..6cb2fef5c3f 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.25.0", + "version": "0.26.0", "private": false, "files": [ "dist"