From 6fc1ad75b552d866d424304c0732b3104bf56fef Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 26 May 2022 11:41:14 -0700 Subject: [PATCH 01/22] Fix working version Signed-off-by: Kevin Zhang --- protos/feast/core/DataFormat.proto | 9 +- protos/feast/core/DataSource.proto | 1 - sdk/python/feast/data_format.py | 24 +++- sdk/python/feast/feature_store.py | 158 ++++----------------- sdk/python/feast/stream_feature_view.py | 50 ++++++- sdk/python/feast/test.py | 174 ++++++++++++++++++++++++ 6 files changed, 271 insertions(+), 145 deletions(-) create mode 100644 sdk/python/feast/test.py diff --git a/protos/feast/core/DataFormat.proto b/protos/feast/core/DataFormat.proto index 9fd01e865c2..c453e5e4c83 100644 --- a/protos/feast/core/DataFormat.proto +++ b/protos/feast/core/DataFormat.proto @@ -26,7 +26,7 @@ option java_package = "feast.proto.core"; message FileFormat { // Defines options for the Parquet data format message ParquetFormat {} - + oneof format { ParquetFormat parquet_format = 1; } @@ -40,7 +40,7 @@ message StreamFormat { // Feature data from the obtained stream message string class_path = 1; } - + // Defines options for the avro data format message AvroFormat { // Optional if used in a File DataSource as schema is embedded in avro file. @@ -48,9 +48,14 @@ message StreamFormat { string schema_json = 1; } + message JsonFormat { + string schema_json = 1; + } + // Specifies the data format and format specific options oneof format { AvroFormat avro_format = 1; ProtoFormat proto_format = 2; + JsonFormat json_format = 3; } } diff --git a/protos/feast/core/DataSource.proto b/protos/feast/core/DataSource.proto index 9e6028ccfa4..e5fe32ab82d 100644 --- a/protos/feast/core/DataSource.proto +++ b/protos/feast/core/DataSource.proto @@ -216,7 +216,6 @@ message DataSource { map deprecated_schema = 2; repeated FeatureSpecV2 schema = 3; - } // Defines options for DataSource that supports pushing data to it. This allows data to be pushed to diff --git a/sdk/python/feast/data_format.py b/sdk/python/feast/data_format.py index b6c7bf94e92..d2eea977c1e 100644 --- a/sdk/python/feast/data_format.py +++ b/sdk/python/feast/data_format.py @@ -89,11 +89,12 @@ def from_proto(cls, proto): fmt = proto.WhichOneof("format") if fmt == "avro_format": return AvroFormat(schema_json=proto.avro_format.schema_json) + if fmt == "json_format": + return JsonFormat(schema_json=proto.json_format.schema_json) if fmt == "proto_format": return ProtoFormat(class_path=proto.proto_format.class_path) raise NotImplementedError(f"StreamFormat is unsupported: {fmt}") - class AvroFormat(StreamFormat): """ Defines the Avro streaming data format that encodes data in Avro format @@ -113,6 +114,27 @@ def to_proto(self): return StreamFormatProto(avro_format=proto) +class JsonFormat(StreamFormat): + """ + Defines the Json streaming data format that encodes data in Json format + """ + + def __init__(self, schema_json: str): + """ + Construct a new Json data format. + + For spark, uses pyspark ddl string format. Example shown here: + https://vincent.doba.fr/posts/20211004_spark_data_description_language_for_defining_spark_schema/ + + Args: + schema_json: Json schema definition + """ + self.schema_json = schema_json + + def to_proto(self): + proto = StreamFormatProto.JsonFormat(schema_json=self.schema_json) + return StreamFormatProto(json_format=proto) + class ProtoFormat(StreamFormat): """ Defines the Protobuf data format diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 2e312240fe1..689defabb9c 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -45,7 +45,6 @@ from feast.data_source import DataSource from feast.diff.infra_diff import InfraDiff, diff_infra_protos from feast.diff.registry_diff import RegistryDiff, apply_diff_to_registry, diff_between -from feast.dqm.errors import ValidationFailed from feast.entity import Entity from feast.errors import ( EntityNotFoundException, @@ -66,11 +65,12 @@ ) from feast.inference import ( update_data_sources_with_inferred_event_timestamp_col, - update_feature_views_with_inferred_features_and_entities, + update_entities_with_inferred_types_from_feature_views, ) from feast.infra.infra_object import Infra from feast.infra.provider import Provider, RetrievalJob, get_provider from feast.on_demand_feature_view import OnDemandFeatureView +from feast.stream_feature_view import StreamFeatureView from feast.online_response import OnlineResponse from feast.protos.feast.core.InfraObject_pb2 import Infra as InfraProto from feast.protos.feast.serving.ServingService_pb2 import ( @@ -83,7 +83,7 @@ from feast.repo_config import RepoConfig, load_repo_config from feast.repo_contents import RepoContents from feast.request_feature_view import RequestFeatureView -from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference +from feast.saved_dataset import SavedDataset, SavedDatasetStorage from feast.type_map import ( feast_value_type_to_python_type, python_values_to_proto_values, @@ -255,7 +255,6 @@ def _list_feature_views( ): if hide_dummy_entity and fv.entities[0] == DUMMY_ENTITY_NAME: fv.entities = [] - fv.entity_columns = [] feature_views.append(fv) return feature_views @@ -480,6 +479,10 @@ def _make_inferences( feature_services_to_update: List[FeatureService], ): """Makes inferences for entities, feature views, odfvs, and feature services.""" + update_entities_with_inferred_types_from_feature_views( + entities_to_update, views_to_update, self.config + ) + update_data_sources_with_inferred_event_timestamp_col( data_sources_to_update, self.config ) @@ -490,7 +493,7 @@ def _make_inferences( # New feature views may reference previously applied entities. entities = self._list_entities() - update_feature_views_with_inferred_features_and_entities( + update_feature_views_with_inferred_features( views_to_update, entities + entities_to_update, self.config ) @@ -520,11 +523,11 @@ def _plan( Examples: Generate a plan adding an Entity and a FeatureView. - >>> from feast import FeatureStore, Entity, FeatureView, Feature, FileSource, RepoConfig + >>> from feast import FeatureStore, Entity, FeatureView, Feature, ValueType, FileSource, RepoConfig >>> from feast.feature_store import RepoContents >>> from datetime import timedelta >>> fs = FeatureStore(repo_path="feature_repo") - >>> driver = Entity(name="driver_id", description="driver id") + >>> driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id") >>> driver_hourly_stats = FileSource( ... path="feature_repo/data/driver_stats.parquet", ... timestamp_field="event_timestamp", @@ -532,7 +535,7 @@ def _plan( ... ) >>> driver_hourly_stats_view = FeatureView( ... name="driver_hourly_stats", - ... entities=[driver], + ... entities=["driver_id"], ... ttl=timedelta(seconds=86400 * 1), ... batch_source=driver_hourly_stats, ... ) @@ -607,8 +610,8 @@ def apply( FeatureView, OnDemandFeatureView, RequestFeatureView, + StreamFeatureView, FeatureService, - ValidationReference, List[FeastObject], ], objects_to_delete: Optional[List[FeastObject]] = None, @@ -634,10 +637,10 @@ def apply( Examples: Register an Entity and a FeatureView. - >>> from feast import FeatureStore, Entity, FeatureView, Feature, FileSource, RepoConfig + >>> from feast import FeatureStore, Entity, FeatureView, Feature, ValueType, FileSource, RepoConfig >>> from datetime import timedelta >>> fs = FeatureStore(repo_path="feature_repo") - >>> driver = Entity(name="driver_id", description="driver id") + >>> driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id") >>> driver_hourly_stats = FileSource( ... path="feature_repo/data/driver_stats.parquet", ... timestamp_field="event_timestamp", @@ -645,7 +648,7 @@ def apply( ... ) >>> driver_hourly_stats_view = FeatureView( ... name="driver_hourly_stats", - ... entities=[driver], + ... entities=["driver_id"], ... ttl=timedelta(seconds=86400 * 1), ... batch_source=driver_hourly_stats, ... ) @@ -670,9 +673,6 @@ def apply( data_sources_set_to_update = { ob for ob in objects if isinstance(ob, DataSource) } - validation_references_to_update = [ - ob for ob in objects if isinstance(ob, ValidationReference) - ] for fv in views_to_update: data_sources_set_to_update.add(fv.batch_source) @@ -695,9 +695,6 @@ def apply( data_sources_to_update = list(data_sources_set_to_update) - # Handle all entityless feature views by using DUMMY_ENTITY as a placeholder entity. - entities_to_update.append(DUMMY_ENTITY) - # Validate all feature views and make inferences. self._validate_all_feature_views( views_to_update, odfvs_to_update, request_views_to_update @@ -710,6 +707,9 @@ def apply( services_to_update, ) + # Handle all entityless feature views by using DUMMY_ENTITY as a placeholder entity. + entities_to_update.append(DUMMY_ENTITY) + # Add all objects to the registry and update the provider's infrastructure. for ds in data_sources_to_update: self._registry.apply_data_source(ds, project=self.project, commit=False) @@ -723,10 +723,6 @@ def apply( self._registry.apply_feature_service( feature_service, project=self.project, commit=False ) - for validation_references in validation_references_to_update: - self._registry.apply_validation_reference( - validation_references, project=self.project, commit=False - ) if not partial: # Delete all registry objects that should not exist. @@ -748,9 +744,6 @@ def apply( data_sources_to_delete = [ ob for ob in objects_to_delete if isinstance(ob, DataSource) ] - validation_references_to_delete = [ - ob for ob in objects_to_delete if isinstance(ob, ValidationReference) - ] for data_source in data_sources_to_delete: self._registry.delete_data_source( @@ -776,10 +769,6 @@ def apply( self._registry.delete_feature_service( service.name, project=self.project, commit=False ) - for validation_references in validation_references_to_delete: - self._registry.delete_validation_reference( - validation_references.name, project=self.project, commit=False - ) self._get_provider().update_infra( project=self.project, @@ -1573,12 +1562,12 @@ def _get_columnar_entity_values( def _get_entity_maps( self, feature_views ) -> Tuple[Dict[str, str], Dict[str, ValueType], Set[str]]: - # TODO(felixwang9817): Support entities that have different types for different feature views. entities = self._list_entities(allow_cache=True, hide_dummy_entity=False) entity_name_to_join_key_map: Dict[str, str] = {} entity_type_map: Dict[str, ValueType] = {} for entity in entities: entity_name_to_join_key_map[entity.name] = entity.join_key + entity_type_map[entity.name] = entity.value_type for feature_view in feature_views: for entity_name in feature_view.entities: entity = self._registry.get_entity( @@ -1593,11 +1582,7 @@ def _get_entity_maps( entity.join_key, entity.join_key ) entity_name_to_join_key_map[entity_name] = join_key - for entity_column in feature_view.entity_columns: - entity_type_map[ - entity_column.name - ] = entity_column.dtype.to_value_type() - + entity_type_map[join_key] = entity.value_type return ( entity_name_to_join_key_map, entity_type_map, @@ -1995,36 +1980,14 @@ def _get_feature_views_to_use( return views_to_use @log_exceptions_and_usage - def serve( - self, - host: str, - port: int, - type_: str, - no_access_log: bool, - no_feature_log: bool, - ) -> None: + def serve(self, host: str, port: int, no_access_log: bool) -> None: """Start the feature consumption server locally on a given port.""" - type_ = type_.lower() if self.config.go_feature_retrieval: # Start go server instead of python if the flag is enabled self._lazy_init_go_server() - if type_ == "http": - self._go_server.start_http_server( - host, port, enable_logging=not no_feature_log - ) - elif type_ == "grpc": - self._go_server.start_grpc_server( - host, port, enable_logging=not no_feature_log - ) - else: - raise ValueError( - f"Unsupported server type '{type_}'. Must be one of 'http' or 'grpc'." - ) + # TODO(tsotne) add http/grpc flag in CLI and call appropriate method here depending on that + self._go_server.start_grpc_server(host, port) else: - if type_ != "http": - raise ValueError( - f"Python server only supports 'http'. Got '{type_}' instead." - ) # Start the python server if go server isn't enabled feature_server.start_server(self, host, port, no_access_log) @@ -2065,7 +2028,6 @@ def serve_transformations(self, port: int) -> None: def _teardown_go_server(self): self._go_server = None - @log_exceptions_and_usage def write_logged_features( self, logs: Union[pa.Table, Path], source: Union[FeatureService] ): @@ -2093,80 +2055,6 @@ def write_logged_features( registry=self._registry, ) - @log_exceptions_and_usage - def validate_logged_features( - self, - source: Union[FeatureService], - start: datetime, - end: datetime, - reference: ValidationReference, - throw_exception: bool = True, - cache_profile: bool = True, - ) -> Optional[ValidationFailed]: - """ - Load logged features from an offline store and validate them against provided validation reference. - - Args: - source: Logs source object (currently only feature services are supported) - start: lower bound for loading logged features - end: upper bound for loading logged features - reference: validation reference - throw_exception: throw exception or return it as a result - cache_profile: store cached profile in Feast registry - - Returns: - Throw or return (depends on parameter) ValidationFailed exception if validation was not successful - or None if successful. - - """ - warnings.warn( - "Logged features validation is an experimental feature. " - "This API is unstable and it could and most probably will be changed in the future. " - "We do not guarantee that future changes will maintain backward compatibility.", - RuntimeWarning, - ) - - if not isinstance(source, FeatureService): - raise ValueError("Only feature service is currently supported as a source") - - j = self._get_provider().retrieve_feature_service_logs( - feature_service=source, - start_date=start, - end_date=end, - config=self.config, - registry=self.registry, - ) - - # read and run validation - try: - j.to_arrow(validation_reference=reference) - except ValidationFailed as exc: - if throw_exception: - raise - - return exc - - if cache_profile: - self.apply(reference) - - return None - - @log_exceptions_and_usage - def get_validation_reference( - self, name: str, allow_cache: bool = False - ) -> ValidationReference: - """ - Retrieves a validation reference. - - Raises: - ValidationReferenceNotFoundException: The validation reference could not be found. - """ - ref = self._registry.get_validation_reference( - name, project=self.project, allow_cache=allow_cache - ) - ref._dataset = self.get_saved_dataset(ref.dataset_name) - return ref - def _validate_entity_values(join_key_values: Dict[str, List[Value]]): set_of_row_lengths = {len(v) for v in join_key_values.values()} diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index cfb3f63d7df..338bd9e1de4 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -1,15 +1,24 @@ +import abc from datetime import timedelta -from typing import Dict, List, Optional, Union +from types import MethodType +from typing import Dict, List, Optional, Union, Callable, Tuple from feast.data_source import DataSource from feast.entity import Entity from feast.feature_view import FeatureView from feast.field import Field from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto +from regex import R SUPPORTED_STREAM_SOURCES = {"KafkaSource", "KinesisSource", "PushSource"} +# class Aggregation(abc.ABC): +# column: str # Column name of the feature we are aggregating. +# function: str # Provide built in aggregations sum, max, min, count mean +# time_windows: Union[(timedelta, timedelta), List[tuple(timedelta, timedelta)]] # The time window and the slide + + class StreamFeatureView(FeatureView): def __init__( self, @@ -18,30 +27,40 @@ def __init__( entities: Optional[Union[List[Entity], List[str]]] = None, ttl: Optional[timedelta] = None, tags: Optional[Dict[str, str]] = None, - online: bool = True, - description: str = "", - owner: str = "", + online: Optional[bool] = True, + description: Optional[str] = "", + owner: Optional[str] = "", schema: Optional[List[Field]] = None, source: Optional[DataSource] = None, + #aggregations: List[Aggregation], + mode: Optional[str] = "spark", # Mode of ingestion/transformation + timestamp_field: Optional[str] = "", # Timestamp for aggregation + udf: Optional[MethodType] = None, ): if source is None: raise ValueError("Feature views need a source specified") + #TODO: There is a bug here with stream_source/batch_source + # source uses the batch_source of the kafkasource in feature_view if ( type(source).__name__ not in SUPPORTED_STREAM_SOURCES - and source.to_proto().type != DataSourceProto.SourceType.CUSTOM_SOURCE + # and source.to_proto().type != DataSourceProto.SourceType.CUSTOM_SOURCE ): raise ValueError( f"Stream feature views need a stream source, expected one of {SUPPORTED_STREAM_SOURCES} " f"or CUSTOM_SOURCE, got {type(source).__name__}: {source.name} instead " ) + # self.aggregations = aggregations + self.mode = mode + self.timestamp_field = timestamp_field + self.udf = udf super().__init__( name=name, entities=entities, ttl=ttl, batch_source=None, - stream_source=None, + stream_source=source, tags=tags, online=online, description=description, @@ -49,3 +68,22 @@ def __init__( schema=schema, source=source, ) + + def __eq__(self, other): + if not isinstance(other, StreamFeatureView): + raise TypeError( + "Comparisons should only involve StreamFeatureViews" + ) + + if not super().__eq__(other): + return False + + if ( self.mode != other.mode or self.timestamp_field != other.timestamp_field + or self.udf.__code__.co_code != other.udf.__code__.co_code): + return False + + return True + + def __hash__(self): + return super().__hash__() + diff --git a/sdk/python/feast/test.py b/sdk/python/feast/test.py new file mode 100644 index 00000000000..012a53d3411 --- /dev/null +++ b/sdk/python/feast/test.py @@ -0,0 +1,174 @@ +import abc +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Mapping, + Optional, + Sequence, + Set, + Tuple, + Union, + cast, +) + +import pandas as pd +from feast import StreamFeatureView, FeatureStore +from feast.data_source import DataSource, KafkaSource +from feast.data_format import AvroFormat, JsonFormat +from datetime import timedelta + +from pyspark.sql import DataFrame, SparkSession +from pyspark.sql.types import StructType, IntegerType, DoubleType, TimestampType +from pyspark.sql.functions import col, from_json +from pyspark.sql.avro.functions import from_avro + +StreamTable = DataFrame # Can add more to this later(change to union). +class StreamProcessor(abc.ABC): + data_source: DataSource + sfv: StreamFeatureView + def __init__(self, sfv: StreamFeatureView, data_source: DataSource): + self.sfv = sfv + self.data_source = data_source + + def _ingest_stream_data(self) -> StreamTable: + """ + Ingests data into StreamTable depending on what type of data it is + """ + pass + + def _construct_transformation_plan(self, table: StreamTable) -> StreamTable: + """ + Applies transformations on top of StreamTable object. Since stream engines use lazy + evaluation, the StreamTable will not be materialized until it is actually evaluated. + For example: df.collect() in spark or tbl.execute() in Flink. + """ + pass + + def _write_to_online_store(self, table: StreamTable): + """ + Returns query for writing stream. + """ + pass + + def transform_stream_data(self) -> StreamTable: + pass + + def ingest_stream_feature_view(self): + pass + + def transform_and_write(self, table: StreamTable): + pass + +def write_row(fs, feature_view, row, join_keys, input_timestamp_field, output_timestamp_column=""): + row: pd.DataFrame = row.toPandas() + + row = row.sort_values(by=join_keys + [input_timestamp_field], ascending=True).groupby(join_keys).nth(0) + if output_timestamp_column and output_timestamp_column != input_timestamp_field: + row = row.rename(columns = {input_timestamp_field, output_timestamp_column}) + row['created'] = pd.to_datetime('now', utc=True) + # print("========================") + # print(row) + fs.write_to_online_store( + feature_view, + row, + ) + + +class SparkStreamKafkaProcessor(StreamProcessor): + # TODO: wrap spark data in some kind of config + # includes session, format, checkpoint location etc. + spark: SparkSession + format: str + fs: FeatureStore + join_keys: List[str] + def __init__( + self, + sfv: StreamFeatureView, + spark_session: SparkSession, + fs: FeatureStore): + if not isinstance(sfv.stream_source, KafkaSource): + raise ValueError("data source is not kafka source") + if not isinstance(sfv.stream_source.kafka_options.message_format, AvroFormat) and not isinstance(sfv.stream_source.kafka_options.message_format, JsonFormat): + raise ValueError("spark streaming currently only supports json or avro format for kafka source schema") + # if not sfv.mode == "spark": + # raise ValueError(f"stream feature view mode is {sfv.mode}, but only supports spark") + self.format = "json" if isinstance(sfv.stream_source.kafka_options.message_format, JsonFormat) else "avro" + self.spark = spark_session + self.fs = fs + self.join_keys = [self.fs.get_entity(entity, allow_registry_cache=True).join_key for entity in sfv.entities] + super().__init__(sfv=sfv, data_source=sfv.stream_source) + + + + def _ingest_stream_data(self) -> StreamTable: + """ + Ingests data into StreamTable depending on what type of data format it is in. + Only supports json and avro formats currently. + """ + if self.format == "json": + streamingDF = ( + self.spark.readStream.format("kafka") + .option("kafka.bootstrap.servers", self.data_source.kafka_options.bootstrap_servers) + .option("subscribe", self.data_source.kafka_options.topic) + .option("startingOffsets", "latest") # Query start + .load() + .selectExpr('CAST(value AS STRING)') + .select(from_json(col('value'), self.data_source.kafka_options.message_format.schema_json).alias("table")) + .select("table.*") + ) + else: + streamingDF = ( + self.spark.readStream.format("kafka") + .option("kafka.bootstrap.servers", self.data_source.kafka_options.bootstrap_servers) + .option("subscribe", self.data_source.kafka_options.topic) + .option("startingOffsets", "latest") # Query start + .load() + .selectExpr('CAST(value AS STRING)') + .select(from_avro(col('value'), self.data_source.kafka_options.message_format.schema_json).alias("table")) + .select("table.*") + ) + return streamingDF + + def _construct_transformation_plan(self, df : StreamTable) -> StreamTable: + """ + Applies transformations on top of StreamTable object. Since stream engines use lazy + evaluation, the StreamTable will not be materialized until it is actually evaluated. + For example: df.collect() in spark or tbl.execute() in Flink. + """ + # if self.sfv.udf == None: + # return table + # else: + # return None + return df + + def _write_to_online_store(self, df: StreamTable): + """ + Returns query for writing stream. + """ + # Validation occurs at the fs.write_to_online_store() phase against the stream feature view schema. + query = df \ + .writeStream \ + .outputMode("update") \ + .option("checkpointLocation", "/tmp/checkpoint/") \ + .trigger(processingTime="30 seconds") \ + .foreachBatch(lambda row, batch_id: write_row(fs=self.fs, feature_view=self.sfv.name, row=row, join_keys=self.join_keys, input_timestamp_field="event_timestamp")) \ + .start() + query.awaitTermination(timeout=30) + return query + + def transform_stream_data(self) -> StreamTable: + df = self._ingest_stream_data() + return self._construct_transformation_plan(df) + + def ingest_stream_feature_view(self): + ingested_stream_df = self._ingest_stream_data() + transformed_df = self._construct_transformation_plan(ingested_stream_df) + online_store_query = self._write_to_online_store(transformed_df) + return online_store_query + + def transform_and_write(self, table: StreamTable): + pass \ No newline at end of file From dbfba2374bf7f135ad7c4879f912b51f75ba2e9c Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 26 May 2022 12:05:21 -0700 Subject: [PATCH 02/22] Working commit Signed-off-by: Kevin Zhang --- sdk/python/feast/feature_store.py | 240 ++++++++++++++++-------------- 1 file changed, 132 insertions(+), 108 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 689defabb9c..0318572c138 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -45,6 +45,7 @@ from feast.data_source import DataSource from feast.diff.infra_diff import InfraDiff, diff_infra_protos from feast.diff.registry_diff import RegistryDiff, apply_diff_to_registry, diff_between +from feast.dqm.errors import ValidationFailed from feast.entity import Entity from feast.errors import ( EntityNotFoundException, @@ -65,12 +66,11 @@ ) from feast.inference import ( update_data_sources_with_inferred_event_timestamp_col, - update_entities_with_inferred_types_from_feature_views, + update_feature_views_with_inferred_features_and_entities, ) from feast.infra.infra_object import Infra from feast.infra.provider import Provider, RetrievalJob, get_provider from feast.on_demand_feature_view import OnDemandFeatureView -from feast.stream_feature_view import StreamFeatureView from feast.online_response import OnlineResponse from feast.protos.feast.core.InfraObject_pb2 import Infra as InfraProto from feast.protos.feast.serving.ServingService_pb2 import ( @@ -83,7 +83,7 @@ from feast.repo_config import RepoConfig, load_repo_config from feast.repo_contents import RepoContents from feast.request_feature_view import RequestFeatureView -from feast.saved_dataset import SavedDataset, SavedDatasetStorage +from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference from feast.type_map import ( feast_value_type_to_python_type, python_values_to_proto_values, @@ -101,7 +101,6 @@ class FeatureStore: """ A FeatureStore object is used to define, create, and retrieve features. - Args: repo_path (optional): Path to a `feature_store.yaml` used to configure the feature store. @@ -120,7 +119,6 @@ def __init__( ): """ Creates a FeatureStore object. - Raises: ValueError: If both or neither of repo_path and config are specified. """ @@ -163,13 +161,11 @@ def _get_provider(self) -> Provider: @log_exceptions_and_usage def refresh_registry(self): """Fetches and caches a copy of the feature registry in memory. - Explicitly calling this method allows for direct control of the state of the registry cache. Every time this method is called the complete registry state will be retrieved from the remote registry store backend (e.g., GCS, S3), and the cache timer will be reset. If refresh_registry() is run before get_online_features() is called, then get_online_features() will use the cached registry instead of retrieving (and caching) the registry itself. - Additionally, the TTL for the registry cache can be set to infinity (by setting it to 0), which means that refresh_registry() will become the only way to update the cached registry. If the TTL is set to a value greater than 0, then once the cache becomes stale (more time than the TTL has passed), a new cache will be @@ -185,10 +181,8 @@ def refresh_registry(self): def list_entities(self, allow_cache: bool = False) -> List[Entity]: """ Retrieves the list of entities from the registry. - Args: allow_cache: Whether to allow returning entities from a cached registry. - Returns: A list of entities. """ @@ -210,7 +204,6 @@ def _list_entities( def list_feature_services(self) -> List[FeatureService]: """ Retrieves the list of feature services from the registry. - Returns: A list of feature services. """ @@ -220,10 +213,8 @@ def list_feature_services(self) -> List[FeatureService]: def list_feature_views(self, allow_cache: bool = False) -> List[FeatureView]: """ Retrieves the list of feature views from the registry. - Args: allow_cache: Whether to allow returning entities from a cached registry. - Returns: A list of feature views. """ @@ -235,10 +226,8 @@ def list_request_feature_views( ) -> List[RequestFeatureView]: """ Retrieves the list of feature views from the registry. - Args: allow_cache: Whether to allow returning entities from a cached registry. - Returns: A list of feature views. """ @@ -255,6 +244,7 @@ def _list_feature_views( ): if hide_dummy_entity and fv.entities[0] == DUMMY_ENTITY_NAME: fv.entities = [] + fv.entity_columns = [] feature_views.append(fv) return feature_views @@ -264,7 +254,6 @@ def list_on_demand_feature_views( ) -> List[OnDemandFeatureView]: """ Retrieves the list of on demand feature views from the registry. - Returns: A list of on demand feature views. """ @@ -276,10 +265,8 @@ def list_on_demand_feature_views( def list_data_sources(self, allow_cache: bool = False) -> List[DataSource]: """ Retrieves the list of data sources from the registry. - Args: allow_cache: Whether to allow returning data sources from a cached registry. - Returns: A list of data sources. """ @@ -289,14 +276,11 @@ def list_data_sources(self, allow_cache: bool = False) -> List[DataSource]: def get_entity(self, name: str, allow_registry_cache: bool = False) -> Entity: """ Retrieves an entity. - Args: name: Name of entity. allow_registry_cache: (Optional) Whether to allow returning this entity from a cached registry - Returns: The specified entity. - Raises: EntityNotFoundException: The entity could not be found. """ @@ -310,14 +294,11 @@ def get_feature_service( ) -> FeatureService: """ Retrieves a feature service. - Args: name: Name of feature service. allow_cache: Whether to allow returning feature services from a cached registry. - Returns: The specified feature service. - Raises: FeatureServiceNotFoundException: The feature service could not be found. """ @@ -329,14 +310,11 @@ def get_feature_view( ) -> FeatureView: """ Retrieves a feature view. - Args: name: Name of feature view. allow_registry_cache: (Optional) Whether to allow returning this entity from a cached registry - Returns: The specified feature view. - Raises: FeatureViewNotFoundException: The feature view could not be found. """ @@ -359,13 +337,10 @@ def _get_feature_view( def get_on_demand_feature_view(self, name: str) -> OnDemandFeatureView: """ Retrieves a feature view. - Args: name: Name of feature view. - Returns: The specified feature view. - Raises: FeatureViewNotFoundException: The feature view could not be found. """ @@ -375,13 +350,10 @@ def get_on_demand_feature_view(self, name: str) -> OnDemandFeatureView: def get_data_source(self, name: str) -> DataSource: """ Retrieves the list of data sources from the registry. - Args: name: Name of the data source. - Returns: The specified data source. - Raises: DataSourceObjectNotFoundException: The data source could not be found. """ @@ -391,10 +363,8 @@ def get_data_source(self, name: str) -> DataSource: def delete_feature_view(self, name: str): """ Deletes a feature view. - Args: name: Name of feature view. - Raises: FeatureViewNotFoundException: The feature view could not be found. """ @@ -404,10 +374,8 @@ def delete_feature_view(self, name: str): def delete_feature_service(self, name: str): """ Deletes a feature service. - Args: name: Name of feature service. - Raises: FeatureServiceNotFoundException: The feature view could not be found. """ @@ -479,10 +447,6 @@ def _make_inferences( feature_services_to_update: List[FeatureService], ): """Makes inferences for entities, feature views, odfvs, and feature services.""" - update_entities_with_inferred_types_from_feature_views( - entities_to_update, views_to_update, self.config - ) - update_data_sources_with_inferred_event_timestamp_col( data_sources_to_update, self.config ) @@ -493,7 +457,7 @@ def _make_inferences( # New feature views may reference previously applied entities. entities = self._list_entities() - update_feature_views_with_inferred_features( + update_feature_views_with_inferred_features_and_entities( views_to_update, entities + entities_to_update, self.config ) @@ -509,25 +473,20 @@ def _plan( self, desired_repo_contents: RepoContents ) -> Tuple[RegistryDiff, InfraDiff, Infra]: """Dry-run registering objects to metadata store. - The plan method dry-runs registering one or more definitions (e.g., Entity, FeatureView), and produces a list of all the changes the that would be introduced in the feature repo. The changes computed by the plan command are for informational purposes, and are not actually applied to the registry. - Args: desired_repo_contents: The desired repo state. - Raises: ValueError: The 'objects' parameter could not be parsed properly. - Examples: Generate a plan adding an Entity and a FeatureView. - - >>> from feast import FeatureStore, Entity, FeatureView, Feature, ValueType, FileSource, RepoConfig + >>> from feast import FeatureStore, Entity, FeatureView, Feature, FileSource, RepoConfig >>> from feast.feature_store import RepoContents >>> from datetime import timedelta >>> fs = FeatureStore(repo_path="feature_repo") - >>> driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id") + >>> driver = Entity(name="driver_id", description="driver id") >>> driver_hourly_stats = FileSource( ... path="feature_repo/data/driver_stats.parquet", ... timestamp_field="event_timestamp", @@ -535,7 +494,7 @@ def _plan( ... ) >>> driver_hourly_stats_view = FeatureView( ... name="driver_hourly_stats", - ... entities=["driver_id"], + ... entities=[driver], ... ttl=timedelta(seconds=86400 * 1), ... batch_source=driver_hourly_stats, ... ) @@ -588,7 +547,6 @@ def _apply_diffs( self, registry_diff: RegistryDiff, infra_diff: InfraDiff, new_infra: Infra ): """Applies the given diffs to the metadata store and infrastructure. - Args: registry_diff: The diff between the current registry and the desired registry. infra_diff: The diff between the current infra and the desired infra. @@ -610,37 +568,32 @@ def apply( FeatureView, OnDemandFeatureView, RequestFeatureView, - StreamFeatureView, FeatureService, + ValidationReference, List[FeastObject], ], objects_to_delete: Optional[List[FeastObject]] = None, partial: bool = True, ): """Register objects to metadata store and update related infrastructure. - The apply method registers one or more definitions (e.g., Entity, FeatureView) and registers or updates these objects in the Feast registry. Once the apply method has updated the infrastructure (e.g., create tables in an online store), it will commit the updated registry. All operations are idempotent, meaning they can safely be rerun. - Args: objects: A single object, or a list of objects that should be registered with the Feature Store. objects_to_delete: A list of objects to be deleted from the registry and removed from the provider's infrastructure. This deletion will only be performed if partial is set to False. partial: If True, apply will only handle the specified objects; if False, apply will also delete all the objects in objects_to_delete, and tear down any associated cloud resources. - Raises: ValueError: The 'objects' parameter could not be parsed properly. - Examples: Register an Entity and a FeatureView. - - >>> from feast import FeatureStore, Entity, FeatureView, Feature, ValueType, FileSource, RepoConfig + >>> from feast import FeatureStore, Entity, FeatureView, Feature, FileSource, RepoConfig >>> from datetime import timedelta >>> fs = FeatureStore(repo_path="feature_repo") - >>> driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id") + >>> driver = Entity(name="driver_id", description="driver id") >>> driver_hourly_stats = FileSource( ... path="feature_repo/data/driver_stats.parquet", ... timestamp_field="event_timestamp", @@ -648,7 +601,7 @@ def apply( ... ) >>> driver_hourly_stats_view = FeatureView( ... name="driver_hourly_stats", - ... entities=["driver_id"], + ... entities=[driver], ... ttl=timedelta(seconds=86400 * 1), ... batch_source=driver_hourly_stats, ... ) @@ -673,6 +626,9 @@ def apply( data_sources_set_to_update = { ob for ob in objects if isinstance(ob, DataSource) } + validation_references_to_update = [ + ob for ob in objects if isinstance(ob, ValidationReference) + ] for fv in views_to_update: data_sources_set_to_update.add(fv.batch_source) @@ -695,6 +651,9 @@ def apply( data_sources_to_update = list(data_sources_set_to_update) + # Handle all entityless feature views by using DUMMY_ENTITY as a placeholder entity. + entities_to_update.append(DUMMY_ENTITY) + # Validate all feature views and make inferences. self._validate_all_feature_views( views_to_update, odfvs_to_update, request_views_to_update @@ -707,9 +666,6 @@ def apply( services_to_update, ) - # Handle all entityless feature views by using DUMMY_ENTITY as a placeholder entity. - entities_to_update.append(DUMMY_ENTITY) - # Add all objects to the registry and update the provider's infrastructure. for ds in data_sources_to_update: self._registry.apply_data_source(ds, project=self.project, commit=False) @@ -723,6 +679,10 @@ def apply( self._registry.apply_feature_service( feature_service, project=self.project, commit=False ) + for validation_references in validation_references_to_update: + self._registry.apply_validation_reference( + validation_references, project=self.project, commit=False + ) if not partial: # Delete all registry objects that should not exist. @@ -744,6 +704,9 @@ def apply( data_sources_to_delete = [ ob for ob in objects_to_delete if isinstance(ob, DataSource) ] + validation_references_to_delete = [ + ob for ob in objects_to_delete if isinstance(ob, ValidationReference) + ] for data_source in data_sources_to_delete: self._registry.delete_data_source( @@ -769,6 +732,10 @@ def apply( self._registry.delete_feature_service( service.name, project=self.project, commit=False ) + for validation_references in validation_references_to_delete: + self._registry.delete_validation_reference( + validation_references.name, project=self.project, commit=False + ) self._get_provider().update_infra( project=self.project, @@ -808,18 +775,14 @@ def get_historical_features( full_feature_names: bool = False, ) -> RetrievalJob: """Enrich an entity dataframe with historical feature values for either training or batch scoring. - This method joins historical feature data from one or more feature views to an entity dataframe by using a time travel join. - Each feature view is joined to the entity dataframe using all entities configured for the respective feature view. All configured entities must be available in the entity dataframe. Therefore, the entity dataframe must contain all entities found in all feature views, but the individual feature views can have different entities. - Time travel is based on the configured TTL for each feature view. A shorter TTL will limit the amount of scanning that will be done in order to find feature data for a specific entity key. Setting a short TTL may result in null values being returned. - Args: entity_df (Union[pd.DataFrame, str]): An entity dataframe is a collection of rows containing all entity columns (e.g., customer_id, driver_id) on which features need to be joined, as well as a event_timestamp @@ -831,16 +794,12 @@ def get_historical_features( full_feature_names: If True, feature names will be prefixed with the corresponding feature view name, changing them from the format "feature" to "feature_view__feature" (e.g. "daily_transactions" changes to "customer_fv__daily_transactions"). - Returns: RetrievalJob which can be used to materialize the results. - Raises: ValueError: Both or neither of features and feature_refs are specified. - Examples: Retrieve historical features from a local offline store. - >>> from feast import FeatureStore, RepoConfig >>> import pandas as pd >>> fs = FeatureStore(repo_path="feature_repo") @@ -943,10 +902,8 @@ def create_saved_dataset( After data successfully persisted saved dataset object with dataset metadata is committed to the registry. Name for the saved dataset should be unique within project, since it's possible to overwrite previously stored dataset with the same name. - Returns: SavedDataset object with attached RetrievalJob - Raises: ValueError if given retrieval job doesn't have metadata """ @@ -992,14 +949,10 @@ def get_saved_dataset(self, name: str) -> SavedDataset: """ Find a saved dataset in the registry by provided name and create a retrieval job to pull whole dataset from storage (offline store). - If dataset couldn't be found by provided name SavedDatasetNotFound exception will be raised. - Data will be retrieved from globally configured offline store. - Returns: SavedDataset with RetrievalJob attached - Raises: SavedDatasetNotFound """ @@ -1024,24 +977,19 @@ def materialize_incremental( ) -> None: """ Materialize incremental new data from the offline store into the online store. - This method loads incremental new feature data up to the specified end time from either the specified feature views, or all feature views if none are specified, into the online store where it is available for online serving. The start time of the interval materialized is either the most recent end time of a prior materialization or (now - ttl) if no such prior materialization exists. - Args: end_date (datetime): End date for time range of data to materialize into the online store feature_views (List[str]): Optional list of feature view names. If selected, will only run materialization for the specified feature views. - Raises: Exception: A feature view being materialized does not have a TTL set. - Examples: Materialize all features into the online store up to 5 minutes ago. - >>> from feast import FeatureStore, RepoConfig >>> from datetime import datetime, timedelta >>> fs = FeatureStore(repo_path="feature_repo") @@ -1128,21 +1076,17 @@ def materialize( ) -> None: """ Materialize data from the offline store into the online store. - This method loads feature data in the specified interval from either the specified feature views, or all feature views if none are specified, into the online store where it is available for online serving. - Args: start_date (datetime): Start date for time range of data to materialize into the online store end_date (datetime): End date for time range of data to materialize into the online store feature_views (List[str]): Optional list of feature view names. If selected, will only run materialization for the specified feature views. - Examples: Materialize all features into the online store over the interval from 3 hours ago to 10 minutes ago. - >>> from feast import FeatureStore, RepoConfig >>> from datetime import datetime, timedelta >>> fs = FeatureStore(repo_path="feature_repo") @@ -1273,7 +1217,6 @@ def get_online_features( ) -> OnlineResponse: """ Retrieves the latest online feature data. - Note: This method will download the full feature registry the first time it is run. If you are using a remote registry like GCS or S3 then that may take a few seconds. The registry remains cached up to a TTL duration (which can be set to infinity). If the cached registry is stale (more time than the TTL has @@ -1281,7 +1224,6 @@ def get_online_features( introduce latency to online feature retrieval. In order to avoid synchronous downloads, please call refresh_registry() prior to the TTL being reached. Remember it is possible to set the cache TTL to infinity (cache forever). - Args: features: The list of features that should be retrieved from the online store. These features can be specified either as a list of string feature references or as a feature service. String feature @@ -1290,16 +1232,12 @@ def get_online_features( full_feature_names: If True, feature names will be prefixed with the corresponding feature view name, changing them from the format "feature" to "feature_view__feature" (e.g. "daily_transactions" changes to "customer_fv__daily_transactions"). - Returns: OnlineResponse containing the feature data in records. - Raises: Exception: No entity with the specified name exists. - Examples: Retrieve online features from an online store. - >>> from feast import FeatureStore, RepoConfig >>> fs = FeatureStore(repo_path="feature_repo") >>> online_response = fs.get_online_features( @@ -1562,12 +1500,12 @@ def _get_columnar_entity_values( def _get_entity_maps( self, feature_views ) -> Tuple[Dict[str, str], Dict[str, ValueType], Set[str]]: + # TODO(felixwang9817): Support entities that have different types for different feature views. entities = self._list_entities(allow_cache=True, hide_dummy_entity=False) entity_name_to_join_key_map: Dict[str, str] = {} entity_type_map: Dict[str, ValueType] = {} for entity in entities: entity_name_to_join_key_map[entity.name] = entity.join_key - entity_type_map[entity.name] = entity.value_type for feature_view in feature_views: for entity_name in feature_view.entities: entity = self._registry.get_entity( @@ -1582,7 +1520,11 @@ def _get_entity_maps( entity.join_key, entity.join_key ) entity_name_to_join_key_map[entity_name] = join_key - entity_type_map[join_key] = entity.value_type + for entity_column in feature_view.entity_columns: + entity_type_map[ + entity_column.name + ] = entity_column.dtype.to_value_type() + return ( entity_name_to_join_key_map, entity_type_map, @@ -1671,7 +1613,6 @@ def _get_unique_entities( entity_name_to_join_key_map: Dict[str, str], ) -> Tuple[Tuple[Dict[str, Value], ...], Tuple[List[int], ...]]: """Return the set of unique composite Entities for a Feature View and the indexes at which they appear. - This method allows us to query the OnlineStore for data we need only once rather than requesting and processing data for the same combination of Entities multiple times. @@ -1711,10 +1652,8 @@ def _read_from_online_store( table: FeatureView, ) -> List[Tuple[List[Timestamp], List["FieldStatus.ValueType"], List[Value]]]: """Read and process data from the OnlineStore for a given FeatureView. - This method guarantees that the order of the data in each element of the List returned is the same as the order of `requested_features`. - This method assumes that `provider.online_read` returns data for each combination of Entities in `entity_rows` in the same order as they are provided. @@ -1775,11 +1714,9 @@ def _populate_response_from_feature_data( table: FeatureView, ): """Populate the GetOnlineFeaturesResponse with feature data. - This method assumes that `_read_from_online_store` returns data for each combination of Entities in `entity_rows` in the same order as they are provided. - Args: feature_data: A list of data in Protobuf form which was retrieved from the OnlineStore. indexes: A list of indexes which should be the same length as `feature_data`. Each list @@ -1827,11 +1764,9 @@ def _augment_response_with_on_demand_transforms( full_feature_names: bool, ): """Computes on demand feature values and adds them to the result rows. - Assumes that 'online_features_response' already contains the necessary request data and input feature views for the on demand feature views. Unneeded feature values such as request data and unrequested input feature views will be removed from 'online_features_response'. - Args: online_features_response: Protobuf object to populate feature_refs: List of all feature references to be returned. @@ -1896,7 +1831,6 @@ def _drop_unneeded_columns( """ Unneeded feature values such as request data and unrequested input feature views will be removed from 'online_features_response'. - Args: online_features_response: Protobuf object to populate requested_result_row_names: Fields from 'result_rows' that have been requested, and @@ -1980,14 +1914,36 @@ def _get_feature_views_to_use( return views_to_use @log_exceptions_and_usage - def serve(self, host: str, port: int, no_access_log: bool) -> None: + def serve( + self, + host: str, + port: int, + type_: str, + no_access_log: bool, + no_feature_log: bool, + ) -> None: """Start the feature consumption server locally on a given port.""" + type_ = type_.lower() if self.config.go_feature_retrieval: # Start go server instead of python if the flag is enabled self._lazy_init_go_server() - # TODO(tsotne) add http/grpc flag in CLI and call appropriate method here depending on that - self._go_server.start_grpc_server(host, port) + if type_ == "http": + self._go_server.start_http_server( + host, port, enable_logging=not no_feature_log + ) + elif type_ == "grpc": + self._go_server.start_grpc_server( + host, port, enable_logging=not no_feature_log + ) + else: + raise ValueError( + f"Unsupported server type '{type_}'. Must be one of 'http' or 'grpc'." + ) else: + if type_ != "http": + raise ValueError( + f"Python server only supports 'http'. Got '{type_}' instead." + ) # Start the python server if go server isn't enabled feature_server.start_server(self, host, port, no_access_log) @@ -2028,13 +1984,13 @@ def serve_transformations(self, port: int) -> None: def _teardown_go_server(self): self._go_server = None + @log_exceptions_and_usage def write_logged_features( self, logs: Union[pa.Table, Path], source: Union[FeatureService] ): """ Write logs produced by a source (currently only feature service is supported as a source) to an offline store. - Args: logs: Arrow Table or path to parquet dataset directory on disk source: Object that produces logs @@ -2055,6 +2011,76 @@ def write_logged_features( registry=self._registry, ) + @log_exceptions_and_usage + def validate_logged_features( + self, + source: Union[FeatureService], + start: datetime, + end: datetime, + reference: ValidationReference, + throw_exception: bool = True, + cache_profile: bool = True, + ) -> Optional[ValidationFailed]: + """ + Load logged features from an offline store and validate them against provided validation reference. + Args: + source: Logs source object (currently only feature services are supported) + start: lower bound for loading logged features + end: upper bound for loading logged features + reference: validation reference + throw_exception: throw exception or return it as a result + cache_profile: store cached profile in Feast registry + Returns: + Throw or return (depends on parameter) ValidationFailed exception if validation was not successful + or None if successful. + """ + warnings.warn( + "Logged features validation is an experimental feature. " + "This API is unstable and it could and most probably will be changed in the future. " + "We do not guarantee that future changes will maintain backward compatibility.", + RuntimeWarning, + ) + + if not isinstance(source, FeatureService): + raise ValueError("Only feature service is currently supported as a source") + + j = self._get_provider().retrieve_feature_service_logs( + feature_service=source, + start_date=start, + end_date=end, + config=self.config, + registry=self.registry, + ) + + # read and run validation + try: + j.to_arrow(validation_reference=reference) + except ValidationFailed as exc: + if throw_exception: + raise + + return exc + + if cache_profile: + self.apply(reference) + + return None + + @log_exceptions_and_usage + def get_validation_reference( + self, name: str, allow_cache: bool = False + ) -> ValidationReference: + """ + Retrieves a validation reference. + Raises: + ValidationReferenceNotFoundException: The validation reference could not be found. + """ + ref = self._registry.get_validation_reference( + name, project=self.project, allow_cache=allow_cache + ) + ref._dataset = self.get_saved_dataset(ref.dataset_name) + return ref + def _validate_entity_values(join_key_values: Dict[str, List[Value]]): set_of_row_lengths = {len(v) for v in join_key_values.values()} @@ -2066,13 +2092,11 @@ def _validate_entity_values(join_key_values: Dict[str, List[Value]]): def _validate_feature_refs(feature_refs: List[str], full_feature_names: bool = False): """ Validates that there are no collisions among the feature references. - Args: feature_refs: List of feature references to validate. Feature references must have format "feature_view:feature", e.g. "customer_fv:daily_transactions". full_feature_names: If True, the full feature references are compared for collisions; if False, only the feature names are compared. - Raises: FeatureNameCollisionError: There is a collision among the feature references. """ @@ -2231,4 +2255,4 @@ def apply_list_mapping( for idx in destinations: output[idx] = elem - return output + return output \ No newline at end of file From 48a4c2901c787b978c76fc2f6625237ea82fdbd5 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 26 May 2022 17:20:31 -0700 Subject: [PATCH 03/22] Fixes Signed-off-by: Kevin Zhang --- protos/feast/core/StreamFeatureView.proto | 95 +++++++++++++++ sdk/python/feast/feature_store.py | 38 +++++- sdk/python/feast/stream_feature_view.py | 108 +++++++++++++++++- .../feast/{test.py => stream_processor.py} | 25 +--- 4 files changed, 242 insertions(+), 24 deletions(-) create mode 100644 protos/feast/core/StreamFeatureView.proto rename sdk/python/feast/{test.py => stream_processor.py} (85%) diff --git a/protos/feast/core/StreamFeatureView.proto b/protos/feast/core/StreamFeatureView.proto new file mode 100644 index 00000000000..01d4fe7bcde --- /dev/null +++ b/protos/feast/core/StreamFeatureView.proto @@ -0,0 +1,95 @@ +// +// Copyright 2020 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + +syntax = "proto3"; +package feast.core; + +option go_package = "github.com/feast-dev/feast/go/protos/feast/core"; +option java_outer_classname = "StreamFeatureViewProto"; +option java_package = "feast.proto.core"; + + +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; +import "feast/core/OnDemandFeatureView.proto"; +import "feast/core/FeatureView.proto"; +import "feast/core/FeatureViewProjection.proto"; +import "feast/core/Feature.proto"; +import "feast/core/DataSource.proto"; + +message StreamFeatureView { + // User-specified specifications of this feature view. + StreamFeatureViewSpec spec = 1; + StreamFeatureViewMeta meta = 2; +} + +// Next available id: 10 +message StreamFeatureViewSpec { + // Name of the feature view. Must be unique. Not updated. + string name = 1; + + // Name of Feast project that this feature view belongs to. + string project = 2; + + // List of names of entities associated with this feature view. + repeated string entities = 3; + + // List of specifications for each feature defined as part of this feature view. + repeated FeatureSpecV2 features = 4; + + // List of specifications for each entity defined as part of this feature view. + repeated FeatureSpecV2 entity_columns = 12; + + // Description of the feature view. + string description = 10; + + // User defined metadata + map tags = 5; + + // Owner of the feature view. + string owner = 11; + + // Features in this feature view can only be retrieved from online serving + // younger than ttl. Ttl is measured as the duration of time between + // the feature's event timestamp and when the feature is retrieved + // Feature values outside ttl will be returned as unset values and indicated to end user + google.protobuf.Duration ttl = 6; + + // Batch/Offline DataSource where this view can retrieve offline feature data. + DataSource batch_source = 7; + // Streaming DataSource from where this view can consume "online" feature data. + DataSource stream_source = 9; + + // Whether these features should be served online or not + bool online = 8; + + UserDefinedFunction user_defined_function = 13; + + // Mode of execution + string mode = 14; + + // Timestamp field for aggregation + string timestamp_field = 15; +} + +message StreamFeatureViewMeta { + // Time where this Feature View is created + google.protobuf.Timestamp created_timestamp = 1; + + // Time where this Feature View is last updated + google.protobuf.Timestamp last_updated_timestamp = 2; +} diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 0318572c138..f5f39ea3084 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -34,6 +34,7 @@ cast, ) +from feast.stream_processor import SparkStreamKafkaProcessor import pandas as pd import pyarrow as pa from colorama import Fore, Style @@ -71,6 +72,7 @@ from feast.infra.infra_object import Infra from feast.infra.provider import Provider, RetrievalJob, get_provider from feast.on_demand_feature_view import OnDemandFeatureView +from feast.stream_feature_view import StreamFeatureView from feast.online_response import OnlineResponse from feast.protos.feast.core.InfraObject_pb2 import Infra as InfraProto from feast.protos.feast.serving.ServingService_pb2 import ( @@ -91,6 +93,8 @@ from feast.usage import log_exceptions, log_exceptions_and_usage, set_usage_attribute from feast.value_type import ValueType from feast.version import get_version +from pyspark.sql import SparkSession +# from feast.test import SparkStreamKafkaProcessor warnings.simplefilter("once", DeprecationWarning) @@ -568,6 +572,7 @@ def apply( FeatureView, OnDemandFeatureView, RequestFeatureView, + # StreamFeatureView, FeatureService, ValidationReference, List[FeastObject], @@ -618,6 +623,7 @@ def apply( # Separate all objects into entities, feature services, and different feature view types. entities_to_update = [ob for ob in objects if isinstance(ob, Entity)] views_to_update = [ob for ob in objects if isinstance(ob, FeatureView)] + sfvs_to_update = [ob for ob in objects if isinstance(ob, StreamFeatureView)] request_views_to_update = [ ob for ob in objects if isinstance(ob, RequestFeatureView) ] @@ -630,7 +636,7 @@ def apply( ob for ob in objects if isinstance(ob, ValidationReference) ] - for fv in views_to_update: + for fv in itertools.chain(views_to_update, sfvs_to_update): data_sources_set_to_update.add(fv.batch_source) if fv.stream_source: data_sources_set_to_update.add(fv.stream_source) @@ -670,7 +676,7 @@ def apply( for ds in data_sources_to_update: self._registry.apply_data_source(ds, project=self.project, commit=False) for view in itertools.chain( - views_to_update, odfvs_to_update, request_views_to_update + views_to_update, odfvs_to_update, request_views_to_update, sfvs_to_update ): self._registry.apply_feature_view(view, project=self.project, commit=False) for ent in entities_to_update: @@ -767,6 +773,34 @@ def teardown(self): self._registry.teardown() self._teardown_go_server() + def _write_stream_row(self, feature_view, row, join_keys, input_timestamp_field, output_timestamp_column=""): + row: pd.DataFrame = row.toPandas() + + row = row.sort_values(by=join_keys + [input_timestamp_field], ascending=True).groupby(join_keys).nth(0) + if output_timestamp_column and output_timestamp_column != input_timestamp_field: + row = row.rename(columns = {input_timestamp_field, output_timestamp_column}) + row['created'] = pd.to_datetime('now', utc=True) + # print("========================") + # print(row)s + self.write_to_online_store( + feature_view, + row, + ) + + def ingest_stream_feature_view(self, sfv_name: str, spark_session: SparkSession) -> bool: + # TODO: Actually write the code to get the stream feature view. + for fv in self.list_feature_views(): + if fv.name == sfv_name: + sfv = fv + + join_keys = [self.get_entity(entity, allow_registry_cache=True).join_key for entity in sfv.entities] + + skp = SparkStreamKafkaProcessor(sfv=sfv, spark_session=spark_session, write_function=lambda row, input_timestamp, output_timestamp: self._write_stream_row(feature_view=sfv.name, row=row, join_keys=join_keys, input_timestamp_field=input_timestamp, output_timestamp_column=output_timestamp)) + query = skp.ingest_stream_feature_view() + # Handle query(set up monitoring thread, etc) + # Return success + return True + @log_exceptions_and_usage def get_historical_features( self, diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 338bd9e1de4..7f3505566a7 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -1,4 +1,4 @@ -import abc +import dill from datetime import timedelta from types import MethodType from typing import Dict, List, Optional, Union, Callable, Tuple @@ -7,7 +7,18 @@ from feast.entity import Entity from feast.feature_view import FeatureView from feast.field import Field -from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto +from google.protobuf.duration_pb2 import Duration +from feast.protos.feast.core.StreamFeatureView_pb2 import StreamFeatureView as StreamFeatureViewProto +from feast.protos.feast.core.StreamFeatureView_pb2 import ( + StreamFeatureViewMeta as StreamFeatureViewMetaProto, +) +from feast.protos.feast.core.StreamFeatureView_pb2 import ( + StreamFeatureViewSpec as StreamFeatureViewSpecProto, +) +from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( + UserDefinedFunction as UserDefinedFunctionProto, +) +from isort import stream from regex import R SUPPORTED_STREAM_SOURCES = {"KafkaSource", "KinesisSource", "PushSource"} @@ -87,3 +98,96 @@ def __eq__(self, other): def __hash__(self): return super().__hash__() + def to_proto(self): + meta = StreamFeatureViewMetaProto() + if self.created_timestamp: + meta.created_timestamp.FromDatetime(self.created_timestamp) + if self.last_updated_timestamp: + meta.last_updated_timestamp.FromDatetime(self.last_updated_timestamp) + + ttl_duration = None + if self.ttl is not None: + ttl_duration = Duration() + ttl_duration.FromTimedelta(self.ttl) + + if self.batch_source: + batch_source_proto = self.batch_source.to_proto() + batch_source_proto.data_source_class_type = f"{self.batch_source.__class__.__module__}.{self.batch_source.__class__.__name__}" + + stream_source_proto = None + if self.stream_source: + stream_source_proto = self.stream_source.to_proto() + stream_source_proto.data_source_class_type = f"{self.stream_source.__class__.__module__}.{self.stream_source.__class__.__name__}" + + spec = StreamFeatureViewSpecProto( + name=self.name, + entities=self.entities, + entity_columns=[field.to_proto() for field in self.entity_columns], + features=[field.to_proto() for field in self.features], + user_defined_function=UserDefinedFunctionProto( + name=self.udf.__name__, body=dill.dumps(self.udf, recurse=True), + ), + description=self.description, + tags=self.tags, + owner=self.owner, + ttl=(ttl_duration if ttl_duration is not None else None), + online=self.online, + batch_source=batch_source_proto, + stream_source=stream_source_proto, + timestamp_field=self.timestamp_field, + ) + + return StreamFeatureViewProto(spec=spec, meta=meta) + + @classmethod + def from_proto(cls, sfv_proto: StreamFeatureViewProto): + batch_source = ( + DataSource.from_proto(sfv_proto.spec.batch_source) + if sfv_proto.spec.HasField("batch_source") + else None + ) + stream_source = ( + DataSource.from_proto(sfv_proto.spec.stream_source) + if sfv_proto.spec.HasField("stream_source") + else None + ) + sfv_feature_view = cls( + name=sfv_proto.spec.name, + description=sfv_proto.spec.description, + tags=dict(sfv_proto.spec.tags), + owner=sfv_proto.spec.owner, + online=sfv_proto.spec.online, + ttl=( + timedelta(days=0) + if sfv_proto.spec.ttl.ToNanoseconds() == 0 + else sfv_proto.spec.ttl.ToTimedelta() + ), + source=stream_source, + udf=dill.loads( + sfv_feature_view.spec.user_defined_function.body + ), + ) + + if batch_source: + sfv_feature_view.batch_source = batch_source + + if stream_source: + sfv_feature_view.stream_source = stream_source + + sfv_feature_view.entities = sfv_proto.spec.entities + + sfv_feature_view.features = [ + Field.from_proto(field_proto) + for field_proto in sfv_proto.spec.features + ] + + if sfv_proto.meta.HasField("created_timestamp"): + sfv_feature_view.created_timestamp = ( + sfv_proto.meta.created_timestamp.ToDatetime() + ) + if sfv_proto.meta.HasField("last_updated_timestamp"): + sfv_feature_view.last_updated_timestamp = ( + sfv_proto.meta.last_updated_timestamp.ToDatetime() + ) + + return sfv_feature_view \ No newline at end of file diff --git a/sdk/python/feast/test.py b/sdk/python/feast/stream_processor.py similarity index 85% rename from sdk/python/feast/test.py rename to sdk/python/feast/stream_processor.py index 012a53d3411..a259e6c0177 100644 --- a/sdk/python/feast/test.py +++ b/sdk/python/feast/stream_processor.py @@ -16,7 +16,7 @@ ) import pandas as pd -from feast import StreamFeatureView, FeatureStore +from feast.stream_feature_view import StreamFeatureView from feast.data_source import DataSource, KafkaSource from feast.data_format import AvroFormat, JsonFormat from datetime import timedelta @@ -63,33 +63,19 @@ def ingest_stream_feature_view(self): def transform_and_write(self, table: StreamTable): pass -def write_row(fs, feature_view, row, join_keys, input_timestamp_field, output_timestamp_column=""): - row: pd.DataFrame = row.toPandas() - - row = row.sort_values(by=join_keys + [input_timestamp_field], ascending=True).groupby(join_keys).nth(0) - if output_timestamp_column and output_timestamp_column != input_timestamp_field: - row = row.rename(columns = {input_timestamp_field, output_timestamp_column}) - row['created'] = pd.to_datetime('now', utc=True) - # print("========================") - # print(row) - fs.write_to_online_store( - feature_view, - row, - ) - class SparkStreamKafkaProcessor(StreamProcessor): # TODO: wrap spark data in some kind of config # includes session, format, checkpoint location etc. spark: SparkSession format: str - fs: FeatureStore + write_function: Callable join_keys: List[str] def __init__( self, sfv: StreamFeatureView, spark_session: SparkSession, - fs: FeatureStore): + write_function: Callable): if not isinstance(sfv.stream_source, KafkaSource): raise ValueError("data source is not kafka source") if not isinstance(sfv.stream_source.kafka_options.message_format, AvroFormat) and not isinstance(sfv.stream_source.kafka_options.message_format, JsonFormat): @@ -98,8 +84,7 @@ def __init__( # raise ValueError(f"stream feature view mode is {sfv.mode}, but only supports spark") self.format = "json" if isinstance(sfv.stream_source.kafka_options.message_format, JsonFormat) else "avro" self.spark = spark_session - self.fs = fs - self.join_keys = [self.fs.get_entity(entity, allow_registry_cache=True).join_key for entity in sfv.entities] + self.write_function = write_function super().__init__(sfv=sfv, data_source=sfv.stream_source) @@ -155,7 +140,7 @@ def _write_to_online_store(self, df: StreamTable): .outputMode("update") \ .option("checkpointLocation", "/tmp/checkpoint/") \ .trigger(processingTime="30 seconds") \ - .foreachBatch(lambda row, batch_id: write_row(fs=self.fs, feature_view=self.sfv.name, row=row, join_keys=self.join_keys, input_timestamp_field="event_timestamp")) \ + .foreachBatch(lambda row, batch_id: self.write_function(row, input_timestamp="event_timestamp", output_timestamp="")) \ .start() query.awaitTermination(timeout=30) return query From ee455303aac0d87acbadc589ca53defa8a2e0b02 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 31 May 2022 09:16:44 -0700 Subject: [PATCH 04/22] Fix stuffs Signed-off-by: Kevin Zhang --- protos/feast/core/StreamFeatureView.proto | 27 ++++--- sdk/python/feast/base_feature_view.py | 1 - sdk/python/feast/feature_view.py | 5 +- sdk/python/feast/stream_feature_view.py | 83 +++++++++++++++++---- sdk/python/tests/unit/test_feature_views.py | 44 ++++++++++- 5 files changed, 131 insertions(+), 29 deletions(-) diff --git a/protos/feast/core/StreamFeatureView.proto b/protos/feast/core/StreamFeatureView.proto index 01d4fe7bcde..2cfb8d0074b 100644 --- a/protos/feast/core/StreamFeatureView.proto +++ b/protos/feast/core/StreamFeatureView.proto @@ -52,38 +52,41 @@ message StreamFeatureViewSpec { repeated FeatureSpecV2 features = 4; // List of specifications for each entity defined as part of this feature view. - repeated FeatureSpecV2 entity_columns = 12; + repeated FeatureSpecV2 entity_columns = 5; // Description of the feature view. - string description = 10; + string description = 6; // User defined metadata - map tags = 5; + map tags = 7; // Owner of the feature view. - string owner = 11; + string owner = 8; // Features in this feature view can only be retrieved from online serving // younger than ttl. Ttl is measured as the duration of time between // the feature's event timestamp and when the feature is retrieved // Feature values outside ttl will be returned as unset values and indicated to end user - google.protobuf.Duration ttl = 6; + google.protobuf.Duration ttl = 9; // Batch/Offline DataSource where this view can retrieve offline feature data. - DataSource batch_source = 7; + DataSource batch_source = 10; // Streaming DataSource from where this view can consume "online" feature data. - DataSource stream_source = 9; + DataSource stream_source = 11; // Whether these features should be served online or not - bool online = 8; + bool online = 12; UserDefinedFunction user_defined_function = 13; // Mode of execution string mode = 14; + // Aggregation definitions + repeated Aggregation aggregations = 15; + // Timestamp field for aggregation - string timestamp_field = 15; + string timestamp_field = 16; } message StreamFeatureViewMeta { @@ -93,3 +96,9 @@ message StreamFeatureViewMeta { // Time where this Feature View is last updated google.protobuf.Timestamp last_updated_timestamp = 2; } + +message Aggregation { + string column = 1; + string function = 2; + repeated string time_windows = 3; +} diff --git a/sdk/python/feast/base_feature_view.py b/sdk/python/feast/base_feature_view.py index 80b3b0cec82..1fd3720df41 100644 --- a/sdk/python/feast/base_feature_view.py +++ b/sdk/python/feast/base_feature_view.py @@ -130,7 +130,6 @@ def __eq__(self, other): raise TypeError( "Comparisons should only involve BaseFeatureView class objects." ) - if ( self.name != other.name or sorted(self.features) != sorted(other.features) diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 12ce9105f76..ec63f19e1ac 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -254,7 +254,7 @@ def __init__( super().__init__( name=_name, - features=_features, + features=self.schema, description=description, tags=tags, owner=owner, @@ -328,6 +328,7 @@ def __eq__(self, other): ) if not super().__eq__(other): + print("ASdfsf") return False if ( @@ -338,6 +339,7 @@ def __eq__(self, other): or self.stream_source != other.stream_source or sorted(self.entity_columns) != sorted(other.entity_columns) ): + print("ASdfsfd") return False return True @@ -494,7 +496,6 @@ def from_proto(cls, feature_view_proto: FeatureViewProto): # FeatureViewProjections are not saved in the FeatureView proto. # Create the default projection. feature_view.projection = FeatureViewProjection.from_definition(feature_view) - if feature_view_proto.meta.HasField("created_timestamp"): feature_view.created_timestamp = ( feature_view_proto.meta.created_timestamp.ToDatetime() diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 7f3505566a7..2d3b5f97ac8 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -1,3 +1,5 @@ +import abc +from time import time import dill from datetime import timedelta from types import MethodType @@ -11,9 +13,8 @@ from feast.protos.feast.core.StreamFeatureView_pb2 import StreamFeatureView as StreamFeatureViewProto from feast.protos.feast.core.StreamFeatureView_pb2 import ( StreamFeatureViewMeta as StreamFeatureViewMetaProto, -) -from feast.protos.feast.core.StreamFeatureView_pb2 import ( StreamFeatureViewSpec as StreamFeatureViewSpecProto, + Aggregation as AggregationProto ) from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( UserDefinedFunction as UserDefinedFunctionProto, @@ -24,11 +25,42 @@ SUPPORTED_STREAM_SOURCES = {"KafkaSource", "KinesisSource", "PushSource"} -# class Aggregation(abc.ABC): -# column: str # Column name of the feature we are aggregating. -# function: str # Provide built in aggregations sum, max, min, count mean -# time_windows: Union[(timedelta, timedelta), List[tuple(timedelta, timedelta)]] # The time window and the slide +class Aggregation(abc.ABC): + column: str # Column name of the feature we are aggregating. + function: str # Provided built in aggregations sum, max, min, count mean + time_windows: List[str] # The time window. Example ["1h", "24h"] + def __init__(self, column: str, function: str, time_windows: List[str]): + self.column = column + self.function = function + self.time_windows = time_windows + + def to_proto(self) -> AggregationProto: + return AggregationProto( + column=self.column, + function=self.function, + time_windows=self.time_windows, + ) + + @classmethod + def from_proto(cls, agg_proto: AggregationProto): + aggregation = cls( + column = agg_proto.column, + function = agg_proto.function, + time_windows = agg_proto.time_windows, + ) + return aggregation + + def __eq__(self, other): + if not isinstance(other, Aggregation): + raise TypeError( + "Comparisons should only involve Aggregation" + ) + + if self.column != other.column or self.function != other.function or self.time_windows != other.time_windows: + return False + + return True class StreamFeatureView(FeatureView): def __init__( @@ -43,15 +75,14 @@ def __init__( owner: Optional[str] = "", schema: Optional[List[Field]] = None, source: Optional[DataSource] = None, - #aggregations: List[Aggregation], + aggregations: List[Aggregation], mode: Optional[str] = "spark", # Mode of ingestion/transformation timestamp_field: Optional[str] = "", # Timestamp for aggregation udf: Optional[MethodType] = None, ): if source is None: - raise ValueError("Feature views need a source specified") - #TODO: There is a bug here with stream_source/batch_source + raise ValueError("Stream Feature views need a source specified") # source uses the batch_source of the kafkasource in feature_view if ( type(source).__name__ not in SUPPORTED_STREAM_SOURCES @@ -65,12 +96,13 @@ def __init__( self.mode = mode self.timestamp_field = timestamp_field self.udf = udf + self.aggregations = aggregations super().__init__( name=name, entities=entities, ttl=ttl, - batch_source=None, + batch_source=source.batch_source or None, stream_source=source, tags=tags, online=online, @@ -90,7 +122,11 @@ def __eq__(self, other): return False if ( self.mode != other.mode or self.timestamp_field != other.timestamp_field - or self.udf.__code__.co_code != other.udf.__code__.co_code): + or self.udf.__code__.co_code != other.udf.__code__.co_code + or self.aggregations != other.aggregations + or self.timestamp_field != other.timestamp_field): + print(self.udf.__code__.co_code != other.udf.__code__.co_code) + print(self.aggregations != other.aggregations) return False return True @@ -119,22 +155,32 @@ def to_proto(self): stream_source_proto = self.stream_source.to_proto() stream_source_proto.data_source_class_type = f"{self.stream_source.__class__.__module__}.{self.stream_source.__class__.__name__}" + aggregation_proto_lst = [] + for aggregations in self.aggregations: + agg_proto = AggregationProto( + column=aggregations.column, + function=aggregations.function, + time_windows=aggregations.time_windows, + ) + aggregation_proto_lst.append(agg_proto) spec = StreamFeatureViewSpecProto( name=self.name, entities=self.entities, entity_columns=[field.to_proto() for field in self.entity_columns], - features=[field.to_proto() for field in self.features], + features=[field.to_proto() for field in self.schema], user_defined_function=UserDefinedFunctionProto( name=self.udf.__name__, body=dill.dumps(self.udf, recurse=True), - ), + ) if self.udf else None, description=self.description, tags=self.tags, owner=self.owner, ttl=(ttl_duration if ttl_duration is not None else None), online=self.online, - batch_source=batch_source_proto, + batch_source=batch_source_proto or None, stream_source=stream_source_proto, timestamp_field=self.timestamp_field, + aggregations=[agg.to_proto() for agg in self.aggregations], + mode=self.mode ) return StreamFeatureViewProto(spec=spec, meta=meta) @@ -157,15 +203,22 @@ def from_proto(cls, sfv_proto: StreamFeatureViewProto): tags=dict(sfv_proto.spec.tags), owner=sfv_proto.spec.owner, online=sfv_proto.spec.online, + schema=[ + Field.from_proto(field_proto) + for field_proto in sfv_proto.spec.features + ], ttl=( timedelta(days=0) if sfv_proto.spec.ttl.ToNanoseconds() == 0 else sfv_proto.spec.ttl.ToTimedelta() ), source=stream_source, + mode=sfv_proto.spec.mode, udf=dill.loads( - sfv_feature_view.spec.user_defined_function.body + sfv_proto.spec.user_defined_function.body ), + aggregations=[Aggregation.from_proto(agg_proto) for agg_proto in sfv_proto.spec.aggregations], + timestamp_field = sfv_proto.spec.timestamp_field, ) if batch_source: diff --git a/sdk/python/tests/unit/test_feature_views.py b/sdk/python/tests/unit/test_feature_views.py index f72ae4fe9cb..bf002f256fd 100644 --- a/sdk/python/tests/unit/test_feature_views.py +++ b/sdk/python/tests/unit/test_feature_views.py @@ -1,13 +1,15 @@ from datetime import timedelta +from numpy import equal import pytest -from feast import PushSource +from feast import PushSource, Field +from feast.types import Float32 from feast.batch_feature_view import BatchFeatureView from feast.data_format import AvroFormat from feast.data_source import KafkaSource from feast.infra.offline_stores.file_source import FileSource -from feast.stream_feature_view import StreamFeatureView +from feast.stream_feature_view import StreamFeatureView, Aggregation def test_create_batch_feature_view(): @@ -79,3 +81,41 @@ def test_create_stream_feature_view(): ttl=timedelta(days=30), source=FileSource(path="some path"), ) + + +def simple_udf(x: int): + return x + 3 + +def test_stream_feature_view_serialization(): + stream_source = KafkaSource( + name="kafka", + timestamp_field="", + bootstrap_servers="", + message_format=AvroFormat(""), + topic="topic", + batch_source=FileSource(path="some path"), + ) + + sfv = StreamFeatureView( + name="test kafka stream feature view", + entities=["driver"], + ttl=timedelta(days=30), + owner="test@example.com", + online=True, + schema=[ + Field(name="dummy_field", dtype=Float32), + ], + description="desc", + aggregations=[ + Aggregation(column="dummy_field", function="max", time_windows=["1h", "24"])], + timestamp_field="event_timestamp", + mode="spark", + source=stream_source, + udf=simple_udf, + tags={}, + ) + + sfv_proto = sfv.to_proto() + + new_sfv = StreamFeatureView.from_proto(sfv_proto=sfv_proto) + assert new_sfv == sfv From 89704b4ed6bc9253883ee67935d5481fd95b2259 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 31 May 2022 10:15:41 -0700 Subject: [PATCH 05/22] Fix lint Signed-off-by: Kevin Zhang --- protos/feast/core/Registry.proto | 4 +- sdk/python/feast/data_format.py | 2 + sdk/python/feast/feature_store.py | 120 +++++++++++++--- sdk/python/feast/inference.py | 5 +- sdk/python/feast/registry.py | 44 +++++- sdk/python/feast/repo_contents.py | 5 + sdk/python/feast/repo_operations.py | 2 + sdk/python/feast/stream_feature_view.py | 130 +++++++++++------- sdk/python/feast/stream_processor.py | 93 +++++++++---- .../integration/registration/test_registry.py | 60 +++++++- sdk/python/tests/unit/test_feature_views.py | 16 +-- 11 files changed, 364 insertions(+), 117 deletions(-) diff --git a/protos/feast/core/Registry.proto b/protos/feast/core/Registry.proto index 2c31101510b..19f17a81589 100644 --- a/protos/feast/core/Registry.proto +++ b/protos/feast/core/Registry.proto @@ -28,12 +28,13 @@ import "feast/core/FeatureView.proto"; import "feast/core/InfraObject.proto"; import "feast/core/OnDemandFeatureView.proto"; import "feast/core/RequestFeatureView.proto"; +import "feast/core/StreamFeatureView.proto"; import "feast/core/DataSource.proto"; import "feast/core/SavedDataset.proto"; import "feast/core/ValidationProfile.proto"; import "google/protobuf/timestamp.proto"; -// Next id: 14 +// Next id: 15 message Registry { repeated Entity entities = 1; repeated FeatureTable feature_tables = 2; @@ -41,6 +42,7 @@ message Registry { repeated DataSource data_sources = 12; repeated OnDemandFeatureView on_demand_feature_views = 8; repeated RequestFeatureView request_feature_views = 9; + repeated StreamFeatureView stream_feature_views = 14; repeated FeatureService feature_services = 7; repeated SavedDataset saved_datasets = 11; repeated ValidationReference validation_references = 13; diff --git a/sdk/python/feast/data_format.py b/sdk/python/feast/data_format.py index d2eea977c1e..8f3b195e3e6 100644 --- a/sdk/python/feast/data_format.py +++ b/sdk/python/feast/data_format.py @@ -95,6 +95,7 @@ def from_proto(cls, proto): return ProtoFormat(class_path=proto.proto_format.class_path) raise NotImplementedError(f"StreamFormat is unsupported: {fmt}") + class AvroFormat(StreamFormat): """ Defines the Avro streaming data format that encodes data in Avro format @@ -135,6 +136,7 @@ def to_proto(self): proto = StreamFormatProto.JsonFormat(schema_json=self.schema_json) return StreamFormatProto(json_format=proto) + class ProtoFormat(StreamFormat): """ Defines the Protobuf data format diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index f5f39ea3084..673e17a9e67 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -34,11 +34,11 @@ cast, ) -from feast.stream_processor import SparkStreamKafkaProcessor import pandas as pd import pyarrow as pa from colorama import Fore, Style from google.protobuf.timestamp_pb2 import Timestamp +from pyspark.sql import SparkSession from tqdm import tqdm from feast import feature_server, flags, flags_helper, ui_server, utils @@ -72,7 +72,6 @@ from feast.infra.infra_object import Infra from feast.infra.provider import Provider, RetrievalJob, get_provider from feast.on_demand_feature_view import OnDemandFeatureView -from feast.stream_feature_view import StreamFeatureView from feast.online_response import OnlineResponse from feast.protos.feast.core.InfraObject_pb2 import Infra as InfraProto from feast.protos.feast.serving.ServingService_pb2 import ( @@ -86,6 +85,8 @@ from feast.repo_contents import RepoContents from feast.request_feature_view import RequestFeatureView from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference +from feast.stream_feature_view import StreamFeatureView +from feast.stream_processor import SparkStreamKafkaProcessor from feast.type_map import ( feast_value_type_to_python_type, python_values_to_proto_values, @@ -93,7 +94,7 @@ from feast.usage import log_exceptions, log_exceptions_and_usage, set_usage_attribute from feast.value_type import ValueType from feast.version import get_version -from pyspark.sql import SparkSession + # from feast.test import SparkStreamKafkaProcessor warnings.simplefilter("once", DeprecationWarning) @@ -265,6 +266,19 @@ def list_on_demand_feature_views( self.project, allow_cache=allow_cache ) + @log_exceptions_and_usage + def list_stream_feature_views( + self, allow_cache: bool = False + ) -> List[StreamFeatureView]: + """ + Retrieves the list of stream feature views from the registry. + Returns: + A list of stream feature views. + """ + return self._registry.list_stream_feature_views( + self.project, allow_cache=allow_cache + ) + @log_exceptions_and_usage def list_data_sources(self, allow_cache: bool = False) -> List[DataSource]: """ @@ -428,6 +442,7 @@ def _validate_all_feature_views( views_to_update: List[FeatureView], odfvs_to_update: List[OnDemandFeatureView], request_views_to_update: List[RequestFeatureView], + sfvs_to_update: List[StreamFeatureView], ): """Validates all feature views.""" if ( @@ -439,7 +454,12 @@ def _validate_all_feature_views( set_usage_attribute("odfv", bool(odfvs_to_update)) _validate_feature_views( - [*views_to_update, *odfvs_to_update, *request_views_to_update] + [ + *views_to_update, + *odfvs_to_update, + *request_views_to_update, + *sfvs_to_update, + ] ) def _make_inferences( @@ -448,6 +468,7 @@ def _make_inferences( entities_to_update: List[Entity], views_to_update: List[FeatureView], odfvs_to_update: List[OnDemandFeatureView], + sfvs_to_update: List[StreamFeatureView], feature_services_to_update: List[FeatureService], ): """Makes inferences for entities, feature views, odfvs, and feature services.""" @@ -459,16 +480,25 @@ def _make_inferences( [view.batch_source for view in views_to_update], self.config ) + update_data_sources_with_inferred_event_timestamp_col( + [view.batch_source for view in sfvs_to_update], self.config + ) + # New feature views may reference previously applied entities. entities = self._list_entities() update_feature_views_with_inferred_features_and_entities( views_to_update, entities + entities_to_update, self.config ) + update_feature_views_with_inferred_features_and_entities( + sfvs_to_update, entities + entities_to_update, self.config + ) for odfv in odfvs_to_update: odfv.infer_features() - fvs_to_update_map = {view.name: view for view in views_to_update} + fvs_to_update_map = { + view.name: view for view in [*views_to_update, *sfvs_to_update] + } for feature_service in feature_services_to_update: feature_service.infer_features(fvs_to_update=fvs_to_update_map) @@ -506,6 +536,7 @@ def _plan( ... data_sources=[driver_hourly_stats], ... feature_views=[driver_hourly_stats_view], ... on_demand_feature_views=list(), + ... stream_feature_views=list(), ... request_feature_views=list(), ... entities=[driver], ... feature_services=list())) # register entity and feature view @@ -515,6 +546,7 @@ def _plan( desired_repo_contents.feature_views, desired_repo_contents.on_demand_feature_views, desired_repo_contents.request_feature_views, + desired_repo_contents.stream_feature_views, ) _validate_data_sources(desired_repo_contents.data_sources) self._make_inferences( @@ -522,6 +554,7 @@ def _plan( desired_repo_contents.entities, desired_repo_contents.feature_views, desired_repo_contents.on_demand_feature_views, + desired_repo_contents.stream_feature_views, desired_repo_contents.feature_services, ) @@ -572,7 +605,7 @@ def apply( FeatureView, OnDemandFeatureView, RequestFeatureView, - # StreamFeatureView, + StreamFeatureView, FeatureService, ValidationReference, List[FeastObject], @@ -662,13 +695,14 @@ def apply( # Validate all feature views and make inferences. self._validate_all_feature_views( - views_to_update, odfvs_to_update, request_views_to_update + views_to_update, odfvs_to_update, request_views_to_update, sfvs_to_update ) self._make_inferences( data_sources_to_update, entities_to_update, views_to_update, odfvs_to_update, + sfvs_to_update, services_to_update, ) @@ -704,6 +738,9 @@ def apply( odfvs_to_delete = [ ob for ob in objects_to_delete if isinstance(ob, OnDemandFeatureView) ] + sfvs_to_delete = [ + ob for ob in objects_to_delete if isinstance(ob, StreamFeatureView) + ] services_to_delete = [ ob for ob in objects_to_delete if isinstance(ob, FeatureService) ] @@ -734,6 +771,10 @@ def apply( self._registry.delete_feature_view( odfv.name, project=self.project, commit=False ) + for sfv in sfvs_to_delete: + self._registry.delete_feature_view( + sfv.name, project=self.project, commit=False + ) for service in services_to_delete: self._registry.delete_feature_service( service.name, project=self.project, commit=False @@ -773,29 +814,54 @@ def teardown(self): self._registry.teardown() self._teardown_go_server() - def _write_stream_row(self, feature_view, row, join_keys, input_timestamp_field, output_timestamp_column=""): + def _write_stream_row( + self, + feature_view, + row, + join_keys, + input_timestamp_field, + output_timestamp_column="", + ): row: pd.DataFrame = row.toPandas() - row = row.sort_values(by=join_keys + [input_timestamp_field], ascending=True).groupby(join_keys).nth(0) + row = ( + row.sort_values(by=join_keys + [input_timestamp_field], ascending=True) + .groupby(join_keys) + .nth(0) + ) if output_timestamp_column and output_timestamp_column != input_timestamp_field: - row = row.rename(columns = {input_timestamp_field, output_timestamp_column}) - row['created'] = pd.to_datetime('now', utc=True) + row = row.rename(columns={input_timestamp_field, output_timestamp_column}) + row["created"] = pd.to_datetime("now", utc=True) # print("========================") # print(row)s self.write_to_online_store( - feature_view, - row, + feature_view, row, ) - def ingest_stream_feature_view(self, sfv_name: str, spark_session: SparkSession) -> bool: + def ingest_stream_feature_view( + self, sfv_name: str, spark_session: SparkSession + ) -> bool: # TODO: Actually write the code to get the stream feature view. for fv in self.list_feature_views(): if fv.name == sfv_name: sfv = fv - join_keys = [self.get_entity(entity, allow_registry_cache=True).join_key for entity in sfv.entities] + join_keys = [ + self.get_entity(entity, allow_registry_cache=True).join_key + for entity in sfv.entities + ] - skp = SparkStreamKafkaProcessor(sfv=sfv, spark_session=spark_session, write_function=lambda row, input_timestamp, output_timestamp: self._write_stream_row(feature_view=sfv.name, row=row, join_keys=join_keys, input_timestamp_field=input_timestamp, output_timestamp_column=output_timestamp)) + skp = SparkStreamKafkaProcessor( + sfv=sfv, + spark_session=spark_session, + write_function=lambda row, input_timestamp, output_timestamp: self._write_stream_row( + feature_view=sfv.name, + row=row, + join_keys=join_keys, + input_timestamp_field=input_timestamp, + output_timestamp_column=output_timestamp, + ), + ) query = skp.ingest_stream_feature_view() # Handle query(set up monitoring thread, etc) # Return success @@ -861,6 +927,7 @@ def get_historical_features( all_feature_views, all_request_feature_views, all_on_demand_feature_views, + all_stream_feature_views, ) = self._get_feature_views_to_use(features) if all_request_feature_views: @@ -1360,6 +1427,7 @@ def _get_online_features( requested_feature_views, requested_request_feature_views, requested_on_demand_feature_views, + request_stream_feature_views, ) = self._get_feature_views_to_use( features=features, allow_cache=True, hide_dummy_entity=False ) @@ -1888,7 +1956,7 @@ def _get_feature_views_to_use( features: Optional[Union[List[str], FeatureService]], allow_cache=False, hide_dummy_entity: bool = True, - ) -> Tuple[List[FeatureView], List[RequestFeatureView], List[OnDemandFeatureView]]: + ) -> Tuple[List[FeatureView], List[RequestFeatureView], List[OnDemandFeatureView], List[StreamFeatureView]]: fvs = { fv.name: fv @@ -1909,8 +1977,15 @@ def _get_feature_views_to_use( ) } + sfvs = { + fv.name: fv + for fv in self._registry.list_stream_feature_views( + project=self.project, allow_cache=allow_cache + ) + } + if isinstance(features, FeatureService): - fvs_to_use, request_fvs_to_use, od_fvs_to_use = [], [], [] + fvs_to_use, request_fvs_to_use, od_fvs_to_use, sfvs_to_use = [], [], [], [] for fv_name, projection in [ (projection.name, projection) for projection in features.feature_view_projections @@ -1931,18 +2006,23 @@ def _get_feature_views_to_use( fv = fvs[projection.name].with_projection(copy.copy(projection)) if fv not in fvs_to_use: fvs_to_use.append(fv) + elif fv_name in sfvs: + sfvs_to_use.append( + sfvs[fv_name].with_projection(copy.copy(projection)) + ) else: raise ValueError( f"The provided feature service {features.name} contains a reference to a feature view" f"{fv_name} which doesn't exist. Please make sure that you have created the feature view" f'{fv_name} and that you have registered it by running "apply".' ) - views_to_use = (fvs_to_use, request_fvs_to_use, od_fvs_to_use) + views_to_use = (fvs_to_use, request_fvs_to_use, od_fvs_to_use, sfvs_to_use) else: views_to_use = ( [*fvs.values()], [*request_fvs.values()], [*od_fvs.values()], + [*sfvs.values()], ) return views_to_use @@ -2289,4 +2369,4 @@ def apply_list_mapping( for idx in destinations: output[idx] = elem - return output \ No newline at end of file + return output diff --git a/sdk/python/feast/inference.py b/sdk/python/feast/inference.py index aed90c4ac83..97d6ce42ec0 100644 --- a/sdk/python/feast/inference.py +++ b/sdk/python/feast/inference.py @@ -1,5 +1,5 @@ import re -from typing import List, Set +from typing import List, Set, Union from feast.data_source import DataSource, PushSource, RequestSource from feast.entity import Entity @@ -11,6 +11,7 @@ from feast.infra.offline_stores.redshift_source import RedshiftSource from feast.infra.offline_stores.snowflake_source import SnowflakeSource from feast.repo_config import RepoConfig +from feast.stream_feature_view import StreamFeatureView from feast.types import String from feast.value_type import ValueType @@ -88,7 +89,7 @@ def update_data_sources_with_inferred_event_timestamp_col( def update_feature_views_with_inferred_features_and_entities( - fvs: List[FeatureView], entities: List[Entity], config: RepoConfig + fvs: List[Union[FeatureView, StreamFeatureView]], entities: List[Entity], config: RepoConfig ) -> None: """ Infers the features and entities associated with each feature view and updates it in place. diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index c46eba8a5d6..a0f730c0c75 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -51,6 +51,7 @@ from feast.repo_contents import RepoContents from feast.request_feature_view import RequestFeatureView from feast.saved_dataset import SavedDataset, ValidationReference +from feast.stream_feature_view import StreamFeatureView REGISTRY_SCHEMA_VERSION = "1" @@ -476,7 +477,11 @@ def apply_feature_view( self._check_conflicting_feature_view_names(feature_view) existing_feature_views_of_same_type: RepeatedCompositeFieldContainer - if isinstance(feature_view, FeatureView): + if isinstance(feature_view, StreamFeatureView): + existing_feature_views_of_same_type = ( + self.cached_registry_proto.stream_feature_views + ) + elif isinstance(feature_view, FeatureView): existing_feature_views_of_same_type = ( self.cached_registry_proto.feature_views ) @@ -506,11 +511,34 @@ def apply_feature_view( else: del existing_feature_views_of_same_type[idx] break - + print(type(existing_feature_views_of_same_type)) + print(feature_view_proto) existing_feature_views_of_same_type.append(feature_view_proto) if commit: self.commit() + def list_stream_feature_views( + self, project: str, allow_cache: bool = False + ) -> List[StreamFeatureView]: + """ + Retrieve a list of stream feature views from the registry + + Args: + project: Filter stream feature views based on project name + allow_cache: Whether to allow returning stream feature views from a cached registry + + Returns: + List of stream feature views + """ + registry = self._get_registry_proto(allow_cache=allow_cache) + stream_feature_views = [] + for stream_feature_view in registry.stream_feature_views: + if stream_feature_view.spec.project == project: + stream_feature_views.append( + StreamFeatureView.from_proto(stream_feature_view) + ) + return stream_feature_views + def list_on_demand_feature_views( self, project: str, allow_cache: bool = False ) -> List[OnDemandFeatureView]: @@ -764,6 +792,18 @@ def delete_feature_view(self, name: str, project: str, commit: bool = True): self.commit() return + for idx, existing_stream_feature_view_proto in enumerate( + self.cached_registry_proto.stream_feature_views + ): + if ( + existing_stream_feature_view_proto.spec.name == name + and existing_stream_feature_view_proto.spec.project == project + ): + del self.cached_registry_proto.stream_feature_views[idx] + if commit: + self.commit() + return + raise FeatureViewNotFoundException(name, project) def delete_entity(self, name: str, project: str, commit: bool = True): diff --git a/sdk/python/feast/repo_contents.py b/sdk/python/feast/repo_contents.py index 4d7c92f2a6d..42f83912c6c 100644 --- a/sdk/python/feast/repo_contents.py +++ b/sdk/python/feast/repo_contents.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from typing import List, NamedTuple +from feast import StreamFeatureView from feast.data_source import DataSource from feast.entity import Entity @@ -31,6 +32,7 @@ class RepoContents(NamedTuple): feature_views: List[FeatureView] on_demand_feature_views: List[OnDemandFeatureView] request_feature_views: List[RequestFeatureView] + stream_feature_views: List[StreamFeatureView] entities: List[Entity] feature_services: List[FeatureService] @@ -50,4 +52,7 @@ def to_registry_proto(self) -> RegistryProto: registry_proto.feature_services.extend( [fs.to_proto() for fs in self.feature_services] ) + registry_proto.stream_feature_views.extend( + [fv.to_proto() for fv in self.stream_feature_views] + ) return registry_proto diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 0e82fdf47ad..5b0487c7795 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -26,6 +26,7 @@ from feast.repo_contents import RepoContents from feast.request_feature_view import RequestFeatureView from feast.usage import log_exceptions_and_usage +from isort import stream def py_path_to_module(path: Path) -> str: @@ -107,6 +108,7 @@ def parse_repo(repo_root: Path) -> RepoContents: feature_views=[], feature_services=[], on_demand_feature_views=[], + stream_feature_views=[], request_feature_views=[], ) diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 2d3b5f97ac8..eee86b31f1d 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -1,34 +1,46 @@ import abc -from time import time -import dill +import warnings from datetime import timedelta from types import MethodType -from typing import Dict, List, Optional, Union, Callable, Tuple +from typing import Dict, List, Optional, Union + +import dill +from google.protobuf.duration_pb2 import Duration from feast.data_source import DataSource from feast.entity import Entity from feast.feature_view import FeatureView from feast.field import Field -from google.protobuf.duration_pb2 import Duration -from feast.protos.feast.core.StreamFeatureView_pb2 import StreamFeatureView as StreamFeatureViewProto +from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto +from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( + UserDefinedFunction as UserDefinedFunctionProto, +) +from feast.protos.feast.core.StreamFeatureView_pb2 import ( + Aggregation as AggregationProto, +) +from feast.protos.feast.core.StreamFeatureView_pb2 import ( + StreamFeatureView as StreamFeatureViewProto, +) from feast.protos.feast.core.StreamFeatureView_pb2 import ( StreamFeatureViewMeta as StreamFeatureViewMetaProto, - StreamFeatureViewSpec as StreamFeatureViewSpecProto, - Aggregation as AggregationProto ) -from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( - UserDefinedFunction as UserDefinedFunctionProto, +from feast.protos.feast.core.StreamFeatureView_pb2 import ( + StreamFeatureViewSpec as StreamFeatureViewSpecProto, ) -from isort import stream -from regex import R -SUPPORTED_STREAM_SOURCES = {"KafkaSource", "KinesisSource", "PushSource"} +warnings.simplefilter("once", RuntimeWarning) + +SUPPORTED_STREAM_SOURCES = {"KafkaSource", "PushSource"} class Aggregation(abc.ABC): - column: str # Column name of the feature we are aggregating. - function: str # Provided built in aggregations sum, max, min, count mean - time_windows: List[str] # The time window. Example ["1h", "24h"] + """ + NOTE: Feast-handled aggregations are not yet supported. This class provides a way to register user-defined aggregations. + """ + + column: str # Column name of the feature we are aggregating. + function: str # Provided built in aggregations sum, max, min, count mean + time_windows: List[str] # The time window. Example ["1h", "24h"] def __init__(self, column: str, function: str, time_windows: List[str]): self.column = column @@ -37,32 +49,38 @@ def __init__(self, column: str, function: str, time_windows: List[str]): def to_proto(self) -> AggregationProto: return AggregationProto( - column=self.column, - function=self.function, - time_windows=self.time_windows, + column=self.column, function=self.function, time_windows=self.time_windows, ) @classmethod def from_proto(cls, agg_proto: AggregationProto): aggregation = cls( - column = agg_proto.column, - function = agg_proto.function, - time_windows = agg_proto.time_windows, + column=agg_proto.column, + function=agg_proto.function, + time_windows=list(agg_proto.time_windows), ) return aggregation def __eq__(self, other): if not isinstance(other, Aggregation): - raise TypeError( - "Comparisons should only involve Aggregation" - ) + raise TypeError("Comparisons should only involve Aggregations.") - if self.column != other.column or self.function != other.function or self.time_windows != other.time_windows: + if ( + self.column != other.column + or self.function != other.function + or self.time_windows != other.time_windows + ): return False return True + class StreamFeatureView(FeatureView): + """ + NOTE: Stream Feature Views are not yet fully implemented and exist to allow users to register their stream sources and + schemas with Feast. + """ + def __init__( self, *, @@ -76,33 +94,39 @@ def __init__( schema: Optional[List[Field]] = None, source: Optional[DataSource] = None, aggregations: List[Aggregation], - mode: Optional[str] = "spark", # Mode of ingestion/transformation - timestamp_field: Optional[str] = "", # Timestamp for aggregation + mode: Optional[str] = "spark", # Mode of ingestion/transformation + timestamp_field: Optional[str] = "", # Timestamp for aggregation udf: Optional[MethodType] = None, ): - + warnings.warn( + "Stream Feature Views are experimental features in alpha development. " + "Some functionality may still be unstable so functionality can change in the future.", + RuntimeWarning, + ) if source is None: raise ValueError("Stream Feature views need a source specified") # source uses the batch_source of the kafkasource in feature_view if ( type(source).__name__ not in SUPPORTED_STREAM_SOURCES - # and source.to_proto().type != DataSourceProto.SourceType.CUSTOM_SOURCE + and source.to_proto().type != DataSourceProto.SourceType.CUSTOM_SOURCE ): raise ValueError( f"Stream feature views need a stream source, expected one of {SUPPORTED_STREAM_SOURCES} " f"or CUSTOM_SOURCE, got {type(source).__name__}: {source.name} instead " ) - # self.aggregations = aggregations + self.aggregations = aggregations self.mode = mode self.timestamp_field = timestamp_field self.udf = udf self.aggregations = aggregations + _batch_source = source.batch_source if source.batch_source else None + super().__init__( name=name, entities=entities, ttl=ttl, - batch_source=source.batch_source or None, + batch_source=_batch_source, stream_source=source, tags=tags, online=online, @@ -114,19 +138,18 @@ def __init__( def __eq__(self, other): if not isinstance(other, StreamFeatureView): - raise TypeError( - "Comparisons should only involve StreamFeatureViews" - ) + raise TypeError("Comparisons should only involve StreamFeatureViews") if not super().__eq__(other): return False - if ( self.mode != other.mode or self.timestamp_field != other.timestamp_field - or self.udf.__code__.co_code != other.udf.__code__.co_code - or self.aggregations != other.aggregations - or self.timestamp_field != other.timestamp_field): - print(self.udf.__code__.co_code != other.udf.__code__.co_code) - print(self.aggregations != other.aggregations) + if ( + self.mode != other.mode + or self.timestamp_field != other.timestamp_field + or self.udf.__code__.co_code != other.udf.__code__.co_code + or self.aggregations != other.aggregations + or self.timestamp_field != other.timestamp_field + ): return False return True @@ -170,7 +193,9 @@ def to_proto(self): features=[field.to_proto() for field in self.schema], user_defined_function=UserDefinedFunctionProto( name=self.udf.__name__, body=dill.dumps(self.udf, recurse=True), - ) if self.udf else None, + ) + if self.udf + else None, description=self.description, tags=self.tags, owner=self.owner, @@ -180,7 +205,7 @@ def to_proto(self): stream_source=stream_source_proto, timestamp_field=self.timestamp_field, aggregations=[agg.to_proto() for agg in self.aggregations], - mode=self.mode + mode=self.mode, ) return StreamFeatureViewProto(spec=spec, meta=meta) @@ -204,8 +229,7 @@ def from_proto(cls, sfv_proto: StreamFeatureViewProto): owner=sfv_proto.spec.owner, online=sfv_proto.spec.online, schema=[ - Field.from_proto(field_proto) - for field_proto in sfv_proto.spec.features + Field.from_proto(field_proto) for field_proto in sfv_proto.spec.features ], ttl=( timedelta(days=0) @@ -214,11 +238,12 @@ def from_proto(cls, sfv_proto: StreamFeatureViewProto): ), source=stream_source, mode=sfv_proto.spec.mode, - udf=dill.loads( - sfv_proto.spec.user_defined_function.body - ), - aggregations=[Aggregation.from_proto(agg_proto) for agg_proto in sfv_proto.spec.aggregations], - timestamp_field = sfv_proto.spec.timestamp_field, + udf=dill.loads(sfv_proto.spec.user_defined_function.body), + aggregations=[ + Aggregation.from_proto(agg_proto) + for agg_proto in sfv_proto.spec.aggregations + ], + timestamp_field=sfv_proto.spec.timestamp_field, ) if batch_source: @@ -227,11 +252,10 @@ def from_proto(cls, sfv_proto: StreamFeatureViewProto): if stream_source: sfv_feature_view.stream_source = stream_source - sfv_feature_view.entities = sfv_proto.spec.entities + sfv_feature_view.entities = list(sfv_proto.spec.entities) sfv_feature_view.features = [ - Field.from_proto(field_proto) - for field_proto in sfv_proto.spec.features + Field.from_proto(field_proto) for field_proto in sfv_proto.spec.features ] if sfv_proto.meta.HasField("created_timestamp"): @@ -243,4 +267,4 @@ def from_proto(cls, sfv_proto: StreamFeatureViewProto): sfv_proto.meta.last_updated_timestamp.ToDatetime() ) - return sfv_feature_view \ No newline at end of file + return sfv_feature_view diff --git a/sdk/python/feast/stream_processor.py b/sdk/python/feast/stream_processor.py index a259e6c0177..32a77c28e83 100644 --- a/sdk/python/feast/stream_processor.py +++ b/sdk/python/feast/stream_processor.py @@ -1,4 +1,5 @@ import abc +from datetime import timedelta from typing import ( TYPE_CHECKING, Any, @@ -16,20 +17,22 @@ ) import pandas as pd -from feast.stream_feature_view import StreamFeatureView -from feast.data_source import DataSource, KafkaSource -from feast.data_format import AvroFormat, JsonFormat -from datetime import timedelta - from pyspark.sql import DataFrame, SparkSession -from pyspark.sql.types import StructType, IntegerType, DoubleType, TimestampType -from pyspark.sql.functions import col, from_json from pyspark.sql.avro.functions import from_avro +from pyspark.sql.functions import col, from_json +from pyspark.sql.types import DoubleType, IntegerType, StructType, TimestampType + +from feast.data_format import AvroFormat, JsonFormat +from feast.data_source import DataSource, KafkaSource +from feast.stream_feature_view import StreamFeatureView + +StreamTable = DataFrame # Can add more to this later(change to union). + -StreamTable = DataFrame # Can add more to this later(change to union). class StreamProcessor(abc.ABC): data_source: DataSource sfv: StreamFeatureView + def __init__(self, sfv: StreamFeatureView, data_source: DataSource): self.sfv = sfv self.data_source = data_source @@ -71,24 +74,34 @@ class SparkStreamKafkaProcessor(StreamProcessor): format: str write_function: Callable join_keys: List[str] + def __init__( self, sfv: StreamFeatureView, spark_session: SparkSession, - write_function: Callable): + write_function: Callable, + ): if not isinstance(sfv.stream_source, KafkaSource): raise ValueError("data source is not kafka source") - if not isinstance(sfv.stream_source.kafka_options.message_format, AvroFormat) and not isinstance(sfv.stream_source.kafka_options.message_format, JsonFormat): - raise ValueError("spark streaming currently only supports json or avro format for kafka source schema") + if not isinstance( + sfv.stream_source.kafka_options.message_format, AvroFormat + ) and not isinstance( + sfv.stream_source.kafka_options.message_format, JsonFormat + ): + raise ValueError( + "spark streaming currently only supports json or avro format for kafka source schema" + ) # if not sfv.mode == "spark": # raise ValueError(f"stream feature view mode is {sfv.mode}, but only supports spark") - self.format = "json" if isinstance(sfv.stream_source.kafka_options.message_format, JsonFormat) else "avro" + self.format = ( + "json" + if isinstance(sfv.stream_source.kafka_options.message_format, JsonFormat) + else "avro" + ) self.spark = spark_session self.write_function = write_function super().__init__(sfv=sfv, data_source=sfv.stream_source) - - def _ingest_stream_data(self) -> StreamTable: """ Ingests data into StreamTable depending on what type of data format it is in. @@ -97,28 +110,44 @@ def _ingest_stream_data(self) -> StreamTable: if self.format == "json": streamingDF = ( self.spark.readStream.format("kafka") - .option("kafka.bootstrap.servers", self.data_source.kafka_options.bootstrap_servers) + .option( + "kafka.bootstrap.servers", + self.data_source.kafka_options.bootstrap_servers, + ) .option("subscribe", self.data_source.kafka_options.topic) - .option("startingOffsets", "latest") # Query start + .option("startingOffsets", "latest") # Query start .load() - .selectExpr('CAST(value AS STRING)') - .select(from_json(col('value'), self.data_source.kafka_options.message_format.schema_json).alias("table")) + .selectExpr("CAST(value AS STRING)") + .select( + from_json( + col("value"), + self.data_source.kafka_options.message_format.schema_json, + ).alias("table") + ) .select("table.*") ) else: streamingDF = ( self.spark.readStream.format("kafka") - .option("kafka.bootstrap.servers", self.data_source.kafka_options.bootstrap_servers) + .option( + "kafka.bootstrap.servers", + self.data_source.kafka_options.bootstrap_servers, + ) .option("subscribe", self.data_source.kafka_options.topic) - .option("startingOffsets", "latest") # Query start + .option("startingOffsets", "latest") # Query start .load() - .selectExpr('CAST(value AS STRING)') - .select(from_avro(col('value'), self.data_source.kafka_options.message_format.schema_json).alias("table")) + .selectExpr("CAST(value AS STRING)") + .select( + from_avro( + col("value"), + self.data_source.kafka_options.message_format.schema_json, + ).alias("table") + ) .select("table.*") ) return streamingDF - def _construct_transformation_plan(self, df : StreamTable) -> StreamTable: + def _construct_transformation_plan(self, df: StreamTable) -> StreamTable: """ Applies transformations on top of StreamTable object. Since stream engines use lazy evaluation, the StreamTable will not be materialized until it is actually evaluated. @@ -135,13 +164,17 @@ def _write_to_online_store(self, df: StreamTable): Returns query for writing stream. """ # Validation occurs at the fs.write_to_online_store() phase against the stream feature view schema. - query = df \ - .writeStream \ - .outputMode("update") \ - .option("checkpointLocation", "/tmp/checkpoint/") \ - .trigger(processingTime="30 seconds") \ - .foreachBatch(lambda row, batch_id: self.write_function(row, input_timestamp="event_timestamp", output_timestamp="")) \ + query = ( + df.writeStream.outputMode("update") + .option("checkpointLocation", "/tmp/checkpoint/") + .trigger(processingTime="30 seconds") + .foreachBatch( + lambda row, batch_id: self.write_function( + row, input_timestamp="event_timestamp", output_timestamp="" + ) + ) .start() + ) query.awaitTermination(timeout=30) return query @@ -156,4 +189,4 @@ def ingest_stream_feature_view(self): return online_store_query def transform_and_write(self, table: StreamTable): - pass \ No newline at end of file + pass diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index bb02f9a9e32..baf3bdceae5 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -20,7 +20,8 @@ from pytest_lazyfixture import lazy_fixture from feast import FileSource -from feast.data_format import ParquetFormat +from feast.data_format import AvroFormat, ParquetFormat +from feast.data_source import KafkaSource from feast.entity import Entity from feast.feature import Feature from feast.feature_view import FeatureView @@ -28,6 +29,7 @@ from feast.on_demand_feature_view import RequestSource, on_demand_feature_view from feast.registry import Registry from feast.repo_config import RegistryConfig +from feast.stream_feature_view import Aggregation, StreamFeatureView from feast.types import Array, Bytes, Float32, Int32, Int64, String from feast.value_type import ValueType @@ -299,6 +301,62 @@ def location_features_from_push(inputs: pd.DataFrame) -> pd.DataFrame: test_registry._get_registry_proto() +@pytest.mark.parametrize( + "test_registry", [lazy_fixture("local_registry")], +) +def test_apply_stream_feature_view_success(test_registry): + # Create Feature Views + def simple_udf(x: int): + return x + 3 + + stream_source = KafkaSource( + name="kafka", + timestamp_field="", + bootstrap_servers="", + message_format=AvroFormat(""), + topic="topic", + batch_source=FileSource(path="some path"), + ) + + sfv = StreamFeatureView( + name="test kafka stream feature view", + entities=["driver"], + ttl=timedelta(days=30), + owner="test@example.com", + online=True, + schema=[Field(name="dummy_field", dtype=Float32),], + description="desc", + aggregations=[ + Aggregation(column="dummy_field", function="max", time_windows=["1h", "24"]) + ], + timestamp_field="event_timestamp", + mode="spark", + source=stream_source, + udf=simple_udf, + tags={}, + ) + + project = "project" + + # Register Feature View + test_registry.apply_feature_view(sfv, project) + + feature_views = test_registry.list_stream_feature_views(project) + + # List Feature Views + assert feature_views[0] == sfv + + test_registry.delete_feature_view("test kafka stream feature view", project) + feature_views = test_registry.list_stream_feature_views(project) + assert len(feature_views) == 0 + + test_registry.teardown() + + # Will try to reload registry, which will fail because the file has been deleted + with pytest.raises(FileNotFoundError): + test_registry._get_registry_proto() + + @pytest.mark.parametrize( "test_registry", [lazy_fixture("local_registry")], ) diff --git a/sdk/python/tests/unit/test_feature_views.py b/sdk/python/tests/unit/test_feature_views.py index bf002f256fd..6ad6b85b9d1 100644 --- a/sdk/python/tests/unit/test_feature_views.py +++ b/sdk/python/tests/unit/test_feature_views.py @@ -1,15 +1,15 @@ from datetime import timedelta -from numpy import equal import pytest +from numpy import equal -from feast import PushSource, Field -from feast.types import Float32 +from feast import Field, PushSource from feast.batch_feature_view import BatchFeatureView from feast.data_format import AvroFormat from feast.data_source import KafkaSource from feast.infra.offline_stores.file_source import FileSource -from feast.stream_feature_view import StreamFeatureView, Aggregation +from feast.stream_feature_view import Aggregation, StreamFeatureView +from feast.types import Float32 def test_create_batch_feature_view(): @@ -86,6 +86,7 @@ def test_create_stream_feature_view(): def simple_udf(x: int): return x + 3 + def test_stream_feature_view_serialization(): stream_source = KafkaSource( name="kafka", @@ -102,12 +103,11 @@ def test_stream_feature_view_serialization(): ttl=timedelta(days=30), owner="test@example.com", online=True, - schema=[ - Field(name="dummy_field", dtype=Float32), - ], + schema=[Field(name="dummy_field", dtype=Float32),], description="desc", aggregations=[ - Aggregation(column="dummy_field", function="max", time_windows=["1h", "24"])], + Aggregation(column="dummy_field", function="max", time_windows=["1h", "24"]) + ], timestamp_field="event_timestamp", mode="spark", source=stream_source, From ebaa527b462326c97c7725fe9e71fee8d81c6298 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 31 May 2022 10:20:03 -0700 Subject: [PATCH 06/22] Fix things Signed-off-by: Kevin Zhang --- protos/feast/core/DataSource.proto | 1 + sdk/python/feast/feature_store.py | 60 +-------- sdk/python/feast/inference.py | 4 +- sdk/python/feast/repo_contents.py | 2 +- sdk/python/feast/repo_operations.py | 2 +- sdk/python/feast/stream_processor.py | 192 --------------------------- 6 files changed, 12 insertions(+), 249 deletions(-) delete mode 100644 sdk/python/feast/stream_processor.py diff --git a/protos/feast/core/DataSource.proto b/protos/feast/core/DataSource.proto index e5fe32ab82d..9e6028ccfa4 100644 --- a/protos/feast/core/DataSource.proto +++ b/protos/feast/core/DataSource.proto @@ -216,6 +216,7 @@ message DataSource { map deprecated_schema = 2; repeated FeatureSpecV2 schema = 3; + } // Defines options for DataSource that supports pushing data to it. This allows data to be pushed to diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 673e17a9e67..ac075962318 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -814,59 +814,6 @@ def teardown(self): self._registry.teardown() self._teardown_go_server() - def _write_stream_row( - self, - feature_view, - row, - join_keys, - input_timestamp_field, - output_timestamp_column="", - ): - row: pd.DataFrame = row.toPandas() - - row = ( - row.sort_values(by=join_keys + [input_timestamp_field], ascending=True) - .groupby(join_keys) - .nth(0) - ) - if output_timestamp_column and output_timestamp_column != input_timestamp_field: - row = row.rename(columns={input_timestamp_field, output_timestamp_column}) - row["created"] = pd.to_datetime("now", utc=True) - # print("========================") - # print(row)s - self.write_to_online_store( - feature_view, row, - ) - - def ingest_stream_feature_view( - self, sfv_name: str, spark_session: SparkSession - ) -> bool: - # TODO: Actually write the code to get the stream feature view. - for fv in self.list_feature_views(): - if fv.name == sfv_name: - sfv = fv - - join_keys = [ - self.get_entity(entity, allow_registry_cache=True).join_key - for entity in sfv.entities - ] - - skp = SparkStreamKafkaProcessor( - sfv=sfv, - spark_session=spark_session, - write_function=lambda row, input_timestamp, output_timestamp: self._write_stream_row( - feature_view=sfv.name, - row=row, - join_keys=join_keys, - input_timestamp_field=input_timestamp, - output_timestamp_column=output_timestamp, - ), - ) - query = skp.ingest_stream_feature_view() - # Handle query(set up monitoring thread, etc) - # Return success - return True - @log_exceptions_and_usage def get_historical_features( self, @@ -1956,7 +1903,12 @@ def _get_feature_views_to_use( features: Optional[Union[List[str], FeatureService]], allow_cache=False, hide_dummy_entity: bool = True, - ) -> Tuple[List[FeatureView], List[RequestFeatureView], List[OnDemandFeatureView], List[StreamFeatureView]]: + ) -> Tuple[ + List[FeatureView], + List[RequestFeatureView], + List[OnDemandFeatureView], + List[StreamFeatureView], + ]: fvs = { fv.name: fv diff --git a/sdk/python/feast/inference.py b/sdk/python/feast/inference.py index 97d6ce42ec0..1f3f6ac3e95 100644 --- a/sdk/python/feast/inference.py +++ b/sdk/python/feast/inference.py @@ -89,7 +89,9 @@ def update_data_sources_with_inferred_event_timestamp_col( def update_feature_views_with_inferred_features_and_entities( - fvs: List[Union[FeatureView, StreamFeatureView]], entities: List[Entity], config: RepoConfig + fvs: List[Union[FeatureView, StreamFeatureView]], + entities: List[Entity], + config: RepoConfig, ) -> None: """ Infers the features and entities associated with each feature view and updates it in place. diff --git a/sdk/python/feast/repo_contents.py b/sdk/python/feast/repo_contents.py index 42f83912c6c..b1074239058 100644 --- a/sdk/python/feast/repo_contents.py +++ b/sdk/python/feast/repo_contents.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. from typing import List, NamedTuple -from feast import StreamFeatureView +from feast import StreamFeatureView from feast.data_source import DataSource from feast.entity import Entity from feast.feature_service import FeatureService diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 5b0487c7795..017220a555e 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -11,6 +11,7 @@ import click from click.exceptions import BadParameter +from isort import stream from feast import PushSource from feast.data_source import DataSource @@ -26,7 +27,6 @@ from feast.repo_contents import RepoContents from feast.request_feature_view import RequestFeatureView from feast.usage import log_exceptions_and_usage -from isort import stream def py_path_to_module(path: Path) -> str: diff --git a/sdk/python/feast/stream_processor.py b/sdk/python/feast/stream_processor.py deleted file mode 100644 index 32a77c28e83..00000000000 --- a/sdk/python/feast/stream_processor.py +++ /dev/null @@ -1,192 +0,0 @@ -import abc -from datetime import timedelta -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Mapping, - Optional, - Sequence, - Set, - Tuple, - Union, - cast, -) - -import pandas as pd -from pyspark.sql import DataFrame, SparkSession -from pyspark.sql.avro.functions import from_avro -from pyspark.sql.functions import col, from_json -from pyspark.sql.types import DoubleType, IntegerType, StructType, TimestampType - -from feast.data_format import AvroFormat, JsonFormat -from feast.data_source import DataSource, KafkaSource -from feast.stream_feature_view import StreamFeatureView - -StreamTable = DataFrame # Can add more to this later(change to union). - - -class StreamProcessor(abc.ABC): - data_source: DataSource - sfv: StreamFeatureView - - def __init__(self, sfv: StreamFeatureView, data_source: DataSource): - self.sfv = sfv - self.data_source = data_source - - def _ingest_stream_data(self) -> StreamTable: - """ - Ingests data into StreamTable depending on what type of data it is - """ - pass - - def _construct_transformation_plan(self, table: StreamTable) -> StreamTable: - """ - Applies transformations on top of StreamTable object. Since stream engines use lazy - evaluation, the StreamTable will not be materialized until it is actually evaluated. - For example: df.collect() in spark or tbl.execute() in Flink. - """ - pass - - def _write_to_online_store(self, table: StreamTable): - """ - Returns query for writing stream. - """ - pass - - def transform_stream_data(self) -> StreamTable: - pass - - def ingest_stream_feature_view(self): - pass - - def transform_and_write(self, table: StreamTable): - pass - - -class SparkStreamKafkaProcessor(StreamProcessor): - # TODO: wrap spark data in some kind of config - # includes session, format, checkpoint location etc. - spark: SparkSession - format: str - write_function: Callable - join_keys: List[str] - - def __init__( - self, - sfv: StreamFeatureView, - spark_session: SparkSession, - write_function: Callable, - ): - if not isinstance(sfv.stream_source, KafkaSource): - raise ValueError("data source is not kafka source") - if not isinstance( - sfv.stream_source.kafka_options.message_format, AvroFormat - ) and not isinstance( - sfv.stream_source.kafka_options.message_format, JsonFormat - ): - raise ValueError( - "spark streaming currently only supports json or avro format for kafka source schema" - ) - # if not sfv.mode == "spark": - # raise ValueError(f"stream feature view mode is {sfv.mode}, but only supports spark") - self.format = ( - "json" - if isinstance(sfv.stream_source.kafka_options.message_format, JsonFormat) - else "avro" - ) - self.spark = spark_session - self.write_function = write_function - super().__init__(sfv=sfv, data_source=sfv.stream_source) - - def _ingest_stream_data(self) -> StreamTable: - """ - Ingests data into StreamTable depending on what type of data format it is in. - Only supports json and avro formats currently. - """ - if self.format == "json": - streamingDF = ( - self.spark.readStream.format("kafka") - .option( - "kafka.bootstrap.servers", - self.data_source.kafka_options.bootstrap_servers, - ) - .option("subscribe", self.data_source.kafka_options.topic) - .option("startingOffsets", "latest") # Query start - .load() - .selectExpr("CAST(value AS STRING)") - .select( - from_json( - col("value"), - self.data_source.kafka_options.message_format.schema_json, - ).alias("table") - ) - .select("table.*") - ) - else: - streamingDF = ( - self.spark.readStream.format("kafka") - .option( - "kafka.bootstrap.servers", - self.data_source.kafka_options.bootstrap_servers, - ) - .option("subscribe", self.data_source.kafka_options.topic) - .option("startingOffsets", "latest") # Query start - .load() - .selectExpr("CAST(value AS STRING)") - .select( - from_avro( - col("value"), - self.data_source.kafka_options.message_format.schema_json, - ).alias("table") - ) - .select("table.*") - ) - return streamingDF - - def _construct_transformation_plan(self, df: StreamTable) -> StreamTable: - """ - Applies transformations on top of StreamTable object. Since stream engines use lazy - evaluation, the StreamTable will not be materialized until it is actually evaluated. - For example: df.collect() in spark or tbl.execute() in Flink. - """ - # if self.sfv.udf == None: - # return table - # else: - # return None - return df - - def _write_to_online_store(self, df: StreamTable): - """ - Returns query for writing stream. - """ - # Validation occurs at the fs.write_to_online_store() phase against the stream feature view schema. - query = ( - df.writeStream.outputMode("update") - .option("checkpointLocation", "/tmp/checkpoint/") - .trigger(processingTime="30 seconds") - .foreachBatch( - lambda row, batch_id: self.write_function( - row, input_timestamp="event_timestamp", output_timestamp="" - ) - ) - .start() - ) - query.awaitTermination(timeout=30) - return query - - def transform_stream_data(self) -> StreamTable: - df = self._ingest_stream_data() - return self._construct_transformation_plan(df) - - def ingest_stream_feature_view(self): - ingested_stream_df = self._ingest_stream_data() - transformed_df = self._construct_transformation_plan(ingested_stream_df) - online_store_query = self._write_to_online_store(transformed_df) - return online_store_query - - def transform_and_write(self, table: StreamTable): - pass From 8c294a7a5ed0be1c774e8a5c2ddd09ee69070a02 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 31 May 2022 10:24:48 -0700 Subject: [PATCH 07/22] Fix Signed-off-by: Kevin Zhang --- sdk/python/feast/feature_store.py | 3 --- sdk/python/feast/repo_contents.py | 2 +- sdk/python/feast/stream_feature_view.py | 2 +- sdk/python/tests/unit/test_feature_views.py | 6 +++++- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index ac075962318..ff4fffc580c 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -86,7 +86,6 @@ from feast.request_feature_view import RequestFeatureView from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference from feast.stream_feature_view import StreamFeatureView -from feast.stream_processor import SparkStreamKafkaProcessor from feast.type_map import ( feast_value_type_to_python_type, python_values_to_proto_values, @@ -95,8 +94,6 @@ from feast.value_type import ValueType from feast.version import get_version -# from feast.test import SparkStreamKafkaProcessor - warnings.simplefilter("once", DeprecationWarning) if TYPE_CHECKING: diff --git a/sdk/python/feast/repo_contents.py b/sdk/python/feast/repo_contents.py index b1074239058..6bb8d99fc55 100644 --- a/sdk/python/feast/repo_contents.py +++ b/sdk/python/feast/repo_contents.py @@ -13,7 +13,7 @@ # limitations under the License. from typing import List, NamedTuple -from feast import StreamFeatureView +from feast.stream_feature_view import StreamFeatureView from feast.data_source import DataSource from feast.entity import Entity from feast.feature_service import FeatureService diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index eee86b31f1d..848970d4776 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -93,7 +93,7 @@ def __init__( owner: Optional[str] = "", schema: Optional[List[Field]] = None, source: Optional[DataSource] = None, - aggregations: List[Aggregation], + aggregations: Optional[List[Aggregation]] = None, mode: Optional[str] = "spark", # Mode of ingestion/transformation timestamp_field: Optional[str] = "", # Timestamp for aggregation udf: Optional[MethodType] = None, diff --git a/sdk/python/tests/unit/test_feature_views.py b/sdk/python/tests/unit/test_feature_views.py index 6ad6b85b9d1..4d4fa1548a4 100644 --- a/sdk/python/tests/unit/test_feature_views.py +++ b/sdk/python/tests/unit/test_feature_views.py @@ -57,6 +57,7 @@ def test_create_stream_feature_view(): entities=[], ttl=timedelta(days=30), source=stream_source, + aggregations=[], ) push_source = PushSource( @@ -67,11 +68,13 @@ def test_create_stream_feature_view(): entities=[], ttl=timedelta(days=30), source=push_source, + aggregations=[], + ) with pytest.raises(ValueError): StreamFeatureView( - name="test batch feature view", entities=[], ttl=timedelta(days=30) + name="test batch feature view", entities=[], ttl=timedelta(days=30), aggregations=[], ) with pytest.raises(ValueError): @@ -80,6 +83,7 @@ def test_create_stream_feature_view(): entities=[], ttl=timedelta(days=30), source=FileSource(path="some path"), + aggregations=[], ) From 1ae6623c354674c591fe2f2791a0f9e30a116777 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 31 May 2022 11:54:17 -0700 Subject: [PATCH 08/22] Fix Signed-off-by: Kevin Zhang --- sdk/python/feast/feature_view.py | 4 +- sdk/python/feast/stream_feature_view.py | 50 ++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index ec63f19e1ac..3494011244a 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -254,7 +254,7 @@ def __init__( super().__init__( name=_name, - features=self.schema, + features=_features, description=description, tags=tags, owner=owner, @@ -328,7 +328,6 @@ def __eq__(self, other): ) if not super().__eq__(other): - print("ASdfsf") return False if ( @@ -339,7 +338,6 @@ def __eq__(self, other): or self.stream_source != other.stream_source or sorted(self.entity_columns) != sorted(other.entity_columns) ): - print("ASdfsfd") return False return True diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 848970d4776..da2733867ff 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -3,7 +3,7 @@ from datetime import timedelta from types import MethodType from typing import Dict, List, Optional, Union - +import functools import dill from google.protobuf.duration_pb2 import Duration @@ -268,3 +268,51 @@ def from_proto(cls, sfv_proto: StreamFeatureViewProto): ) return sfv_feature_view + + +def stream_feature_view( + *, + entities: Optional[Union[List[Entity], List[str]]] = None, + ttl: Optional[timedelta] = None, + tags: Optional[Dict[str, str]] = None, + online: Optional[bool] = True, + description: Optional[str] = "", + owner: Optional[str] = "", + schema: Optional[List[Field]] = None, + source: Optional[DataSource] = None, + aggregations: Optional[List[Aggregation]] = None, + mode: Optional[str] = "spark", # Mode of ingestion/transformation + timestamp_field: Optional[str] = "", # Timestamp for aggregation +): + """ + Creates an StreamFeatureView object with the given user function as udf. + """ + def mainify(obj): + # Needed to allow dill to properly serialize the udf. Otherwise, clients will need to have a file with the same + # name as the original file defining the ODFV. + if obj.__module__ != "__main__": + obj.__module__ = "__main__" + + def decorator(user_function): + mainify(user_function) + stream_feature_view_obj = StreamFeatureView( + name=user_function.__name__, + entities=entities, + ttl=ttl, + source=source, + schema=schema, + udf=user_function, + description=description, + tags=tags, + online=online, + owner=owner, + aggregations=aggregations, + mode=mode, + timestamp_field=timestamp_field + ) + functools.update_wrapper( + wrapper=stream_feature_view_obj, wrapped=user_function + ) + return stream_feature_view_obj + + return decorator \ No newline at end of file From 958ed8365286406a0487ca8f81f89a2b0ef3ff29 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 31 May 2022 11:58:31 -0700 Subject: [PATCH 09/22] Fix lint Signed-off-by: Kevin Zhang --- sdk/python/feast/feature_store.py | 38 ++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index ff4fffc580c..22aaea25742 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -38,7 +38,6 @@ import pyarrow as pa from colorama import Fore, Style from google.protobuf.timestamp_pb2 import Timestamp -from pyspark.sql import SparkSession from tqdm import tqdm from feast import feature_server, flags, flags_helper, ui_server, utils @@ -103,6 +102,7 @@ class FeatureStore: """ A FeatureStore object is used to define, create, and retrieve features. + Args: repo_path (optional): Path to a `feature_store.yaml` used to configure the feature store. @@ -183,6 +183,7 @@ def refresh_registry(self): def list_entities(self, allow_cache: bool = False) -> List[Entity]: """ Retrieves the list of entities from the registry. + Args: allow_cache: Whether to allow returning entities from a cached registry. Returns: @@ -206,6 +207,7 @@ def _list_entities( def list_feature_services(self) -> List[FeatureService]: """ Retrieves the list of feature services from the registry. + Returns: A list of feature services. """ @@ -215,6 +217,7 @@ def list_feature_services(self) -> List[FeatureService]: def list_feature_views(self, allow_cache: bool = False) -> List[FeatureView]: """ Retrieves the list of feature views from the registry. + Args: allow_cache: Whether to allow returning entities from a cached registry. Returns: @@ -228,6 +231,7 @@ def list_request_feature_views( ) -> List[RequestFeatureView]: """ Retrieves the list of feature views from the registry. + Args: allow_cache: Whether to allow returning entities from a cached registry. Returns: @@ -256,6 +260,7 @@ def list_on_demand_feature_views( ) -> List[OnDemandFeatureView]: """ Retrieves the list of on demand feature views from the registry. + Returns: A list of on demand feature views. """ @@ -269,6 +274,7 @@ def list_stream_feature_views( ) -> List[StreamFeatureView]: """ Retrieves the list of stream feature views from the registry. + Returns: A list of stream feature views. """ @@ -280,6 +286,7 @@ def list_stream_feature_views( def list_data_sources(self, allow_cache: bool = False) -> List[DataSource]: """ Retrieves the list of data sources from the registry. + Args: allow_cache: Whether to allow returning data sources from a cached registry. Returns: @@ -291,6 +298,7 @@ def list_data_sources(self, allow_cache: bool = False) -> List[DataSource]: def get_entity(self, name: str, allow_registry_cache: bool = False) -> Entity: """ Retrieves an entity. + Args: name: Name of entity. allow_registry_cache: (Optional) Whether to allow returning this entity from a cached registry @@ -309,6 +317,7 @@ def get_feature_service( ) -> FeatureService: """ Retrieves a feature service. + Args: name: Name of feature service. allow_cache: Whether to allow returning feature services from a cached registry. @@ -325,6 +334,7 @@ def get_feature_view( ) -> FeatureView: """ Retrieves a feature view. + Args: name: Name of feature view. allow_registry_cache: (Optional) Whether to allow returning this entity from a cached registry @@ -352,6 +362,7 @@ def _get_feature_view( def get_on_demand_feature_view(self, name: str) -> OnDemandFeatureView: """ Retrieves a feature view. + Args: name: Name of feature view. Returns: @@ -365,6 +376,7 @@ def get_on_demand_feature_view(self, name: str) -> OnDemandFeatureView: def get_data_source(self, name: str) -> DataSource: """ Retrieves the list of data sources from the registry. + Args: name: Name of the data source. Returns: @@ -378,6 +390,7 @@ def get_data_source(self, name: str) -> DataSource: def delete_feature_view(self, name: str): """ Deletes a feature view. + Args: name: Name of feature view. Raises: @@ -389,6 +402,7 @@ def delete_feature_view(self, name: str): def delete_feature_service(self, name: str): """ Deletes a feature service. + Args: name: Name of feature service. Raises: @@ -507,6 +521,7 @@ def _plan( The plan method dry-runs registering one or more definitions (e.g., Entity, FeatureView), and produces a list of all the changes the that would be introduced in the feature repo. The changes computed by the plan command are for informational purposes, and are not actually applied to the registry. + Args: desired_repo_contents: The desired repo state. Raises: @@ -581,6 +596,7 @@ def _apply_diffs( self, registry_diff: RegistryDiff, infra_diff: InfraDiff, new_infra: Infra ): """Applies the given diffs to the metadata store and infrastructure. + Args: registry_diff: The diff between the current registry and the desired registry. infra_diff: The diff between the current infra and the desired infra. @@ -615,6 +631,7 @@ def apply( objects in the Feast registry. Once the apply method has updated the infrastructure (e.g., create tables in an online store), it will commit the updated registry. All operations are idempotent, meaning they can safely be rerun. + Args: objects: A single object, or a list of objects that should be registered with the Feature Store. objects_to_delete: A list of objects to be deleted from the registry and removed from the @@ -827,6 +844,7 @@ def get_historical_features( Time travel is based on the configured TTL for each feature view. A shorter TTL will limit the amount of scanning that will be done in order to find feature data for a specific entity key. Setting a short TTL may result in null values being returned. + Args: entity_df (Union[pd.DataFrame, str]): An entity dataframe is a collection of rows containing all entity columns (e.g., customer_id, driver_id) on which features need to be joined, as well as a event_timestamp @@ -947,6 +965,7 @@ def create_saved_dataset( After data successfully persisted saved dataset object with dataset metadata is committed to the registry. Name for the saved dataset should be unique within project, since it's possible to overwrite previously stored dataset with the same name. + Returns: SavedDataset object with attached RetrievalJob Raises: @@ -996,6 +1015,7 @@ def get_saved_dataset(self, name: str) -> SavedDataset: create a retrieval job to pull whole dataset from storage (offline store). If dataset couldn't be found by provided name SavedDatasetNotFound exception will be raised. Data will be retrieved from globally configured offline store. + Returns: SavedDataset with RetrievalJob attached Raises: @@ -1027,6 +1047,7 @@ def materialize_incremental( into the online store where it is available for online serving. The start time of the interval materialized is either the most recent end time of a prior materialization or (now - ttl) if no such prior materialization exists. + Args: end_date (datetime): End date for time range of data to materialize into the online store feature_views (List[str]): Optional list of feature view names. If selected, will only run @@ -1124,6 +1145,7 @@ def materialize( This method loads feature data in the specified interval from either the specified feature views, or all feature views if none are specified, into the online store where it is available for online serving. + Args: start_date (datetime): Start date for time range of data to materialize into the online store end_date (datetime): End date for time range of data to materialize into the online store @@ -1201,6 +1223,7 @@ def push( ): """ Push features to a push source. This updates all the feature views that have the push source as stream source. + Args: push_source_name: The name of the push source we want to push data to. df: the data being pushed. @@ -1269,6 +1292,7 @@ def get_online_features( introduce latency to online feature retrieval. In order to avoid synchronous downloads, please call refresh_registry() prior to the TTL being reached. Remember it is possible to set the cache TTL to infinity (cache forever). + Args: features: The list of features that should be retrieved from the online store. These features can be specified either as a list of string feature references or as a feature service. String feature @@ -1277,12 +1301,16 @@ def get_online_features( full_feature_names: If True, feature names will be prefixed with the corresponding feature view name, changing them from the format "feature" to "feature_view__feature" (e.g. "daily_transactions" changes to "customer_fv__daily_transactions"). + Returns: OnlineResponse containing the feature data in records. + Raises: Exception: No entity with the specified name exists. + Examples: Retrieve online features from an online store. + >>> from feast import FeatureStore, RepoConfig >>> fs = FeatureStore(repo_path="feature_repo") >>> online_response = fs.get_online_features( @@ -1763,6 +1791,7 @@ def _populate_response_from_feature_data( This method assumes that `_read_from_online_store` returns data for each combination of Entities in `entity_rows` in the same order as they are provided. + Args: feature_data: A list of data in Protobuf form which was retrieved from the OnlineStore. indexes: A list of indexes which should be the same length as `feature_data`. Each list @@ -1813,6 +1842,7 @@ def _augment_response_with_on_demand_transforms( Assumes that 'online_features_response' already contains the necessary request data and input feature views for the on demand feature views. Unneeded feature values such as request data and unrequested input feature views will be removed from 'online_features_response'. + Args: online_features_response: Protobuf object to populate feature_refs: List of all feature references to be returned. @@ -1877,6 +1907,7 @@ def _drop_unneeded_columns( """ Unneeded feature values such as request data and unrequested input feature views will be removed from 'online_features_response'. + Args: online_features_response: Protobuf object to populate requested_result_row_names: Fields from 'result_rows' that have been requested, and @@ -2054,6 +2085,7 @@ def write_logged_features( """ Write logs produced by a source (currently only feature service is supported as a source) to an offline store. + Args: logs: Arrow Table or path to parquet dataset directory on disk source: Object that produces logs @@ -2086,6 +2118,7 @@ def validate_logged_features( ) -> Optional[ValidationFailed]: """ Load logged features from an offline store and validate them against provided validation reference. + Args: source: Logs source object (currently only feature services are supported) start: lower bound for loading logged features @@ -2093,6 +2126,7 @@ def validate_logged_features( reference: validation reference throw_exception: throw exception or return it as a result cache_profile: store cached profile in Feast registry + Returns: Throw or return (depends on parameter) ValidationFailed exception if validation was not successful or None if successful. @@ -2155,11 +2189,13 @@ def _validate_entity_values(join_key_values: Dict[str, List[Value]]): def _validate_feature_refs(feature_refs: List[str], full_feature_names: bool = False): """ Validates that there are no collisions among the feature references. + Args: feature_refs: List of feature references to validate. Feature references must have format "feature_view:feature", e.g. "customer_fv:daily_transactions". full_feature_names: If True, the full feature references are compared for collisions; if False, only the feature names are compared. + Raises: FeatureNameCollisionError: There is a collision among the feature references. """ From e36a97603d44a88c50acfdfd25997e19975e0381 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 31 May 2022 12:00:57 -0700 Subject: [PATCH 10/22] Fix lint Signed-off-by: Kevin Zhang --- sdk/python/feast/feature_store.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 22aaea25742..b3f85fe9fc1 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -121,6 +121,7 @@ def __init__( ): """ Creates a FeatureStore object. + Raises: ValueError: If both or neither of repo_path and config are specified. """ @@ -163,11 +164,13 @@ def _get_provider(self) -> Provider: @log_exceptions_and_usage def refresh_registry(self): """Fetches and caches a copy of the feature registry in memory. + Explicitly calling this method allows for direct control of the state of the registry cache. Every time this method is called the complete registry state will be retrieved from the remote registry store backend (e.g., GCS, S3), and the cache timer will be reset. If refresh_registry() is run before get_online_features() is called, then get_online_features() will use the cached registry instead of retrieving (and caching) the registry itself. + Additionally, the TTL for the registry cache can be set to infinity (by setting it to 0), which means that refresh_registry() will become the only way to update the cached registry. If the TTL is set to a value greater than 0, then once the cache becomes stale (more time than the TTL has passed), a new cache will be @@ -186,6 +189,7 @@ def list_entities(self, allow_cache: bool = False) -> List[Entity]: Args: allow_cache: Whether to allow returning entities from a cached registry. + Returns: A list of entities. """ @@ -220,6 +224,7 @@ def list_feature_views(self, allow_cache: bool = False) -> List[FeatureView]: Args: allow_cache: Whether to allow returning entities from a cached registry. + Returns: A list of feature views. """ @@ -234,6 +239,7 @@ def list_request_feature_views( Args: allow_cache: Whether to allow returning entities from a cached registry. + Returns: A list of feature views. """ @@ -289,6 +295,7 @@ def list_data_sources(self, allow_cache: bool = False) -> List[DataSource]: Args: allow_cache: Whether to allow returning data sources from a cached registry. + Returns: A list of data sources. """ @@ -302,6 +309,7 @@ def get_entity(self, name: str, allow_registry_cache: bool = False) -> Entity: Args: name: Name of entity. allow_registry_cache: (Optional) Whether to allow returning this entity from a cached registry + Returns: The specified entity. Raises: @@ -321,8 +329,10 @@ def get_feature_service( Args: name: Name of feature service. allow_cache: Whether to allow returning feature services from a cached registry. + Returns: The specified feature service. + Raises: FeatureServiceNotFoundException: The feature service could not be found. """ @@ -338,8 +348,10 @@ def get_feature_view( Args: name: Name of feature view. allow_registry_cache: (Optional) Whether to allow returning this entity from a cached registry + Returns: The specified feature view. + Raises: FeatureViewNotFoundException: The feature view could not be found. """ @@ -365,8 +377,10 @@ def get_on_demand_feature_view(self, name: str) -> OnDemandFeatureView: Args: name: Name of feature view. + Returns: The specified feature view. + Raises: FeatureViewNotFoundException: The feature view could not be found. """ @@ -379,8 +393,10 @@ def get_data_source(self, name: str) -> DataSource: Args: name: Name of the data source. + Returns: The specified data source. + Raises: DataSourceObjectNotFoundException: The data source could not be found. """ @@ -393,6 +409,7 @@ def delete_feature_view(self, name: str): Args: name: Name of feature view. + Raises: FeatureViewNotFoundException: The feature view could not be found. """ @@ -405,6 +422,7 @@ def delete_feature_service(self, name: str): Args: name: Name of feature service. + Raises: FeatureServiceNotFoundException: The feature view could not be found. """ @@ -858,10 +876,13 @@ def get_historical_features( changes to "customer_fv__daily_transactions"). Returns: RetrievalJob which can be used to materialize the results. + Raises: ValueError: Both or neither of features and feature_refs are specified. + Examples: Retrieve historical features from a local offline store. + >>> from feast import FeatureStore, RepoConfig >>> import pandas as pd >>> fs = FeatureStore(repo_path="feature_repo") @@ -968,6 +989,7 @@ def create_saved_dataset( Returns: SavedDataset object with attached RetrievalJob + Raises: ValueError if given retrieval job doesn't have metadata """ @@ -1018,6 +1040,7 @@ def get_saved_dataset(self, name: str) -> SavedDataset: Returns: SavedDataset with RetrievalJob attached + Raises: SavedDatasetNotFound """ From b594559b068995107ed2cf183376a96e3dadafd6 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 31 May 2022 12:04:08 -0700 Subject: [PATCH 11/22] Fix lihnt Signed-off-by: Kevin Zhang --- sdk/python/feast/feature_store.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index b3f85fe9fc1..9d37eb287fd 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -536,14 +536,17 @@ def _plan( self, desired_repo_contents: RepoContents ) -> Tuple[RegistryDiff, InfraDiff, Infra]: """Dry-run registering objects to metadata store. + The plan method dry-runs registering one or more definitions (e.g., Entity, FeatureView), and produces a list of all the changes the that would be introduced in the feature repo. The changes computed by the plan command are for informational purposes, and are not actually applied to the registry. Args: desired_repo_contents: The desired repo state. + Raises: ValueError: The 'objects' parameter could not be parsed properly. + Examples: Generate a plan adding an Entity and a FeatureView. >>> from feast import FeatureStore, Entity, FeatureView, Feature, FileSource, RepoConfig @@ -645,6 +648,7 @@ def apply( partial: bool = True, ): """Register objects to metadata store and update related infrastructure. + The apply method registers one or more definitions (e.g., Entity, FeatureView) and registers or updates these objects in the Feast registry. Once the apply method has updated the infrastructure (e.g., create tables in an online store), it will commit the updated registry. All operations are idempotent, meaning they can safely @@ -656,10 +660,13 @@ def apply( provider's infrastructure. This deletion will only be performed if partial is set to False. partial: If True, apply will only handle the specified objects; if False, apply will also delete all the objects in objects_to_delete, and tear down any associated cloud resources. + Raises: ValueError: The 'objects' parameter could not be parsed properly. + Examples: Register an Entity and a FeatureView. + >>> from feast import FeatureStore, Entity, FeatureView, Feature, FileSource, RepoConfig >>> from datetime import timedelta >>> fs = FeatureStore(repo_path="feature_repo") @@ -854,11 +861,14 @@ def get_historical_features( full_feature_names: bool = False, ) -> RetrievalJob: """Enrich an entity dataframe with historical feature values for either training or batch scoring. + This method joins historical feature data from one or more feature views to an entity dataframe by using a time travel join. + Each feature view is joined to the entity dataframe using all entities configured for the respective feature view. All configured entities must be available in the entity dataframe. Therefore, the entity dataframe must contain all entities found in all feature views, but the individual feature views can have different entities. + Time travel is based on the configured TTL for each feature view. A shorter TTL will limit the amount of scanning that will be done in order to find feature data for a specific entity key. Setting a short TTL may result in null values being returned. @@ -874,6 +884,7 @@ def get_historical_features( full_feature_names: If True, feature names will be prefixed with the corresponding feature view name, changing them from the format "feature" to "feature_view__feature" (e.g. "daily_transactions" changes to "customer_fv__daily_transactions"). + Returns: RetrievalJob which can be used to materialize the results. @@ -1035,7 +1046,9 @@ def get_saved_dataset(self, name: str) -> SavedDataset: """ Find a saved dataset in the registry by provided name and create a retrieval job to pull whole dataset from storage (offline store). + If dataset couldn't be found by provided name SavedDatasetNotFound exception will be raised. + Data will be retrieved from globally configured offline store. Returns: @@ -1065,6 +1078,7 @@ def materialize_incremental( ) -> None: """ Materialize incremental new data from the offline store into the online store. + This method loads incremental new feature data up to the specified end time from either the specified feature views, or all feature views if none are specified, into the online store where it is available for online serving. The start time of @@ -1075,10 +1089,13 @@ def materialize_incremental( end_date (datetime): End date for time range of data to materialize into the online store feature_views (List[str]): Optional list of feature view names. If selected, will only run materialization for the specified feature views. + Raises: Exception: A feature view being materialized does not have a TTL set. + Examples: Materialize all features into the online store up to 5 minutes ago. + >>> from feast import FeatureStore, RepoConfig >>> from datetime import datetime, timedelta >>> fs = FeatureStore(repo_path="feature_repo") @@ -1165,6 +1182,7 @@ def materialize( ) -> None: """ Materialize data from the offline store into the online store. + This method loads feature data in the specified interval from either the specified feature views, or all feature views if none are specified, into the online store where it is available for online serving. @@ -1174,8 +1192,10 @@ def materialize( end_date (datetime): End date for time range of data to materialize into the online store feature_views (List[str]): Optional list of feature view names. If selected, will only run materialization for the specified feature views. + Examples: Materialize all features into the online store over the interval + from 3 hours ago to 10 minutes ago. >>> from feast import FeatureStore, RepoConfig >>> from datetime import datetime, timedelta @@ -1308,6 +1328,7 @@ def get_online_features( ) -> OnlineResponse: """ Retrieves the latest online feature data. + Note: This method will download the full feature registry the first time it is run. If you are using a remote registry like GCS or S3 then that may take a few seconds. The registry remains cached up to a TTL duration (which can be set to infinity). If the cached registry is stale (more time than the TTL has @@ -1710,6 +1731,7 @@ def _get_unique_entities( entity_name_to_join_key_map: Dict[str, str], ) -> Tuple[Tuple[Dict[str, Value], ...], Tuple[List[int], ...]]: """Return the set of unique composite Entities for a Feature View and the indexes at which they appear. + This method allows us to query the OnlineStore for data we need only once rather than requesting and processing data for the same combination of Entities multiple times. @@ -1749,9 +1771,11 @@ def _read_from_online_store( table: FeatureView, ) -> List[Tuple[List[Timestamp], List["FieldStatus.ValueType"], List[Value]]]: """Read and process data from the OnlineStore for a given FeatureView. + This method guarantees that the order of the data in each element of the List returned is the same as the order of `requested_features`. - This method assumes that `provider.online_read` returns data for each + + This method assumes that `provider.online_read` returns data for each combination of Entities in `entity_rows` in the same order as they are provided. """ @@ -1862,6 +1886,7 @@ def _augment_response_with_on_demand_transforms( full_feature_names: bool, ): """Computes on demand feature values and adds them to the result rows. + Assumes that 'online_features_response' already contains the necessary request data and input feature views for the on demand feature views. Unneeded feature values such as request data and unrequested input feature views will be removed from 'online_features_response'. @@ -2153,6 +2178,7 @@ def validate_logged_features( Returns: Throw or return (depends on parameter) ValidationFailed exception if validation was not successful or None if successful. + """ warnings.warn( "Logged features validation is an experimental feature. " @@ -2192,6 +2218,7 @@ def get_validation_reference( ) -> ValidationReference: """ Retrieves a validation reference. + Raises: ValidationReferenceNotFoundException: The validation reference could not be found. """ From 945afd0abb6e32022013964ab285985f292e4bbc Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 31 May 2022 13:40:43 -0700 Subject: [PATCH 12/22] Fix stuff Signed-off-by: Kevin Zhang --- sdk/python/feast/base_feature_view.py | 1 + sdk/python/feast/feature_store.py | 3 ++- sdk/python/feast/feature_view.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/base_feature_view.py b/sdk/python/feast/base_feature_view.py index 1fd3720df41..80b3b0cec82 100644 --- a/sdk/python/feast/base_feature_view.py +++ b/sdk/python/feast/base_feature_view.py @@ -130,6 +130,7 @@ def __eq__(self, other): raise TypeError( "Comparisons should only involve BaseFeatureView class objects." ) + if ( self.name != other.name or sorted(self.features) != sorted(other.features) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 9d37eb287fd..e4c1060cf9b 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1775,7 +1775,7 @@ def _read_from_online_store( This method guarantees that the order of the data in each element of the List returned is the same as the order of `requested_features`. - This method assumes that `provider.online_read` returns data for each + This method assumes that `provider.online_read` returns data for each combination of Entities in `entity_rows` in the same order as they are provided. """ @@ -1835,6 +1835,7 @@ def _populate_response_from_feature_data( table: FeatureView, ): """Populate the GetOnlineFeaturesResponse with feature data. + This method assumes that `_read_from_online_store` returns data for each combination of Entities in `entity_rows` in the same order as they are provided. diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 3494011244a..12ce9105f76 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -494,6 +494,7 @@ def from_proto(cls, feature_view_proto: FeatureViewProto): # FeatureViewProjections are not saved in the FeatureView proto. # Create the default projection. feature_view.projection = FeatureViewProjection.from_definition(feature_view) + if feature_view_proto.meta.HasField("created_timestamp"): feature_view.created_timestamp = ( feature_view_proto.meta.created_timestamp.ToDatetime() From a86b3c7aceee07aefb91f968579078b1e6e35265 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 31 May 2022 15:29:24 -0700 Subject: [PATCH 13/22] Fix Signed-off-by: Kevin Zhang --- sdk/python/feast/registry.py | 2 -- sdk/python/feast/repo_contents.py | 2 +- sdk/python/feast/stream_feature_view.py | 3 ++- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index a0f730c0c75..7f298b19b82 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -511,8 +511,6 @@ def apply_feature_view( else: del existing_feature_views_of_same_type[idx] break - print(type(existing_feature_views_of_same_type)) - print(feature_view_proto) existing_feature_views_of_same_type.append(feature_view_proto) if commit: self.commit() diff --git a/sdk/python/feast/repo_contents.py b/sdk/python/feast/repo_contents.py index 6bb8d99fc55..fe5cbd284bc 100644 --- a/sdk/python/feast/repo_contents.py +++ b/sdk/python/feast/repo_contents.py @@ -13,7 +13,6 @@ # limitations under the License. from typing import List, NamedTuple -from feast.stream_feature_view import StreamFeatureView from feast.data_source import DataSource from feast.entity import Entity from feast.feature_service import FeatureService @@ -21,6 +20,7 @@ from feast.on_demand_feature_view import OnDemandFeatureView from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.request_feature_view import RequestFeatureView +from feast.stream_feature_view import StreamFeatureView class RepoContents(NamedTuple): diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index da2733867ff..a1167e41861 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -1,9 +1,10 @@ import abc +import functools import warnings from datetime import timedelta from types import MethodType from typing import Dict, List, Optional, Union -import functools + import dill from google.protobuf.duration_pb2 import Duration From 836abaaa944befea3389cbea895382fc66e95692 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 31 May 2022 16:21:00 -0700 Subject: [PATCH 14/22] Fix lint Signed-off-by: Kevin Zhang --- protos/feast/core/FeatureService.proto | 1 - protos/feast/core/ValidationProfile.proto | 2 -- sdk/python/feast/inference.py | 2 +- sdk/python/feast/repo_operations.py | 1 - sdk/python/feast/stream_feature_view.py | 17 ++++++++--------- .../integration/registration/test_registry.py | 2 +- sdk/python/tests/unit/test_feature_views.py | 9 +++++---- 7 files changed, 15 insertions(+), 19 deletions(-) diff --git a/protos/feast/core/FeatureService.proto b/protos/feast/core/FeatureService.proto index 2654703cc59..51b9c6c02a2 100644 --- a/protos/feast/core/FeatureService.proto +++ b/protos/feast/core/FeatureService.proto @@ -5,7 +5,6 @@ option go_package = "github.com/feast-dev/feast/go/protos/feast/core"; option java_outer_classname = "FeatureServiceProto"; option java_package = "feast.proto.core"; -import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; import "feast/core/FeatureViewProjection.proto"; diff --git a/protos/feast/core/ValidationProfile.proto b/protos/feast/core/ValidationProfile.proto index b660e449bd2..10027995859 100644 --- a/protos/feast/core/ValidationProfile.proto +++ b/protos/feast/core/ValidationProfile.proto @@ -22,8 +22,6 @@ option java_package = "feast.proto.core"; option java_outer_classname = "ValidationProfile"; option go_package = "github.com/feast-dev/feast/go/protos/feast/core"; -import "feast/core/SavedDataset.proto"; - message GEValidationProfiler { message UserDefinedProfiler { // The python-syntax function body (serialized by dill) diff --git a/sdk/python/feast/inference.py b/sdk/python/feast/inference.py index 1f3f6ac3e95..a7f8c7df38f 100644 --- a/sdk/python/feast/inference.py +++ b/sdk/python/feast/inference.py @@ -89,7 +89,7 @@ def update_data_sources_with_inferred_event_timestamp_col( def update_feature_views_with_inferred_features_and_entities( - fvs: List[Union[FeatureView, StreamFeatureView]], + fvs: Union[List[FeatureView], List[StreamFeatureView]], entities: List[Entity], config: RepoConfig, ) -> None: diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 017220a555e..8b81a71bae4 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -11,7 +11,6 @@ import click from click.exceptions import BadParameter -from isort import stream from feast import PushSource from feast.data_source import DataSource diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index a1167e41861..116172a5f6d 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -8,7 +8,7 @@ import dill from google.protobuf.duration_pb2 import Duration -from feast.data_source import DataSource +from feast.data_source import DataSource, KafkaSource from feast.entity import Entity from feast.feature_view import FeatureView from feast.field import Field @@ -120,8 +120,8 @@ def __init__( self.timestamp_field = timestamp_field self.udf = udf self.aggregations = aggregations - - _batch_source = source.batch_source if source.batch_source else None + if isinstance(source, KafkaSource): + _batch_source = source.batch_source if source.batch_source else None super().__init__( name=name, @@ -212,7 +212,7 @@ def to_proto(self): return StreamFeatureViewProto(spec=spec, meta=meta) @classmethod - def from_proto(cls, sfv_proto: StreamFeatureViewProto): + def from_proto(cls, sfv_proto): batch_source = ( DataSource.from_proto(sfv_proto.spec.batch_source) if sfv_proto.spec.HasField("batch_source") @@ -288,6 +288,7 @@ def stream_feature_view( """ Creates an StreamFeatureView object with the given user function as udf. """ + def mainify(obj): # Needed to allow dill to properly serialize the udf. Otherwise, clients will need to have a file with the same # name as the original file defining the ODFV. @@ -309,11 +310,9 @@ def decorator(user_function): owner=owner, aggregations=aggregations, mode=mode, - timestamp_field=timestamp_field - ) - functools.update_wrapper( - wrapper=stream_feature_view_obj, wrapped=user_function + timestamp_field=timestamp_field, ) + functools.update_wrapper(wrapper=stream_feature_view_obj, wrapped=user_function) return stream_feature_view_obj - return decorator \ No newline at end of file + return decorator diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index baf3bdceae5..cbf0e7c3667 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -324,7 +324,7 @@ def simple_udf(x: int): ttl=timedelta(days=30), owner="test@example.com", online=True, - schema=[Field(name="dummy_field", dtype=Float32),], + schema=[Field(name="dummy_field", dtype=Float32)], description="desc", aggregations=[ Aggregation(column="dummy_field", function="max", time_windows=["1h", "24"]) diff --git a/sdk/python/tests/unit/test_feature_views.py b/sdk/python/tests/unit/test_feature_views.py index 4d4fa1548a4..bb823e0d5de 100644 --- a/sdk/python/tests/unit/test_feature_views.py +++ b/sdk/python/tests/unit/test_feature_views.py @@ -1,7 +1,6 @@ from datetime import timedelta import pytest -from numpy import equal from feast import Field, PushSource from feast.batch_feature_view import BatchFeatureView @@ -69,12 +68,14 @@ def test_create_stream_feature_view(): ttl=timedelta(days=30), source=push_source, aggregations=[], - ) with pytest.raises(ValueError): StreamFeatureView( - name="test batch feature view", entities=[], ttl=timedelta(days=30), aggregations=[], + name="test batch feature view", + entities=[], + ttl=timedelta(days=30), + aggregations=[], ) with pytest.raises(ValueError): @@ -107,7 +108,7 @@ def test_stream_feature_view_serialization(): ttl=timedelta(days=30), owner="test@example.com", online=True, - schema=[Field(name="dummy_field", dtype=Float32),], + schema=[Field(name="dummy_field", dtype=Float32)], description="desc", aggregations=[ Aggregation(column="dummy_field", function="max", time_windows=["1h", "24"]) From fb4c4e29572c98c29d47f10829429356cffd5a2e Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 31 May 2022 16:44:31 -0700 Subject: [PATCH 15/22] Fix unit tests Signed-off-by: Kevin Zhang --- sdk/python/feast/stream_feature_view.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 116172a5f6d..09ede5e4189 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -120,6 +120,7 @@ def __init__( self.timestamp_field = timestamp_field self.udf = udf self.aggregations = aggregations + _batch_source = None if isinstance(source, KafkaSource): _batch_source = source.batch_source if source.batch_source else None From 45353bc343fa23ff9785cbd65daf5af61e9da55b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 1 Jun 2022 11:22:23 -0700 Subject: [PATCH 16/22] Address review comments Signed-off-by: Kevin Zhang --- protos/feast/core/Aggregation.proto | 12 +++++ protos/feast/core/StreamFeatureView.proto | 12 ++--- sdk/python/feast/aggregation.py | 54 +++++++++++++++++++ sdk/python/feast/feature_store.py | 3 +- sdk/python/feast/stream_feature_view.py | 54 +++---------------- .../integration/registration/test_registry.py | 12 +++-- sdk/python/tests/unit/test_feature_views.py | 7 ++- 7 files changed, 89 insertions(+), 65 deletions(-) create mode 100644 protos/feast/core/Aggregation.proto create mode 100644 sdk/python/feast/aggregation.py diff --git a/protos/feast/core/Aggregation.proto b/protos/feast/core/Aggregation.proto new file mode 100644 index 00000000000..721963dfd37 --- /dev/null +++ b/protos/feast/core/Aggregation.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package feast.core; + +option go_package = "github.com/feast-dev/feast/go/protos/feast/core"; +option java_outer_classname = "AggregationProto"; +option java_package = "feast.proto.core"; + +message Aggregation { + string column = 1; + string function = 2; + repeated string time_windows = 3; +} \ No newline at end of file diff --git a/protos/feast/core/StreamFeatureView.proto b/protos/feast/core/StreamFeatureView.proto index 2cfb8d0074b..3be9dc866af 100644 --- a/protos/feast/core/StreamFeatureView.proto +++ b/protos/feast/core/StreamFeatureView.proto @@ -26,10 +26,9 @@ option java_package = "feast.proto.core"; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; import "feast/core/OnDemandFeatureView.proto"; -import "feast/core/FeatureView.proto"; -import "feast/core/FeatureViewProjection.proto"; import "feast/core/Feature.proto"; import "feast/core/DataSource.proto"; +import "feast/core/Aggregation.proto"; message StreamFeatureView { // User-specified specifications of this feature view. @@ -37,7 +36,7 @@ message StreamFeatureView { StreamFeatureViewMeta meta = 2; } -// Next available id: 10 +// Next available id: 17 message StreamFeatureViewSpec { // Name of the feature view. Must be unique. Not updated. string name = 1; @@ -77,6 +76,7 @@ message StreamFeatureViewSpec { // Whether these features should be served online or not bool online = 12; + // Serialized function that is encoded in the streamfeatureview UserDefinedFunction user_defined_function = 13; // Mode of execution @@ -96,9 +96,3 @@ message StreamFeatureViewMeta { // Time where this Feature View is last updated google.protobuf.Timestamp last_updated_timestamp = 2; } - -message Aggregation { - string column = 1; - string function = 2; - repeated string time_windows = 3; -} diff --git a/sdk/python/feast/aggregation.py b/sdk/python/feast/aggregation.py new file mode 100644 index 00000000000..0bd97cd7171 --- /dev/null +++ b/sdk/python/feast/aggregation.py @@ -0,0 +1,54 @@ +import abc +from datetime import timedelta +from typing import List, Union + +from feast.protos.feast.core.Aggregation_pb2 import ( + Aggregation as AggregationProto, +) + +class Aggregation(abc.ABC): + """ + NOTE: Feast-handled aggregations are not yet supported. This class provides a way to register user-defined aggregations. + + Attributes: + column: str # Column name of the feature we are aggregating. + function: str # Provided built in aggregations sum, max, min, count mean + time_windows: Union[timedelta, List[timedelta]] # The time windows for aggregations. + """ + + column: str + function: str + time_windows: List[timedelta] + + def __init__(self, column: str, function: str, time_windows: Union[timedelta, List[timedelta]]): + self.column = column + self.function = function + _time_windows = [time_windows] if not isinstance(time_windows, list) else time_windows + self.time_windows = _time_windows + + def to_proto(self) -> AggregationProto: + return AggregationProto( + column=self.column, function=self.function, time_windows=self.time_windows, + ) + + @classmethod + def from_proto(cls, agg_proto: AggregationProto): + aggregation = cls( + column=agg_proto.column, + function=agg_proto.function, + time_windows=list(agg_proto.time_windows), + ) + return aggregation + + def __eq__(self, other): + if not isinstance(other, Aggregation): + raise TypeError("Comparisons should only involve Aggregations.") + + if ( + self.column != other.column + or self.function != other.function + or sorted(self.time_windows) != sorted(other.time_windows) + ): + return False + + return True \ No newline at end of file diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index e4c1060cf9b..6b06b89c197 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -312,6 +312,7 @@ def get_entity(self, name: str, allow_registry_cache: bool = False) -> Entity: Returns: The specified entity. + Raises: EntityNotFoundException: The entity could not be found. """ @@ -549,6 +550,7 @@ def _plan( Examples: Generate a plan adding an Entity and a FeatureView. + >>> from feast import FeatureStore, Entity, FeatureView, Feature, FileSource, RepoConfig >>> from feast.feature_store import RepoContents >>> from datetime import timedelta @@ -1195,7 +1197,6 @@ def materialize( Examples: Materialize all features into the online store over the interval - from 3 hours ago to 10 minutes ago. >>> from feast import FeatureStore, RepoConfig >>> from datetime import datetime, timedelta diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 09ede5e4189..e417bb56858 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -1,4 +1,3 @@ -import abc import functools import warnings from datetime import timedelta @@ -12,13 +11,14 @@ from feast.entity import Entity from feast.feature_view import FeatureView from feast.field import Field +from feast.aggregation import Aggregation from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto +from feast.protos.feast.core.Aggregation_pb2 import ( + Aggregation as AggregationProto +) from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( UserDefinedFunction as UserDefinedFunctionProto, ) -from feast.protos.feast.core.StreamFeatureView_pb2 import ( - Aggregation as AggregationProto, -) from feast.protos.feast.core.StreamFeatureView_pb2 import ( StreamFeatureView as StreamFeatureViewProto, ) @@ -34,46 +34,6 @@ SUPPORTED_STREAM_SOURCES = {"KafkaSource", "PushSource"} -class Aggregation(abc.ABC): - """ - NOTE: Feast-handled aggregations are not yet supported. This class provides a way to register user-defined aggregations. - """ - - column: str # Column name of the feature we are aggregating. - function: str # Provided built in aggregations sum, max, min, count mean - time_windows: List[str] # The time window. Example ["1h", "24h"] - - def __init__(self, column: str, function: str, time_windows: List[str]): - self.column = column - self.function = function - self.time_windows = time_windows - - def to_proto(self) -> AggregationProto: - return AggregationProto( - column=self.column, function=self.function, time_windows=self.time_windows, - ) - - @classmethod - def from_proto(cls, agg_proto: AggregationProto): - aggregation = cls( - column=agg_proto.column, - function=agg_proto.function, - time_windows=list(agg_proto.time_windows), - ) - return aggregation - - def __eq__(self, other): - if not isinstance(other, Aggregation): - raise TypeError("Comparisons should only involve Aggregations.") - - if ( - self.column != other.column - or self.function != other.function - or self.time_windows != other.time_windows - ): - return False - - return True class StreamFeatureView(FeatureView): @@ -119,7 +79,6 @@ def __init__( self.mode = mode self.timestamp_field = timestamp_field self.udf = udf - self.aggregations = aggregations _batch_source = None if isinstance(source, KafkaSource): _batch_source = source.batch_source if source.batch_source else None @@ -150,7 +109,6 @@ def __eq__(self, other): or self.timestamp_field != other.timestamp_field or self.udf.__code__.co_code != other.udf.__code__.co_code or self.aggregations != other.aggregations - or self.timestamp_field != other.timestamp_field ): return False @@ -180,14 +138,14 @@ def to_proto(self): stream_source_proto = self.stream_source.to_proto() stream_source_proto.data_source_class_type = f"{self.stream_source.__class__.__module__}.{self.stream_source.__class__.__name__}" - aggregation_proto_lst = [] + aggregation_proto_list = [] for aggregations in self.aggregations: agg_proto = AggregationProto( column=aggregations.column, function=aggregations.function, time_windows=aggregations.time_windows, ) - aggregation_proto_lst.append(agg_proto) + aggregation_proto_list.append(agg_proto) spec = StreamFeatureViewSpecProto( name=self.name, entities=self.entities, diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index cbf0e7c3667..57623c08c9e 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -29,7 +29,8 @@ from feast.on_demand_feature_view import RequestSource, on_demand_feature_view from feast.registry import Registry from feast.repo_config import RegistryConfig -from feast.stream_feature_view import Aggregation, StreamFeatureView +from feast.stream_feature_view import StreamFeatureView +from feast.aggregation import Aggregation from feast.types import Array, Bytes, Float32, Int32, Int64, String from feast.value_type import ValueType @@ -341,14 +342,15 @@ def simple_udf(x: int): # Register Feature View test_registry.apply_feature_view(sfv, project) - feature_views = test_registry.list_stream_feature_views(project) + stream_feature_views = test_registry.list_stream_feature_views(project) # List Feature Views - assert feature_views[0] == sfv + assert len(stream_feature_views) == 1 + assert stream_feature_views[0] == sfv test_registry.delete_feature_view("test kafka stream feature view", project) - feature_views = test_registry.list_stream_feature_views(project) - assert len(feature_views) == 0 + stream_feature_views = test_registry.list_stream_feature_views(project) + assert len(stream_feature_views) == 0 test_registry.teardown() diff --git a/sdk/python/tests/unit/test_feature_views.py b/sdk/python/tests/unit/test_feature_views.py index bb823e0d5de..1797cd791a3 100644 --- a/sdk/python/tests/unit/test_feature_views.py +++ b/sdk/python/tests/unit/test_feature_views.py @@ -2,12 +2,13 @@ import pytest -from feast import Field, PushSource +from feast import Field, PushSource, Entity from feast.batch_feature_view import BatchFeatureView from feast.data_format import AvroFormat from feast.data_source import KafkaSource from feast.infra.offline_stores.file_source import FileSource -from feast.stream_feature_view import Aggregation, StreamFeatureView +from feast.stream_feature_view import StreamFeatureView +from feast.aggregation import Aggregation from feast.types import Float32 @@ -93,6 +94,8 @@ def simple_udf(x: int): def test_stream_feature_view_serialization(): + + entity = Entity(name="driver_entity", join_keys=["test_key"]) stream_source = KafkaSource( name="kafka", timestamp_field="", From dc87a8db8861248b6eb9073f959dfcff37db47b5 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 1 Jun 2022 11:36:59 -0700 Subject: [PATCH 17/22] Fix Signed-off-by: Kevin Zhang --- protos/feast/core/Aggregation.proto | 4 +- sdk/python/feast/aggregation.py | 38 +++++++++++++++---- sdk/python/feast/stream_feature_view.py | 15 +------- .../integration/registration/test_registry.py | 8 +++- sdk/python/tests/unit/test_feature_views.py | 10 +++-- 5 files changed, 46 insertions(+), 29 deletions(-) diff --git a/protos/feast/core/Aggregation.proto b/protos/feast/core/Aggregation.proto index 721963dfd37..d4166ffb140 100644 --- a/protos/feast/core/Aggregation.proto +++ b/protos/feast/core/Aggregation.proto @@ -5,8 +5,10 @@ option go_package = "github.com/feast-dev/feast/go/protos/feast/core"; option java_outer_classname = "AggregationProto"; option java_package = "feast.proto.core"; +import "google/protobuf/duration.proto"; + message Aggregation { string column = 1; string function = 2; - repeated string time_windows = 3; + repeated google.protobuf.Duration time_windows = 3; } \ No newline at end of file diff --git a/sdk/python/feast/aggregation.py b/sdk/python/feast/aggregation.py index 0bd97cd7171..5ad7f789aa9 100644 --- a/sdk/python/feast/aggregation.py +++ b/sdk/python/feast/aggregation.py @@ -2,9 +2,10 @@ from datetime import timedelta from typing import List, Union -from feast.protos.feast.core.Aggregation_pb2 import ( - Aggregation as AggregationProto, -) +from google.protobuf.duration_pb2 import Duration + +from feast.protos.feast.core.Aggregation_pb2 import Aggregation as AggregationProto + class Aggregation(abc.ABC): """ @@ -20,23 +21,44 @@ class Aggregation(abc.ABC): function: str time_windows: List[timedelta] - def __init__(self, column: str, function: str, time_windows: Union[timedelta, List[timedelta]]): + def __init__( + self, + column: str, + function: str, + time_windows: Union[timedelta, List[timedelta]], + ): self.column = column self.function = function - _time_windows = [time_windows] if not isinstance(time_windows, list) else time_windows + _time_windows = ( + [time_windows] if not isinstance(time_windows, list) else time_windows + ) self.time_windows = _time_windows def to_proto(self) -> AggregationProto: + duration_windows = [] + for time_window in self.time_windows: + ttl_duration = Duration() + ttl_duration.FromTimedelta(time_window) + duration_windows.append(ttl_duration) + return AggregationProto( - column=self.column, function=self.function, time_windows=self.time_windows, + column=self.column, function=self.function, time_windows=duration_windows, ) @classmethod def from_proto(cls, agg_proto: AggregationProto): + time_windows = [] + for duration in list(agg_proto.time_windows): + time_windows.append( + timedelta(days=0) + if duration.ToNanoseconds() == 0 + else duration.ToTimedelta() + ) + aggregation = cls( column=agg_proto.column, function=agg_proto.function, - time_windows=list(agg_proto.time_windows), + time_windows=time_windows, ) return aggregation @@ -51,4 +73,4 @@ def __eq__(self, other): ): return False - return True \ No newline at end of file + return True diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index e417bb56858..89a8ab6939d 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -7,15 +7,12 @@ import dill from google.protobuf.duration_pb2 import Duration +from feast.aggregation import Aggregation from feast.data_source import DataSource, KafkaSource from feast.entity import Entity from feast.feature_view import FeatureView from feast.field import Field -from feast.aggregation import Aggregation from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto -from feast.protos.feast.core.Aggregation_pb2 import ( - Aggregation as AggregationProto -) from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( UserDefinedFunction as UserDefinedFunctionProto, ) @@ -34,8 +31,6 @@ SUPPORTED_STREAM_SOURCES = {"KafkaSource", "PushSource"} - - class StreamFeatureView(FeatureView): """ NOTE: Stream Feature Views are not yet fully implemented and exist to allow users to register their stream sources and @@ -138,14 +133,6 @@ def to_proto(self): stream_source_proto = self.stream_source.to_proto() stream_source_proto.data_source_class_type = f"{self.stream_source.__class__.__module__}.{self.stream_source.__class__.__name__}" - aggregation_proto_list = [] - for aggregations in self.aggregations: - agg_proto = AggregationProto( - column=aggregations.column, - function=aggregations.function, - time_windows=aggregations.time_windows, - ) - aggregation_proto_list.append(agg_proto) spec = StreamFeatureViewSpecProto( name=self.name, entities=self.entities, diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 57623c08c9e..8507a27598a 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -20,6 +20,7 @@ from pytest_lazyfixture import lazy_fixture from feast import FileSource +from feast.aggregation import Aggregation from feast.data_format import AvroFormat, ParquetFormat from feast.data_source import KafkaSource from feast.entity import Entity @@ -30,7 +31,6 @@ from feast.registry import Registry from feast.repo_config import RegistryConfig from feast.stream_feature_view import StreamFeatureView -from feast.aggregation import Aggregation from feast.types import Array, Bytes, Float32, Int32, Int64, String from feast.value_type import ValueType @@ -328,7 +328,11 @@ def simple_udf(x: int): schema=[Field(name="dummy_field", dtype=Float32)], description="desc", aggregations=[ - Aggregation(column="dummy_field", function="max", time_windows=["1h", "24"]) + Aggregation( + column="dummy_field", + function="max", + time_windows=[timedelta(days=1), timedelta(days=7)], + ) ], timestamp_field="event_timestamp", mode="spark", diff --git a/sdk/python/tests/unit/test_feature_views.py b/sdk/python/tests/unit/test_feature_views.py index 1797cd791a3..c403d6806fc 100644 --- a/sdk/python/tests/unit/test_feature_views.py +++ b/sdk/python/tests/unit/test_feature_views.py @@ -2,13 +2,13 @@ import pytest -from feast import Field, PushSource, Entity +from feast import Entity, Field, PushSource +from feast.aggregation import Aggregation from feast.batch_feature_view import BatchFeatureView from feast.data_format import AvroFormat from feast.data_source import KafkaSource from feast.infra.offline_stores.file_source import FileSource from feast.stream_feature_view import StreamFeatureView -from feast.aggregation import Aggregation from feast.types import Float32 @@ -107,14 +107,16 @@ def test_stream_feature_view_serialization(): sfv = StreamFeatureView( name="test kafka stream feature view", - entities=["driver"], + entities=[entity], ttl=timedelta(days=30), owner="test@example.com", online=True, schema=[Field(name="dummy_field", dtype=Float32)], description="desc", aggregations=[ - Aggregation(column="dummy_field", function="max", time_windows=["1h", "24"]) + Aggregation( + column="dummy_field", function="max", time_windows=[timedelta(days=1)] + ) ], timestamp_field="event_timestamp", mode="spark", From 6fdc95985ab86917f4c05867b355b7b457d205a4 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 1 Jun 2022 14:51:14 -0700 Subject: [PATCH 18/22] Fix Signed-off-by: Kevin Zhang --- protos/feast/core/Aggregation.proto | 5 +- sdk/python/feast/aggregation.py | 46 ++++++------- sdk/python/feast/data_source.py | 3 + sdk/python/feast/feature_store.py | 11 +-- sdk/python/feast/inference.py | 1 - .../integration/registration/test_registry.py | 12 +++- .../test_stream_feature_view_apply.py | 68 +++++++++++++++++++ sdk/python/tests/unit/test_feature_views.py | 14 ++-- 8 files changed, 118 insertions(+), 42 deletions(-) create mode 100644 sdk/python/tests/integration/registration/test_stream_feature_view_apply.py diff --git a/protos/feast/core/Aggregation.proto b/protos/feast/core/Aggregation.proto index d4166ffb140..52164160b67 100644 --- a/protos/feast/core/Aggregation.proto +++ b/protos/feast/core/Aggregation.proto @@ -10,5 +10,8 @@ import "google/protobuf/duration.proto"; message Aggregation { string column = 1; string function = 2; - repeated google.protobuf.Duration time_windows = 3; + google.protobuf.Duration time_window = 3; + + // Column name after aggregation + string name = 4; } \ No newline at end of file diff --git a/sdk/python/feast/aggregation.py b/sdk/python/feast/aggregation.py index 5ad7f789aa9..5988f264f21 100644 --- a/sdk/python/feast/aggregation.py +++ b/sdk/python/feast/aggregation.py @@ -1,64 +1,58 @@ -import abc +from curses import window from datetime import timedelta -from typing import List, Union +from typing import List, Union, Optional from google.protobuf.duration_pb2 import Duration from feast.protos.feast.core.Aggregation_pb2 import Aggregation as AggregationProto -class Aggregation(abc.ABC): +class Aggregation(): """ NOTE: Feast-handled aggregations are not yet supported. This class provides a way to register user-defined aggregations. Attributes: column: str # Column name of the feature we are aggregating. function: str # Provided built in aggregations sum, max, min, count mean - time_windows: Union[timedelta, List[timedelta]] # The time windows for aggregations. + time_window: timedelta # The time window for this aggregation. """ column: str function: str - time_windows: List[timedelta] + time_window: timedelta def __init__( self, - column: str, - function: str, - time_windows: Union[timedelta, List[timedelta]], + column: Optional[str] = "", + function: Optional[str] = "", + time_window: Optional[timedelta] = None, ): self.column = column self.function = function - _time_windows = ( - [time_windows] if not isinstance(time_windows, list) else time_windows - ) - self.time_windows = _time_windows + self.time_window = time_window def to_proto(self) -> AggregationProto: - duration_windows = [] - for time_window in self.time_windows: - ttl_duration = Duration() - ttl_duration.FromTimedelta(time_window) - duration_windows.append(ttl_duration) + window_duration = None + if self.time_window is not None: + window_duration = Duration() + window_duration.FromTimedelta(self.time_window) return AggregationProto( - column=self.column, function=self.function, time_windows=duration_windows, + column=self.column, function=self.function, time_window=window_duration ) @classmethod def from_proto(cls, agg_proto: AggregationProto): - time_windows = [] - for duration in list(agg_proto.time_windows): - time_windows.append( + time_window = ( timedelta(days=0) - if duration.ToNanoseconds() == 0 - else duration.ToTimedelta() - ) + if agg_proto.time_window.ToNanoseconds() == 0 + else agg_proto.time_window.ToTimedelta() + ) aggregation = cls( column=agg_proto.column, function=agg_proto.function, - time_windows=time_windows, + time_window=time_window, ) return aggregation @@ -69,7 +63,7 @@ def __eq__(self, other): if ( self.column != other.column or self.function != other.function - or sorted(self.time_windows) != sorted(other.time_windows) + or self.time_window != other.time_window ): return False diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 4392314bb8d..eecf1882369 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -410,6 +410,9 @@ def __init__( if _message_format is None: raise ValueError("Message format must be specified for Kafka source") + if not timestamp_field and not _event_timestamp_column: + raise ValueError("Timestamp field must be specified for Kafka source") + super().__init__( event_timestamp_column=_event_timestamp_column, created_timestamp_column=created_timestamp_column, diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 6b06b89c197..430a04557db 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -85,6 +85,7 @@ from feast.request_feature_view import RequestFeatureView from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference from feast.stream_feature_view import StreamFeatureView +from feast.batch_feature_view import BatchFeatureView from feast.type_map import ( feast_value_type_to_python_type, python_values_to_proto_values, @@ -519,9 +520,11 @@ def _make_inferences( update_feature_views_with_inferred_features_and_entities( views_to_update, entities + entities_to_update, self.config ) - update_feature_views_with_inferred_features_and_entities( - sfvs_to_update, entities + entities_to_update, self.config - ) + #TODO(kevjumba): Update schema inferrence + for sfv in sfvs_to_update: + if not sfv.schema: + raise ValueError( + f"schema inference not yet supported for stream feature views. please define schema for stream feature view: {sfv.name}") for odfv in odfvs_to_update: odfv.infer_features() @@ -696,7 +699,7 @@ def apply( # Separate all objects into entities, feature services, and different feature view types. entities_to_update = [ob for ob in objects if isinstance(ob, Entity)] - views_to_update = [ob for ob in objects if isinstance(ob, FeatureView)] + views_to_update = [ob for ob in objects if (isinstance(ob, FeatureView) and not isinstance(ob, StreamFeatureView) and not isinstance(ob, BatchFeatureView))] sfvs_to_update = [ob for ob in objects if isinstance(ob, StreamFeatureView)] request_views_to_update = [ ob for ob in objects if isinstance(ob, RequestFeatureView) diff --git a/sdk/python/feast/inference.py b/sdk/python/feast/inference.py index a7f8c7df38f..37f0cb8b05e 100644 --- a/sdk/python/feast/inference.py +++ b/sdk/python/feast/inference.py @@ -20,7 +20,6 @@ def update_data_sources_with_inferred_event_timestamp_col( data_sources: List[DataSource], config: RepoConfig ) -> None: ERROR_MSG_PREFIX = "Unable to infer DataSource timestamp_field" - for data_source in data_sources: if isinstance(data_source, RequestSource): continue diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 8507a27598a..25947f56ffe 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -309,10 +309,11 @@ def test_apply_stream_feature_view_success(test_registry): # Create Feature Views def simple_udf(x: int): return x + 3 + entity = Entity(name="driver_entity", join_keys=["test_key"]) stream_source = KafkaSource( name="kafka", - timestamp_field="", + timestamp_field="event_timestamp", bootstrap_servers="", message_format=AvroFormat(""), topic="topic", @@ -321,7 +322,7 @@ def simple_udf(x: int): sfv = StreamFeatureView( name="test kafka stream feature view", - entities=["driver"], + entities=[entity], ttl=timedelta(days=30), owner="test@example.com", online=True, @@ -331,7 +332,12 @@ def simple_udf(x: int): Aggregation( column="dummy_field", function="max", - time_windows=[timedelta(days=1), timedelta(days=7)], + time_window=timedelta(days=1), + ), + Aggregation( + column="dummy_field2", + function="count", + time_window=timedelta(days=24), ) ], timestamp_field="event_timestamp", diff --git a/sdk/python/tests/integration/registration/test_stream_feature_view_apply.py b/sdk/python/tests/integration/registration/test_stream_feature_view_apply.py new file mode 100644 index 00000000000..02ea1184875 --- /dev/null +++ b/sdk/python/tests/integration/registration/test_stream_feature_view_apply.py @@ -0,0 +1,68 @@ +import pytest +from datetime import timedelta + +from feast.types import Float32 +from feast import StreamFeatureView, FileSource, Entity, Field +from feast.data_source import KafkaSource +from feast.data_format import AvroFormat +from feast.aggregation import Aggregation + +@pytest.mark.integration +def test_read_pre_applied(environment) -> None: + """ + Test apply of StreamFeatureView. + """ + fs = environment.feature_store + + # Create Feature Views + entity = Entity(name="driver_entity", join_keys=["test_key"]) + + def simple_udf(x: int): + return x + 3 + + stream_source = KafkaSource( + name="kafka", + timestamp_field="event_timestamp", + bootstrap_servers="", + message_format=AvroFormat(""), + topic="topic", + batch_source=FileSource( + path="test_path", + timestamp_field="event_timestamp" + ) + ) + + sfv = StreamFeatureView( + name="test kafka stream feature view", + entities=[entity], + ttl=timedelta(days=30), + owner="test@example.com", + online=True, + schema=[Field(name="dummy_field", dtype=Float32)], + description="desc", + aggregations=[ + Aggregation( + column="dummy_field", + function="max", + time_window=timedelta(days=1), + ), + Aggregation( + column="dummy_field2", + function="count", + time_window=timedelta(days=24), + ) + ], + timestamp_field="event_timestamp", + mode="spark", + source=stream_source, + udf=simple_udf, + tags={}, + ) + fs.apply([entity, sfv]) + stream_feature_views = fs.list_stream_feature_views() + assert len(stream_feature_views) == 1 + assert stream_feature_views[0] == sfv + + entities = fs.list_entities() + assert len(entities) == 1 + assert entities[0] == entity \ No newline at end of file diff --git a/sdk/python/tests/unit/test_feature_views.py b/sdk/python/tests/unit/test_feature_views.py index c403d6806fc..331c6178127 100644 --- a/sdk/python/tests/unit/test_feature_views.py +++ b/sdk/python/tests/unit/test_feature_views.py @@ -2,11 +2,12 @@ import pytest -from feast import Entity, Field, PushSource +from feast.entity import Entity +from feast.field import Field from feast.aggregation import Aggregation from feast.batch_feature_view import BatchFeatureView from feast.data_format import AvroFormat -from feast.data_source import KafkaSource +from feast.data_source import KafkaSource, PushSource from feast.infra.offline_stores.file_source import FileSource from feast.stream_feature_view import StreamFeatureView from feast.types import Float32 @@ -28,7 +29,7 @@ def test_create_batch_feature_view(): stream_source = KafkaSource( name="kafka", - timestamp_field="", + timestamp_field="event_timestamp", bootstrap_servers="", message_format=AvroFormat(""), topic="topic", @@ -46,7 +47,7 @@ def test_create_batch_feature_view(): def test_create_stream_feature_view(): stream_source = KafkaSource( name="kafka", - timestamp_field="", + timestamp_field="event_timestamp", bootstrap_servers="", message_format=AvroFormat(""), topic="topic", @@ -94,11 +95,10 @@ def simple_udf(x: int): def test_stream_feature_view_serialization(): - entity = Entity(name="driver_entity", join_keys=["test_key"]) stream_source = KafkaSource( name="kafka", - timestamp_field="", + timestamp_field="event_timestamp", bootstrap_servers="", message_format=AvroFormat(""), topic="topic", @@ -115,7 +115,7 @@ def test_stream_feature_view_serialization(): description="desc", aggregations=[ Aggregation( - column="dummy_field", function="max", time_windows=[timedelta(days=1)] + column="dummy_field", function="max", time_window=timedelta(days=1), ) ], timestamp_field="event_timestamp", From 86dc73df009d6277c62fec163ca578b294578c18 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 1 Jun 2022 15:11:29 -0700 Subject: [PATCH 19/22] Fix lint Signed-off-by: Kevin Zhang --- sdk/python/feast/aggregation.py | 12 ++--- sdk/python/feast/feature_store.py | 17 +++++-- .../integration/registration/test_registry.py | 11 ++--- .../test_stream_feature_view_apply.py | 44 ++++++++----------- sdk/python/tests/unit/test_feature_views.py | 4 +- 5 files changed, 43 insertions(+), 45 deletions(-) diff --git a/sdk/python/feast/aggregation.py b/sdk/python/feast/aggregation.py index 5988f264f21..a6405a8da74 100644 --- a/sdk/python/feast/aggregation.py +++ b/sdk/python/feast/aggregation.py @@ -1,13 +1,13 @@ from curses import window from datetime import timedelta -from typing import List, Union, Optional +from typing import List, Optional, Union from google.protobuf.duration_pb2 import Duration from feast.protos.feast.core.Aggregation_pb2 import Aggregation as AggregationProto -class Aggregation(): +class Aggregation: """ NOTE: Feast-handled aggregations are not yet supported. This class provides a way to register user-defined aggregations. @@ -19,7 +19,7 @@ class Aggregation(): column: str function: str - time_window: timedelta + time_window: Optional[timedelta] def __init__( self, @@ -27,8 +27,8 @@ def __init__( function: Optional[str] = "", time_window: Optional[timedelta] = None, ): - self.column = column - self.function = function + self.column = column or "" + self.function = function or "" self.time_window = time_window def to_proto(self) -> AggregationProto: @@ -44,7 +44,7 @@ def to_proto(self) -> AggregationProto: @classmethod def from_proto(cls, agg_proto: AggregationProto): time_window = ( - timedelta(days=0) + timedelta(days=0) if agg_proto.time_window.ToNanoseconds() == 0 else agg_proto.time_window.ToTimedelta() ) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 430a04557db..1279c470ee1 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -42,6 +42,7 @@ from feast import feature_server, flags, flags_helper, ui_server, utils from feast.base_feature_view import BaseFeatureView +from feast.batch_feature_view import BatchFeatureView from feast.data_source import DataSource from feast.diff.infra_diff import InfraDiff, diff_infra_protos from feast.diff.registry_diff import RegistryDiff, apply_diff_to_registry, diff_between @@ -85,7 +86,6 @@ from feast.request_feature_view import RequestFeatureView from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference from feast.stream_feature_view import StreamFeatureView -from feast.batch_feature_view import BatchFeatureView from feast.type_map import ( feast_value_type_to_python_type, python_values_to_proto_values, @@ -520,11 +520,12 @@ def _make_inferences( update_feature_views_with_inferred_features_and_entities( views_to_update, entities + entities_to_update, self.config ) - #TODO(kevjumba): Update schema inferrence + # TODO(kevjumba): Update schema inferrence for sfv in sfvs_to_update: if not sfv.schema: raise ValueError( - f"schema inference not yet supported for stream feature views. please define schema for stream feature view: {sfv.name}") + f"schema inference not yet supported for stream feature views. please define schema for stream feature view: {sfv.name}" + ) for odfv in odfvs_to_update: odfv.infer_features() @@ -699,7 +700,15 @@ def apply( # Separate all objects into entities, feature services, and different feature view types. entities_to_update = [ob for ob in objects if isinstance(ob, Entity)] - views_to_update = [ob for ob in objects if (isinstance(ob, FeatureView) and not isinstance(ob, StreamFeatureView) and not isinstance(ob, BatchFeatureView))] + views_to_update = [ + ob + for ob in objects + if ( + isinstance(ob, FeatureView) + and not isinstance(ob, StreamFeatureView) + and not isinstance(ob, BatchFeatureView) + ) + ] sfvs_to_update = [ob for ob in objects if isinstance(ob, StreamFeatureView)] request_views_to_update = [ ob for ob in objects if isinstance(ob, RequestFeatureView) diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index 25947f56ffe..222eb116d26 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -309,6 +309,7 @@ def test_apply_stream_feature_view_success(test_registry): # Create Feature Views def simple_udf(x: int): return x + 3 + entity = Entity(name="driver_entity", join_keys=["test_key"]) stream_source = KafkaSource( @@ -330,15 +331,11 @@ def simple_udf(x: int): description="desc", aggregations=[ Aggregation( - column="dummy_field", - function="max", - time_window=timedelta(days=1), + column="dummy_field", function="max", time_window=timedelta(days=1), ), Aggregation( - column="dummy_field2", - function="count", - time_window=timedelta(days=24), - ) + column="dummy_field2", function="count", time_window=timedelta(days=24), + ), ], timestamp_field="event_timestamp", mode="spark", diff --git a/sdk/python/tests/integration/registration/test_stream_feature_view_apply.py b/sdk/python/tests/integration/registration/test_stream_feature_view_apply.py index 02ea1184875..e55573f2169 100644 --- a/sdk/python/tests/integration/registration/test_stream_feature_view_apply.py +++ b/sdk/python/tests/integration/registration/test_stream_feature_view_apply.py @@ -1,11 +1,13 @@ -import pytest from datetime import timedelta -from feast.types import Float32 -from feast import StreamFeatureView, FileSource, Entity, Field -from feast.data_source import KafkaSource -from feast.data_format import AvroFormat +import pytest + +from feast import Entity, Field, FileSource, StreamFeatureView +from feast.stream_feature_view import stream_feature_view from feast.aggregation import Aggregation +from feast.data_format import AvroFormat +from feast.data_source import KafkaSource +from feast.types import Float32 @pytest.mark.integration def test_read_pre_applied(environment) -> None: @@ -17,23 +19,15 @@ def test_read_pre_applied(environment) -> None: # Create Feature Views entity = Entity(name="driver_entity", join_keys=["test_key"]) - def simple_udf(x: int): - return x + 3 - stream_source = KafkaSource( name="kafka", timestamp_field="event_timestamp", bootstrap_servers="", message_format=AvroFormat(""), topic="topic", - batch_source=FileSource( - path="test_path", - timestamp_field="event_timestamp" - ) + batch_source=FileSource(path="test_path", timestamp_field="event_timestamp"), ) - - sfv = StreamFeatureView( - name="test kafka stream feature view", + @stream_feature_view( entities=[entity], ttl=timedelta(days=30), owner="test@example.com", @@ -42,27 +36,25 @@ def simple_udf(x: int): description="desc", aggregations=[ Aggregation( - column="dummy_field", - function="max", - time_window=timedelta(days=1), + column="dummy_field", function="max", time_window=timedelta(days=1), ), Aggregation( - column="dummy_field2", - function="count", - time_window=timedelta(days=24), - ) + column="dummy_field2", function="count", time_window=timedelta(days=24), + ), ], timestamp_field="event_timestamp", mode="spark", source=stream_source, - udf=simple_udf, tags={}, ) - fs.apply([entity, sfv]) + def simple_sfv(df): + return df + + fs.apply([entity, simple_sfv]) stream_feature_views = fs.list_stream_feature_views() assert len(stream_feature_views) == 1 - assert stream_feature_views[0] == sfv + assert stream_feature_views[0] == simple_sfv entities = fs.list_entities() assert len(entities) == 1 - assert entities[0] == entity \ No newline at end of file + assert entities[0] == entity diff --git a/sdk/python/tests/unit/test_feature_views.py b/sdk/python/tests/unit/test_feature_views.py index 331c6178127..904260dfe61 100644 --- a/sdk/python/tests/unit/test_feature_views.py +++ b/sdk/python/tests/unit/test_feature_views.py @@ -2,12 +2,12 @@ import pytest -from feast.entity import Entity -from feast.field import Field from feast.aggregation import Aggregation from feast.batch_feature_view import BatchFeatureView from feast.data_format import AvroFormat from feast.data_source import KafkaSource, PushSource +from feast.entity import Entity +from feast.field import Field from feast.infra.offline_stores.file_source import FileSource from feast.stream_feature_view import StreamFeatureView from feast.types import Float32 From e80ce38eb246ae29377860845b622c2790919b9e Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 1 Jun 2022 15:12:39 -0700 Subject: [PATCH 20/22] Fix Signed-off-by: Kevin Zhang --- sdk/python/feast/aggregation.py | 3 +-- .../registration/test_stream_feature_view_apply.py | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/aggregation.py b/sdk/python/feast/aggregation.py index a6405a8da74..0a5fe845659 100644 --- a/sdk/python/feast/aggregation.py +++ b/sdk/python/feast/aggregation.py @@ -1,6 +1,5 @@ -from curses import window from datetime import timedelta -from typing import List, Optional, Union +from typing import Optional from google.protobuf.duration_pb2 import Duration diff --git a/sdk/python/tests/integration/registration/test_stream_feature_view_apply.py b/sdk/python/tests/integration/registration/test_stream_feature_view_apply.py index e55573f2169..b01ca434fa8 100644 --- a/sdk/python/tests/integration/registration/test_stream_feature_view_apply.py +++ b/sdk/python/tests/integration/registration/test_stream_feature_view_apply.py @@ -2,13 +2,14 @@ import pytest -from feast import Entity, Field, FileSource, StreamFeatureView -from feast.stream_feature_view import stream_feature_view +from feast import Entity, Field, FileSource from feast.aggregation import Aggregation from feast.data_format import AvroFormat from feast.data_source import KafkaSource +from feast.stream_feature_view import stream_feature_view from feast.types import Float32 + @pytest.mark.integration def test_read_pre_applied(environment) -> None: """ @@ -27,6 +28,7 @@ def test_read_pre_applied(environment) -> None: topic="topic", batch_source=FileSource(path="test_path", timestamp_field="event_timestamp"), ) + @stream_feature_view( entities=[entity], ttl=timedelta(days=30), From 3e745bf328ab038a2dccd4954d6035b2ecbee32d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 1 Jun 2022 15:22:10 -0700 Subject: [PATCH 21/22] Fixed Signed-off-by: Kevin Zhang --- protos/feast/core/Aggregation.proto | 3 --- 1 file changed, 3 deletions(-) diff --git a/protos/feast/core/Aggregation.proto b/protos/feast/core/Aggregation.proto index 52164160b67..d848ce69721 100644 --- a/protos/feast/core/Aggregation.proto +++ b/protos/feast/core/Aggregation.proto @@ -11,7 +11,4 @@ message Aggregation { string column = 1; string function = 2; google.protobuf.Duration time_window = 3; - - // Column name after aggregation - string name = 4; } \ No newline at end of file From e7a324e545382d07c3de8c5e09d659ac2322b913 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 1 Jun 2022 15:23:54 -0700 Subject: [PATCH 22/22] Unsaved changes Signed-off-by: Kevin Zhang --- sdk/python/feast/stream_feature_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 89a8ab6939d..bba16e2627f 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -237,7 +237,7 @@ def stream_feature_view( def mainify(obj): # Needed to allow dill to properly serialize the udf. Otherwise, clients will need to have a file with the same - # name as the original file defining the ODFV. + # name as the original file defining the sfv. if obj.__module__ != "__main__": obj.__module__ = "__main__"