From 57cd3682cffe3eb0794c44a7c32d9a854787d5c2 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Wed, 8 Apr 2020 12:19:26 +0800 Subject: [PATCH 01/12] Add support for historical feature statistics on bigquery --- core/pom.xml | 18 + ....java => FeatureStatisticsRepository.java} | 16 +- .../java/feast/core/grpc/CoreServiceImpl.java | 45 +- .../main/java/feast/core/model/Entity.java | 123 +++++ .../main/java/feast/core/model/Feature.java | 153 ++++++ .../java/feast/core/model/FeatureSet.java | 148 +++--- .../feast/core/model/FeatureStatistics.java | 265 +++++++++++ .../src/main/java/feast/core/model/Field.java | 211 +-------- .../main/java/feast/core/model/FieldId.java | 68 +++ core/src/main/java/feast/core/model/Job.java | 28 -- .../main/java/feast/core/model/Metrics.java | 64 --- .../java/feast/core/service/StatsService.java | 436 ++++++++++++++++++ .../feast/core/service/JobServiceTest.java | 7 +- .../feast/core/service/SpecServiceTest.java | 34 +- .../feast/core/service/StatsServiceTest.java | 286 ++++++++++++ ingestion/pom.xml | 24 - protos/feast/core/CoreService.proto | 46 ++ protos/feast/core/Store.proto | 1 + protos/feast/types/FeatureRow.proto | 4 + .../proto/v0/statistics.proto | 426 +++++++++++++++++ sdk/python/feast/client.py | 99 +++- sdk/python/feast/loaders/ingest.py | 18 +- .../api/statistics/FeatureSetStatistics.java | 48 ++ .../api/statistics/StatisticsRetriever.java | 46 ++ .../bigquery/retriever/QueryTemplater.java | 2 + .../stats/BigQueryStatisticsRetriever.java | 152 ++++++ .../stats/FeatureSetStatisticsQueryInfo.java | 92 ++++ .../stats/FeatureStatisticsQueryInfo.java | 64 +++ .../bigquery/stats/StatsQueryTemplater.java | 78 ++++ .../connectors/bigquery/stats/StatsUtil.java | 234 ++++++++++ .../bigquery/writer/BigQueryFeatureSink.java | 11 +- .../bigquery/writer/FeatureRowToTableRow.java | 2 + .../main/resources/templates/basic_stats.sql | 93 ++++ .../main/resources/templates/hist_stats.sql | 41 ++ .../BigQueryStatisticsRetrieverTest.java | 95 ++++ .../bigquery/stats/StatsUtilTest.java | 218 +++++++++ 36 files changed, 3230 insertions(+), 466 deletions(-) rename core/src/main/java/feast/core/dao/{MetricsRepository.java => FeatureStatisticsRepository.java} (59%) create mode 100644 core/src/main/java/feast/core/model/Entity.java create mode 100644 core/src/main/java/feast/core/model/Feature.java create mode 100644 core/src/main/java/feast/core/model/FeatureStatistics.java create mode 100644 core/src/main/java/feast/core/model/FieldId.java delete mode 100644 core/src/main/java/feast/core/model/Metrics.java create mode 100644 core/src/main/java/feast/core/service/StatsService.java create mode 100644 core/src/test/java/feast/core/service/StatsServiceTest.java create mode 100644 protos/tensorflow_metadata/proto/v0/statistics.proto create mode 100644 storage/api/src/main/java/feast/storage/api/statistics/FeatureSetStatistics.java create mode 100644 storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java create mode 100644 storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetriever.java create mode 100644 storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureSetStatisticsQueryInfo.java create mode 100644 storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureStatisticsQueryInfo.java create mode 100644 storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsQueryTemplater.java create mode 100644 storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsUtil.java create mode 100644 storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql create mode 100644 storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql create mode 100644 storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetrieverTest.java create mode 100644 storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/StatsUtilTest.java diff --git a/core/pom.xml b/core/pom.xml index f4fb6c659c0..d7d57d560c8 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -57,6 +57,24 @@ ${project.version} + + dev.feast + feast-storage-api + ${project.version} + + + + dev.feast + feast-storage-connector-bigquery + ${project.version} + + + org.apache.beam + * + + + + diff --git a/core/src/main/java/feast/core/dao/MetricsRepository.java b/core/src/main/java/feast/core/dao/FeatureStatisticsRepository.java similarity index 59% rename from core/src/main/java/feast/core/dao/MetricsRepository.java rename to core/src/main/java/feast/core/dao/FeatureStatisticsRepository.java index 7146e1e3ecb..f31df3803b5 100644 --- a/core/src/main/java/feast/core/dao/MetricsRepository.java +++ b/core/src/main/java/feast/core/dao/FeatureStatisticsRepository.java @@ -16,12 +16,16 @@ */ package feast.core.dao; -import feast.core.model.Metrics; -import java.util.List; +import feast.core.model.Feature; +import feast.core.model.FeatureStatistics; +import java.util.Date; +import java.util.Optional; import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.stereotype.Repository; -@Repository -public interface MetricsRepository extends JpaRepository { - List findByJob_Id(String id); +/** JPA repository supplying Statistics objects keyed by id. */ +public interface FeatureStatisticsRepository extends JpaRepository { + Optional findFeatureStatisticsByFeatureAndDatasetId( + Feature featureName, String datasetId); + + Optional findFeatureStatisticsByFeatureAndDate(Feature featureName, Date date); } diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index 42bc0ba23de..513fe23c79f 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -19,39 +19,18 @@ import com.google.api.gax.rpc.InvalidArgumentException; import com.google.protobuf.InvalidProtocolBufferException; import feast.core.CoreServiceGrpc.CoreServiceImplBase; -import feast.core.CoreServiceProto.ApplyFeatureSetRequest; -import feast.core.CoreServiceProto.ApplyFeatureSetResponse; -import feast.core.CoreServiceProto.ArchiveProjectRequest; -import feast.core.CoreServiceProto.ArchiveProjectResponse; -import feast.core.CoreServiceProto.CreateProjectRequest; -import feast.core.CoreServiceProto.CreateProjectResponse; -import feast.core.CoreServiceProto.GetFeastCoreVersionRequest; -import feast.core.CoreServiceProto.GetFeastCoreVersionResponse; -import feast.core.CoreServiceProto.GetFeatureSetRequest; -import feast.core.CoreServiceProto.GetFeatureSetResponse; -import feast.core.CoreServiceProto.ListFeatureSetsRequest; -import feast.core.CoreServiceProto.ListFeatureSetsResponse; -import feast.core.CoreServiceProto.ListIngestionJobsRequest; -import feast.core.CoreServiceProto.ListIngestionJobsResponse; -import feast.core.CoreServiceProto.ListProjectsRequest; -import feast.core.CoreServiceProto.ListProjectsResponse; -import feast.core.CoreServiceProto.ListStoresRequest; -import feast.core.CoreServiceProto.ListStoresResponse; -import feast.core.CoreServiceProto.RestartIngestionJobRequest; -import feast.core.CoreServiceProto.RestartIngestionJobResponse; -import feast.core.CoreServiceProto.StopIngestionJobRequest; -import feast.core.CoreServiceProto.StopIngestionJobResponse; -import feast.core.CoreServiceProto.UpdateStoreRequest; -import feast.core.CoreServiceProto.UpdateStoreResponse; +import feast.core.CoreServiceProto.*; import feast.core.exception.RetrievalException; import feast.core.grpc.interceptors.MonitoringInterceptor; import feast.core.model.Project; import feast.core.service.AccessManagementService; import feast.core.service.JobService; import feast.core.service.SpecService; +import feast.core.service.StatsService; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; +import java.io.IOException; import java.util.List; import java.util.NoSuchElementException; import java.util.stream.Collectors; @@ -65,15 +44,18 @@ public class CoreServiceImpl extends CoreServiceImplBase { private SpecService specService; + private StatsService statsService; private AccessManagementService accessManagementService; private JobService jobService; @Autowired public CoreServiceImpl( SpecService specService, + StatsService statsService, AccessManagementService accessManagementService, JobService jobService) { this.specService = specService; + this.statsService = statsService; this.accessManagementService = accessManagementService; this.jobService = jobService; } @@ -113,6 +95,21 @@ public void listFeatureSets( } } + @Override + public void getFeatureStatistics( + GetFeatureStatisticsRequest request, + StreamObserver responseObserver) { + try { + GetFeatureStatisticsResponse response = statsService.getFeatureStatistics(request); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } catch (RetrievalException | IllegalArgumentException | IOException e) { + log.error("Exception has occurred in GetFeatureStatistics method: ", e); + responseObserver.onError( + Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); + } + } + @Override public void listStores( ListStoresRequest request, StreamObserver responseObserver) { diff --git a/core/src/main/java/feast/core/model/Entity.java b/core/src/main/java/feast/core/model/Entity.java new file mode 100644 index 00000000000..574abb0c4e9 --- /dev/null +++ b/core/src/main/java/feast/core/model/Entity.java @@ -0,0 +1,123 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.model; + +import feast.core.FeatureSetProto.EntitySpec; +import feast.types.ValueProto.ValueType; +import java.util.Objects; +import javax.persistence.*; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@javax.persistence.Entity +public class Entity extends Field { + + public Entity() {} + + public Entity(String name, ValueType.Enum type) { + this.setId(new FieldId()); + this.setName(name); + this.setType(type.toString()); + } + + public static Entity fromProto(EntitySpec entitySpec) { + Entity entity = new Entity(entitySpec.getName(), entitySpec.getValueType()); + + switch (entitySpec.getPresenceConstraintsCase()) { + case PRESENCE: + entity.setPresence(entitySpec.getPresence().toByteArray()); + break; + case GROUP_PRESENCE: + entity.setGroupPresence(entitySpec.getGroupPresence().toByteArray()); + break; + case PRESENCECONSTRAINTS_NOT_SET: + break; + } + + switch (entitySpec.getShapeTypeCase()) { + case SHAPE: + entity.setShape(entitySpec.getShape().toByteArray()); + break; + case VALUE_COUNT: + entity.setValueCount(entitySpec.getValueCount().toByteArray()); + break; + case SHAPETYPE_NOT_SET: + break; + } + + switch (entitySpec.getDomainInfoCase()) { + case DOMAIN: + entity.setDomain(entitySpec.getDomain()); + break; + case INT_DOMAIN: + entity.setIntDomain(entitySpec.getIntDomain().toByteArray()); + break; + case FLOAT_DOMAIN: + entity.setFloatDomain(entitySpec.getFloatDomain().toByteArray()); + break; + case STRING_DOMAIN: + entity.setStringDomain(entitySpec.getStringDomain().toByteArray()); + break; + case BOOL_DOMAIN: + entity.setBoolDomain(entitySpec.getBoolDomain().toByteArray()); + break; + case STRUCT_DOMAIN: + entity.setStructDomain(entitySpec.getStructDomain().toByteArray()); + break; + case NATURAL_LANGUAGE_DOMAIN: + entity.setNaturalLanguageDomain(entitySpec.getNaturalLanguageDomain().toByteArray()); + break; + case IMAGE_DOMAIN: + entity.setImageDomain(entitySpec.getImageDomain().toByteArray()); + break; + case MID_DOMAIN: + entity.setMidDomain(entitySpec.getMidDomain().toByteArray()); + break; + case URL_DOMAIN: + entity.setUrlDomain(entitySpec.getUrlDomain().toByteArray()); + break; + case TIME_DOMAIN: + entity.setTimeDomain(entitySpec.getTimeDomain().toByteArray()); + break; + case TIME_OF_DAY_DOMAIN: + entity.setTimeOfDayDomain(entitySpec.getTimeOfDayDomain().toByteArray()); + break; + case DOMAININFO_NOT_SET: + break; + } + return entity; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Entity feature = (Entity) o; + return getId().equals(feature.getId()) && getType().equals(feature.getType()); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), getId(), getType()); + } +} diff --git a/core/src/main/java/feast/core/model/Feature.java b/core/src/main/java/feast/core/model/Feature.java new file mode 100644 index 00000000000..ccfbbe53895 --- /dev/null +++ b/core/src/main/java/feast/core/model/Feature.java @@ -0,0 +1,153 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.model; + +import feast.core.FeatureSetProto.FeatureSpec; +import feast.types.ValueProto.ValueType; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import javax.persistence.*; +import javax.persistence.Entity; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@Entity +public class Feature extends Field { + + @OneToMany(cascade = CascadeType.ALL, fetch = FetchType.LAZY) + private List statistics; + + public Feature() {} + + public Feature(FieldId fieldId) { + this.setId(fieldId); + } + + public Feature(String name, ValueType.Enum type) { + this.setId(new FieldId()); + this.setName(name); + this.setType(type.toString()); + } + + public static Feature fromProto(FeatureSpec featureSpec) { + Feature feature = new Feature(featureSpec.getName(), featureSpec.getValueType()); + + switch (featureSpec.getPresenceConstraintsCase()) { + case PRESENCE: + feature.setPresence(featureSpec.getPresence().toByteArray()); + break; + case GROUP_PRESENCE: + feature.setGroupPresence(featureSpec.getGroupPresence().toByteArray()); + break; + case PRESENCECONSTRAINTS_NOT_SET: + break; + } + + switch (featureSpec.getShapeTypeCase()) { + case SHAPE: + feature.setShape(featureSpec.getShape().toByteArray()); + break; + case VALUE_COUNT: + feature.setValueCount(featureSpec.getValueCount().toByteArray()); + break; + case SHAPETYPE_NOT_SET: + break; + } + + switch (featureSpec.getDomainInfoCase()) { + case DOMAIN: + feature.setDomain(featureSpec.getDomain()); + break; + case INT_DOMAIN: + feature.setIntDomain(featureSpec.getIntDomain().toByteArray()); + break; + case FLOAT_DOMAIN: + feature.setFloatDomain(featureSpec.getFloatDomain().toByteArray()); + break; + case STRING_DOMAIN: + feature.setStringDomain(featureSpec.getStringDomain().toByteArray()); + break; + case BOOL_DOMAIN: + feature.setBoolDomain(featureSpec.getBoolDomain().toByteArray()); + break; + case STRUCT_DOMAIN: + feature.setStructDomain(featureSpec.getStructDomain().toByteArray()); + break; + case NATURAL_LANGUAGE_DOMAIN: + feature.setNaturalLanguageDomain(featureSpec.getNaturalLanguageDomain().toByteArray()); + break; + case IMAGE_DOMAIN: + feature.setImageDomain(featureSpec.getImageDomain().toByteArray()); + break; + case MID_DOMAIN: + feature.setMidDomain(featureSpec.getMidDomain().toByteArray()); + break; + case URL_DOMAIN: + feature.setUrlDomain(featureSpec.getUrlDomain().toByteArray()); + break; + case TIME_DOMAIN: + feature.setTimeDomain(featureSpec.getTimeDomain().toByteArray()); + break; + case TIME_OF_DAY_DOMAIN: + feature.setTimeOfDayDomain(featureSpec.getTimeOfDayDomain().toByteArray()); + break; + case DOMAININFO_NOT_SET: + break; + } + return feature; + } + + public void addStatistics(FeatureStatistics newStatistic) { + this.statistics.add(newStatistic); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Feature feature = (Feature) o; + return Objects.equals(getId(), feature.getId()) + && Arrays.equals(getPresence(), feature.getPresence()) + && Arrays.equals(getGroupPresence(), feature.getGroupPresence()) + && Arrays.equals(getShape(), feature.getShape()) + && Arrays.equals(getValueCount(), feature.getValueCount()) + && Objects.equals(getDomain(), feature.getDomain()) + && Arrays.equals(getIntDomain(), feature.getIntDomain()) + && Arrays.equals(getFloatDomain(), feature.getFloatDomain()) + && Arrays.equals(getStringDomain(), feature.getStringDomain()) + && Arrays.equals(getBoolDomain(), feature.getBoolDomain()) + && Arrays.equals(getStructDomain(), feature.getStructDomain()) + && Arrays.equals(getNaturalLanguageDomain(), feature.getNaturalLanguageDomain()) + && Arrays.equals(getImageDomain(), feature.getImageDomain()) + && Arrays.equals(getMidDomain(), feature.getMidDomain()) + && Arrays.equals(getUrlDomain(), feature.getUrlDomain()) + && Arrays.equals(getTimeDomain(), feature.getTimeDomain()) + && Arrays.equals(getTimeDomain(), feature.getTimeOfDayDomain()); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), getId(), getType()); + } +} diff --git a/core/src/main/java/feast/core/model/FeatureSet.java b/core/src/main/java/feast/core/model/FeatureSet.java index 232a5f67d14..6d9c3dec079 100644 --- a/core/src/main/java/feast/core/model/FeatureSet.java +++ b/core/src/main/java/feast/core/model/FeatureSet.java @@ -20,53 +20,18 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Timestamp; import feast.core.FeatureSetProto; -import feast.core.FeatureSetProto.EntitySpec; -import feast.core.FeatureSetProto.FeatureSetMeta; -import feast.core.FeatureSetProto.FeatureSetSpec; -import feast.core.FeatureSetProto.FeatureSetStatus; -import feast.core.FeatureSetProto.FeatureSpec; +import feast.core.FeatureSetProto.*; import feast.types.ValueProto.ValueType.Enum; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import javax.persistence.CascadeType; -import javax.persistence.CollectionTable; -import javax.persistence.Column; -import javax.persistence.ElementCollection; -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.Id; -import javax.persistence.JoinColumn; -import javax.persistence.ManyToOne; -import javax.persistence.Table; -import javax.persistence.UniqueConstraint; +import java.util.*; +import javax.persistence.*; import lombok.Getter; import lombok.Setter; import org.apache.commons.lang3.builder.HashCodeBuilder; -import org.hibernate.annotations.Fetch; -import org.hibernate.annotations.FetchMode; -import org.tensorflow.metadata.v0.BoolDomain; -import org.tensorflow.metadata.v0.FeaturePresence; -import org.tensorflow.metadata.v0.FeaturePresenceWithinGroup; -import org.tensorflow.metadata.v0.FixedShape; -import org.tensorflow.metadata.v0.FloatDomain; -import org.tensorflow.metadata.v0.ImageDomain; -import org.tensorflow.metadata.v0.IntDomain; -import org.tensorflow.metadata.v0.MIDDomain; -import org.tensorflow.metadata.v0.NaturalLanguageDomain; -import org.tensorflow.metadata.v0.StringDomain; -import org.tensorflow.metadata.v0.StructDomain; -import org.tensorflow.metadata.v0.TimeDomain; -import org.tensorflow.metadata.v0.TimeOfDayDomain; -import org.tensorflow.metadata.v0.URLDomain; -import org.tensorflow.metadata.v0.ValueCount; +import org.tensorflow.metadata.v0.*; @Getter @Setter -@Entity +@javax.persistence.Entity @Table(name = "feature_sets") public class FeatureSet extends AbstractTimestampEntity implements Comparable { @@ -93,19 +58,12 @@ public class FeatureSet extends AbstractTimestampEntity implements Comparable entities; + @OneToMany(cascade = CascadeType.ALL, fetch = FetchType.EAGER) + private Set entities; // Feature fields inside this feature set - @ElementCollection(fetch = FetchType.EAGER) - @CollectionTable( - name = "features", - joinColumns = @JoinColumn(name = "feature_set_id"), - uniqueConstraints = @UniqueConstraint(columnNames = {"name", "project", "version"})) - @Fetch(FetchMode.SUBSELECT) - private Set features; + @OneToMany(cascade = CascadeType.ALL, fetch = FetchType.EAGER) + private Set features; // Source on which feature rows can be found @ManyToOne(cascade = CascadeType.ALL, fetch = FetchType.EAGER) @@ -125,8 +83,8 @@ public FeatureSet( String project, int version, long maxAgeSeconds, - List entities, - List features, + List entities, + List features, Source source, FeatureSetStatus status) { this.maxAgeSeconds = maxAgeSeconds; @@ -173,14 +131,14 @@ public static FeatureSet fromProto(FeatureSetProto.FeatureSet featureSetProto) { FeatureSetSpec featureSetSpec = featureSetProto.getSpec(); Source source = Source.fromProto(featureSetSpec.getSource()); - List featureSpecs = new ArrayList<>(); + List featureSpecs = new ArrayList<>(); for (FeatureSpec featureSpec : featureSetSpec.getFeaturesList()) { - featureSpecs.add(new Field(featureSpec)); + featureSpecs.add(Feature.fromProto(featureSpec)); } - List entitySpecs = new ArrayList<>(); + List entitySpecs = new ArrayList<>(); for (EntitySpec entitySpec : featureSetSpec.getEntitiesList()) { - entitySpecs.add(new Field(entitySpec)); + entitySpecs.add(Entity.fromProto(entitySpec)); } return new FeatureSet( @@ -194,40 +152,42 @@ public static FeatureSet fromProto(FeatureSetProto.FeatureSet featureSetProto) { featureSetProto.getMeta().getStatus()); } - public void addEntities(List fields) { - for (Field field : fields) { - addEntity(field); + public void addEntities(List entities) { + for (Entity entity : entities) { + addEntity(entity); } } - public void addEntity(Field field) { - field.setProject(this.project.getName()); - field.setVersion(this.getVersion()); - entities.add(field); + public void addEntity(Entity entity) { + entity.setProject(this.project.getName()); + entity.setFeatureSet(this.getName()); + entity.setVersion(this.getVersion()); + entities.add(entity); } - public void addFeatures(List fields) { - for (Field field : fields) { - addFeature(field); + public void addFeatures(List features) { + for (Feature feature : features) { + addFeature(feature); } } - public void addFeature(Field field) { - field.setProject(this.project.getName()); - field.setVersion(this.getVersion()); - features.add(field); + public void addFeature(Feature feature) { + feature.setProject(this.project.getName()); + feature.setFeatureSet(this.getName()); + feature.setVersion(this.getVersion()); + features.add(feature); } public FeatureSetProto.FeatureSet toProto() throws InvalidProtocolBufferException { List entitySpecs = new ArrayList<>(); - for (Field entityField : entities) { + for (Entity entityField : entities) { EntitySpec.Builder entitySpecBuilder = EntitySpec.newBuilder(); setEntitySpecFields(entitySpecBuilder, entityField); entitySpecs.add(entitySpecBuilder.build()); } List featureSpecs = new ArrayList<>(); - for (Field featureField : features) { + for (Feature featureField : features) { FeatureSpec.Builder featureSpecBuilder = FeatureSpec.newBuilder(); setFeatureSpecFields(featureSpecBuilder, featureField); featureSpecs.add(featureSpecBuilder.build()); @@ -252,14 +212,10 @@ public FeatureSetProto.FeatureSet toProto() throws InvalidProtocolBufferExceptio return FeatureSetProto.FeatureSet.newBuilder().setMeta(meta).setSpec(spec).build(); } - // setEntitySpecFields and setFeatureSpecFields methods contain duplicated code because - // Feast internally treat EntitySpec and FeatureSpec as Field class. However, the proto message - // builder for EntitySpec and FeatureSpec are of different class. - @SuppressWarnings("DuplicatedCode") - private void setEntitySpecFields(EntitySpec.Builder entitySpecBuilder, Field entityField) + private void setEntitySpecFields(EntitySpec.Builder entitySpecBuilder, Entity entityField) throws InvalidProtocolBufferException { entitySpecBuilder - .setName(entityField.getName()) + .setName(entityField.getId().getName()) .setValueType(Enum.valueOf(entityField.getType())); if (entityField.getPresence() != null) { @@ -304,12 +260,10 @@ private void setEntitySpecFields(EntitySpec.Builder entitySpecBuilder, Field ent } } - // Refer to setEntitySpecFields method for the reason for code duplication. - @SuppressWarnings("DuplicatedCode") - private void setFeatureSpecFields(FeatureSpec.Builder featureSpecBuilder, Field featureField) + private void setFeatureSpecFields(FeatureSpec.Builder featureSpecBuilder, Feature featureField) throws InvalidProtocolBufferException { featureSpecBuilder - .setName(featureField.getName()) + .setName(featureField.getId().getName()) .setValueType(Enum.valueOf(featureField.getType())); if (featureField.getPresence() != null) { @@ -378,36 +332,40 @@ public boolean equalTo(FeatureSet other) { } // Create a map of all fields in this feature set - Map fields = new HashMap<>(); + Map entitiesMap = new HashMap<>(); + Map featuresMap = new HashMap<>(); - for (Field e : entities) { - fields.putIfAbsent(e.getName(), e); + for (Entity e : entities) { + entitiesMap.putIfAbsent(e.getId().getName(), e); } - for (Field f : features) { - fields.putIfAbsent(f.getName(), f); + for (Feature f : features) { + featuresMap.putIfAbsent(f.getId().getName(), f); } // Ensure map size is consistent with existing fields - if (fields.size() != other.getFeatures().size() + other.getEntities().size()) { + if (entitiesMap.size() != other.getEntities().size()) { + return false; + } + if (featuresMap.size() != other.getFeatures().size()) { return false; } // Ensure the other entities and features exist in the field map - for (Field e : other.getEntities()) { - if (!fields.containsKey(e.getName())) { + for (Entity e : other.getEntities()) { + if (!entitiesMap.containsKey(e.getId().getName())) { return false; } - if (!e.equals(fields.get(e.getName()))) { + if (!e.equals(entitiesMap.get(e.getId().getName()))) { return false; } } - for (Field f : other.getFeatures()) { - if (!fields.containsKey(f.getName())) { + for (Feature f : other.getFeatures()) { + if (!featuresMap.containsKey(f.getId().getName())) { return false; } - if (!f.equals(fields.get(f.getName()))) { + if (!f.equals(featuresMap.get(f.getId().getName()))) { return false; } } diff --git a/core/src/main/java/feast/core/model/FeatureStatistics.java b/core/src/main/java/feast/core/model/FeatureStatistics.java new file mode 100644 index 00000000000..75bca3b91fd --- /dev/null +++ b/core/src/main/java/feast/core/model/FeatureStatistics.java @@ -0,0 +1,265 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.model; + +import com.google.protobuf.InvalidProtocolBufferException; +import java.io.*; +import java.util.Date; +import java.util.List; +import javax.persistence.*; +import javax.persistence.Entity; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.tensorflow.metadata.v0.*; + +@NoArgsConstructor +@Getter +@Setter +@Entity +@Table( + name = "statistics", + indexes = { + @Index(name = "idx_statistics_feature", columnList = "project,feature_set,version,name"), + @Index(name = "idx_statistics_dataset_id", columnList = "datasetId"), + @Index(name = "idx_statistics_date", columnList = "date"), + }) +public class FeatureStatistics { + @Id + @GeneratedValue(strategy = GenerationType.AUTO) + private int id; + + @ManyToOne + @JoinColumns({ + @JoinColumn(name = "project", referencedColumnName = "project"), + @JoinColumn(name = "feature_set", referencedColumnName = "feature_set"), + @JoinColumn(name = "version", referencedColumnName = "version"), + @JoinColumn(name = "name", referencedColumnName = "name") + }) + private Feature feature; + + // Only one of these fields should be populated. + private String datasetId; + private Date date; + + // General statistics + private String featureType; + private long count; + private long numMissing; + private long minNumValues; + private long maxNumValues; + private float avgNumValues; + private long totalNumValues; + private byte[] numValuesHistogram; + + // Numeric statistics + private double mean; + private double stdev; + private long zeroes; + private double min; + private double max; + private double median; + private byte[] numericValueHistogram; + private byte[] numericValueQuantiles; + + // String statistics + @Column(name = "n_unique") + private long unique; + + private float averageLength; + private byte[] rankHistogram; + private byte[] topValues; + + // Byte statistics + private float minBytes; + private float maxBytes; + private float avgBytes; + + // Instantiates a Statistics object from a tensorflow metadata FeatureNameStatistics object and a + // dataset ID. + public static FeatureStatistics fromProto( + String project, + String featureSetName, + int version, + FeatureNameStatistics featureNameStatistics, + String datasetId) + throws IOException { + FeatureStatistics featureStatistics = FeatureStatistics.fromProto(featureNameStatistics); + Feature feature = new Feature(); + feature.setId( + new FieldId(project, featureSetName, version, featureNameStatistics.getPath().getStep(0))); + featureStatistics.setFeature(feature); + featureStatistics.setDatasetId(datasetId); + return featureStatistics; + } + + // Instantiates a Statistics object from a tensorflow metadata FeatureNameStatistics object and a + // date. + public static FeatureStatistics fromProto( + String project, + String featureSetName, + int version, + FeatureNameStatistics featureNameStatistics, + Date date) + throws IOException { + FeatureStatistics featureStatistics = FeatureStatistics.fromProto(featureNameStatistics); + featureStatistics.setDate(date); + Feature feature = new Feature(); + feature.setId( + new FieldId(project, featureSetName, version, featureNameStatistics.getPath().getStep(0))); + featureStatistics.setFeature(feature); + return featureStatistics; + } + + public FeatureNameStatistics toProto() throws InvalidProtocolBufferException { + FeatureNameStatistics.Builder featureNameStatisticsBuilder = + FeatureNameStatistics.newBuilder() + .setType(FeatureNameStatistics.Type.valueOf(featureType)) + .setPath(Path.newBuilder().addStep(feature.getId().getName())); + CommonStatistics commonStatistics = + CommonStatistics.newBuilder() + .setNumNonMissing(count - numMissing) + .setNumMissing(numMissing) + .setMaxNumValues(maxNumValues) + .setMinNumValues(minNumValues) + .setTotNumValues(totalNumValues) + .setNumValuesHistogram(Histogram.parseFrom(numValuesHistogram)) + .build(); + + switch (featureNameStatisticsBuilder.getType()) { + case INT: + case FLOAT: + NumericStatistics numStats = + NumericStatistics.newBuilder() + .setCommonStats(commonStatistics) + .setMean(mean) + .setStdDev(stdev) + .setNumZeros(zeroes) + .setMin(min) + .setMax(max) + .setMedian(median) + .addHistograms(Histogram.parseFrom(numericValueHistogram)) + .addHistograms(Histogram.parseFrom(numericValueQuantiles)) + .build(); + featureNameStatisticsBuilder.setNumStats(numStats); + break; + case STRING: + StringStatistics.Builder stringStats = + StringStatistics.newBuilder() + .setCommonStats(commonStatistics) + .setUnique(unique) + .setAvgLength(averageLength); + if (rankHistogram == null) { + stringStats.setRankHistogram(RankHistogram.getDefaultInstance()); + } else { + stringStats.setRankHistogram(RankHistogram.parseFrom(rankHistogram)); + } + try (ByteArrayInputStream bis = new ByteArrayInputStream(topValues)) { + ObjectInputStream ois = new ObjectInputStream(bis); + List freqAndValueList = + (List) ois.readObject(); + stringStats.addAllTopValues(freqAndValueList); + } catch (IOException | ClassNotFoundException e) { + throw new InvalidProtocolBufferException( + "Failed to parse field: StringStatistics.TopValues. Check if the value is malformed."); + } + featureNameStatisticsBuilder.setStringStats(stringStats); + break; + case BYTES: + BytesStatistics bytesStats = + BytesStatistics.newBuilder() + .setCommonStats(commonStatistics) + .setAvgNumBytes(avgBytes) + .setMinNumBytes(minBytes) + .setMaxNumBytes(maxBytes) + .build(); + featureNameStatisticsBuilder.setBytesStats(bytesStats); + break; + case STRUCT: + StructStatistics structStats = + StructStatistics.newBuilder().setCommonStats(commonStatistics).build(); + featureNameStatisticsBuilder.setStructStats(structStats); + break; + } + return featureNameStatisticsBuilder.build(); + } + + private static FeatureStatistics fromProto(FeatureNameStatistics featureNameStatistics) + throws IOException, IllegalArgumentException { + FeatureStatistics featureStatistics = new FeatureStatistics(); + featureStatistics.setFeatureType(featureNameStatistics.getType().toString()); + CommonStatistics commonStats; + switch (featureNameStatistics.getType()) { + case FLOAT: + case INT: + NumericStatistics numStats = featureNameStatistics.getNumStats(); + commonStats = numStats.getCommonStats(); + featureStatistics.setMean(numStats.getMean()); + featureStatistics.setStdev(numStats.getStdDev()); + featureStatistics.setZeroes(numStats.getNumZeros()); + featureStatistics.setMin(numStats.getMin()); + featureStatistics.setMax(numStats.getMax()); + featureStatistics.setMedian(numStats.getMedian()); + for (Histogram histogram : numStats.getHistogramsList()) { + switch (histogram.getType()) { + case STANDARD: + featureStatistics.setNumericValueHistogram(histogram.toByteArray()); + case QUANTILES: + featureStatistics.setNumericValueQuantiles(histogram.toByteArray()); + default: + // invalid type, dropping the values + } + } + break; + case STRING: + StringStatistics stringStats = featureNameStatistics.getStringStats(); + commonStats = stringStats.getCommonStats(); + featureStatistics.setUnique(stringStats.getUnique()); + featureStatistics.setAverageLength(stringStats.getAvgLength()); + featureStatistics.setRankHistogram(stringStats.getRankHistogram().toByteArray()); + try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) { + ObjectOutputStream oos = new ObjectOutputStream(bos); + oos.writeObject(stringStats.getTopValuesList()); + featureStatistics.setTopValues(bos.toByteArray()); + } + break; + case BYTES: + BytesStatistics bytesStats = featureNameStatistics.getBytesStats(); + commonStats = bytesStats.getCommonStats(); + featureStatistics.setUnique(bytesStats.getUnique()); + featureStatistics.setMaxBytes(bytesStats.getMaxNumBytes()); + featureStatistics.setMinBytes(bytesStats.getMinNumBytes()); + featureStatistics.setAvgBytes(bytesStats.getAvgNumBytes()); + break; + case STRUCT: + StructStatistics structStats = featureNameStatistics.getStructStats(); + commonStats = structStats.getCommonStats(); + break; + default: + throw new IllegalArgumentException("Feature statistics provided were of unknown type."); + } + featureStatistics.setCount(commonStats.getNumMissing() + commonStats.getNumNonMissing()); + featureStatistics.setNumMissing(commonStats.getNumMissing()); + featureStatistics.setMinNumValues(commonStats.getMinNumValues()); + featureStatistics.setMaxNumValues(commonStats.getMaxNumValues()); + featureStatistics.setAvgNumValues(commonStats.getAvgNumValues()); + featureStatistics.setTotalNumValues(commonStats.getTotNumValues()); + featureStatistics.setNumValuesHistogram(commonStats.getNumValuesHistogram().toByteArray()); + + return featureStatistics; + } +} diff --git a/core/src/main/java/feast/core/model/Field.java b/core/src/main/java/feast/core/model/Field.java index cb23e4eceb7..66dff6ec8f5 100644 --- a/core/src/main/java/feast/core/model/Field.java +++ b/core/src/main/java/feast/core/model/Field.java @@ -1,6 +1,6 @@ /* * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors + * Copyright 2018-2020 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,37 +16,22 @@ */ package feast.core.model; -import feast.core.FeatureSetProto.EntitySpec; -import feast.core.FeatureSetProto.FeatureSpec; -import feast.types.ValueProto.ValueType; -import java.util.Arrays; -import java.util.Objects; -import javax.persistence.Column; -import javax.persistence.Embeddable; +import javax.persistence.EmbeddedId; +import javax.persistence.MappedSuperclass; import lombok.Getter; import lombok.Setter; +// A field in a feature set, which may or may not contain value constraints +// for validation purposes. @Getter @Setter -@Embeddable -public class Field { +@MappedSuperclass +public abstract class Field { + @EmbeddedId private FieldId id; - // Name of the feature - @Column(name = "name", nullable = false) - private String name; - - // Type of the feature, should correspond with feast.types.ValueType - @Column(name = "type", nullable = false) + // Type of the field private String type; - // Version of the field - @Column(name = "version") - private int version; - - // Project that this field belongs to - @Column(name = "project") - private String project; - // Presence constraints (refer to proto feast.core.FeatureSet.FeatureSpec) // Only one of them can be set. private byte[] presence; @@ -72,181 +57,19 @@ public class Field { private byte[] timeDomain; private byte[] timeOfDayDomain; - public Field() {} - - public Field(String name, ValueType.Enum type) { - this.name = name; - this.type = type.toString(); - } - - public Field(FeatureSpec featureSpec) { - this.name = featureSpec.getName(); - this.type = featureSpec.getValueType().toString(); - - switch (featureSpec.getPresenceConstraintsCase()) { - case PRESENCE: - this.presence = featureSpec.getPresence().toByteArray(); - break; - case GROUP_PRESENCE: - this.groupPresence = featureSpec.getGroupPresence().toByteArray(); - break; - case PRESENCECONSTRAINTS_NOT_SET: - break; - } - - switch (featureSpec.getShapeTypeCase()) { - case SHAPE: - this.shape = featureSpec.getShape().toByteArray(); - break; - case VALUE_COUNT: - this.valueCount = featureSpec.getValueCount().toByteArray(); - break; - case SHAPETYPE_NOT_SET: - break; - } - - switch (featureSpec.getDomainInfoCase()) { - case DOMAIN: - this.domain = featureSpec.getDomain(); - break; - case INT_DOMAIN: - this.intDomain = featureSpec.getIntDomain().toByteArray(); - break; - case FLOAT_DOMAIN: - this.floatDomain = featureSpec.getFloatDomain().toByteArray(); - break; - case STRING_DOMAIN: - this.stringDomain = featureSpec.getStringDomain().toByteArray(); - break; - case BOOL_DOMAIN: - this.boolDomain = featureSpec.getBoolDomain().toByteArray(); - break; - case STRUCT_DOMAIN: - this.structDomain = featureSpec.getStructDomain().toByteArray(); - break; - case NATURAL_LANGUAGE_DOMAIN: - this.naturalLanguageDomain = featureSpec.getNaturalLanguageDomain().toByteArray(); - break; - case IMAGE_DOMAIN: - this.imageDomain = featureSpec.getImageDomain().toByteArray(); - break; - case MID_DOMAIN: - this.midDomain = featureSpec.getMidDomain().toByteArray(); - break; - case URL_DOMAIN: - this.urlDomain = featureSpec.getUrlDomain().toByteArray(); - break; - case TIME_DOMAIN: - this.timeDomain = featureSpec.getTimeDomain().toByteArray(); - break; - case TIME_OF_DAY_DOMAIN: - this.timeOfDayDomain = featureSpec.getTimeOfDayDomain().toByteArray(); - break; - case DOMAININFO_NOT_SET: - break; - } + public void setName(String name) { + this.id.setName(name); } - public Field(EntitySpec entitySpec) { - this.name = entitySpec.getName(); - this.type = entitySpec.getValueType().toString(); - - switch (entitySpec.getPresenceConstraintsCase()) { - case PRESENCE: - this.presence = entitySpec.getPresence().toByteArray(); - break; - case GROUP_PRESENCE: - this.groupPresence = entitySpec.getGroupPresence().toByteArray(); - break; - case PRESENCECONSTRAINTS_NOT_SET: - break; - } - - switch (entitySpec.getShapeTypeCase()) { - case SHAPE: - this.shape = entitySpec.getShape().toByteArray(); - break; - case VALUE_COUNT: - this.valueCount = entitySpec.getValueCount().toByteArray(); - break; - case SHAPETYPE_NOT_SET: - break; - } - - switch (entitySpec.getDomainInfoCase()) { - case DOMAIN: - this.domain = entitySpec.getDomain(); - break; - case INT_DOMAIN: - this.intDomain = entitySpec.getIntDomain().toByteArray(); - break; - case FLOAT_DOMAIN: - this.floatDomain = entitySpec.getFloatDomain().toByteArray(); - break; - case STRING_DOMAIN: - this.stringDomain = entitySpec.getStringDomain().toByteArray(); - break; - case BOOL_DOMAIN: - this.boolDomain = entitySpec.getBoolDomain().toByteArray(); - break; - case STRUCT_DOMAIN: - this.structDomain = entitySpec.getStructDomain().toByteArray(); - break; - case NATURAL_LANGUAGE_DOMAIN: - this.naturalLanguageDomain = entitySpec.getNaturalLanguageDomain().toByteArray(); - break; - case IMAGE_DOMAIN: - this.imageDomain = entitySpec.getImageDomain().toByteArray(); - break; - case MID_DOMAIN: - this.midDomain = entitySpec.getMidDomain().toByteArray(); - break; - case URL_DOMAIN: - this.urlDomain = entitySpec.getUrlDomain().toByteArray(); - break; - case TIME_DOMAIN: - this.timeDomain = entitySpec.getTimeDomain().toByteArray(); - break; - case TIME_OF_DAY_DOMAIN: - this.timeOfDayDomain = entitySpec.getTimeOfDayDomain().toByteArray(); - break; - case DOMAININFO_NOT_SET: - break; - } + public void setProject(String project) { + this.id.setProject(project); } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Field field = (Field) o; - return Objects.equals(name, field.name) - && Objects.equals(type, field.type) - && Objects.equals(project, field.project) - && Arrays.equals(presence, field.presence) - && Arrays.equals(groupPresence, field.groupPresence) - && Arrays.equals(shape, field.shape) - && Arrays.equals(valueCount, field.valueCount) - && Objects.equals(domain, field.domain) - && Arrays.equals(intDomain, field.intDomain) - && Arrays.equals(floatDomain, field.floatDomain) - && Arrays.equals(stringDomain, field.stringDomain) - && Arrays.equals(boolDomain, field.boolDomain) - && Arrays.equals(structDomain, field.structDomain) - && Arrays.equals(naturalLanguageDomain, field.naturalLanguageDomain) - && Arrays.equals(imageDomain, field.imageDomain) - && Arrays.equals(midDomain, field.midDomain) - && Arrays.equals(urlDomain, field.urlDomain) - && Arrays.equals(timeDomain, field.timeDomain) - && Arrays.equals(timeOfDayDomain, field.timeOfDayDomain); + public void setVersion(int version) { + this.id.setVersion(version); } - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), name, type); + public void setFeatureSet(String featureSet) { + this.id.setFeatureSet(featureSet); } } diff --git a/core/src/main/java/feast/core/model/FieldId.java b/core/src/main/java/feast/core/model/FieldId.java new file mode 100644 index 00000000000..8f65f1f2091 --- /dev/null +++ b/core/src/main/java/feast/core/model/FieldId.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.model; + +import java.io.Serializable; +import java.util.Objects; +import javax.persistence.Column; +import javax.persistence.Embeddable; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Embeddable +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +public class FieldId implements Serializable { + // Project the field belongs to + @Column(nullable = false) + private String project; + + // Feature set the field belongs to + @Column(name = "feature_set", nullable = false) + private String featureSet; + + // Version of the feature set this field belongs to + @Column(nullable = false) + private int version; + + // Name of the field + @Column(nullable = false) + private String name; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + FieldId fieldId = (FieldId) o; + return Objects.equals(name, fieldId.getName()) + && Objects.equals(project, fieldId.getProject()) + && Objects.equals(featureSet, fieldId.getFeatureSet()); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), project, featureSet, name); + } +} diff --git a/core/src/main/java/feast/core/model/Job.java b/core/src/main/java/feast/core/model/Job.java index 738a16db2d1..5a29c8aa4a3 100644 --- a/core/src/main/java/feast/core/model/Job.java +++ b/core/src/main/java/feast/core/model/Job.java @@ -21,7 +21,6 @@ import feast.core.IngestionJobProto; import java.util.ArrayList; import java.util.List; -import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.EnumType; @@ -32,7 +31,6 @@ import javax.persistence.JoinTable; import javax.persistence.ManyToMany; import javax.persistence.ManyToOne; -import javax.persistence.OneToMany; import javax.persistence.Table; import lombok.AllArgsConstructor; import lombok.Getter; @@ -81,10 +79,6 @@ public class Job extends AbstractTimestampEntity { }) private List featureSets; - // Job Metrics - @OneToMany(mappedBy = "job", cascade = CascadeType.ALL) - private List metrics; - @Enumerated(EnumType.STRING) @Column(name = "status", length = 16) private JobStatus status; @@ -93,28 +87,6 @@ public Job() { super(); } - public Job( - String id, - String extId, - String runner, - Source source, - Store sink, - List featureSets, - JobStatus jobStatus) { - this.id = id; - this.extId = extId; - this.source = source; - this.runner = runner; - this.store = sink; - this.featureSets = featureSets; - this.status = jobStatus; - } - - public void updateMetrics(List newMetrics) { - metrics.clear(); - metrics.addAll(newMetrics); - } - public String getSinkName() { return store.getName(); } diff --git a/core/src/main/java/feast/core/model/Metrics.java b/core/src/main/java/feast/core/model/Metrics.java deleted file mode 100644 index 0b7514816fa..00000000000 --- a/core/src/main/java/feast/core/model/Metrics.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.JoinColumn; -import javax.persistence.ManyToOne; -import javax.persistence.Table; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -@NoArgsConstructor -@Getter -@Setter -@Entity -@Table(name = "metrics") -public class Metrics extends AbstractTimestampEntity { - - @Id - @GeneratedValue(strategy = GenerationType.AUTO) - private long id; - - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "job_id") - private Job job; - - /** Metrics name */ - private String name; - - /** Metrics value */ - private double value; - - /** - * Create a metrics owned by a {@code job}. - * - * @param job owner of this metrics. - * @param metricsName metrics name. - * @param value metrics value. - */ - public Metrics(Job job, String metricsName, double value) { - this.job = job; - this.name = metricsName; - this.value = value; - } -} diff --git a/core/src/main/java/feast/core/service/StatsService.java b/core/src/main/java/feast/core/service/StatsService.java new file mode 100644 index 00000000000..614fa22a330 --- /dev/null +++ b/core/src/main/java/feast/core/service/StatsService.java @@ -0,0 +1,436 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.service; + +import static java.lang.Math.*; + +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Timestamp; +import feast.core.CoreServiceProto.GetFeatureSetRequest; +import feast.core.CoreServiceProto.GetFeatureStatisticsRequest; +import feast.core.CoreServiceProto.GetFeatureStatisticsResponse; +import feast.core.FeatureSetProto.FeatureSet; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.core.FeatureSetProto.FeatureSpec; +import feast.core.StoreProto.Store; +import feast.core.StoreProto.Store.StoreType; +import feast.core.dao.FeatureStatisticsRepository; +import feast.core.dao.StoreRepository; +import feast.core.model.Feature; +import feast.core.model.FeatureStatistics; +import feast.core.model.FieldId; +import feast.storage.api.statistics.FeatureSetStatistics; +import feast.storage.api.statistics.StatisticsRetriever; +import feast.storage.connectors.bigquery.stats.BigQueryStatisticsRetriever; +import java.io.IOException; +import java.time.Instant; +import java.util.*; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.tensorflow.metadata.v0.*; +import org.tensorflow.metadata.v0.FeatureNameStatistics.Builder; + +@Slf4j +@Service +public class StatsService { + + private StoreRepository storeRepository; + private SpecService specService; + private FeatureStatisticsRepository featureStatisticsRepository; + + @Autowired + public StatsService( + StoreRepository storeRepository, + SpecService specService, + FeatureStatisticsRepository featureStatisticsRepository) { + this.storeRepository = storeRepository; + this.specService = specService; + this.featureStatisticsRepository = featureStatisticsRepository; + } + + @Transactional + public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsRequest request) + throws IOException { + StatisticsRetriever statisticsRetriever = getStatisticsRetriever(request.getStore()); + FeatureSetSpec featureSetSpec = getFeatureSetSpec(request.getFeatureSetId()); + List features = request.getFeatureIdsList(); + if (features.size() == 0) { + features = + featureSetSpec.getFeaturesList().stream() + .map(FeatureSpec::getName) + .collect(Collectors.toList()); + } + List> featureNameStatisticsList = new ArrayList<>(); + if (request.getDatasetIdsCount() == 0) { + // retrieve by date + long timestamp = request.getStartDate().getSeconds(); + while (timestamp < request.getEndDate().getSeconds()) { + List featureNameStatistics = + getFeatureNameStatisticsByDate( + statisticsRetriever, featureSetSpec, features, timestamp); + featureNameStatisticsList.add(featureNameStatistics); + timestamp += 86400; // advance by a day + } + } else { + // retrieve by dataset + for (String datasetId : request.getDatasetIdsList()) { + List featureNameStatistics = + getFeatureNameStatisticsByDataset( + statisticsRetriever, featureSetSpec, features, datasetId); + featureNameStatisticsList.add(featureNameStatistics); + } + } + List featureNameStatistics = mergeStatistics(featureNameStatisticsList); + return GetFeatureStatisticsResponse.newBuilder() + .setDatasetFeatureStatisticsList( + DatasetFeatureStatisticsList.newBuilder() + .addDatasets( + DatasetFeatureStatistics.newBuilder().addAllFeatures(featureNameStatistics))) + .build(); + } + + private List getFeatureNameStatisticsByDataset( + StatisticsRetriever statisticsRetriever, + FeatureSetSpec featureSetSpec, + List features, + String datasetId) + throws IOException { + List featureNameStatistics = new ArrayList<>(); + List featuresMissingStats = new ArrayList<>(); + for (String featureName : features) { + Feature feature = + new Feature( + new FieldId( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + featureName)); + Optional cachedFeatureStatistics = + featureStatisticsRepository.findFeatureStatisticsByFeatureAndDatasetId( + feature, datasetId); + if (cachedFeatureStatistics.isPresent()) { + featureNameStatistics.add(cachedFeatureStatistics.get().toProto()); + } else { + featuresMissingStats.add(featureName); + } + } + if (featuresMissingStats.size() > 0) { + FeatureSetStatistics featureStatistics = + statisticsRetriever.getFeatureStatistics(featureSetSpec, featuresMissingStats, datasetId); + for (FeatureNameStatistics stat : featureStatistics.getFeatureNameStatistics()) { + featureStatisticsRepository.save( + FeatureStatistics.fromProto( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + stat, + datasetId)); + } + featureNameStatistics.addAll(featureStatistics.getFeatureNameStatistics()); + } + return featureNameStatistics; + } + + private List getFeatureNameStatisticsByDate( + StatisticsRetriever statisticsRetriever, + FeatureSetSpec featureSetSpec, + List features, + long timestamp) + throws IOException { + Date date = Date.from(Instant.ofEpochSecond(timestamp)); + List featureNameStatistics = new ArrayList<>(); + List featuresMissingStats = new ArrayList<>(); + for (String featureName : features) { + Feature feature = + new Feature( + new FieldId( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + featureName)); + Optional cachedFeatureStatistics = + featureStatisticsRepository.findFeatureStatisticsByFeatureAndDate(feature, date); + if (cachedFeatureStatistics.isPresent()) { + featureNameStatistics.add(cachedFeatureStatistics.get().toProto()); + } else { + featuresMissingStats.add(featureName); + } + } + if (featuresMissingStats.size() > 0) { + FeatureSetStatistics featureStatistics = + statisticsRetriever.getFeatureStatistics( + featureSetSpec, + featuresMissingStats, + Timestamp.newBuilder().setSeconds(timestamp).build()); + for (FeatureNameStatistics stat : featureStatistics.getFeatureNameStatistics()) { + featureStatisticsRepository.save( + FeatureStatistics.fromProto( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + stat, + date)); + } + featureNameStatistics.addAll(featureStatistics.getFeatureNameStatistics()); + } + return featureNameStatistics; + } + + private StatisticsRetriever getStatisticsRetriever(String storeName) + throws InvalidProtocolBufferException { + Store store = storeRepository.getOne(storeName).toProto(); + if (store.getType() != StoreType.BIGQUERY) { + throw new IllegalArgumentException("Batch statistics are only supported for BigQuery stores"); + } + return BigQueryStatisticsRetriever.newBuilder() + .setProjectId(store.getBigqueryConfig().getProjectId()) + .setDatasetId(store.getBigqueryConfig().getDatasetId()) + .setBigquery(BigQueryOptions.getDefaultInstance().getService()) + .build(); + } + + private FeatureSetSpec getFeatureSetSpec(String featureSetId) + throws InvalidProtocolBufferException { + String[] split = featureSetId.split("/"); + String project = split[0]; + split = split[1].split(":"); + FeatureSet featureSet = + specService + .getFeatureSet( + GetFeatureSetRequest.newBuilder() + .setProject(project) + .setName(split[0]) + .setVersion(Integer.parseInt(split[1])) + .build()) + .getFeatureSet(); + return featureSet.getSpec(); + } + + @VisibleForTesting + public List mergeStatistics( + List> featureNameStatistics) { + List unnestedList = new ArrayList<>(); + + featureNameStatistics.forEach(unnestedList::addAll); + Map> groupByPath = + unnestedList.stream() + .collect(Collectors.groupingBy(FeatureNameStatistics::getPath, Collectors.toList())); + + List merged = new ArrayList<>(); + for (Path key : groupByPath.keySet()) { + List featureNameStatisticsForKey = groupByPath.get(key); + if (featureNameStatisticsForKey.size() == 1) { + merged.add(featureNameStatisticsForKey.get(0)); + } else { + switch (featureNameStatisticsForKey.get(0).getType()) { + case INT: + case FLOAT: + merged.add(mergeNumStatistics(featureNameStatisticsForKey)); + break; + case STRING: + merged.add(mergeCategoricalStatistics(groupByPath.get(key))); + break; + case BYTES: + merged.add(mergeByteStatistics(groupByPath.get(key))); + break; + case STRUCT: + merged.add(mergeStructStats(groupByPath.get(key))); + break; + default: + throw new IllegalArgumentException( + "Statistics are only supported for string, boolean, bytes and numeric features"); + } + } + } + return merged; + } + + private FeatureNameStatistics mergeStructStats( + List featureNameStatisticsList) { + Builder mergedFeatureNameStatistics = + FeatureNameStatistics.newBuilder() + .setPath(featureNameStatisticsList.get(0).getPath()) + .setType(featureNameStatisticsList.get(0).getType()); + + long totalCount = 0; + long missingCount = 0; + long totalNumValues = 0; + long maxNumValues = + featureNameStatisticsList.get(0).getStructStats().getCommonStats().getMaxNumValues(); + long minNumValues = + featureNameStatisticsList.get(0).getStructStats().getCommonStats().getMinNumValues(); + + for (FeatureNameStatistics featureNameStatistics : featureNameStatisticsList) { + StructStatistics structStats = featureNameStatistics.getStructStats(); + totalCount += structStats.getCommonStats().getNumNonMissing(); + missingCount += structStats.getCommonStats().getNumMissing(); + totalNumValues += + structStats.getCommonStats().getAvgNumValues() + * structStats.getCommonStats().getNumNonMissing(); + maxNumValues = max(maxNumValues, structStats.getCommonStats().getMaxNumValues()); + minNumValues = min(minNumValues, structStats.getCommonStats().getMinNumValues()); + } + + StructStatistics mergedStructStatistics = + StructStatistics.newBuilder() + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(totalCount) + .setNumNonMissing(totalCount) + .setAvgNumValues((float) totalNumValues / totalCount) + .setMaxNumValues(maxNumValues) + .setMinNumValues(minNumValues) + .setNumMissing(missingCount)) + .build(); + + return mergedFeatureNameStatistics.setStructStats(mergedStructStatistics).build(); + } + + private FeatureNameStatistics mergeNumStatistics( + List featureNameStatisticsList) { + Builder mergedFeatureNameStatistics = + FeatureNameStatistics.newBuilder() + .setPath(featureNameStatisticsList.get(0).getPath()) + .setType(featureNameStatisticsList.get(0).getType()); + + FeatureNameStatistics first = featureNameStatisticsList.remove(0); + double max = first.getNumStats().getMax(); + double min = first.getNumStats().getMin(); + double var = pow(first.getNumStats().getStdDev(), 2); + long totalCount = first.getNumStats().getCommonStats().getNumNonMissing(); + double totalVal = totalCount * first.getNumStats().getMean(); + long missingCount = first.getNumStats().getCommonStats().getNumMissing(); + long zeroes = first.getNumStats().getNumZeros(); + + for (FeatureNameStatistics featureNameStatistics : featureNameStatisticsList) { + NumericStatistics numStats = featureNameStatistics.getNumStats(); + max = max(numStats.getMax(), max); + min = min(numStats.getMin(), min); + long count = numStats.getCommonStats().getNumNonMissing(); + double sampleVar = pow(numStats.getStdDev(), 2); + float aggMean = (float) totalVal / totalCount; + var = getVar(var, totalCount, aggMean, sampleVar, count, numStats.getMean()); + totalVal += numStats.getMean() * count; + totalCount += count; + missingCount += numStats.getCommonStats().getNumMissing(); + zeroes += numStats.getNumZeros(); + } + NumericStatistics mergedNumericStatistics = + NumericStatistics.newBuilder() + .setMax(max) + .setMin(min) + .setMean(totalVal / totalCount) + .setNumZeros(zeroes) + .setStdDev(sqrt(var)) + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(totalCount) + .setNumNonMissing(totalCount) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(missingCount)) + .build(); + return mergedFeatureNameStatistics.setNumStats(mergedNumericStatistics).build(); + } + + // Aggregation of sample variance follows the formula described here: + // https://www.tandfonline.com/doi/abs/10.1080/00031305.2014.966589 + private double getVar( + double s1Var, long s1Count, double s1Mean, double s2Var, long s2Count, double s2Mean) { + long totalCount = s1Count + s2Count; + return ((s1Count - 1) * s1Var + + (s2Count - 1) * s2Var + + ((float) s1Count * s2Count / totalCount) * pow(s1Mean - s2Mean, 2)) + / (s1Count + s2Count - 1); + } + + private FeatureNameStatistics mergeCategoricalStatistics( + List featureNameStatisticsList) { + Builder mergedFeatureNameStatistics = + FeatureNameStatistics.newBuilder() + .setPath(featureNameStatisticsList.get(0).getPath()) + .setType(featureNameStatisticsList.get(0).getType()); + long totalCount = 0; + long missingCount = 0; + long totalLen = 0; + for (FeatureNameStatistics featureNameStatistics : featureNameStatisticsList) { + StringStatistics stringStats = featureNameStatistics.getStringStats(); + totalCount += stringStats.getCommonStats().getNumNonMissing(); + missingCount += stringStats.getCommonStats().getNumMissing(); + totalLen += stringStats.getAvgLength() * stringStats.getCommonStats().getNumNonMissing(); + } + StringStatistics mergedStringStatistics = + StringStatistics.newBuilder() + .setAvgLength((float) totalLen / totalCount) + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(totalCount) + .setNumNonMissing(totalCount) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(missingCount)) + .build(); + return mergedFeatureNameStatistics.setStringStats(mergedStringStatistics).build(); + } + + private FeatureNameStatistics mergeByteStatistics( + List featureNameStatisticsList) { + Builder mergedFeatureNameStatistics = + FeatureNameStatistics.newBuilder() + .setPath(featureNameStatisticsList.get(0).getPath()) + .setType(featureNameStatisticsList.get(0).getType()); + + long totalCount = 0; + long missingCount = 0; + float totalNumBytes = 0; + float maxNumBytes = featureNameStatisticsList.get(0).getBytesStats().getMaxNumBytes(); + float minNumBytes = featureNameStatisticsList.get(0).getBytesStats().getMinNumBytes(); + + for (FeatureNameStatistics featureNameStatistics : featureNameStatisticsList) { + BytesStatistics bytesStats = featureNameStatistics.getBytesStats(); + totalCount += bytesStats.getCommonStats().getNumNonMissing(); + missingCount += bytesStats.getCommonStats().getNumMissing(); + totalNumBytes += bytesStats.getAvgNumBytes() * bytesStats.getCommonStats().getNumNonMissing(); + maxNumBytes = max(maxNumBytes, bytesStats.getMaxNumBytes()); + minNumBytes = min(minNumBytes, bytesStats.getMinNumBytes()); + } + + BytesStatistics mergedBytesStatistics = + BytesStatistics.newBuilder() + .setAvgNumBytes(totalNumBytes / totalCount) + .setMinNumBytes(minNumBytes) + .setMaxNumBytes(maxNumBytes) + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(totalCount) + .setNumNonMissing(totalCount) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(missingCount)) + .build(); + + return mergedFeatureNameStatistics.setBytesStats(mergedBytesStatistics).build(); + } +} diff --git a/core/src/test/java/feast/core/service/JobServiceTest.java b/core/src/test/java/feast/core/service/JobServiceTest.java index c0e90ca43f4..f5963282b37 100644 --- a/core/src/test/java/feast/core/service/JobServiceTest.java +++ b/core/src/test/java/feast/core/service/JobServiceTest.java @@ -45,7 +45,8 @@ import feast.core.job.JobManager; import feast.core.job.Runner; import feast.core.model.FeatureSet; -import feast.core.model.Field; +import feast.core.model.Feature; +import feast.core.model.Entity; import feast.core.model.Job; import feast.core.model.JobStatus; import feast.core.model.Source; @@ -158,8 +159,8 @@ public void setupJobManager() { // dummy model constructorss private FeatureSet newDummyFeatureSet(String name, int version, String project) { - Field feature = new Field(name + "_feature", Enum.INT64); - Field entity = new Field(name + "_entity", Enum.STRING); + Feature feature = new Feature(name + "_feature", Enum.INT64); + Entity entity = new Entity(name + "_entity", Enum.STRING); FeatureSet fs = new FeatureSet( diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index 43a66135dce..cd0e2681b3a 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -51,11 +51,7 @@ import feast.core.dao.ProjectRepository; import feast.core.dao.StoreRepository; import feast.core.exception.RetrievalException; -import feast.core.model.FeatureSet; -import feast.core.model.Field; -import feast.core.model.Project; -import feast.core.model.Source; -import feast.core.model.Store; +import feast.core.model.*; import feast.types.ValueProto.ValueType.Enum; import java.sql.Date; import java.time.Instant; @@ -124,9 +120,9 @@ public void setUp() { FeatureSet featureSet1v3 = newDummyFeatureSet("f1", 3, "project1"); FeatureSet featureSet2v1 = newDummyFeatureSet("f2", 1, "project1"); - Field f3f1 = new Field("f3f1", Enum.INT64); - Field f3f2 = new Field("f3f2", Enum.INT64); - Field f3e1 = new Field("f3e1", Enum.STRING); + Feature f3f1 = new Feature("f3f1", Enum.INT64); + Feature f3f2 = new Feature("f3f2", Enum.INT64); + Entity f3e1 = new Entity("f3e1", Enum.STRING); FeatureSet featureSet3v1 = new FeatureSet( "f3", @@ -490,9 +486,9 @@ public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists() public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered() throws InvalidProtocolBufferException { - Field f3f1 = new Field("f3f1", Enum.INT64); - Field f3f2 = new Field("f3f2", Enum.INT64); - Field f3e1 = new Field("f3e1", Enum.STRING); + Feature f3f1 = new Feature("f3f1", Enum.INT64); + Feature f3f2 = new Feature("f3f2", Enum.INT64); + Entity f3e1 = new Entity("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = (new FeatureSet( "f3", @@ -713,9 +709,9 @@ public void applyFeatureSetShouldUpdateFeatureSetWhenConstraintsAreUpdated() @Test public void applyFeatureSetShouldCreateProjectWhenNotAlreadyExists() throws InvalidProtocolBufferException { - Field f3f1 = new Field("f3f1", Enum.INT64); - Field f3f2 = new Field("f3f2", Enum.INT64); - Field f3e1 = new Field("f3e1", Enum.STRING); + Feature f3f1 = new Feature("f3f1", Enum.INT64); + Feature f3f2 = new Feature("f3f2", Enum.INT64); + Entity f3e1 = new Entity("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = (new FeatureSet( "f3", @@ -739,9 +735,9 @@ public void applyFeatureSetShouldCreateProjectWhenNotAlreadyExists() @Test public void applyFeatureSetShouldFailWhenProjectIsArchived() throws InvalidProtocolBufferException { - Field f3f1 = new Field("f3f1", Enum.INT64); - Field f3f2 = new Field("f3f2", Enum.INT64); - Field f3e1 = new Field("f3e1", Enum.STRING); + Feature f3f1 = new Feature("f3f1", Enum.INT64); + Feature f3f2 = new Feature("f3f2", Enum.INT64); + Entity f3e1 = new Entity("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = (new FeatureSet( "f3", @@ -806,8 +802,8 @@ public void shouldFailIfGetFeatureSetWithoutProject() throws InvalidProtocolBuff } private FeatureSet newDummyFeatureSet(String name, int version, String project) { - Field feature = new Field("feature", Enum.INT64); - Field entity = new Field("entity", Enum.STRING); + Feature feature = new Feature("feature", Enum.INT64); + Entity entity = new Entity("entity", Enum.STRING); FeatureSet fs = new FeatureSet( diff --git a/core/src/test/java/feast/core/service/StatsServiceTest.java b/core/src/test/java/feast/core/service/StatsServiceTest.java new file mode 100644 index 00000000000..e3db263e4c2 --- /dev/null +++ b/core/src/test/java/feast/core/service/StatsServiceTest.java @@ -0,0 +1,286 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.service; + +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.*; + +import feast.core.dao.FeatureStatisticsRepository; +import feast.core.dao.StoreRepository; +import java.util.Arrays; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.tensorflow.metadata.v0.*; +import org.tensorflow.metadata.v0.FeatureNameStatistics.Type; + +public class StatsServiceTest { + + private StatsService statsService; + @Mock private StoreRepository storeRepository; + @Mock private FeatureStatisticsRepository featureStatisticsRepository; + @Mock private SpecService specService; + + @Before + public void setUp() { + statsService = new StatsService(storeRepository, specService, featureStatisticsRepository); + } + + @Test + public void shouldAggregateNumericStatistics() { + FeatureNameStatistics stat1 = + FeatureNameStatistics.newBuilder() + .setNumStats( + NumericStatistics.newBuilder() + .setMax(20) + .setMin(1) + .setMean(6) + .setNumZeros(0) + .setStdDev(7.90569415) + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(5) + .setNumNonMissing(5) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(0))) + .setType(Type.INT) + .setPath(Path.newBuilder().addStep("feature").build()) + .build(); + FeatureNameStatistics stat2 = + FeatureNameStatistics.newBuilder() + .setNumStats( + NumericStatistics.newBuilder() + .setMax(10) + .setMin(0) + .setMean(4) + .setNumZeros(1) + .setStdDev(3.807886553) + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(5) + .setNumNonMissing(5) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(1))) + .setPath(Path.newBuilder().addStep("feature").build()) + .setType(Type.INT) + .build(); + + FeatureNameStatistics expected = + FeatureNameStatistics.newBuilder() + .setNumStats( + NumericStatistics.newBuilder() + .setMax(20) + .setMin(0) + .setMean(5) + .setNumZeros(1) + .setStdDev(5.944184833146219) + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(10) + .setNumNonMissing(10) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(1))) + .setPath(Path.newBuilder().addStep("feature").build()) + .setType(Type.INT) + .build(); + + assertThat( + statsService.mergeStatistics(Arrays.asList(Arrays.asList(stat1, stat2))), + equalTo(Arrays.asList(expected))); + } + + @Test + public void shouldAggregateCategoricalStatistics() { + FeatureNameStatistics stat1 = + FeatureNameStatistics.newBuilder() + .setStringStats( + StringStatistics.newBuilder() + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(5) + .setNumNonMissing(5) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(0)) + .setUnique(4) + .setAvgLength(6)) + .setType(Type.STRING) + .setPath(Path.newBuilder().addStep("feature").build()) + .build(); + FeatureNameStatistics stat2 = + FeatureNameStatistics.newBuilder() + .setStringStats( + StringStatistics.newBuilder() + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(5) + .setNumNonMissing(5) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(1)) + .setUnique(4) + .setAvgLength(4)) + .setType(Type.STRING) + .setPath(Path.newBuilder().addStep("feature").build()) + .build(); + + FeatureNameStatistics expected = + FeatureNameStatistics.newBuilder() + .setStringStats( + StringStatistics.newBuilder() + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(10) + .setNumNonMissing(10) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(1)) + .setAvgLength(5)) + .setType(Type.STRING) + .setPath(Path.newBuilder().addStep("feature").build()) + .build(); + assertThat( + statsService.mergeStatistics(Arrays.asList(Arrays.asList(stat1, stat2))), + equalTo(Arrays.asList(expected))); + } + + @Test + public void shouldAggregateBytesStatistics() { + FeatureNameStatistics stat1 = + FeatureNameStatistics.newBuilder() + .setBytesStats( + BytesStatistics.newBuilder() + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(5) + .setNumNonMissing(5) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(0)) + .setUnique(4) + .setAvgNumBytes(6) + .setMaxNumBytes(10) + .setMinNumBytes(0)) + .setType(Type.BYTES) + .setPath(Path.newBuilder().addStep("feature").build()) + .build(); + FeatureNameStatistics stat2 = + FeatureNameStatistics.newBuilder() + .setBytesStats( + BytesStatistics.newBuilder() + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(5) + .setNumNonMissing(5) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(1)) + .setUnique(4) + .setAvgNumBytes(4) + .setMaxNumBytes(20) + .setMinNumBytes(1)) + .setType(Type.BYTES) + .setPath(Path.newBuilder().addStep("feature").build()) + .build(); + + FeatureNameStatistics expected = + FeatureNameStatistics.newBuilder() + .setBytesStats( + BytesStatistics.newBuilder() + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(10) + .setNumNonMissing(10) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(1)) + .setAvgNumBytes(5) + .setMaxNumBytes(20) + .setMinNumBytes(0)) + .setType(Type.BYTES) + .setPath(Path.newBuilder().addStep("feature").build()) + .build(); + assertThat( + statsService.mergeStatistics(Arrays.asList(Arrays.asList(stat1, stat2))), + equalTo(Arrays.asList(expected))); + } + + @Test + public void shouldAggregateStructStatistics() { + FeatureNameStatistics stat1 = + FeatureNameStatistics.newBuilder() + .setStructStats( + StructStatistics.newBuilder() + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(5) + .setNumNonMissing(5) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(0))) + .setType(Type.STRUCT) + .setPath(Path.newBuilder().addStep("feature").build()) + .build(); + FeatureNameStatistics stat2 = + FeatureNameStatistics.newBuilder() + .setStructStats( + StructStatistics.newBuilder() + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(5) + .setNumNonMissing(5) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(1))) + .setType(Type.STRUCT) + .setPath(Path.newBuilder().addStep("feature").build()) + .build(); + + FeatureNameStatistics expected = + FeatureNameStatistics.newBuilder() + .setStructStats( + StructStatistics.newBuilder() + .setCommonStats( + CommonStatistics.newBuilder() + .setTotNumValues(10) + .setNumNonMissing(10) + .setAvgNumValues(1) + .setMaxNumValues(1) + .setMinNumValues(1) + .setNumMissing(1))) + .setType(Type.STRUCT) + .setPath(Path.newBuilder().addStep("feature").build()) + .build(); + assertThat( + statsService.mergeStatistics(Arrays.asList(Arrays.asList(stat1, stat2))), + equalTo(Arrays.asList(expected))); + } +} diff --git a/ingestion/pom.xml b/ingestion/pom.xml index 9386d066bfd..9ba371f719c 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -63,30 +63,6 @@ reference.conf - - - - org.springframework - org.springframework.vendor - - - io.grpc - io.grpc.vendor - - - io.opencensus - io.opencensus.vendor - - - feast.core - feast.core.vendor - - - com.google.cloud.bigquery - com.google.cloud.bigquery.vendor - - diff --git a/protos/feast/core/CoreService.proto b/protos/feast/core/CoreService.proto index b7760d0b9aa..08a2af4ce68 100644 --- a/protos/feast/core/CoreService.proto +++ b/protos/feast/core/CoreService.proto @@ -22,6 +22,8 @@ option go_package = "github.com/gojek/feast/sdk/go/protos/feast/core"; option java_outer_classname = "CoreServiceProto"; option java_package = "feast.core"; +import "google/protobuf/timestamp.proto"; +import "tensorflow_metadata/proto/v0/statistics.proto"; import "feast/core/FeatureSet.proto"; import "feast/core/Store.proto"; import "feast/core/FeatureSetReference.proto"; @@ -42,6 +44,11 @@ service CoreService { // sets currently stored in the registry. rpc ListFeatureSets (ListFeatureSetsRequest) returns (ListFeatureSetsResponse); + // Get feature statistics computed over the data in the batch stores. + // + // Returns a dataset containing TFDV statistics mapped to each valid historical store. + rpc GetFeatureStatistics (GetFeatureStatisticsRequest) returns (GetFeatureStatisticsResponse); + // Retrieve store details given a filter. // // Returns all stores matching that filter. If none are found, an empty list will be returned. @@ -274,3 +281,42 @@ message StopIngestionJobRequest { // Request from stopping an ingestion job message StopIngestionJobResponse {} + +message GetFeatureStatisticsRequest { + // Feature set to retrieve the statistics for. A fully qualified feature set + // id in the format of project/feature_set:version must be provided. + string feature_set_id = 1; + + // Optional filter which filters returned statistics by selected features. These + // features must be present in the data that is being processed. + repeated string feature_ids = 2; + + // Optional filter to select store over which the statistics will retrieved. + // Only historical stores are allowed. + string store = 3; + + // Optional start and end dates over which to filter statistical data + // Start date is inclusive, but end date is not. + // Only dates are supported, not times. + // Cannot be used with dataset_ids. + // If this period spans multiple days, unaggregatable statistics will be dropped. + google.protobuf.Timestamp start_date = 4; + google.protobuf.Timestamp end_date = 5; + + // Optional list of dataset Ids by which to filter data before + // retrieving statistics. + // Cannot be used with the date ranges + // If multiple dataset ids are provided, unaggregatable statistics will be dropped. + repeated string dataset_ids = 6; + + // Setting this flag to true will force a recalculation of statistics and overwrite results currently in the + // cache, if any. + bool force_refresh = 7; +} + +message GetFeatureStatisticsResponse { + // Contains statistics for the requested data. + // Due to the limitations of TFDV and Facets, only a single dataset can be returned in, + // despite the message being of list type. + tensorflow.metadata.v0.DatasetFeatureStatisticsList dataset_feature_statistics_list = 1; +} diff --git a/protos/feast/core/Store.proto b/protos/feast/core/Store.proto index de9af0a99fe..4ffdbcdf58f 100644 --- a/protos/feast/core/Store.proto +++ b/protos/feast/core/Store.proto @@ -69,6 +69,7 @@ message Store { // ====================|==================|================================ // - event_timestamp | TIMESTAMP | event time of the FeatureRow // - created_timestamp | TIMESTAMP | processing time of the ingestion of the FeatureRow + // - dataset_id | STRING | identifier of the batch dataset a row belongs to // - job_id | STRING | identifier for the job that writes the FeatureRow to the corresponding BigQuery table // // BigQuery table created will be partitioned by the field "event_timestamp" diff --git a/protos/feast/types/FeatureRow.proto b/protos/feast/types/FeatureRow.proto index c170cd5d502..c3614e00274 100644 --- a/protos/feast/types/FeatureRow.proto +++ b/protos/feast/types/FeatureRow.proto @@ -39,4 +39,8 @@ message FeatureRow { // /:. This value will be used by the feast ingestion job to filter // rows, and write the values to the correct tables. string feature_set = 6; + + // Identifier tying this feature row to a specific ingestion dataset. For + // batch loads, this dataset id can be attributed to a single ingestion job. + string dataset_id = 7; } diff --git a/protos/tensorflow_metadata/proto/v0/statistics.proto b/protos/tensorflow_metadata/proto/v0/statistics.proto new file mode 100644 index 00000000000..6258d06dfcf --- /dev/null +++ b/protos/tensorflow_metadata/proto/v0/statistics.proto @@ -0,0 +1,426 @@ +// Copyright 2017 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +// Definitions for aggregated feature statistics for datasets. +// TODO(b/80075690): make a Javascript build rule for this. +// TODO(b/80075691): migrate Facets to use this. +syntax = "proto3"; +option cc_enable_arenas = true; + +package tensorflow.metadata.v0; + +option java_package = "org.tensorflow.metadata.v0"; +option java_multiple_files = true; + +import "tensorflow_metadata/proto/v0/path.proto"; + +// Copied from Facets feature_statistics.proto +// Must be kept binary-compatible with the original, until all usages +// are updated to use this version, or we write a proto-to-proto converter. + +// A list of features statistics for different datasets. If you wish to compare +// different datasets using this list, then the DatasetFeatureStatistics +// entries should all contain the same list of features. +message DatasetFeatureStatisticsList { + repeated DatasetFeatureStatistics datasets = 1; +} + +// The feature statistics for a single dataset. +message DatasetFeatureStatistics { + // The name of the dataset. + string name = 1; + // The number of examples in the dataset. + uint64 num_examples = 2; + + // Only valid if the weight feature was specified. + // Treats a missing weighted feature as zero. + double weighted_num_examples = 4; + // The feature statistics for the dataset. + repeated FeatureNameStatistics features = 3; + + // Cross feature statistics for the dataset. + repeated CrossFeatureStatistics cross_features = 5; +} + +message CrossFeatureStatistics { + // The path of feature x. + Path path_x = 1; + // The path of feature y. + Path path_y = 2; + + // Number of occurrences of this feature cross in the data. If any of + // the features in the cross is missing, the example is ignored. + uint64 count = 3; + + oneof cross_stats { + NumericCrossStatistics num_cross_stats = 4; + CategoricalCrossStatistics categorical_cross_stats = 5; + } +} + +message NumericCrossStatistics { + // Pearson product-moment correlation coefficient. + float correlation = 1; + // Standard covariance. E[(X-E[X])*(Y-E[Y])] + float covariance = 2; +} + +message CategoricalCrossStatistics { + LiftStatistics lift = 1; +} + +message LiftStatistics { + // Lift information for each value of path_y. Lift is defined for each pair of + // values (x,y) as P(path_y=y|path_x=x)/P(path_y=y). + repeated LiftSeries lift_series = 1; + // Weighted lift information for each value of path_y. Weighted lift is + // defined for each pair of values (x,y) as P(path_y=y|path_x=x)/P(path_y=y) + // where probabilities are computed over weighted example space. + repeated LiftSeries weighted_lift_series = 2; +} + +// Container for lift information for a specific y-value. +message LiftSeries { + // A bucket for referring to binned numeric features. + message Bucket { + // The low value of the bucket, inclusive. + double low_value = 1; + // The high value of the bucket, exclusive (unless the high_value is + // positive infinity). + double high_value = 2; + } + + // The particular value of path_y corresponding to this LiftSeries. Each + // element in lift_values corresponds to the lift a different x_value and + // this specific y_value. + oneof y_value { + int32 y_int = 1; + string y_string = 2; + Bucket y_bucket = 3; + } + + // The number of examples in which y_value appears. + oneof y_count_value { + uint64 y_count = 4; + double weighted_y_count = 5; + } + + // A container for lift information about a specific value of path_x. + message LiftValue { + oneof x_value { + int32 x_int = 1; + string x_string = 2; + } + // P(path_y=y|path_x=x) / P(path_y=y) for x_value and the enclosing y_value. + // In terms of concrete fields, this number represents: + // (x_and_y_count / x_count) / (y_count / num_examples) + double lift = 3; + // The number of examples in which x_value appears. + oneof x_count_value { + uint64 x_count = 4; + double weighted_x_count = 5; + } + // The number of examples in which x_value appears and y_value appears. + oneof x_and_y_count_value { + uint64 x_and_y_count = 6; + double weighted_x_and_y_count = 7; + } + } + + // The lifts for a each path_x value and this y_value. + repeated LiftValue lift_values = 6; +} + +// The complete set of statistics for a given feature name for a dataset. +message FeatureNameStatistics { + // The types supported by the feature statistics. When aggregating + // tf.Examples, if the bytelist contains a string, it is recommended to encode + // it here as STRING instead of BYTES in order to calculate string-specific + // statistical measures. + enum Type { + INT = 0; + FLOAT = 1; + STRING = 2; + BYTES = 3; + STRUCT = 4; + } + + // One can identify a field either by the name (for simple fields), or by + // a path (for structured fields). Note that: + // name: "foo" + // is equivalent to: + // path: {step:"foo"} + // Note: this oneof must be consistently either name or path across all + // FeatureNameStatistics in one DatasetFeatureStatistics. + oneof field_id { + // The feature name + string name = 1; + + // The path of the feature. + Path path = 8; + } + + // The data type of the feature + Type type = 2; + + // The statistics of the values of the feature. + oneof stats { + NumericStatistics num_stats = 3; + StringStatistics string_stats = 4; + BytesStatistics bytes_stats = 5; + StructStatistics struct_stats = 7; + } + + // Any custom statistics can be stored in this list. + repeated CustomStatistic custom_stats = 6; +} + +// Common weighted statistics for all feature types. Statistics counting number +// of values (i.e., avg_num_values and tot_num_values) include NaNs. +// If the weighted column is missing, then this counts as a weight of 1 +// for that example. +message WeightedCommonStatistics { + // Weighted number of examples not missing. + double num_non_missing = 1; + // Weighted number of examples missing. + // Note that if the weighted column is zero, this does not count + // as missing. + double num_missing = 2; + // average number of values, weighted by the number of examples. + double avg_num_values = 3; + // tot_num_values = avg_num_values * num_non_missing. + // This is calculated directly, so should have less numerical error. + double tot_num_values = 4; +} + +// Stores the name and value of any custom statistic. The value can be a string, +// double, or histogram. +message CustomStatistic { + string name = 1; + oneof val { + double num = 2; + string str = 3; + Histogram histogram = 4; + RankHistogram rank_histogram = 5; + } +} + +// Statistics for a numeric feature in a dataset. +message NumericStatistics { + CommonStatistics common_stats = 1; + // The mean of the values + double mean = 2; + // The standard deviation of the values + double std_dev = 3; + // The number of values that equal 0 + uint64 num_zeros = 4; + // The minimum value + double min = 5; + // The median value + double median = 6; + // The maximum value + double max = 7; + // The histogram(s) of the feature values. + repeated Histogram histograms = 8; + + // Weighted statistics for the feature, if the values have weights. + WeightedNumericStatistics weighted_numeric_stats = 9; +} + +// Statistics for a string feature in a dataset. +message StringStatistics { + CommonStatistics common_stats = 1; + // The number of unique values + uint64 unique = 2; + + message FreqAndValue { + string value = 2; + + // The number of times the value occurs. Stored as a double to be able to + // handle weighted features. + double frequency = 3; + + // Deleted fields. + reserved 1; + } + // A sorted list of the most-frequent values and their frequencies, with + // the most-frequent being first. + repeated FreqAndValue top_values = 3; + + // The average length of the values + float avg_length = 4; + + // The rank histogram for the values of the feature. + // The rank is used to measure of how commonly the value is found in the + // dataset. The most common value would have a rank of 1, with the second-most + // common value having a rank of 2, and so on. + RankHistogram rank_histogram = 5; + + // Weighted statistics for the feature, if the values have weights. + WeightedStringStatistics weighted_string_stats = 6; + + // A vocabulary file, used for vocabularies too large to store in the proto + // itself. Note that the file may be relative to some context-dependent + // directory. E.g. in TFX the feature statistics will live in a PPP and + // vocabulary file names will be relative to this PPP. + string vocabulary_file = 7; +} + +// Statistics for a weighted numeric feature in a dataset. +message WeightedNumericStatistics { + // The weighted mean of the values + double mean = 1; + // The weighted standard deviation of the values + double std_dev = 2; + // The weighted median of the values + double median = 3; + + // The histogram(s) of the weighted feature values. + repeated Histogram histograms = 4; +} + +// Statistics for a weighted string feature in a dataset. +message WeightedStringStatistics { + // A sorted list of the most-frequent values and their weighted frequencies, + // with the most-frequent being first. + repeated StringStatistics.FreqAndValue top_values = 1; + + // The rank histogram for the weighted values of the feature. + RankHistogram rank_histogram = 2; +} + +// Statistics for a bytes feature in a dataset. +message BytesStatistics { + CommonStatistics common_stats = 1; + // The number of unique values + uint64 unique = 2; + + // The average number of bytes in a value + float avg_num_bytes = 3; + // The minimum number of bytes in a value + float min_num_bytes = 4; + // The maximum number of bytes in a value + float max_num_bytes = 5; +} + +message StructStatistics { + CommonStatistics common_stats = 1; +} + +// Common statistics for all feature types. Statistics counting number of values +// (i.e., min_num_values, max_num_values, avg_num_values, and tot_num_values) +// include NaNs. +message CommonStatistics { + // The number of examples with at least one value for this feature. + uint64 num_non_missing = 1; + // The number of examples with no values for this feature. + uint64 num_missing = 2; + // The minimum number of values in a single example for this feature. + uint64 min_num_values = 3; + // The maximum number of values in a single example for this feature. + uint64 max_num_values = 4; + // The average number of values in a single example for this feature. + float avg_num_values = 5; + // tot_num_values = avg_num_values * num_non_missing. + // This is calculated directly, so should have less numerical error. + uint64 tot_num_values = 8; + // The quantiles histogram for the number of values in this feature. + Histogram num_values_histogram = 6; + WeightedCommonStatistics weighted_common_stats = 7; + // The histogram for the number of features in the feature list (only set if + // this feature is a non-context feature from a tf.SequenceExample). + // This is different from num_values_histogram, as num_values_histogram tracks + // the count of all values for a feature in an example, whereas this tracks + // the length of the feature list for this feature in an example (where each + // feature list can contain multiple values). + Histogram feature_list_length_histogram = 9; +} + +// The data used to create a histogram of a numeric feature for a dataset. +message Histogram { + // Each bucket defines its low and high values along with its count. The + // low and high values must be a real number or positive or negative + // infinity. They cannot be NaN or undefined. Counts of those special values + // can be found in the numNaN and numUndefined fields. + message Bucket { + // The low value of the bucket, inclusive. + double low_value = 1; + // The high value of the bucket, exclusive (unless the highValue is + // positive infinity). + double high_value = 2; + + // The number of items in the bucket. Stored as a double to be able to + // handle weighted histograms. + double sample_count = 4; + + // Deleted fields. + reserved 3; + } + + // The number of NaN values in the dataset. + uint64 num_nan = 1; + // The number of undefined values in the dataset. + uint64 num_undefined = 2; + + // A list of buckets in the histogram, sorted from lowest bucket to highest + // bucket. + repeated Bucket buckets = 3; + + // The type of the histogram. A standard histogram has equal-width buckets. + // The quantiles type is used for when the histogram message is used to store + // quantile information (by using equal-count buckets with variable widths). + enum HistogramType { + STANDARD = 0; + QUANTILES = 1; + } + + // The type of the histogram. + HistogramType type = 4; + + // An optional descriptive name of the histogram, to be used for labeling. + string name = 5; +} + +// The data used to create a rank histogram of a non-numeric feature of a +// dataset. The rank of a value in a feature can be used as a measure of how +// commonly the value is found in the entire dataset. With bucket sizes of one, +// this becomes a distribution function of all feature values. +message RankHistogram { + // Each bucket defines its start and end ranks along with its count. + message Bucket { + // The low rank of the bucket, inclusive. + uint64 low_rank = 1; + // The high rank of the bucket, exclusive. + uint64 high_rank = 2; + + // The label for the bucket. Can be used to list or summarize the values in + // this rank bucket. + string label = 4; + + // The number of items in the bucket. Stored as a double to be able to + // handle weighted histograms. + double sample_count = 5; + + // Deleted fields. + reserved 3; + } + + // A list of buckets in the histogram, sorted from lowest-ranked bucket to + // highest-ranked bucket. + repeated Bucket buckets = 1; + + // An optional descriptive name of the histogram, to be used for labeling. + string name = 2; +} \ No newline at end of file diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 0a38236a510..1c4de9edd03 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -17,7 +17,9 @@ import os import shutil import tempfile +import datetime import time +import uuid from collections import OrderedDict from math import ceil from typing import Dict, List, Optional, Tuple, Union @@ -26,6 +28,8 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq +from tensorflow_metadata.proto.v0 import statistics_pb2 +from google.protobuf.timestamp_pb2 import Timestamp from feast.config import Config from feast.constants import ( @@ -46,6 +50,8 @@ GetFeastCoreVersionRequest, GetFeatureSetRequest, GetFeatureSetResponse, + GetFeatureStatisticsRequest, + GetFeatureStatisticsResponse, ListFeatureSetsRequest, ListFeatureSetsResponse, ListIngestionJobsRequest, @@ -659,6 +665,7 @@ def get_online_features( return response + def list_ingest_jobs( self, job_id: str = None, @@ -735,7 +742,7 @@ def ingest( max_workers: int = max(CPU_COUNT - 1, 1), disable_progress_bar: bool = False, timeout: int = KAFKA_CHUNK_PRODUCTION_TIMEOUT, - ) -> None: + ) -> str: """ Loads feature data into Feast for a specific feature set. @@ -771,8 +778,8 @@ def ingest( Timeout in seconds to wait for completion. Returns: - None: - None + str: + dataset id of the ingested dataset """ if isinstance(feature_set, FeatureSet): @@ -825,6 +832,7 @@ def ingest( # Loop optimization declarations produce = producer.produce flush = producer.flush + dataset_id = _generate_dataset_id(feature_set) # Transform and push data to Kafka if feature_set.source.source_type == "Kafka": @@ -832,6 +840,7 @@ def ingest( file=dest_path, row_groups=list(range(pq_file.num_row_groups)), fs=feature_set, + dataset_id=dataset_id, max_workers=max_workers, ): @@ -859,6 +868,76 @@ def ingest( print("Removing temporary file(s)...") shutil.rmtree(dir_path) + return dataset_id + + + def get_statistics( + self, + feature_set_id: str, + store: str, + features: List[str] = [], + dataset_ids: Optional[List[str]] = None, + start_date: Optional[datetime.datetime] = None, + end_date: Optional[datetime.datetime] = None, + force_refresh: bool = False, + default_project: Optional[str] = None, + ) -> statistics_pb2.DatasetFeatureStatisticsList: + """ + Retrieves the feature featureStatistics computed over the data in the batch + stores. + + Args: + feature_set_id: Fully qualified feature set id in the format + project/feature_set:version to retrieve batch featureStatistics for. + store: Name of the store to retrieve feature featureStatistics over. This + store must be a historical store. + features: Optional list of feature names to filter from the results. + dataset_ids: Optional list of dataset Ids by which to filter data + before retrieving featureStatistics. Cannot be used with start_date + and end_date. + If multiple dataset ids are provided, unaggregatable featureStatistics + will be dropped. + start_date: Optional start date over which to filter statistical data. + Data from this date will be included. + Cannot be used with dataset_ids. If the provided period spans + multiple days, unaggregatable featureStatistics will be dropped. + end_date: Optional end date over which to filter statistical data. + Data from this data will not be included. + Cannot be used with dataset_ids. If the provided period spans + multiple days, unaggregatable featureStatistics will be dropped. + force_refresh: Setting this flag to true will force a recalculation + of featureStatistics and overwrite results currently in the cache, if any. + default_project: This project will be used if the project name is + not provided in the feature reference + + Returns: + Returns a tensorflow DatasetFeatureStatisticsList containing TFDV featureStatistics. + """ + + self._connect_core() + if dataset_ids is not None and (start_date is not None or end_date is not None): + raise ValueError( + "Only one of dataset_id or [start_date, end_date] can be provided." + ) + + request = GetFeatureStatisticsRequest( + feature_set_id=feature_set_id, feature_ids=features, store=store, force_refresh=force_refresh + ) + if dataset_ids is not None: + request.dataset_ids.extend(dataset_ids) + else: + if start_date is not None: + request.start_date.CopyFrom( + Timestamp(seconds=int(start_date.strftime("%s"))) + ) + if end_date is not None: + request.end_date.CopyFrom( + Timestamp(seconds=int(end_date.strftime("%s"))) + ) + + return self._core_service_stub.GetFeatureStatistics( + request + ).dataset_feature_statistics_list return None @@ -916,6 +995,20 @@ def _build_feature_references( return features +def _generate_dataset_id(feature_set: FeatureSet) -> str: + """ + Generates a UUID from the feature set name, version, and the current time. + + Args: + feature_set: Feature set of the dataset to be ingested. + + Returns: + UUID unique to current time and the feature set provided. + """ + uuid_str = f"{feature_set.name}_{feature_set.version}_{int(time.time())}" + return str(uuid.uuid3(uuid.NAMESPACE_DNS, uuid_str)) + + def _read_table_from_source( source: Union[pd.DataFrame, str], chunk_size: int, max_workers: int ) -> Tuple[str, str]: diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py index b4490f025c5..dc8ad48a746 100644 --- a/sdk/python/feast/loaders/ingest.py +++ b/sdk/python/feast/loaders/ingest.py @@ -25,7 +25,9 @@ KAFKA_CHUNK_PRODUCTION_TIMEOUT = 120 # type: int -def _encode_pa_tables(file: str, fs: FeatureSet, row_group_idx: int) -> List[bytes]: +def _encode_pa_tables( + file: str, fs: FeatureSet, dataset_id: str, row_group_idx: int +) -> List[bytes]: """ Helper function to encode a PyArrow table(s) read from parquet file(s) into FeatureRows. @@ -44,6 +46,9 @@ def _encode_pa_tables(file: str, fs: FeatureSet, row_group_idx: int) -> List[byt fs (feast.feature_set.FeatureSet): FeatureSet describing parquet files. + dataset_id (str): + UUID unique to this dataset. + row_group_idx(int): Row group index to read and encode into byte like FeatureRow protobuf objects. @@ -78,7 +83,9 @@ def _encode_pa_tables(file: str, fs: FeatureSet, row_group_idx: int) -> List[byt # Iterate through the rows for row_idx in range(table.num_rows): feature_row = FeatureRow( - event_timestamp=datetime_col[row_idx], feature_set=feature_set + event_timestamp=datetime_col[row_idx], + feature_set=feature_set, + dataset_id=dataset_id, ) # Loop optimization declaration ext = feature_row.fields.extend @@ -94,7 +101,7 @@ def _encode_pa_tables(file: str, fs: FeatureSet, row_group_idx: int) -> List[byt def get_feature_row_chunks( - file: str, row_groups: List[int], fs: FeatureSet, max_workers: int + file: str, row_groups: List[int], fs: FeatureSet, dataset_id: str, max_workers: int ) -> Iterable[List[bytes]]: """ Iterator function to encode a PyArrow table read from a parquet file to @@ -112,6 +119,9 @@ def get_feature_row_chunks( fs (feast.feature_set.FeatureSet): FeatureSet describing parquet files. + dataset_id (str): + UUID unique to this dataset. + max_workers (int): Maximum number of workers to spawn. @@ -121,7 +131,7 @@ def get_feature_row_chunks( """ pool = Pool(max_workers) - func = partial(_encode_pa_tables, file, fs) + func = partial(_encode_pa_tables, file, fs, dataset_id) for chunk in pool.imap(func, row_groups): yield chunk return diff --git a/storage/api/src/main/java/feast/storage/api/statistics/FeatureSetStatistics.java b/storage/api/src/main/java/feast/storage/api/statistics/FeatureSetStatistics.java new file mode 100644 index 00000000000..a328df46b97 --- /dev/null +++ b/storage/api/src/main/java/feast/storage/api/statistics/FeatureSetStatistics.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.api.statistics; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import org.tensorflow.metadata.v0.FeatureNameStatistics; + +/** Feature statistics for a feature set over a bounded set of data. */ +@AutoValue +public abstract class FeatureSetStatistics { + + public abstract long getNumExamples(); + + public abstract ImmutableList getFeatureNameStatistics(); + + public static Builder newBuilder() { + return new AutoValue_FeatureSetStatistics.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setNumExamples(long numExamples); + + protected abstract ImmutableList.Builder featureNameStatisticsBuilder(); + + public Builder addFeatureNameStatistics(FeatureNameStatistics featureNameStatistics) { + featureNameStatisticsBuilder().add(featureNameStatistics); + return this; + } + + public abstract FeatureSetStatistics build(); + } +} diff --git a/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java b/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java new file mode 100644 index 00000000000..c527b3c80cd --- /dev/null +++ b/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.api.statistics; + +import com.google.protobuf.Timestamp; +import feast.core.FeatureSetProto.FeatureSetSpec; +import java.util.List; + +public interface StatisticsRetriever { + + /** + * Get feature set statistics for a single feature set, for a single dataset id. + * + * @param featureSetSpec feature set spec of the requested feature set + * @param features subset of features to retrieve. + * @param dataset dataset id to filter the data by + * @return {@link FeatureSetStatistics} containing statistics for the requested features. + */ + FeatureSetStatistics getFeatureStatistics( + FeatureSetSpec featureSetSpec, List features, String dataset); + + /** + * Get feature set statistics for a single feature set, for a single day. + * + * @param featureSetSpec feature set spec of the requested feature set + * @param features subset of features to retrieve. + * @param date date to filter the data by + * @return {@link FeatureSetStatistics} containing statistics for the requested features. + */ + FeatureSetStatistics getFeatureStatistics( + FeatureSetSpec featureSetSpec, List features, Timestamp date); +} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java index cba997b6ab0..0c7ef4c5aed 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java @@ -38,6 +38,8 @@ public class QueryTemplater { private static final PebbleEngine engine = new PebbleEngine.Builder().build(); private static final String FEATURESET_TEMPLATE_NAME = "templates/single_featureset_pit_join.sql"; private static final String JOIN_TEMPLATE_NAME = "templates/join_featuresets.sql"; + private static final String BASIC_STATS_TEMPLATE_NAME = "templates/basic_stats.sql"; + private static final String HIST_STATS_TEMPLATE_NAME = "templates/hist_stats.sql"; /** * Get the query for retrieving the earliest and latest timestamps in the entity dataset. diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetriever.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetriever.java new file mode 100644 index 00000000000..85d0f2dc13d --- /dev/null +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetriever.java @@ -0,0 +1,152 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigquery.stats; + +import static feast.storage.connectors.bigquery.stats.StatsUtil.toFeatureNameStatistics; + +import com.google.auto.value.AutoValue; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.FieldValueList; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.TableResult; +import com.google.common.collect.Streams; +import com.google.protobuf.Timestamp; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.core.FeatureSetProto.FeatureSpec; +import feast.storage.api.statistics.FeatureSetStatistics; +import feast.storage.api.statistics.StatisticsRetriever; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.tensorflow.metadata.v0.FeatureNameStatistics; + +@AutoValue +public abstract class BigQueryStatisticsRetriever implements StatisticsRetriever { + + public abstract String projectId(); + + public abstract String datasetId(); + + public abstract BigQuery bigquery(); + + public static Builder newBuilder() { + return new AutoValue_BigQueryStatisticsRetriever.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setProjectId(String projectId); + + public abstract Builder setDatasetId(String datasetId); + + public abstract Builder setBigquery(BigQuery bigquery); + + public abstract BigQueryStatisticsRetriever build(); + } + + @Override + public FeatureSetStatistics getFeatureStatistics( + FeatureSetSpec featureSetSpec, List features, String dataset) { + FeatureSetStatisticsQueryInfo featureSetStatisticsQueryInfo = + new FeatureSetStatisticsQueryInfo( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + dataset); + return getFeatureSetStatistics(featureSetSpec, features, featureSetStatisticsQueryInfo); + } + + @Override + public FeatureSetStatistics getFeatureStatistics( + FeatureSetSpec featureSetSpec, List features, Timestamp date) { + FeatureSetStatisticsQueryInfo featureSetStatisticsQueryInfo = + new FeatureSetStatisticsQueryInfo( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + date); + return getFeatureSetStatistics(featureSetSpec, features, featureSetStatisticsQueryInfo); + } + + private FeatureSetStatistics getFeatureSetStatistics( + FeatureSetSpec featureSetSpec, + List features, + FeatureSetStatisticsQueryInfo featureSetStatisticsQueryInfo) { + List featuresList = featureSetSpec.getFeaturesList(); + + FeatureSetSpec.Builder featureSetSpecBuilder = featureSetSpec.toBuilder().clearFeatures(); + for (FeatureSpec featureSpec : featuresList) { + if (features.contains(featureSpec.getName())) { + featureSetStatisticsQueryInfo.addFeature(featureSpec); + featureSetSpecBuilder = featureSetSpecBuilder.addFeatures(featureSpec); + } + } + featureSetSpec = featureSetSpecBuilder.build(); + + try { + String getFeatureSetStatsQuery = + StatsQueryTemplater.createGetFeatureSetStatsQuery( + featureSetStatisticsQueryInfo, projectId(), datasetId()); + String getFeatureSetHistQuery = + StatsQueryTemplater.createGetFeatureSetHistQuery( + featureSetStatisticsQueryInfo, projectId(), datasetId()); + QueryJobConfiguration queryJobConfiguration = + QueryJobConfiguration.newBuilder(getFeatureSetStatsQuery).build(); + TableResult basicStats = bigquery().query(queryJobConfiguration); + queryJobConfiguration = QueryJobConfiguration.newBuilder(getFeatureSetHistQuery).build(); + TableResult hist = bigquery().query(queryJobConfiguration); + + Map basicStatsValues = + Streams.stream(basicStats.getValues()) + .collect( + Collectors.toMap( + fieldValueList -> fieldValueList.get(0).getStringValue(), + fieldValueList -> fieldValueList)); + Map histValues = + Streams.stream(hist.getValues()) + .collect( + Collectors.toMap( + fieldValueList -> fieldValueList.get(0).getStringValue(), + fieldValueList -> fieldValueList)); + + int totalCountIndex = basicStats.getSchema().getFields().getIndex("total_count"); + FeatureSetStatistics.Builder featureSetStatisticsBuilder = + FeatureSetStatistics.newBuilder() + .setNumExamples( + basicStatsValues.get(features.get(0)).get(totalCountIndex).getLongValue()); + + for (FeatureSpec featureSpec : featureSetSpec.getFeaturesList()) { + FeatureNameStatistics featureNameStatistics = + toFeatureNameStatistics( + featureSpec, + basicStats.getSchema(), + basicStatsValues.get(featureSpec.getName()), + hist.getSchema(), + histValues.get(featureSpec.getName())); + featureSetStatisticsBuilder.addFeatureNameStatistics(featureNameStatistics); + } + return featureSetStatisticsBuilder.build(); + } catch (IOException | InterruptedException e) { + throw new RuntimeException( + String.format( + "Unable to retrieve statistics from BigQuery for Feature set %s, features %s", + featureSetSpec.getName(), features), + e); + } + } +} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureSetStatisticsQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureSetStatisticsQueryInfo.java new file mode 100644 index 00000000000..5809db74559 --- /dev/null +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureSetStatisticsQueryInfo.java @@ -0,0 +1,92 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigquery.stats; + +import com.google.protobuf.Timestamp; +import feast.core.FeatureSetProto.FeatureSpec; +import java.util.ArrayList; +import java.util.List; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +public class FeatureSetStatisticsQueryInfo { + private final String project; + private final String name; + private final int version; + private String datasetId = ""; + private String date = ""; + private final List features; + + public FeatureSetStatisticsQueryInfo( + String project, + String name, + int version, + String datasetId, + String date, + List features) { + this.project = project; + this.name = name; + this.version = version; + this.datasetId = datasetId; + this.date = date; + this.features = features; + } + + public FeatureSetStatisticsQueryInfo(String project, String name, int version, String datasetId) { + this.project = project; + this.name = name; + this.version = version; + this.features = new ArrayList<>(); + this.datasetId = datasetId; + } + + public FeatureSetStatisticsQueryInfo(String project, String name, int version, Timestamp date) { + this.project = project; + this.name = name; + this.version = version; + this.features = new ArrayList<>(); + DateTime dateTime = new DateTime(date.getSeconds() * 1000, DateTimeZone.UTC); + DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd"); + this.date = fmt.print(dateTime); + } + + public void addFeature(FeatureSpec featureSpec) { + this.features.add(FeatureStatisticsQueryInfo.fromProto(featureSpec)); + } + + public String getProject() { + return project; + } + + public String getName() { + return name; + } + + public int getVersion() { + return version; + } + + public String getDatasetId() { + return datasetId; + } + + public List getFeatures() { + return features; + } +} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureStatisticsQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureStatisticsQueryInfo.java new file mode 100644 index 00000000000..73bbdfbac2c --- /dev/null +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureStatisticsQueryInfo.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigquery.stats; + +import feast.core.FeatureSetProto.FeatureSpec; +import feast.types.ValueProto.ValueType.Enum; + +public class FeatureStatisticsQueryInfo { + private final String name; + private final String type; + + private FeatureStatisticsQueryInfo(String name, String type) { + this.name = name; + this.type = type; + } + + public static FeatureStatisticsQueryInfo fromProto(FeatureSpec featureSpec) { + Enum valueType = featureSpec.getValueType(); + switch (valueType) { + case FLOAT: + case DOUBLE: + case INT32: + case INT64: + case BOOL: + return new FeatureStatisticsQueryInfo(featureSpec.getName(), "NUMERIC"); + case STRING: + return new FeatureStatisticsQueryInfo(featureSpec.getName(), "CATEGORICAL"); + case BYTES: + return new FeatureStatisticsQueryInfo(featureSpec.getName(), "BYTES"); + case BYTES_LIST: + case BOOL_LIST: + case FLOAT_LIST: + case INT32_LIST: + case INT64_LIST: + case DOUBLE_LIST: + case STRING_LIST: + return new FeatureStatisticsQueryInfo(featureSpec.getName(), "LIST"); + default: + throw new IllegalArgumentException("Invalid feature type provided"); + } + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } +} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsQueryTemplater.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsQueryTemplater.java new file mode 100644 index 00000000000..1552a1f4f52 --- /dev/null +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsQueryTemplater.java @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigquery.stats; + +import com.mitchellbosecke.pebble.PebbleEngine; +import com.mitchellbosecke.pebble.template.PebbleTemplate; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.util.HashMap; +import java.util.Map; + +public class StatsQueryTemplater { + + private static final PebbleEngine engine = new PebbleEngine.Builder().build(); + private static final String BASIC_STATS_TEMPLATE_NAME = "templates/basic_stats.sql"; + private static final String HIST_STATS_TEMPLATE_NAME = "templates/hist_stats.sql"; + + /** + * Generate the query for getting basic statistics about a given feature set + * + * @param featureSetInfo Information about the feature set necessary for the query templating + * @param projectId google project ID + * @param datasetId feast bigquery dataset ID + * @return point in time correctness join BQ SQL query + */ + public static String createGetFeatureSetStatsQuery( + FeatureSetStatisticsQueryInfo featureSetInfo, String projectId, String datasetId) + throws IOException { + + PebbleTemplate template = engine.getTemplate(BASIC_STATS_TEMPLATE_NAME); + Map context = new HashMap<>(); + context.put("featureSet", featureSetInfo); + context.put("projectId", projectId); + context.put("datasetId", datasetId); + + Writer writer = new StringWriter(); + template.evaluate(writer, context); + return writer.toString(); + } + + /** + * Generate the query for getting histograms for features in a given feature set + * + * @param featureSetInfo Information about the feature set necessary for the query templating + * @param projectId google project ID + * @param datasetId feast bigquery dataset ID + * @return point in time correctness join BQ SQL query + */ + public static String createGetFeatureSetHistQuery( + FeatureSetStatisticsQueryInfo featureSetInfo, String projectId, String datasetId) + throws IOException { + + PebbleTemplate template = engine.getTemplate(HIST_STATS_TEMPLATE_NAME); + Map context = new HashMap<>(); + context.put("featureSet", featureSetInfo); + context.put("projectId", projectId); + context.put("datasetId", datasetId); + + Writer writer = new StringWriter(); + template.evaluate(writer, context); + return writer.toString(); + } +} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsUtil.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsUtil.java new file mode 100644 index 00000000000..40dca0d7df6 --- /dev/null +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsUtil.java @@ -0,0 +1,234 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigquery.stats; + +import com.google.cloud.bigquery.FieldList; +import com.google.cloud.bigquery.FieldValue; +import com.google.cloud.bigquery.FieldValueList; +import com.google.cloud.bigquery.Schema; +import feast.core.FeatureSetProto.FeatureSpec; +import feast.types.ValueProto.ValueType; +import feast.types.ValueProto.ValueType.Enum; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.tensorflow.metadata.v0.*; +import org.tensorflow.metadata.v0.FeatureNameStatistics.Builder; +import org.tensorflow.metadata.v0.FeatureNameStatistics.Type; +import org.tensorflow.metadata.v0.Histogram.Bucket; +import org.tensorflow.metadata.v0.Histogram.HistogramType; +import org.tensorflow.metadata.v0.StringStatistics.FreqAndValue; + +public class StatsUtil { + private static final Map TFDV_TYPE_MAP = new HashMap<>(); + + static { + TFDV_TYPE_MAP.put(Enum.INT64, Type.INT); + TFDV_TYPE_MAP.put(Enum.INT32, Type.INT); + TFDV_TYPE_MAP.put(Enum.BOOL, Type.INT); + TFDV_TYPE_MAP.put(Enum.FLOAT, Type.FLOAT); + TFDV_TYPE_MAP.put(Enum.DOUBLE, Type.FLOAT); + TFDV_TYPE_MAP.put(Enum.STRING, Type.STRING); + TFDV_TYPE_MAP.put(Enum.BYTES, Type.BYTES); + TFDV_TYPE_MAP.put(Enum.BYTES_LIST, Type.STRUCT); + TFDV_TYPE_MAP.put(Enum.STRING_LIST, Type.STRUCT); + TFDV_TYPE_MAP.put(Enum.INT32_LIST, Type.STRUCT); + TFDV_TYPE_MAP.put(Enum.INT64_LIST, Type.STRUCT); + TFDV_TYPE_MAP.put(Enum.BOOL_LIST, Type.STRUCT); + TFDV_TYPE_MAP.put(Enum.FLOAT_LIST, Type.STRUCT); + TFDV_TYPE_MAP.put(Enum.DOUBLE_LIST, Type.STRUCT); + } + + public static FeatureNameStatistics toFeatureNameStatistics( + FeatureSpec featureSpec, + Schema basicStatsSchema, + FieldValueList basicStatsValues, + Schema histSchema, + FieldValueList histValues) { + Map valuesMap = new HashMap<>(); + + FieldList basicStatsfields = basicStatsSchema.getFields(); + for (int i = 0; i < basicStatsSchema.getFields().size(); i++) { + valuesMap.put(basicStatsfields.get(i).getName(), basicStatsValues.get(i)); + } + + FieldList histFields = histSchema.getFields(); + for (int i = 0; i < histSchema.getFields().size(); i++) { + valuesMap.put(histFields.get(i).getName(), histValues.get(i)); + } + + Builder featureNameStatisticsBuilder = + FeatureNameStatistics.newBuilder() + .setPath(Path.newBuilder().addStep(valuesMap.get("feature_name").getStringValue())) + .setType(TFDV_TYPE_MAP.get(featureSpec.getValueType())); + + switch (featureSpec.getValueType()) { + case FLOAT: + case BOOL: + case DOUBLE: + case INT32: + case INT64: + NumericStatistics numStats = getNumericStatistics(valuesMap); + featureNameStatisticsBuilder.setNumStats(numStats); + break; + case STRING: + StringStatistics stringStats = getStringStatistics(valuesMap); + featureNameStatisticsBuilder.setStringStats(stringStats); + break; + case BYTES: + BytesStatistics bytesStats = getBytesStatistics(valuesMap); + featureNameStatisticsBuilder.setBytesStats(bytesStats); + break; + case BYTES_LIST: + case BOOL_LIST: + case FLOAT_LIST: + case INT32_LIST: + case INT64_LIST: + case DOUBLE_LIST: + case STRING_LIST: + StructStatistics structStats = getStructStatistics(valuesMap); + featureNameStatisticsBuilder.setStructStats(structStats); + break; + default: + throw new IllegalArgumentException( + "Invalid feature type provided. Only statistics for numeric, bytes, string, boolean and list features are supported."); + } + return featureNameStatisticsBuilder.build(); + } + + private static BytesStatistics getBytesStatistics(Map valuesMap) { + return BytesStatistics.newBuilder() + .setCommonStats( + CommonStatistics.newBuilder() + .setNumMissing(valuesMap.get("missing_count").getLongValue()) + .setNumNonMissing(valuesMap.get("feature_count").getLongValue()) + .setMinNumValues(1) + .setMaxNumValues(1) + .setAvgNumValues(1) + .setTotNumValues(valuesMap.get("total_count").getLongValue())) + .setUnique(valuesMap.get("unique").getLongValue()) + .setMaxNumBytes((float) valuesMap.get("max").getDoubleValue()) + .setMinNumBytes((float) valuesMap.get("min").getDoubleValue()) + .setAvgNumBytes((float) valuesMap.get("mean").getDoubleValue()) + .build(); + } + + private static StringStatistics getStringStatistics(Map valuesMap) { + List topCount = + valuesMap.get("top_count").getRepeatedValue().stream() + .map( + tc -> { + FieldValueList recordValue = tc.getRecordValue(); + return FreqAndValue.newBuilder() + .setValue(recordValue.get(0).getStringValue()) + .setFrequency(recordValue.get(1).getLongValue()) + .build(); + }) + .collect(Collectors.toList()); + + RankHistogram.Builder rankHistogram = RankHistogram.newBuilder(); + valuesMap + .get("cat_hist") + .getRepeatedValue() + .forEach( + v -> { + FieldValueList recordValue = v.getRecordValue(); + rankHistogram.addBuckets( + RankHistogram.Bucket.newBuilder() + .setLabel(recordValue.get(0).getStringValue()) + .setSampleCount(recordValue.get(1).getLongValue())); + }); + + return StringStatistics.newBuilder() + .setUnique(valuesMap.get("unique").getLongValue()) + .setCommonStats( + CommonStatistics.newBuilder() + .setNumMissing(valuesMap.get("missing_count").getLongValue()) + .setNumNonMissing(valuesMap.get("feature_count").getLongValue()) + .setMinNumValues(1) + .setMaxNumValues(1) + .setAvgNumValues(1) + .setTotNumValues(valuesMap.get("total_count").getLongValue())) + .setRankHistogram(rankHistogram) + .addAllTopValues(topCount) + .build(); + } + + private static NumericStatistics getNumericStatistics(Map valuesMap) { + // Build quantiles + long quantileCount = valuesMap.get("feature_count").getLongValue() / 10; + Histogram.Builder quantilesBuilder = Histogram.newBuilder().setType(HistogramType.QUANTILES); + + List quantilesRaw = valuesMap.get("quantiles").getRepeatedValue(); + for (int i = 0; i < quantilesRaw.size() - 1; i++) { + quantilesBuilder.addBuckets( + Bucket.newBuilder() + .setLowValue(quantilesRaw.get(i).getDoubleValue()) + .setHighValue(quantilesRaw.get(i + 1).getDoubleValue()) + .setSampleCount(quantileCount)); + } + // Build histogram + Histogram.Builder histBuilder = Histogram.newBuilder().setType(HistogramType.STANDARD); + + // Order of histogram records is defined in the query hist_stats.sql:L35 + valuesMap + .get("num_hist") + .getRepeatedValue() + .forEach( + v -> { + FieldValueList recordValue = v.getRecordValue(); + histBuilder.addBuckets( + Bucket.newBuilder() + .setHighValue(recordValue.get(2).getDoubleValue()) + .setLowValue(recordValue.get(1).getDoubleValue()) + .setSampleCount(recordValue.get(0).getLongValue())); + }); + + return NumericStatistics.newBuilder() + .setMax(valuesMap.get("max").getDoubleValue()) + .setMin(valuesMap.get("min").getDoubleValue()) + .setMedian(quantilesRaw.get(5).getDoubleValue()) + .setNumZeros(valuesMap.get("zeroes").getLongValue()) + .setStdDev(valuesMap.get("stdev").getDoubleValue()) + .setMean(valuesMap.get("mean").getDoubleValue()) + .setCommonStats( + CommonStatistics.newBuilder() + .setNumMissing(valuesMap.get("missing_count").getLongValue()) + .setNumNonMissing(valuesMap.get("feature_count").getLongValue()) + .setMinNumValues(1) + .setMaxNumValues(1) + .setAvgNumValues(1) + .setTotNumValues(valuesMap.get("feature_count").getLongValue())) + .addHistograms(quantilesBuilder) + .addHistograms(histBuilder) + .build(); + } + + private static StructStatistics getStructStatistics(Map valuesMap) { + return StructStatistics.newBuilder() + .setCommonStats( + CommonStatistics.newBuilder() + .setNumMissing(valuesMap.get("missing_count").getLongValue()) + .setNumNonMissing(valuesMap.get("feature_count").getLongValue()) + .setMinNumValues(valuesMap.get("min").getLongValue()) + .setMaxNumValues(valuesMap.get("max").getLongValue()) + .setAvgNumValues(valuesMap.get("mean").getLongValue()) + .setTotNumValues(valuesMap.get("total_count").getLongValue())) + .build(); + } +} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java index d155d3f1f50..33f79f8d97a 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java @@ -42,6 +42,8 @@ public abstract class BigQueryFeatureSink implements FeatureSink { "Event time for the FeatureRow"; public static final String BIGQUERY_CREATED_TIMESTAMP_FIELD_DESCRIPTION = "Processing time of the FeatureRow ingestion in Feast\""; + public static final String BIGQUERY_DATASET_ID_FIELD_DESCRIPTION = + "Identifier of the batch dataset a row belongs to"; public static final String BIGQUERY_JOB_ID_FIELD_DESCRIPTION = "Feast import job ID for the FeatureRow"; @@ -108,10 +110,13 @@ public void prepareWrite(FeatureSetProto.FeatureSet featureSet) { Table table = bigquery.getTable(tableId); if (table != null) { log.info( - "Writing to existing BigQuery table '{}:{}.{}'", - getProjectId(), + "Updating and writing to existing BigQuery table '{}:{}.{}'", + datasetId.getProject(), datasetId.getDataset(), tableName); + TableDefinition tableDefinition = createBigQueryTableDefinition(featureSet.getSpec()); + TableInfo tableInfo = TableInfo.of(tableId, tableDefinition); + bigquery.update(tableInfo); return; } @@ -166,6 +171,8 @@ private TableDefinition createBigQueryTableDefinition(FeatureSetProto.FeatureSet "created_timestamp", Pair.of( StandardSQLTypeName.TIMESTAMP, BIGQUERY_CREATED_TIMESTAMP_FIELD_DESCRIPTION), + "dataset_id", + Pair.of(StandardSQLTypeName.STRING, BIGQUERY_DATASET_ID_FIELD_DESCRIPTION), "job_id", Pair.of(StandardSQLTypeName.STRING, BIGQUERY_JOB_ID_FIELD_DESCRIPTION)); for (Map.Entry> entry : diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureRowToTableRow.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureRowToTableRow.java index 12833b31b85..9eaf504558e 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureRowToTableRow.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureRowToTableRow.java @@ -31,6 +31,7 @@ public class FeatureRowToTableRow implements SerializableFunction { private static final String EVENT_TIMESTAMP_COLUMN = "event_timestamp"; private static final String CREATED_TIMESTAMP_COLUMN = "created_timestamp"; + private static final String DATASET_ID_COLUMN = "dataset_id"; private static final String JOB_ID_COLUMN = "job_id"; private final String jobId; @@ -47,6 +48,7 @@ public TableRow apply(FeatureRow featureRow) { TableRow tableRow = new TableRow(); tableRow.set(EVENT_TIMESTAMP_COLUMN, Timestamps.toString(featureRow.getEventTimestamp())); tableRow.set(CREATED_TIMESTAMP_COLUMN, Instant.now().toString()); + tableRow.set(DATASET_ID_COLUMN, featureRow.getDatasetId()); tableRow.set(JOB_ID_COLUMN, jobId); for (Field field : featureRow.getFieldsList()) { diff --git a/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql b/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql new file mode 100644 index 00000000000..e07810affc7 --- /dev/null +++ b/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql @@ -0,0 +1,93 @@ +WITH subset AS ( +SELECT * FROM `{{ projectId }}.{{ datasetId }}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}` +{% if featureSet.datasetId == "" %} +WHERE event_timestamp >= '{{ featureSet.date }} 00:00:00 UTC' AND event_timestamp < DATETIME_ADD('{{ featureSet.date }} 00:00:00 UTC', INTERVAL 1 DAY) +{% else %} +WHERE dataset_id='{{ featureSet.datasetId }}' +{% endif %} +) +{% for feature in featureSet.features %} +SELECT + "{{ feature.name }}" as feature_name, + -- total count + COUNT(*) AS total_count, + -- count + COUNT({{ feature.name }}) as feature_count, + -- missing + COUNT(*) - COUNT({{ feature.name }}) as missing_count, + {% if feature.type equals "NUMERIC" %} + -- mean + AVG({{ feature.name }}) as mean, + -- stdev + STDDEV({{ feature.name }}) as stdev, + -- zeroes + COUNTIF({{ feature.name }} = 0) as zeroes, + -- min + MIN({{ feature.name }}) as min, + -- max + MAX({{ feature.name }}) as max, + -- hist will have to be called separately + -- quantiles + APPROX_QUANTILES(CAST({{ feature.name }} AS FLOAT64), 10) AS quantiles, + -- unique + null as unique, + -- top count + ARRAY>[] as top_count + {% elseif feature.type equals "CATEGORICAL" %} + -- mean + null as mean, + -- stdev + null as stdev, + -- zeroes + null as zeroes, + -- min + null as min, + -- max + null as max, + -- quantiles + ARRAY[] AS quantiles, + -- unique + APPROX_COUNT_DISTINCT({{ feature.name }}) as unique, + -- top count + APPROX_TOP_COUNT({{ feature.name }}, 5) as top_count, + {% elseif feature.type equals "BYTES" %} + -- mean + AVG(BIT_COUNT({{ feature.name }})) as mean, + -- stdev + null as stdev, + -- zeroes + null as zeroes, + -- min + MIN(BIT_COUNT({{ feature.name }})) as min, + -- max + MAX(BIT_COUNT({{ feature.name }})) as max, + -- hist will have to be called separately + -- quantiles + ARRAY[] AS quantiles, + -- unique + APPROX_COUNT_DISTINCT({{ feature.name }}) as unique, + -- top count + ARRAY>[] as top_count + {% elseif feature.type equals "LIST" %} + -- mean + AVG(ARRAY_LENGTH({{ feature.name }})) as mean, + -- stdev + null as stdev, + -- zeroes + null as zeroes, + -- min + MIN(ARRAY_LENGTH({{ feature.name }})) as min, + -- max + MAX(ARRAY_LENGTH({{ feature.name }})) as max, + -- hist will have to be called separately + -- quantiles + ARRAY[] AS quantiles, + -- unique + null as unique, + -- top count + ARRAY>[] as top_count + {% endif %} +FROM subset +{% if loop.last %}{% else %}UNION ALL {% endif %} +{% endfor %} + diff --git a/storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql b/storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql new file mode 100644 index 00000000000..d611cb3c36c --- /dev/null +++ b/storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql @@ -0,0 +1,41 @@ +WITH subset AS ( +SELECT * FROM `{{ projectId }}.{{ datasetId }}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}` +{% if featureSet.datasetId == "" %} +WHERE event_timestamp >= '{{ featureSet.date }} 00:00:00 UTC' AND event_timestamp < DATETIME_ADD('{{ featureSet.date }} 00:00:00 UTC', INTERVAL 1 DAY) +{% else %} +WHERE dataset_id='{{ featureSet.datasetId }}' +{% endif %} +) +{% for feature in featureSet.features %} +, {{ feature.name }}_stats AS ( +{% if feature.type == 'NUMERIC' %} + WITH stats AS ( + SELECT min+step*i as min, min+step*(i+1) as max + FROM ( + SELECT MIN({{ feature.name }}) as min, MAX({{ feature.name }}) as max, (MAX({{ feature.name }})-MIN({{ feature.name }}))/10 step, GENERATE_ARRAY(0, 10, 1) i + FROM subset + ), UNNEST(i) i + ), counts as ( + SELECT COUNT(*) as count, min, max, + FROM subset + JOIN stats + ON subset.{{ feature.name }} >= stats.min AND subset.{{ feature.name }}>[] as cat_hist FROM counts +{% elseif feature.type == 'CATEGORICAL' %} + WITH counts AS ( + SELECT {{ feature.name }}, COUNT({{ feature.name }}) AS count FROM subset GROUP BY {{ feature.name }} + ) + SELECT '{{ feature.name }}' as feature, ARRAY>[] as num_hist, ARRAY_AGG(STRUCT({{ feature.name }} as value, count as count)) as cat_hist FROM counts +{% elseif feature.type == 'BYTES' %} + SELECT '{{ feature.name }}' as feature, ARRAY>[] as num_hist, ARRAY>[] as cat_hist +{% elseif feature.type == 'LIST' %} + SELECT '{{ feature.name }}' as feature, ARRAY>[] as num_hist, ARRAY>[] as cat_hist +{% endif %} +) +{% endfor %} +{% for feature in featureSet.features %} +SELECT * FROM {{ feature.name }}_stats +{% if loop.last %}{% else %}UNION ALL {% endif %} +{% endfor %} \ No newline at end of file diff --git a/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetrieverTest.java b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetrieverTest.java new file mode 100644 index 00000000000..28134251cb5 --- /dev/null +++ b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetrieverTest.java @@ -0,0 +1,95 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigquery.stats; + +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.storage.api.statistics.FeatureSetStatistics; +import java.util.Arrays; +import org.junit.Test; + +public class BigQueryStatisticsRetrieverTest { + + @Test + public void shouldRun() throws InvalidProtocolBufferException { + BigQueryStatisticsRetriever retriever = + BigQueryStatisticsRetriever.newBuilder() + .setBigquery(BigQueryOptions.getDefaultInstance().getService()) + .setDatasetId("feast_test_20200202") + .setProjectId("aliz-development") + .build(); + + // FeatureSetSpec featureSetSpec = FeatureSetSpec.newBuilder() + // .setProject("metrics_test") + // .setName("customer_transactions") + // .setVersion(1) + // + // .addEntities(EntitySpec.newBuilder().setName("customer_id").setValueType(Enum.INT64)) + // + // .addFeatures(FeatureSpec.newBuilder().setName("total_transactions").setValueType(Enum.INT64)) + // + // .addFeatures(FeatureSpec.newBuilder().setName("daily_transactions").setValueType(Enum.FLOAT)) + // .build(); + + FeatureSetSpec.Builder featureSetSpec = FeatureSetSpec.newBuilder(); + String bigStatsJson = + "{\"project\":\"metrics_test\",\"maxAge\":\"345599s\",\"name\":\"big\",\"entities\":[{\"name\":\"driver\",\"valueType\":\"STRING\"}],\"features\":[{\"name\":\"ride_driver_id_num_completed\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_prop_completed\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_distance_completed\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_customer_distance_completed\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_distance_cancelled\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_customer_distance_cancelled\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_standing_completed_1\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_standing_completed_2\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_standing_completed_3\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_standing_cancelled_1\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_standing_cancelled_2\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_standing_cancelled_3\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_origin_completed_1\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_origin_completed_2\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_origin_completed_3\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_origin_cancelled_1\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_origin_cancelled_2\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_origin_cancelled_3\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_destination_completed_1\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_destination_completed_2\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_destination_completed_3\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_destination_cancelled_1\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_destination_cancelled_2\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_destination_cancelled_3\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_donut_count_completed\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_donut_count_cancelled\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_num_completed_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_prop_completed_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_distance_completed_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_customer_distance_completed_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_distance_cancelled_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_customer_distance_cancelled_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_donut_count_completed_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_donut_count_cancelled_recent\",\"valueType\":\"FLOAT\"}]}"; + JsonFormat.parser().merge(bigStatsJson, featureSetSpec); + featureSetSpec.setVersion(1); + FeatureSetStatistics featureStatistics = + retriever.getFeatureStatistics( + featureSetSpec.build(), + Arrays.asList( + "ride_driver_id_destination_cancelled_3", + "ride_driver_id_avg_customer_distance_completed_recent", + "ride_driver_id_avg_distance_completed_recent", + "ride_driver_id_origin_cancelled_2", + "ride_driver_id_donut_count_cancelled", + "ride_driver_id_destination_completed_3", + "ride_driver_id_prop_completed", + "ride_driver_id_origin_completed_2", + "ride_driver_id_standing_cancelled_1", + "ride_driver_id_avg_customer_distance_cancelled_recent", + "ride_driver_id_avg_customer_distance_cancelled", + "ride_driver_id_donut_count_completed_recent", + "ride_driver_id_standing_completed_1", + "ride_driver_id_avg_distance_cancelled_recent", + "ride_driver_id_destination_cancelled_1", + "ride_driver_id_standing_cancelled_2", + "ride_driver_id_origin_completed_3", + "ride_driver_id_standing_completed_2", + "ride_driver_id_donut_count_completed", + "ride_driver_id_origin_cancelled_1", + "ride_driver_id_origin_cancelled_3", + "ride_driver_id_origin_completed_1", + "ride_driver_id_num_completed", + "ride_driver_id_destination_cancelled_2", + "ride_driver_id_prop_completed_recent", + "ride_driver_id_standing_cancelled_3", + "ride_driver_id_avg_distance_cancelled", + "ride_driver_id_avg_customer_distance_completed", + "ride_driver_id_donut_count_cancelled_recent", + "ride_driver_id_destination_completed_2", + "ride_driver_id_avg_distance_completed", + "ride_driver_id_num_completed_recent", + "ride_driver_id_standing_completed_3", + "ride_driver_id_destination_completed_1"), + "dataset"); + } +} diff --git a/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/StatsUtilTest.java b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/StatsUtilTest.java new file mode 100644 index 00000000000..1ab1778b3f0 --- /dev/null +++ b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/StatsUtilTest.java @@ -0,0 +1,218 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigquery.stats; + +import static feast.storage.connectors.bigquery.stats.StatsUtil.toFeatureNameStatistics; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.google.cloud.bigquery.FieldValue; +import com.google.cloud.bigquery.FieldValue.Attribute; +import com.google.cloud.bigquery.FieldValueList; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.Schema; +import com.google.common.collect.Lists; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import feast.core.FeatureSetProto.FeatureSpec; +import feast.types.ValueProto.ValueType.Enum; +import org.junit.Test; +import org.tensorflow.metadata.v0.FeatureNameStatistics; + +public class StatsUtilTest { + + private Schema basicStatsSchema = + Schema.of( + com.google.cloud.bigquery.Field.of("feature_name", LegacySQLTypeName.STRING), + com.google.cloud.bigquery.Field.of("total_count", LegacySQLTypeName.INTEGER), + com.google.cloud.bigquery.Field.of("feature_count", LegacySQLTypeName.INTEGER), + com.google.cloud.bigquery.Field.of("missing_count", LegacySQLTypeName.INTEGER), + com.google.cloud.bigquery.Field.of("mean", LegacySQLTypeName.FLOAT), + com.google.cloud.bigquery.Field.of("stdev", LegacySQLTypeName.FLOAT), + com.google.cloud.bigquery.Field.of("zeroes", LegacySQLTypeName.INTEGER), + com.google.cloud.bigquery.Field.of("min", LegacySQLTypeName.FLOAT), + com.google.cloud.bigquery.Field.of("max", LegacySQLTypeName.FLOAT), + com.google.cloud.bigquery.Field.of("quantiles", LegacySQLTypeName.NUMERIC), + com.google.cloud.bigquery.Field.of("unique", LegacySQLTypeName.INTEGER), + com.google.cloud.bigquery.Field.of( + "top_count", + LegacySQLTypeName.RECORD, + com.google.cloud.bigquery.Field.of("value", LegacySQLTypeName.STRING), + com.google.cloud.bigquery.Field.of("count", LegacySQLTypeName.INTEGER))); + + private Schema histStatsSchema = + Schema.of( + com.google.cloud.bigquery.Field.of("feature", LegacySQLTypeName.STRING), + com.google.cloud.bigquery.Field.of( + "num_hist", + LegacySQLTypeName.RECORD, + com.google.cloud.bigquery.Field.of("count", LegacySQLTypeName.INTEGER), + com.google.cloud.bigquery.Field.of("low_value", LegacySQLTypeName.FLOAT), + com.google.cloud.bigquery.Field.of("high_value", LegacySQLTypeName.FLOAT)), + com.google.cloud.bigquery.Field.of( + "cat_hist", + LegacySQLTypeName.RECORD, + com.google.cloud.bigquery.Field.of("value", LegacySQLTypeName.STRING), + com.google.cloud.bigquery.Field.of("count", LegacySQLTypeName.INTEGER))); + + @Test + public void voidShouldConvertNumericStatsToFeatureNameStatistics() + throws InvalidProtocolBufferException { + FieldValueList numericFieldValueList = + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "floats"), + FieldValue.of(Attribute.PRIMITIVE, "20"), + FieldValue.of(Attribute.PRIMITIVE, "20"), + FieldValue.of(Attribute.PRIMITIVE, "0"), + FieldValue.of(Attribute.PRIMITIVE, "1"), + FieldValue.of(Attribute.PRIMITIVE, "6"), + FieldValue.of(Attribute.PRIMITIVE, "0"), + FieldValue.of(Attribute.PRIMITIVE, "-8.5"), + FieldValue.of(Attribute.PRIMITIVE, "10.5"), + FieldValue.of( + Attribute.REPEATED, + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "-8.5"), + FieldValue.of(Attribute.PRIMITIVE, "-7.5"), + FieldValue.of(Attribute.PRIMITIVE, "-5.5"), + FieldValue.of(Attribute.PRIMITIVE, "-3.5"), + FieldValue.of(Attribute.PRIMITIVE, "-1.5"), + FieldValue.of(Attribute.PRIMITIVE, "0.5"), + FieldValue.of(Attribute.PRIMITIVE, "2.5"), + FieldValue.of(Attribute.PRIMITIVE, "4.5"), + FieldValue.of(Attribute.PRIMITIVE, "6.5"), + FieldValue.of(Attribute.PRIMITIVE, "8.5"), + FieldValue.of(Attribute.PRIMITIVE, "10.5")))), + FieldValue.of(Attribute.PRIMITIVE, null), + FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())))); + + FieldValueList numericHistFieldValueList = + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "floats"), + FieldValue.of( + Attribute.REPEATED, + FieldValueList.of( + Lists.newArrayList( + FieldValue.of( + Attribute.RECORD, + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "1"), + FieldValue.of(Attribute.PRIMITIVE, "1"), + FieldValue.of(Attribute.PRIMITIVE, "2")))), + FieldValue.of( + Attribute.RECORD, + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "2"), + FieldValue.of(Attribute.PRIMITIVE, "2"), + FieldValue.of(Attribute.PRIMITIVE, "3"))))))), + FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())))); + + FeatureSpec featureSpec = + FeatureSpec.newBuilder().setName("floats").setValueType(Enum.DOUBLE).build(); + + FeatureNameStatistics actual = + toFeatureNameStatistics( + featureSpec, + basicStatsSchema, + numericFieldValueList, + histStatsSchema, + numericHistFieldValueList); + String expectedJson = + "{\"type\":\"FLOAT\",\"numStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"1\",\"maxNumValues\":\"1\",\"avgNumValues\":1,\"totNumValues\":\"20\"},\"mean\":1,\"stdDev\":6,\"min\":-8.5,\"median\":0.5,\"max\":10.5,\"histograms\":[{\"buckets\":[{\"lowValue\":-8.5,\"highValue\":-7.5,\"sampleCount\":2},{\"lowValue\":-7.5,\"highValue\":-5.5,\"sampleCount\":2},{\"lowValue\":-5.5,\"highValue\":-3.5,\"sampleCount\":2},{\"lowValue\":-3.5,\"highValue\":-1.5,\"sampleCount\":2},{\"lowValue\":-1.5,\"highValue\":0.5,\"sampleCount\":2},{\"lowValue\":0.5,\"highValue\":2.5,\"sampleCount\":2},{\"lowValue\":2.5,\"highValue\":4.5,\"sampleCount\":2},{\"lowValue\":4.5,\"highValue\":6.5,\"sampleCount\":2},{\"lowValue\":6.5,\"highValue\":8.5,\"sampleCount\":2},{\"lowValue\":8.5,\"highValue\":10.5,\"sampleCount\":2}],\"type\":\"QUANTILES\"},{\"buckets\":[{\"lowValue\":1,\"highValue\":2,\"sampleCount\":1},{\"lowValue\":2,\"highValue\":3,\"sampleCount\":2}]}]},\"path\":{\"step\":[\"floats\"]}}"; + FeatureNameStatistics.Builder expected = FeatureNameStatistics.newBuilder(); + JsonFormat.parser().merge(expectedJson, expected); + assertThat(actual, equalTo(expected.build())); + } + + @Test + public void voidShouldConvertStringStatsToFeatureNameStatistics() + throws InvalidProtocolBufferException { + FieldValueList stringFieldValueList = + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "strings"), + FieldValue.of(Attribute.PRIMITIVE, "20"), + FieldValue.of(Attribute.PRIMITIVE, "20"), + FieldValue.of(Attribute.PRIMITIVE, "0"), + FieldValue.of(Attribute.PRIMITIVE, null), + FieldValue.of(Attribute.PRIMITIVE, null), + FieldValue.of(Attribute.PRIMITIVE, null), + FieldValue.of(Attribute.PRIMITIVE, null), + FieldValue.of(Attribute.PRIMITIVE, null), + FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())), + FieldValue.of(Attribute.PRIMITIVE, "2"), + FieldValue.of( + Attribute.REPEATED, + FieldValueList.of( + Lists.newArrayList( + FieldValue.of( + Attribute.RECORD, + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "a"), + FieldValue.of(Attribute.PRIMITIVE, "1")))), + FieldValue.of( + Attribute.RECORD, + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "b"), + FieldValue.of(Attribute.PRIMITIVE, "2"))))))))); + + FieldValueList stringHistFieldValueList = + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "strings"), + FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())), + FieldValue.of( + Attribute.REPEATED, + FieldValueList.of( + Lists.newArrayList( + FieldValue.of( + Attribute.RECORD, + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "a"), + FieldValue.of(Attribute.PRIMITIVE, "1")))), + FieldValue.of( + Attribute.RECORD, + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "b"), + FieldValue.of(Attribute.PRIMITIVE, "2"))))))))); + + FeatureSpec featureSpec = + FeatureSpec.newBuilder().setName("strings").setValueType(Enum.STRING).build(); + + FeatureNameStatistics actual = + toFeatureNameStatistics( + featureSpec, + basicStatsSchema, + stringFieldValueList, + histStatsSchema, + stringHistFieldValueList); + String expectedJson = + "{\"type\":\"STRING\",\"stringStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"1\",\"maxNumValues\":\"1\",\"avgNumValues\":1,\"totNumValues\":\"20\"},\"unique\":\"2\",\"topValues\":[{\"value\":\"a\",\"frequency\":1},{\"value\":\"b\",\"frequency\":2}],\"rankHistogram\":{\"buckets\":[{\"label\":\"a\",\"sampleCount\":1},{\"label\":\"b\",\"sampleCount\":2}]}},\"path\":{\"step\":[\"strings\"]}}"; + FeatureNameStatistics.Builder expected = FeatureNameStatistics.newBuilder(); + JsonFormat.parser().merge(expectedJson, expected); + assertThat(actual, equalTo(expected.build())); + } +} From 254c366d4c1508237c9cabac7cc3d4b232d35181 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Wed, 8 Apr 2020 16:03:25 +0800 Subject: [PATCH 02/12] Add end-to-end test --- .../feast/core/service/JobServiceTest.java | 4 +- infra/scripts/test-end-to-end-batch.sh | 2 +- tests/e2e/feature-validation.py | 99 ++++++ .../expected_output_basic_dataset.json | 310 ++++++++++++++++++ 4 files changed, 412 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/feature-validation.py create mode 100644 tests/e2e/statistics/expected_output_basic_dataset.json diff --git a/core/src/test/java/feast/core/service/JobServiceTest.java b/core/src/test/java/feast/core/service/JobServiceTest.java index f5963282b37..d21ad8e360d 100644 --- a/core/src/test/java/feast/core/service/JobServiceTest.java +++ b/core/src/test/java/feast/core/service/JobServiceTest.java @@ -44,9 +44,9 @@ import feast.core.dao.JobRepository; import feast.core.job.JobManager; import feast.core.job.Runner; -import feast.core.model.FeatureSet; -import feast.core.model.Feature; import feast.core.model.Entity; +import feast.core.model.Feature; +import feast.core.model.FeatureSet; import feast.core.model.Job; import feast.core.model.JobStatus; import feast.core.model.Source; diff --git a/infra/scripts/test-end-to-end-batch.sh b/infra/scripts/test-end-to-end-batch.sh index 0e7bfe8bf8d..81a314315b7 100755 --- a/infra/scripts/test-end-to-end-batch.sh +++ b/infra/scripts/test-end-to-end-batch.sh @@ -251,7 +251,7 @@ ORIGINAL_DIR=$(pwd) cd tests/e2e set +e -pytest bq-batch-retrieval.py --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml +pytest bq-batch-retrieval.py feature-validation.py --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml TEST_EXIT_CODE=$? if [[ ${TEST_EXIT_CODE} != 0 ]]; then diff --git a/tests/e2e/feature-validation.py b/tests/e2e/feature-validation.py new file mode 100644 index 00000000000..bac2aef5d93 --- /dev/null +++ b/tests/e2e/feature-validation.py @@ -0,0 +1,99 @@ +import pandas as pd +import pytest +import pytz +import uuid +import time +from datetime import datetime + +from feast.client import Client +from feast.entity import Entity +from feast.feature import Feature +from feast.feature_set import FeatureSet +from feast.type_map import ValueType +from google.protobuf import json_format +from google.protobuf.duration_pb2 import Duration +from tensorflow_metadata.proto.v0 import statistics_pb2 + +pd.set_option("display.max_columns", None) + +PROJECT_NAME = "batch_" + uuid.uuid4().hex.upper()[0:6] + + +@pytest.fixture(scope="module") +def client(core_url, allow_dirty): + # Get client for core and serving + client = Client(core_url=core_url) + client.create_project(PROJECT_NAME) + client.set_project(PROJECT_NAME) + + # Ensure Feast core is active, but empty + if not allow_dirty: + feature_sets = client.list_feature_sets() + if len(feature_sets) > 0: + raise Exception( + "Feast cannot have existing feature sets registered. Exiting tests." + ) + + return client + + +@pytest.fixture(scope="module") +def dataset_basic(client): + fv_fs = FeatureSet( + "feature_validation", + features=[ + Feature("strings", ValueType.STRING), + Feature("ints", ValueType.INT64), + Feature("floats", ValueType.FLOAT), + ], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) + client.apply(fv_fs) + time.sleep(20) + + N_ROWS = 20 + + time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) + df = pd.DataFrame( + { + "datetime": [time_offset] * N_ROWS, + "entity_id": [i for i in range(N_ROWS)], + "strings": ["a", "b"] * int(N_ROWS / 2), + "ints": [int(i) for i in range(N_ROWS)], + "floats": [10.5 - i for i in range(N_ROWS)], + } + ) + + expected_stats = statistics_pb2.DatasetFeatureStatisticsList() + with open("statistics/expected_output_basic_dataset.json", "r") as fo: + contents = fo.read() + json_format.Parse(contents, expected_stats) + + return { + "id": client.ingest(fv_fs, df), + "date": datetime.datetime( + time_offset.year, time_offset.month, time_offset.day + ).replace(tzinfo=pytz.utc), + "stats": expected_stats, + } + + +def test_basic_retrieval_by_single_dataset(client, dataset_basic): + stats = client.get_statistics( + feature_refs=["strings", "ints", "floats"], + store="bigquery", + dataset_ids=[dataset_basic["id"]], + ) + + assert stats == dataset_basic["stats"] + + +def test_basic_by_date(client, dataset_basic): + stats = client.get_statistics( + feature_refs=["strings", "ints", "floats"], + store="bigquery", + start_date=dataset_basic["date"], + end_date=dataset_basic["date"], + ) + assert stats == dataset_basic["stats"] \ No newline at end of file diff --git a/tests/e2e/statistics/expected_output_basic_dataset.json b/tests/e2e/statistics/expected_output_basic_dataset.json new file mode 100644 index 00000000000..0cf4a9fbab2 --- /dev/null +++ b/tests/e2e/statistics/expected_output_basic_dataset.json @@ -0,0 +1,310 @@ +{ + "datasets": [ + { + "type": "FLOAT", + "numStats": { + "commonStats": { + "numNonMissing": "20", + "minNumValues": "1", + "maxNumValues": "1", + "avgNumValues": 1.0, + "totNumValues": "20" + }, + "mean": 1.0, + "stdDev": 5.766281297335398, + "min": -8.5, + "median": 1.5, + "max": 10.5, + "histograms": [ + { + "buckets": [ + { + "lowValue": -8.5, + "highValue": -6.6, + "sampleCount": 1.998 + }, + { + "lowValue": -6.6, + "highValue": -4.7, + "sampleCount": 1.998 + }, + { + "lowValue": -4.7, + "highValue": -2.8000000000000007, + "sampleCount": 1.998 + }, + { + "lowValue": -2.8000000000000007, + "highValue": -0.9000000000000004, + "sampleCount": 1.998 + }, + { + "lowValue": -0.9000000000000004, + "highValue": 1.0, + "sampleCount": 1.998 + }, + { + "lowValue": 1.0, + "highValue": 2.8999999999999986, + "sampleCount": 1.998 + }, + { + "lowValue": 2.8999999999999986, + "highValue": 4.799999999999999, + "sampleCount": 1.998 + }, + { + "lowValue": 4.799999999999999, + "highValue": 6.699999999999999, + "sampleCount": 1.998 + }, + { + "lowValue": 6.699999999999999, + "highValue": 8.599999999999998, + "sampleCount": 1.998 + }, + { + "lowValue": 8.599999999999998, + "highValue": 10.5, + "sampleCount": 2.0180000000000002 + } + ] + }, + { + "buckets": [ + { + "lowValue": -8.5, + "highValue": -6.5, + "sampleCount": 2.0 + }, + { + "lowValue": -6.5, + "highValue": -4.5, + "sampleCount": 2.0 + }, + { + "lowValue": -4.5, + "highValue": -2.5, + "sampleCount": 2.0 + }, + { + "lowValue": -2.5, + "highValue": -0.5, + "sampleCount": 2.0 + }, + { + "lowValue": -0.5, + "highValue": 1.5, + "sampleCount": 2.0 + }, + { + "lowValue": 1.5, + "highValue": 3.5, + "sampleCount": 2.0 + }, + { + "lowValue": 3.5, + "highValue": 5.5, + "sampleCount": 2.0 + }, + { + "lowValue": 5.5, + "highValue": 7.5, + "sampleCount": 2.0 + }, + { + "lowValue": 7.5, + "highValue": 9.5, + "sampleCount": 2.0 + }, + { + "lowValue": 9.5, + "highValue": 10.5, + "sampleCount": 2.0 + } + ], + "type": "QUANTILES" + } + ] + }, + "path": { + "step": [ + "floats" + ] + } + }, + { + "numStats": { + "commonStats": { + "numNonMissing": "20", + "minNumValues": "1", + "maxNumValues": "1", + "avgNumValues": 1.0, + "totNumValues": "20" + }, + "mean": 9.5, + "stdDev": 5.766281297335398, + "numZeros": "1", + "median": 10.0, + "max": 19.0, + "histograms": [ + { + "buckets": [ + { + "highValue": 1.9, + "sampleCount": 1.998 + }, + { + "lowValue": 1.9, + "highValue": 3.8, + "sampleCount": 1.998 + }, + { + "lowValue": 3.8, + "highValue": 5.699999999999999, + "sampleCount": 1.998 + }, + { + "lowValue": 5.699999999999999, + "highValue": 7.6, + "sampleCount": 1.998 + }, + { + "lowValue": 7.6, + "highValue": 9.5, + "sampleCount": 1.998 + }, + { + "lowValue": 9.5, + "highValue": 11.399999999999999, + "sampleCount": 1.998 + }, + { + "lowValue": 11.399999999999999, + "highValue": 13.299999999999999, + "sampleCount": 1.998 + }, + { + "lowValue": 13.299999999999999, + "highValue": 15.2, + "sampleCount": 1.998 + }, + { + "lowValue": 15.2, + "highValue": 17.099999999999998, + "sampleCount": 1.998 + }, + { + "lowValue": 17.099999999999998, + "highValue": 19.0, + "sampleCount": 2.0180000000000002 + } + ] + }, + { + "buckets": [ + { + "highValue": 2.0, + "sampleCount": 2.0 + }, + { + "lowValue": 2.0, + "highValue": 4.0, + "sampleCount": 2.0 + }, + { + "lowValue": 4.0, + "highValue": 6.0, + "sampleCount": 2.0 + }, + { + "lowValue": 6.0, + "highValue": 8.0, + "sampleCount": 2.0 + }, + { + "lowValue": 8.0, + "highValue": 10.0, + "sampleCount": 2.0 + }, + { + "lowValue": 10.0, + "highValue": 12.0, + "sampleCount": 2.0 + }, + { + "lowValue": 12.0, + "highValue": 14.0, + "sampleCount": 2.0 + }, + { + "lowValue": 14.0, + "highValue": 16.0, + "sampleCount": 2.0 + }, + { + "lowValue": 16.0, + "highValue": 18.0, + "sampleCount": 2.0 + }, + { + "lowValue": 18.0, + "highValue": 19.0, + "sampleCount": 2.0 + } + ], + "type": "QUANTILES" + } + ] + }, + "path": { + "step": [ + "ints" + ] + } + }, + { + "type": "STRING", + "stringStats": { + "commonStats": { + "numNonMissing": "20", + "minNumValues": "1", + "maxNumValues": "1", + "avgNumValues": 1.0, + "totNumValues": "20" + }, + "unique": "2", + "topValues": [ + { + "value": "b", + "frequency": 10.0 + }, + { + "value": "a", + "frequency": 10.0 + } + ], + "avgLength": 1.0, + "rankHistogram": { + "buckets": [ + { + "label": "b", + "sampleCount": 10.0 + }, + { + "label": "a", + "sampleCount": 10.0 + } + ] + } + }, + "path": { + "step": [ + "strings" + ] + } + } + ] + } + ] +} \ No newline at end of file From 639158e0f3c51cddd22d9d2028cb738cae060849 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Wed, 8 Apr 2020 18:39:28 +0800 Subject: [PATCH 03/12] Remove reference at feature level. Stats retrieval should be from StatsRepository --- .../core/dao/EntityStatisticsRepository.java | 31 ++ .../core/dao/FeatureStatisticsRepository.java | 6 +- .../main/java/feast/core/model/Entity.java | 6 +- .../feast/core/model/EntityStatistics.java | 266 ++++++++++++++++++ .../main/java/feast/core/model/Feature.java | 12 +- .../feast/core/model/FeatureStatistics.java | 10 +- .../java/feast/core/service/StatsService.java | 208 +++++++++++--- .../feast/core/service/StatsServiceTest.java | 6 +- .../api/statistics/StatisticsRetriever.java | 6 +- .../BigQueryStatisticsRetriever.java | 78 +++-- .../FeatureSetStatisticsQueryInfo.java | 19 +- .../statistics/FieldStatisticsQueryInfo.java | 94 +++++++ .../StatsQueryTemplater.java | 2 +- .../{stats => statistics}/StatsUtil.java | 70 +++-- .../stats/FeatureStatisticsQueryInfo.java | 64 ----- .../main/resources/templates/basic_stats.sql | 16 +- .../{stats => statistics}/StatsUtilTest.java | 38 +-- .../BigQueryStatisticsRetrieverTest.java | 95 ------- 18 files changed, 724 insertions(+), 303 deletions(-) create mode 100644 core/src/main/java/feast/core/dao/EntityStatisticsRepository.java create mode 100644 core/src/main/java/feast/core/model/EntityStatistics.java rename storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/{stats => statistics}/BigQueryStatisticsRetriever.java (65%) rename storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/{stats => statistics}/FeatureSetStatisticsQueryInfo.java (80%) create mode 100644 storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FieldStatisticsQueryInfo.java rename storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/{stats => statistics}/StatsQueryTemplater.java (98%) rename storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/{stats => statistics}/StatsUtil.java (80%) delete mode 100644 storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureStatisticsQueryInfo.java rename storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/{stats => statistics}/StatsUtilTest.java (85%) delete mode 100644 storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetrieverTest.java diff --git a/core/src/main/java/feast/core/dao/EntityStatisticsRepository.java b/core/src/main/java/feast/core/dao/EntityStatisticsRepository.java new file mode 100644 index 00000000000..a65daafb0bc --- /dev/null +++ b/core/src/main/java/feast/core/dao/EntityStatisticsRepository.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.dao; + +import feast.core.model.Entity; +import feast.core.model.EntityStatistics; +import java.util.Date; +import java.util.Optional; +import org.springframework.data.jpa.repository.JpaRepository; + +/** JPA repository supplying Statistics objects for entities keyed by id. */ +public interface EntityStatisticsRepository extends JpaRepository { + Optional findEntityStatisticsByEntityAndDatasetId( + Entity entity, String datasetId); + + Optional findEntityStatisticsByEntityAndDate(Entity entity, Date date); +} diff --git a/core/src/main/java/feast/core/dao/FeatureStatisticsRepository.java b/core/src/main/java/feast/core/dao/FeatureStatisticsRepository.java index f31df3803b5..4046295b75e 100644 --- a/core/src/main/java/feast/core/dao/FeatureStatisticsRepository.java +++ b/core/src/main/java/feast/core/dao/FeatureStatisticsRepository.java @@ -22,10 +22,10 @@ import java.util.Optional; import org.springframework.data.jpa.repository.JpaRepository; -/** JPA repository supplying Statistics objects keyed by id. */ +/** JPA repository supplying Statistics objects for features keyed by id. */ public interface FeatureStatisticsRepository extends JpaRepository { Optional findFeatureStatisticsByFeatureAndDatasetId( - Feature featureName, String datasetId); + Feature feature, String datasetId); - Optional findFeatureStatisticsByFeatureAndDate(Feature featureName, Date date); + Optional findFeatureStatisticsByFeatureAndDate(Feature feature, Date date); } diff --git a/core/src/main/java/feast/core/model/Entity.java b/core/src/main/java/feast/core/model/Entity.java index 574abb0c4e9..1b827aef6aa 100644 --- a/core/src/main/java/feast/core/model/Entity.java +++ b/core/src/main/java/feast/core/model/Entity.java @@ -19,10 +19,10 @@ import feast.core.FeatureSetProto.EntitySpec; import feast.types.ValueProto.ValueType; import java.util.Objects; -import javax.persistence.*; import lombok.Getter; import lombok.Setter; +/** Feast entity object. Contains name, type as well as domain metadata about the entity. */ @Getter @Setter @javax.persistence.Entity @@ -30,6 +30,10 @@ public class Entity extends Field { public Entity() {} + public Entity(FieldId fieldId) { + this.setId(fieldId); + } + public Entity(String name, ValueType.Enum type) { this.setId(new FieldId()); this.setName(name); diff --git a/core/src/main/java/feast/core/model/EntityStatistics.java b/core/src/main/java/feast/core/model/EntityStatistics.java new file mode 100644 index 00000000000..2b46a5ae67e --- /dev/null +++ b/core/src/main/java/feast/core/model/EntityStatistics.java @@ -0,0 +1,266 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.model; + +import com.google.protobuf.InvalidProtocolBufferException; +import java.io.*; +import java.util.Date; +import java.util.List; +import javax.persistence.*; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.tensorflow.metadata.v0.*; + +@NoArgsConstructor +@Getter +@Setter +@javax.persistence.Entity +@Table( + name = "entity_statistics", + indexes = { + @Index( + name = "idx_entity_statistics_entity", + columnList = "project,feature_set,version,name"), + @Index(name = "idx_entity_statistics_dataset_id", columnList = "datasetId"), + @Index(name = "idx_entity_statistics_date", columnList = "date"), + }) +public class EntityStatistics { + @Id + @GeneratedValue(strategy = GenerationType.AUTO) + private int id; + + @ManyToOne + @JoinColumns({ + @JoinColumn(name = "project", referencedColumnName = "project"), + @JoinColumn(name = "feature_set", referencedColumnName = "feature_set"), + @JoinColumn(name = "version", referencedColumnName = "version"), + @JoinColumn(name = "name", referencedColumnName = "name") + }) + private Entity entity; + + // Only one of these fields should be populated. + private String datasetId; + private Date date; + + // General statistics + private String featureType; + private long count; + private long numMissing; + private long minNumValues; + private long maxNumValues; + private float avgNumValues; + private long totalNumValues; + private byte[] numValuesHistogram; + + // Numeric statistics + private double mean; + private double stdev; + private long zeroes; + private double min; + private double max; + private double median; + private byte[] numericValueHistogram; + private byte[] numericValueQuantiles; + + // String statistics + @Column(name = "n_unique") + private long unique; + + private float averageLength; + private byte[] rankHistogram; + private byte[] topValues; + + // Byte statistics + private float minBytes; + private float maxBytes; + private float avgBytes; + + // Instantiates a Statistics object from a tensorflow metadata FeatureNameStatistics object and a + // dataset ID. + public static EntityStatistics fromProto( + String project, + String featureSetName, + int version, + FeatureNameStatistics featureNameStatistics, + String datasetId) + throws IOException { + EntityStatistics featureStatistics = EntityStatistics.fromProto(featureNameStatistics); + Entity entity = new Entity(); + entity.setId( + new FieldId(project, featureSetName, version, featureNameStatistics.getPath().getStep(0))); + featureStatistics.setEntity(entity); + featureStatistics.setDatasetId(datasetId); + return featureStatistics; + } + + // Instantiates a Statistics object from a tensorflow metadata FeatureNameStatistics object and a + // date. + public static EntityStatistics fromProto( + String project, + String featureSetName, + int version, + FeatureNameStatistics featureNameStatistics, + Date date) + throws IOException { + EntityStatistics entityStatistics = EntityStatistics.fromProto(featureNameStatistics); + entityStatistics.setDate(date); + Entity entity = new Entity(); + entity.setId( + new FieldId(project, featureSetName, version, featureNameStatistics.getPath().getStep(0))); + entityStatistics.setEntity(entity); + return entityStatistics; + } + + public FeatureNameStatistics toProto() throws InvalidProtocolBufferException { + FeatureNameStatistics.Builder featureNameStatisticsBuilder = + FeatureNameStatistics.newBuilder() + .setType(FeatureNameStatistics.Type.valueOf(featureType)) + .setPath(Path.newBuilder().addStep(entity.getId().getName())); + CommonStatistics commonStatistics = + CommonStatistics.newBuilder() + .setNumNonMissing(count - numMissing) + .setNumMissing(numMissing) + .setMaxNumValues(maxNumValues) + .setMinNumValues(minNumValues) + .setTotNumValues(totalNumValues) + .setNumValuesHistogram(Histogram.parseFrom(numValuesHistogram)) + .build(); + + switch (featureNameStatisticsBuilder.getType()) { + case INT: + case FLOAT: + NumericStatistics numStats = + NumericStatistics.newBuilder() + .setCommonStats(commonStatistics) + .setMean(mean) + .setStdDev(stdev) + .setNumZeros(zeroes) + .setMin(min) + .setMax(max) + .setMedian(median) + .addHistograms(Histogram.parseFrom(numericValueHistogram)) + .addHistograms(Histogram.parseFrom(numericValueQuantiles)) + .build(); + featureNameStatisticsBuilder.setNumStats(numStats); + break; + case STRING: + StringStatistics.Builder stringStats = + StringStatistics.newBuilder() + .setCommonStats(commonStatistics) + .setUnique(unique) + .setAvgLength(averageLength); + if (rankHistogram == null) { + stringStats.setRankHistogram(RankHistogram.getDefaultInstance()); + } else { + stringStats.setRankHistogram(RankHistogram.parseFrom(rankHistogram)); + } + try (ByteArrayInputStream bis = new ByteArrayInputStream(topValues)) { + ObjectInputStream ois = new ObjectInputStream(bis); + List freqAndValueList = + (List) ois.readObject(); + stringStats.addAllTopValues(freqAndValueList); + } catch (IOException | ClassNotFoundException e) { + throw new InvalidProtocolBufferException( + "Failed to parse field: StringStatistics.TopValues. Check if the value is malformed."); + } + featureNameStatisticsBuilder.setStringStats(stringStats); + break; + case BYTES: + BytesStatistics bytesStats = + BytesStatistics.newBuilder() + .setCommonStats(commonStatistics) + .setAvgNumBytes(avgBytes) + .setMinNumBytes(minBytes) + .setMaxNumBytes(maxBytes) + .build(); + featureNameStatisticsBuilder.setBytesStats(bytesStats); + break; + case STRUCT: + StructStatistics structStats = + StructStatistics.newBuilder().setCommonStats(commonStatistics).build(); + featureNameStatisticsBuilder.setStructStats(structStats); + break; + } + return featureNameStatisticsBuilder.build(); + } + + private static EntityStatistics fromProto(FeatureNameStatistics featureNameStatistics) + throws IOException, IllegalArgumentException { + EntityStatistics featureStatistics = new EntityStatistics(); + featureStatistics.setFeatureType(featureNameStatistics.getType().toString()); + CommonStatistics commonStats; + switch (featureNameStatistics.getType()) { + case FLOAT: + case INT: + NumericStatistics numStats = featureNameStatistics.getNumStats(); + commonStats = numStats.getCommonStats(); + featureStatistics.setMean(numStats.getMean()); + featureStatistics.setStdev(numStats.getStdDev()); + featureStatistics.setZeroes(numStats.getNumZeros()); + featureStatistics.setMin(numStats.getMin()); + featureStatistics.setMax(numStats.getMax()); + featureStatistics.setMedian(numStats.getMedian()); + for (Histogram histogram : numStats.getHistogramsList()) { + switch (histogram.getType()) { + case STANDARD: + featureStatistics.setNumericValueHistogram(histogram.toByteArray()); + case QUANTILES: + featureStatistics.setNumericValueQuantiles(histogram.toByteArray()); + default: + // invalid type, dropping the values + } + } + break; + case STRING: + StringStatistics stringStats = featureNameStatistics.getStringStats(); + commonStats = stringStats.getCommonStats(); + featureStatistics.setUnique(stringStats.getUnique()); + featureStatistics.setAverageLength(stringStats.getAvgLength()); + featureStatistics.setRankHistogram(stringStats.getRankHistogram().toByteArray()); + try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) { + ObjectOutputStream oos = new ObjectOutputStream(bos); + oos.writeObject(stringStats.getTopValuesList()); + featureStatistics.setTopValues(bos.toByteArray()); + } + break; + case BYTES: + BytesStatistics bytesStats = featureNameStatistics.getBytesStats(); + commonStats = bytesStats.getCommonStats(); + featureStatistics.setUnique(bytesStats.getUnique()); + featureStatistics.setMaxBytes(bytesStats.getMaxNumBytes()); + featureStatistics.setMinBytes(bytesStats.getMinNumBytes()); + featureStatistics.setAvgBytes(bytesStats.getAvgNumBytes()); + break; + case STRUCT: + StructStatistics structStats = featureNameStatistics.getStructStats(); + commonStats = structStats.getCommonStats(); + break; + default: + throw new IllegalArgumentException("Feature statistics provided were of unknown type."); + } + featureStatistics.setCount(commonStats.getNumMissing() + commonStats.getNumNonMissing()); + featureStatistics.setNumMissing(commonStats.getNumMissing()); + featureStatistics.setMinNumValues(commonStats.getMinNumValues()); + featureStatistics.setMaxNumValues(commonStats.getMaxNumValues()); + featureStatistics.setAvgNumValues(commonStats.getAvgNumValues()); + featureStatistics.setTotalNumValues(commonStats.getTotNumValues()); + featureStatistics.setNumValuesHistogram(commonStats.getNumValuesHistogram().toByteArray()); + + return featureStatistics; + } +} diff --git a/core/src/main/java/feast/core/model/Feature.java b/core/src/main/java/feast/core/model/Feature.java index ccfbbe53895..a2a840fc6cf 100644 --- a/core/src/main/java/feast/core/model/Feature.java +++ b/core/src/main/java/feast/core/model/Feature.java @@ -19,21 +19,21 @@ import feast.core.FeatureSetProto.FeatureSpec; import feast.types.ValueProto.ValueType; import java.util.Arrays; -import java.util.List; import java.util.Objects; import javax.persistence.*; import javax.persistence.Entity; import lombok.Getter; import lombok.Setter; +/** + * Feature belonging to a featureset. Contains name, type as well as domain metadata about the + * feature. + */ @Getter @Setter @Entity public class Feature extends Field { - @OneToMany(cascade = CascadeType.ALL, fetch = FetchType.LAZY) - private List statistics; - public Feature() {} public Feature(FieldId fieldId) { @@ -114,10 +114,6 @@ public static Feature fromProto(FeatureSpec featureSpec) { return feature; } - public void addStatistics(FeatureStatistics newStatistic) { - this.statistics.add(newStatistic); - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/core/src/main/java/feast/core/model/FeatureStatistics.java b/core/src/main/java/feast/core/model/FeatureStatistics.java index 75bca3b91fd..1cb9ebb46b2 100644 --- a/core/src/main/java/feast/core/model/FeatureStatistics.java +++ b/core/src/main/java/feast/core/model/FeatureStatistics.java @@ -32,11 +32,13 @@ @Setter @Entity @Table( - name = "statistics", + name = "feature_statistics", indexes = { - @Index(name = "idx_statistics_feature", columnList = "project,feature_set,version,name"), - @Index(name = "idx_statistics_dataset_id", columnList = "datasetId"), - @Index(name = "idx_statistics_date", columnList = "date"), + @Index( + name = "idx_feature_statistics_feature", + columnList = "project,feature_set,version,name"), + @Index(name = "idx_feature_statistics_dataset_id", columnList = "datasetId"), + @Index(name = "idx_feature_statistics_date", columnList = "date"), }) public class FeatureStatistics { @Id diff --git a/core/src/main/java/feast/core/service/StatsService.java b/core/src/main/java/feast/core/service/StatsService.java index 614fa22a330..ad2ae912c73 100644 --- a/core/src/main/java/feast/core/service/StatsService.java +++ b/core/src/main/java/feast/core/service/StatsService.java @@ -25,19 +25,20 @@ import feast.core.CoreServiceProto.GetFeatureSetRequest; import feast.core.CoreServiceProto.GetFeatureStatisticsRequest; import feast.core.CoreServiceProto.GetFeatureStatisticsResponse; +import feast.core.FeatureSetProto.EntitySpec; import feast.core.FeatureSetProto.FeatureSet; import feast.core.FeatureSetProto.FeatureSetSpec; import feast.core.FeatureSetProto.FeatureSpec; import feast.core.StoreProto.Store; import feast.core.StoreProto.Store.StoreType; +import feast.core.dao.EntityStatisticsRepository; import feast.core.dao.FeatureStatisticsRepository; import feast.core.dao.StoreRepository; +import feast.core.model.*; import feast.core.model.Feature; -import feast.core.model.FeatureStatistics; -import feast.core.model.FieldId; import feast.storage.api.statistics.FeatureSetStatistics; import feast.storage.api.statistics.StatisticsRetriever; -import feast.storage.connectors.bigquery.stats.BigQueryStatisticsRetriever; +import feast.storage.connectors.bigquery.statistics.BigQueryStatisticsRetriever; import java.io.IOException; import java.time.Instant; import java.util.*; @@ -49,6 +50,7 @@ import org.tensorflow.metadata.v0.*; import org.tensorflow.metadata.v0.FeatureNameStatistics.Builder; +/** Facilitates the retrieval of feature set statistics from historical stores. */ @Slf4j @Service public class StatsService { @@ -56,17 +58,40 @@ public class StatsService { private StoreRepository storeRepository; private SpecService specService; private FeatureStatisticsRepository featureStatisticsRepository; + private EntityStatisticsRepository entityStatisticsRepository; @Autowired public StatsService( StoreRepository storeRepository, SpecService specService, + EntityStatisticsRepository entityStatisticsRepository, FeatureStatisticsRepository featureStatisticsRepository) { this.storeRepository = storeRepository; this.specService = specService; + this.entityStatisticsRepository = entityStatisticsRepository; this.featureStatisticsRepository = featureStatisticsRepository; } + /** + * Get {@link DatasetFeatureStatistics} for the requested feature set in the provided datasets or + * date range for the store provided. The {@link DatasetFeatureStatistics} will contain a list of + * {@link FeatureNameStatistics} for each feature requested. Results retrieved will be cached + * indefinitely. To force Feast to recompute the statistics, set forceRefresh to true. + * + *

Only one of datasetIds or startDate/endDate should be provided. If both are provided, the + * former will be used over the latter. + * + *

If multiple datasetIds or if the date ranges over a few days, statistics will be retrieved + * for each single unit (dataset id or day) and results aggregated across that set. As a result of + * this, in such a scenario, statistics that cannot be aggregated will be dropped. This includes + * all histograms and quantiles, unique values, and top value counts. + * + * @param request {@link GetFeatureStatisticsRequest} containing feature set name, subset of + * features, dataset ids or date range, and store to retrieve the data from. + * @return {@link GetFeatureStatisticsResponse} containing {@link DatasetFeatureStatistics} with + * the feature statistics requested. + * @throws IOException + */ @Transactional public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsRequest request) throws IOException { @@ -79,6 +104,10 @@ public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsReq .map(FeatureSpec::getName) .collect(Collectors.toList()); } + List entities = + featureSetSpec.getEntitiesList().stream() + .map(EntitySpec::getName) + .collect(Collectors.toList()); List> featureNameStatisticsList = new ArrayList<>(); if (request.getDatasetIdsCount() == 0) { // retrieve by date @@ -86,7 +115,12 @@ public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsReq while (timestamp < request.getEndDate().getSeconds()) { List featureNameStatistics = getFeatureNameStatisticsByDate( - statisticsRetriever, featureSetSpec, features, timestamp); + statisticsRetriever, + featureSetSpec, + entities, + features, + timestamp, + request.getForceRefresh()); featureNameStatisticsList.add(featureNameStatistics); timestamp += 86400; // advance by a day } @@ -95,27 +129,38 @@ public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsReq for (String datasetId : request.getDatasetIdsList()) { List featureNameStatistics = getFeatureNameStatisticsByDataset( - statisticsRetriever, featureSetSpec, features, datasetId); + statisticsRetriever, + featureSetSpec, + entities, + features, + datasetId, + request.getForceRefresh()); featureNameStatisticsList.add(featureNameStatistics); } } List featureNameStatistics = mergeStatistics(featureNameStatisticsList); + long totalCount = getTotalCount(featureNameStatistics.get(0)); return GetFeatureStatisticsResponse.newBuilder() .setDatasetFeatureStatisticsList( DatasetFeatureStatisticsList.newBuilder() .addDatasets( - DatasetFeatureStatistics.newBuilder().addAllFeatures(featureNameStatistics))) + DatasetFeatureStatistics.newBuilder() + .setNumExamples(totalCount) + .addAllFeatures(featureNameStatistics))) .build(); } private List getFeatureNameStatisticsByDataset( StatisticsRetriever statisticsRetriever, FeatureSetSpec featureSetSpec, + List entities, List features, - String datasetId) + String datasetId, + boolean forceRefresh) throws IOException { List featureNameStatistics = new ArrayList<>(); List featuresMissingStats = new ArrayList<>(); + List entitiesMissingStats = new ArrayList<>(); for (String featureName : features) { Feature feature = new Feature( @@ -124,28 +169,62 @@ private List getFeatureNameStatisticsByDataset( featureSetSpec.getName(), featureSetSpec.getVersion(), featureName)); - Optional cachedFeatureStatistics = - featureStatisticsRepository.findFeatureStatisticsByFeatureAndDatasetId( - feature, datasetId); + Optional cachedFeatureStatistics = Optional.empty(); + if (!forceRefresh) { + cachedFeatureStatistics = + featureStatisticsRepository.findFeatureStatisticsByFeatureAndDatasetId( + feature, datasetId); + } if (cachedFeatureStatistics.isPresent()) { featureNameStatistics.add(cachedFeatureStatistics.get().toProto()); } else { featuresMissingStats.add(featureName); } } - if (featuresMissingStats.size() > 0) { - FeatureSetStatistics featureStatistics = - statisticsRetriever.getFeatureStatistics(featureSetSpec, featuresMissingStats, datasetId); - for (FeatureNameStatistics stat : featureStatistics.getFeatureNameStatistics()) { - featureStatisticsRepository.save( - FeatureStatistics.fromProto( - featureSetSpec.getProject(), - featureSetSpec.getName(), - featureSetSpec.getVersion(), - stat, - datasetId)); + for (String entityName : entities) { + Entity entity = + new Entity( + new FieldId( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + entityName)); + Optional cachedEntityStatistics = Optional.empty(); + if (!forceRefresh) { + cachedEntityStatistics = + entityStatisticsRepository.findEntityStatisticsByEntityAndDatasetId(entity, datasetId); + } + if (cachedEntityStatistics.isPresent()) { + featureNameStatistics.add(cachedEntityStatistics.get().toProto()); + } else { + entitiesMissingStats.add(entityName); } - featureNameStatistics.addAll(featureStatistics.getFeatureNameStatistics()); + } + if (featuresMissingStats.size() > 0 || entitiesMissingStats.size() > 0) { + FeatureSetStatistics featureSetStatistics = + statisticsRetriever.getFeatureStatistics( + featureSetSpec, entitiesMissingStats, featuresMissingStats, datasetId); + for (FeatureNameStatistics stat : featureSetStatistics.getFeatureNameStatistics()) { + String name = stat.getPath().getStep(0); + if (features.contains(name)) { + featureStatisticsRepository.save( + FeatureStatistics.fromProto( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + stat, + datasetId)); + } else if (entities.contains(name)) { + entityStatisticsRepository.save( + EntityStatistics.fromProto( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + stat, + datasetId)); + } + } + featureNameStatistics.addAll(featureSetStatistics.getFeatureNameStatistics()); } return featureNameStatistics; } @@ -153,12 +232,15 @@ private List getFeatureNameStatisticsByDataset( private List getFeatureNameStatisticsByDate( StatisticsRetriever statisticsRetriever, FeatureSetSpec featureSetSpec, + List entities, List features, - long timestamp) + long timestamp, + boolean forceRefresh) throws IOException { Date date = Date.from(Instant.ofEpochSecond(timestamp)); List featureNameStatistics = new ArrayList<>(); List featuresMissingStats = new ArrayList<>(); + List entitiesMissingStats = new ArrayList<>(); for (String featureName : features) { Feature feature = new Feature( @@ -167,30 +249,64 @@ private List getFeatureNameStatisticsByDate( featureSetSpec.getName(), featureSetSpec.getVersion(), featureName)); - Optional cachedFeatureStatistics = - featureStatisticsRepository.findFeatureStatisticsByFeatureAndDate(feature, date); + Optional cachedFeatureStatistics = Optional.empty(); + if (!forceRefresh) { + cachedFeatureStatistics = + featureStatisticsRepository.findFeatureStatisticsByFeatureAndDate(feature, date); + } if (cachedFeatureStatistics.isPresent()) { featureNameStatistics.add(cachedFeatureStatistics.get().toProto()); } else { featuresMissingStats.add(featureName); } } + for (String entityName : entities) { + Entity entity = + new Entity( + new FieldId( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + entityName)); + Optional cachedEntityStatistics = Optional.empty(); + if (!forceRefresh) { + cachedEntityStatistics = + entityStatisticsRepository.findEntityStatisticsByEntityAndDate(entity, date); + } + if (cachedEntityStatistics.isPresent()) { + featureNameStatistics.add(cachedEntityStatistics.get().toProto()); + } else { + entitiesMissingStats.add(entityName); + } + } if (featuresMissingStats.size() > 0) { - FeatureSetStatistics featureStatistics = + FeatureSetStatistics featureSetStatistics = statisticsRetriever.getFeatureStatistics( featureSetSpec, featuresMissingStats, + entitiesMissingStats, Timestamp.newBuilder().setSeconds(timestamp).build()); - for (FeatureNameStatistics stat : featureStatistics.getFeatureNameStatistics()) { - featureStatisticsRepository.save( - FeatureStatistics.fromProto( - featureSetSpec.getProject(), - featureSetSpec.getName(), - featureSetSpec.getVersion(), - stat, - date)); + for (FeatureNameStatistics stat : featureSetStatistics.getFeatureNameStatistics()) { + String name = stat.getPath().getStep(0); + if (features.contains(name)) { + featureStatisticsRepository.save( + FeatureStatistics.fromProto( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + stat, + date)); + } else if (entities.contains(name)) { + entityStatisticsRepository.save( + EntityStatistics.fromProto( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + stat, + date)); + } } - featureNameStatistics.addAll(featureStatistics.getFeatureNameStatistics()); + featureNameStatistics.addAll(featureSetStatistics.getFeatureNameStatistics()); } return featureNameStatistics; } @@ -433,4 +549,26 @@ private FeatureNameStatistics mergeByteStatistics( return mergedFeatureNameStatistics.setBytesStats(mergedBytesStatistics).build(); } + + private long getTotalCount(FeatureNameStatistics featureNameStatistics) { + CommonStatistics commonStats; + switch (featureNameStatistics.getType()) { + case STRUCT: + commonStats = featureNameStatistics.getStructStats().getCommonStats(); + break; + case STRING: + commonStats = featureNameStatistics.getStringStats().getCommonStats(); + break; + case BYTES: + commonStats = featureNameStatistics.getBytesStats().getCommonStats(); + break; + case FLOAT: + case INT: + commonStats = featureNameStatistics.getNumStats().getCommonStats(); + break; + default: + throw new RuntimeException("Unable to extract dataset size; Invalid type provided"); + } + return commonStats.getNumNonMissing() + commonStats.getNumMissing(); + } } diff --git a/core/src/test/java/feast/core/service/StatsServiceTest.java b/core/src/test/java/feast/core/service/StatsServiceTest.java index e3db263e4c2..764af9be0c9 100644 --- a/core/src/test/java/feast/core/service/StatsServiceTest.java +++ b/core/src/test/java/feast/core/service/StatsServiceTest.java @@ -19,6 +19,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.*; +import feast.core.dao.EntityStatisticsRepository; import feast.core.dao.FeatureStatisticsRepository; import feast.core.dao.StoreRepository; import java.util.Arrays; @@ -33,11 +34,14 @@ public class StatsServiceTest { private StatsService statsService; @Mock private StoreRepository storeRepository; @Mock private FeatureStatisticsRepository featureStatisticsRepository; + @Mock private EntityStatisticsRepository entityStatisticsRepository; @Mock private SpecService specService; @Before public void setUp() { - statsService = new StatsService(storeRepository, specService, featureStatisticsRepository); + statsService = + new StatsService( + storeRepository, specService, entityStatisticsRepository, featureStatisticsRepository); } @Test diff --git a/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java b/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java index c527b3c80cd..251108304d6 100644 --- a/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java +++ b/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java @@ -26,21 +26,23 @@ public interface StatisticsRetriever { * Get feature set statistics for a single feature set, for a single dataset id. * * @param featureSetSpec feature set spec of the requested feature set + * @param entities subset of entities to retrieve. * @param features subset of features to retrieve. * @param dataset dataset id to filter the data by * @return {@link FeatureSetStatistics} containing statistics for the requested features. */ FeatureSetStatistics getFeatureStatistics( - FeatureSetSpec featureSetSpec, List features, String dataset); + FeatureSetSpec featureSetSpec, List entities, List features, String dataset); /** * Get feature set statistics for a single feature set, for a single day. * * @param featureSetSpec feature set spec of the requested feature set + * @param entities subset of entities to retrieve. * @param features subset of features to retrieve. * @param date date to filter the data by * @return {@link FeatureSetStatistics} containing statistics for the requested features. */ FeatureSetStatistics getFeatureStatistics( - FeatureSetSpec featureSetSpec, List features, Timestamp date); + FeatureSetSpec featureSetSpec, List entities, List features, Timestamp date); } diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetriever.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/BigQueryStatisticsRetriever.java similarity index 65% rename from storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetriever.java rename to storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/BigQueryStatisticsRetriever.java index 85d0f2dc13d..8509a7e3c90 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetriever.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/BigQueryStatisticsRetriever.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.storage.connectors.bigquery.stats; +package feast.storage.connectors.bigquery.statistics; -import static feast.storage.connectors.bigquery.stats.StatsUtil.toFeatureNameStatistics; +import static feast.storage.connectors.bigquery.statistics.StatsUtil.toFeatureNameStatistics; import com.google.auto.value.AutoValue; import com.google.cloud.bigquery.BigQuery; @@ -25,6 +25,7 @@ import com.google.cloud.bigquery.TableResult; import com.google.common.collect.Streams; import com.google.protobuf.Timestamp; +import feast.core.FeatureSetProto.EntitySpec; import feast.core.FeatureSetProto.FeatureSetSpec; import feast.core.FeatureSetProto.FeatureSpec; import feast.storage.api.statistics.FeatureSetStatistics; @@ -61,85 +62,102 @@ public abstract static class Builder { @Override public FeatureSetStatistics getFeatureStatistics( - FeatureSetSpec featureSetSpec, List features, String dataset) { + FeatureSetSpec featureSetSpec, List entities, List features, String dataset) { FeatureSetStatisticsQueryInfo featureSetStatisticsQueryInfo = new FeatureSetStatisticsQueryInfo( featureSetSpec.getProject(), featureSetSpec.getName(), featureSetSpec.getVersion(), dataset); - return getFeatureSetStatistics(featureSetSpec, features, featureSetStatisticsQueryInfo); + return getFeatureSetStatistics( + featureSetSpec, entities, features, featureSetStatisticsQueryInfo); } @Override public FeatureSetStatistics getFeatureStatistics( - FeatureSetSpec featureSetSpec, List features, Timestamp date) { + FeatureSetSpec featureSetSpec, List entities, List features, Timestamp date) { FeatureSetStatisticsQueryInfo featureSetStatisticsQueryInfo = new FeatureSetStatisticsQueryInfo( featureSetSpec.getProject(), featureSetSpec.getName(), featureSetSpec.getVersion(), date); - return getFeatureSetStatistics(featureSetSpec, features, featureSetStatisticsQueryInfo); + return getFeatureSetStatistics( + featureSetSpec, entities, features, featureSetStatisticsQueryInfo); } private FeatureSetStatistics getFeatureSetStatistics( FeatureSetSpec featureSetSpec, + List entities, List features, FeatureSetStatisticsQueryInfo featureSetStatisticsQueryInfo) { List featuresList = featureSetSpec.getFeaturesList(); + List entitiesList = featureSetSpec.getEntitiesList(); - FeatureSetSpec.Builder featureSetSpecBuilder = featureSetSpec.toBuilder().clearFeatures(); + FeatureSetSpec.Builder featureSetSpecBuilder = + featureSetSpec.toBuilder().clearFeatures().clearEntities(); for (FeatureSpec featureSpec : featuresList) { if (features.contains(featureSpec.getName())) { featureSetStatisticsQueryInfo.addFeature(featureSpec); - featureSetSpecBuilder = featureSetSpecBuilder.addFeatures(featureSpec); + featureSetSpecBuilder.addFeatures(featureSpec); + } + } + for (EntitySpec entitySpec : entitiesList) { + if (entities.contains(entitySpec.getName())) { + featureSetStatisticsQueryInfo.addEntity(entitySpec); + featureSetSpecBuilder.addEntities(entitySpec); } } featureSetSpec = featureSetSpecBuilder.build(); try { + // Generate SQL for and retrieve non-histogram statistics String getFeatureSetStatsQuery = StatsQueryTemplater.createGetFeatureSetStatsQuery( featureSetStatisticsQueryInfo, projectId(), datasetId()); - String getFeatureSetHistQuery = - StatsQueryTemplater.createGetFeatureSetHistQuery( - featureSetStatisticsQueryInfo, projectId(), datasetId()); QueryJobConfiguration queryJobConfiguration = QueryJobConfiguration.newBuilder(getFeatureSetStatsQuery).build(); TableResult basicStats = bigquery().query(queryJobConfiguration); + + // Generate SQL for and retrieve histogram statistics + String getFeatureSetHistQuery = + StatsQueryTemplater.createGetFeatureSetHistQuery( + featureSetStatisticsQueryInfo, projectId(), datasetId()); queryJobConfiguration = QueryJobConfiguration.newBuilder(getFeatureSetHistQuery).build(); TableResult hist = bigquery().query(queryJobConfiguration); - Map basicStatsValues = - Streams.stream(basicStats.getValues()) - .collect( - Collectors.toMap( - fieldValueList -> fieldValueList.get(0).getStringValue(), - fieldValueList -> fieldValueList)); - Map histValues = - Streams.stream(hist.getValues()) - .collect( - Collectors.toMap( - fieldValueList -> fieldValueList.get(0).getStringValue(), - fieldValueList -> fieldValueList)); + // Convert to map of feature_name:row containing the statistics + Map basicStatsValues = getTableResultByFeatureName(basicStats); + Map histValues = getTableResultByFeatureName(hist); int totalCountIndex = basicStats.getSchema().getFields().getIndex("total_count"); + String ref = (features.size() > 0) ? features.get(0) : entities.get(0); + ; FeatureSetStatistics.Builder featureSetStatisticsBuilder = FeatureSetStatistics.newBuilder() - .setNumExamples( - basicStatsValues.get(features.get(0)).get(totalCountIndex).getLongValue()); + .setNumExamples(basicStatsValues.get(ref).get(totalCountIndex).getLongValue()); + // Convert BQ rows to FeatureNameStatistics for (FeatureSpec featureSpec : featureSetSpec.getFeaturesList()) { FeatureNameStatistics featureNameStatistics = toFeatureNameStatistics( - featureSpec, + featureSpec.getValueType(), basicStats.getSchema(), basicStatsValues.get(featureSpec.getName()), hist.getSchema(), histValues.get(featureSpec.getName())); featureSetStatisticsBuilder.addFeatureNameStatistics(featureNameStatistics); } + for (EntitySpec entitySpec : featureSetSpec.getEntitiesList()) { + FeatureNameStatistics featureNameStatistics = + toFeatureNameStatistics( + entitySpec.getValueType(), + basicStats.getSchema(), + basicStatsValues.get(entitySpec.getName()), + hist.getSchema(), + histValues.get(entitySpec.getName())); + featureSetStatisticsBuilder.addFeatureNameStatistics(featureNameStatistics); + } return featureSetStatisticsBuilder.build(); } catch (IOException | InterruptedException e) { throw new RuntimeException( @@ -149,4 +167,12 @@ private FeatureSetStatistics getFeatureSetStatistics( e); } } + + private Map getTableResultByFeatureName(TableResult basicStats) { + return Streams.stream(basicStats.getValues()) + .collect( + Collectors.toMap( + fieldValueList -> fieldValueList.get(0).getStringValue(), + fieldValueList -> fieldValueList)); + } } diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureSetStatisticsQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java similarity index 80% rename from storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureSetStatisticsQueryInfo.java rename to storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java index 5809db74559..e6f0db7f062 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureSetStatisticsQueryInfo.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java @@ -14,9 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.storage.connectors.bigquery.stats; +package feast.storage.connectors.bigquery.statistics; import com.google.protobuf.Timestamp; +import feast.core.FeatureSetProto.EntitySpec; import feast.core.FeatureSetProto.FeatureSpec; import java.util.ArrayList; import java.util.List; @@ -25,13 +26,17 @@ import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; +/** + * Value class for Feature Sets containing information necessary to template stats-retrieving + * queries. + */ public class FeatureSetStatisticsQueryInfo { private final String project; private final String name; private final int version; private String datasetId = ""; private String date = ""; - private final List features; + private final List features; public FeatureSetStatisticsQueryInfo( String project, @@ -39,7 +44,7 @@ public FeatureSetStatisticsQueryInfo( int version, String datasetId, String date, - List features) { + List features) { this.project = project; this.name = name; this.version = version; @@ -67,7 +72,11 @@ public FeatureSetStatisticsQueryInfo(String project, String name, int version, T } public void addFeature(FeatureSpec featureSpec) { - this.features.add(FeatureStatisticsQueryInfo.fromProto(featureSpec)); + this.features.add(FieldStatisticsQueryInfo.fromProto(featureSpec)); + } + + public void addEntity(EntitySpec entitySpec) { + this.features.add(FieldStatisticsQueryInfo.fromProto(entitySpec)); } public String getProject() { @@ -86,7 +95,7 @@ public String getDatasetId() { return datasetId; } - public List getFeatures() { + public List getFeatures() { return features; } } diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FieldStatisticsQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FieldStatisticsQueryInfo.java new file mode 100644 index 00000000000..4f1e085ff24 --- /dev/null +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FieldStatisticsQueryInfo.java @@ -0,0 +1,94 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.bigquery.statistics; + +import feast.core.FeatureSetProto.EntitySpec; +import feast.core.FeatureSetProto.FeatureSpec; +import feast.types.ValueProto.ValueType.Enum; + +/** + * Value class for Features containing information necessary to template stats-retrieving queries. + */ +public class FieldStatisticsQueryInfo { + private final String name; + private final String type; + + private FieldStatisticsQueryInfo(String name, String type) { + this.name = name; + this.type = type; + } + + public static FieldStatisticsQueryInfo fromProto(FeatureSpec featureSpec) { + Enum valueType = featureSpec.getValueType(); + switch (valueType) { + case FLOAT: + case DOUBLE: + case INT32: + case INT64: + case BOOL: + return new FieldStatisticsQueryInfo(featureSpec.getName(), "NUMERIC"); + case STRING: + return new FieldStatisticsQueryInfo(featureSpec.getName(), "CATEGORICAL"); + case BYTES: + return new FieldStatisticsQueryInfo(featureSpec.getName(), "BYTES"); + case BYTES_LIST: + case BOOL_LIST: + case FLOAT_LIST: + case INT32_LIST: + case INT64_LIST: + case DOUBLE_LIST: + case STRING_LIST: + return new FieldStatisticsQueryInfo(featureSpec.getName(), "LIST"); + default: + throw new IllegalArgumentException("Invalid feature type provided"); + } + } + + public static FieldStatisticsQueryInfo fromProto(EntitySpec entitySpec) { + Enum valueType = entitySpec.getValueType(); + switch (valueType) { + case FLOAT: + case DOUBLE: + case INT32: + case INT64: + case BOOL: + return new FieldStatisticsQueryInfo(entitySpec.getName(), "NUMERIC"); + case STRING: + return new FieldStatisticsQueryInfo(entitySpec.getName(), "CATEGORICAL"); + case BYTES: + return new FieldStatisticsQueryInfo(entitySpec.getName(), "BYTES"); + case BYTES_LIST: + case BOOL_LIST: + case FLOAT_LIST: + case INT32_LIST: + case INT64_LIST: + case DOUBLE_LIST: + case STRING_LIST: + return new FieldStatisticsQueryInfo(entitySpec.getName(), "LIST"); + default: + throw new IllegalArgumentException("Invalid entity type provided"); + } + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } +} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsQueryTemplater.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryTemplater.java similarity index 98% rename from storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsQueryTemplater.java rename to storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryTemplater.java index 1552a1f4f52..8103b2cb48a 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsQueryTemplater.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryTemplater.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.storage.connectors.bigquery.stats; +package feast.storage.connectors.bigquery.statistics; import com.mitchellbosecke.pebble.PebbleEngine; import com.mitchellbosecke.pebble.template.PebbleTemplate; diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsUtil.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsUtil.java similarity index 80% rename from storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsUtil.java rename to storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsUtil.java index 40dca0d7df6..012959f8951 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/StatsUtil.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsUtil.java @@ -14,13 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.storage.connectors.bigquery.stats; +package feast.storage.connectors.bigquery.statistics; import com.google.cloud.bigquery.FieldList; import com.google.cloud.bigquery.FieldValue; import com.google.cloud.bigquery.FieldValueList; import com.google.cloud.bigquery.Schema; -import feast.core.FeatureSetProto.FeatureSpec; +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.Ordering; import feast.types.ValueProto.ValueType; import feast.types.ValueProto.ValueType.Enum; import java.util.HashMap; @@ -54,8 +55,24 @@ public class StatsUtil { TFDV_TYPE_MAP.put(Enum.DOUBLE_LIST, Type.STRUCT); } + /** + * Convert BQ-retrieved statistics to the corresponding TFDV {@link FeatureNameStatistics} + * specific to the feature type. + * + * @param valueType {@link ValueType.Enum} denoting the value type of the feature + * @param basicStatsSchema BigQuery {@link Schema} of the retrieved statistics row for the + * non-histogram statistics. Used to retrieve the column names corresponding to each value in + * the row. + * @param basicStatsValues BigQuery {@link FieldValueList} containing a single row of + * non-histogram statistics retrieved from BigQuery + * @param histSchema BigQuery {@link Schema} of the retrieved statistics row for the histogram + * statistics. Used to retrieve the column names corresponding to each value in the row. + * @param histValues BigQuery {@link FieldValueList} containing a single row of histogram + * statistics retrieved from BigQuery + * @return {@link FeatureNameStatistics} + */ public static FeatureNameStatistics toFeatureNameStatistics( - FeatureSpec featureSpec, + ValueType.Enum valueType, Schema basicStatsSchema, FieldValueList basicStatsValues, Schema histSchema, @@ -75,9 +92,9 @@ public static FeatureNameStatistics toFeatureNameStatistics( Builder featureNameStatisticsBuilder = FeatureNameStatistics.newBuilder() .setPath(Path.newBuilder().addStep(valuesMap.get("feature_name").getStringValue())) - .setType(TFDV_TYPE_MAP.get(featureSpec.getValueType())); + .setType(TFDV_TYPE_MAP.get(valueType)); - switch (featureSpec.getValueType()) { + switch (valueType) { case FLOAT: case BOOL: case DOUBLE: @@ -112,6 +129,10 @@ public static FeatureNameStatistics toFeatureNameStatistics( } private static BytesStatistics getBytesStatistics(Map valuesMap) { + if (valuesMap.get("total_count").getLongValue() == 0) { + return BytesStatistics.getDefaultInstance(); + } + return BytesStatistics.newBuilder() .setCommonStats( CommonStatistics.newBuilder() @@ -129,17 +150,9 @@ private static BytesStatistics getBytesStatistics(Map values } private static StringStatistics getStringStatistics(Map valuesMap) { - List topCount = - valuesMap.get("top_count").getRepeatedValue().stream() - .map( - tc -> { - FieldValueList recordValue = tc.getRecordValue(); - return FreqAndValue.newBuilder() - .setValue(recordValue.get(0).getStringValue()) - .setFrequency(recordValue.get(1).getLongValue()) - .build(); - }) - .collect(Collectors.toList()); + if (valuesMap.get("total_count").getLongValue() == 0) { + return StringStatistics.getDefaultInstance(); + } RankHistogram.Builder rankHistogram = RankHistogram.newBuilder(); valuesMap @@ -154,6 +167,23 @@ private static StringStatistics getStringStatistics(Map valu .setSampleCount(recordValue.get(1).getLongValue())); }); + List topCount = + rankHistogram.getBucketsList().stream() + .sorted( + (a, b) -> + ComparisonChain.start() + .compare( + a.getSampleCount(), b.getSampleCount(), Ordering.natural().reverse()) + .result()) + .limit(5) + .map( + bucket -> + FreqAndValue.newBuilder() + .setValue(bucket.getLabel()) + .setFrequency(bucket.getSampleCount()) + .build()) + .collect(Collectors.toList()); + return StringStatistics.newBuilder() .setUnique(valuesMap.get("unique").getLongValue()) .setCommonStats( @@ -170,6 +200,10 @@ private static StringStatistics getStringStatistics(Map valu } private static NumericStatistics getNumericStatistics(Map valuesMap) { + if (valuesMap.get("total_count").getLongValue() == 0) { + return NumericStatistics.getDefaultInstance(); + } + // Build quantiles long quantileCount = valuesMap.get("feature_count").getLongValue() / 10; Histogram.Builder quantilesBuilder = Histogram.newBuilder().setType(HistogramType.QUANTILES); @@ -220,6 +254,10 @@ private static NumericStatistics getNumericStatistics(Map va } private static StructStatistics getStructStatistics(Map valuesMap) { + if (valuesMap.get("total_count").getLongValue() == 0) { + return StructStatistics.getDefaultInstance(); + } + return StructStatistics.newBuilder() .setCommonStats( CommonStatistics.newBuilder() diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureStatisticsQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureStatisticsQueryInfo.java deleted file mode 100644 index 73bbdfbac2c..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/stats/FeatureStatisticsQueryInfo.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.stats; - -import feast.core.FeatureSetProto.FeatureSpec; -import feast.types.ValueProto.ValueType.Enum; - -public class FeatureStatisticsQueryInfo { - private final String name; - private final String type; - - private FeatureStatisticsQueryInfo(String name, String type) { - this.name = name; - this.type = type; - } - - public static FeatureStatisticsQueryInfo fromProto(FeatureSpec featureSpec) { - Enum valueType = featureSpec.getValueType(); - switch (valueType) { - case FLOAT: - case DOUBLE: - case INT32: - case INT64: - case BOOL: - return new FeatureStatisticsQueryInfo(featureSpec.getName(), "NUMERIC"); - case STRING: - return new FeatureStatisticsQueryInfo(featureSpec.getName(), "CATEGORICAL"); - case BYTES: - return new FeatureStatisticsQueryInfo(featureSpec.getName(), "BYTES"); - case BYTES_LIST: - case BOOL_LIST: - case FLOAT_LIST: - case INT32_LIST: - case INT64_LIST: - case DOUBLE_LIST: - case STRING_LIST: - return new FeatureStatisticsQueryInfo(featureSpec.getName(), "LIST"); - default: - throw new IllegalArgumentException("Invalid feature type provided"); - } - } - - public String getName() { - return name; - } - - public String getType() { - return type; - } -} diff --git a/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql b/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql index e07810affc7..c4c51ef0636 100644 --- a/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql +++ b/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql @@ -30,9 +30,7 @@ SELECT -- quantiles APPROX_QUANTILES(CAST({{ feature.name }} AS FLOAT64), 10) AS quantiles, -- unique - null as unique, - -- top count - ARRAY>[] as top_count + null as unique {% elseif feature.type equals "CATEGORICAL" %} -- mean null as mean, @@ -47,9 +45,7 @@ SELECT -- quantiles ARRAY[] AS quantiles, -- unique - APPROX_COUNT_DISTINCT({{ feature.name }}) as unique, - -- top count - APPROX_TOP_COUNT({{ feature.name }}, 5) as top_count, + COUNT(DISTINCT({{ feature.name }})) as unique {% elseif feature.type equals "BYTES" %} -- mean AVG(BIT_COUNT({{ feature.name }})) as mean, @@ -65,9 +61,7 @@ SELECT -- quantiles ARRAY[] AS quantiles, -- unique - APPROX_COUNT_DISTINCT({{ feature.name }}) as unique, - -- top count - ARRAY>[] as top_count + COUNT(DISTINCT({{ feature.name }})) as unique {% elseif feature.type equals "LIST" %} -- mean AVG(ARRAY_LENGTH({{ feature.name }})) as mean, @@ -83,9 +77,7 @@ SELECT -- quantiles ARRAY[] AS quantiles, -- unique - null as unique, - -- top count - ARRAY>[] as top_count + null as unique {% endif %} FROM subset {% if loop.last %}{% else %}UNION ALL {% endif %} diff --git a/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/StatsUtilTest.java b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsUtilTest.java similarity index 85% rename from storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/StatsUtilTest.java rename to storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsUtilTest.java index 1ab1778b3f0..813dcc7c704 100644 --- a/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/StatsUtilTest.java +++ b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsUtilTest.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.storage.connectors.bigquery.stats; +package feast.storage.connectors.bigquery.statistics; -import static feast.storage.connectors.bigquery.stats.StatsUtil.toFeatureNameStatistics; +import static feast.storage.connectors.bigquery.statistics.StatsUtil.toFeatureNameStatistics; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; @@ -47,12 +47,7 @@ public class StatsUtilTest { com.google.cloud.bigquery.Field.of("min", LegacySQLTypeName.FLOAT), com.google.cloud.bigquery.Field.of("max", LegacySQLTypeName.FLOAT), com.google.cloud.bigquery.Field.of("quantiles", LegacySQLTypeName.NUMERIC), - com.google.cloud.bigquery.Field.of("unique", LegacySQLTypeName.INTEGER), - com.google.cloud.bigquery.Field.of( - "top_count", - LegacySQLTypeName.RECORD, - com.google.cloud.bigquery.Field.of("value", LegacySQLTypeName.STRING), - com.google.cloud.bigquery.Field.of("count", LegacySQLTypeName.INTEGER))); + com.google.cloud.bigquery.Field.of("unique", LegacySQLTypeName.INTEGER)); private Schema histStatsSchema = Schema.of( @@ -99,8 +94,7 @@ public void voidShouldConvertNumericStatsToFeatureNameStatistics() FieldValue.of(Attribute.PRIMITIVE, "6.5"), FieldValue.of(Attribute.PRIMITIVE, "8.5"), FieldValue.of(Attribute.PRIMITIVE, "10.5")))), - FieldValue.of(Attribute.PRIMITIVE, null), - FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())))); + FieldValue.of(Attribute.PRIMITIVE, null))); FieldValueList numericHistFieldValueList = FieldValueList.of( @@ -131,7 +125,7 @@ public void voidShouldConvertNumericStatsToFeatureNameStatistics() FeatureNameStatistics actual = toFeatureNameStatistics( - featureSpec, + featureSpec.getValueType(), basicStatsSchema, numericFieldValueList, histStatsSchema, @@ -159,23 +153,7 @@ public void voidShouldConvertStringStatsToFeatureNameStatistics() FieldValue.of(Attribute.PRIMITIVE, null), FieldValue.of(Attribute.PRIMITIVE, null), FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())), - FieldValue.of(Attribute.PRIMITIVE, "2"), - FieldValue.of( - Attribute.REPEATED, - FieldValueList.of( - Lists.newArrayList( - FieldValue.of( - Attribute.RECORD, - FieldValueList.of( - Lists.newArrayList( - FieldValue.of(Attribute.PRIMITIVE, "a"), - FieldValue.of(Attribute.PRIMITIVE, "1")))), - FieldValue.of( - Attribute.RECORD, - FieldValueList.of( - Lists.newArrayList( - FieldValue.of(Attribute.PRIMITIVE, "b"), - FieldValue.of(Attribute.PRIMITIVE, "2"))))))))); + FieldValue.of(Attribute.PRIMITIVE, "2"))); FieldValueList stringHistFieldValueList = FieldValueList.of( @@ -204,13 +182,13 @@ public void voidShouldConvertStringStatsToFeatureNameStatistics() FeatureNameStatistics actual = toFeatureNameStatistics( - featureSpec, + featureSpec.getValueType(), basicStatsSchema, stringFieldValueList, histStatsSchema, stringHistFieldValueList); String expectedJson = - "{\"type\":\"STRING\",\"stringStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"1\",\"maxNumValues\":\"1\",\"avgNumValues\":1,\"totNumValues\":\"20\"},\"unique\":\"2\",\"topValues\":[{\"value\":\"a\",\"frequency\":1},{\"value\":\"b\",\"frequency\":2}],\"rankHistogram\":{\"buckets\":[{\"label\":\"a\",\"sampleCount\":1},{\"label\":\"b\",\"sampleCount\":2}]}},\"path\":{\"step\":[\"strings\"]}}"; + "{\"type\":\"STRING\",\"stringStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"1\",\"maxNumValues\":\"1\",\"avgNumValues\":1,\"totNumValues\":\"20\"},\"unique\":\"2\",\"topValues\":[{\"value\":\"b\",\"frequency\":2},{\"value\":\"a\",\"frequency\":1}],\"rankHistogram\":{\"buckets\":[{\"label\":\"a\",\"sampleCount\":1},{\"label\":\"b\",\"sampleCount\":2}]}},\"path\":{\"step\":[\"strings\"]}}"; FeatureNameStatistics.Builder expected = FeatureNameStatistics.newBuilder(); JsonFormat.parser().merge(expectedJson, expected); assertThat(actual, equalTo(expected.build())); diff --git a/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetrieverTest.java b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetrieverTest.java deleted file mode 100644 index 28134251cb5..00000000000 --- a/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/stats/BigQueryStatisticsRetrieverTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.stats; - -import com.google.cloud.bigquery.BigQueryOptions; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import feast.core.FeatureSetProto.FeatureSetSpec; -import feast.storage.api.statistics.FeatureSetStatistics; -import java.util.Arrays; -import org.junit.Test; - -public class BigQueryStatisticsRetrieverTest { - - @Test - public void shouldRun() throws InvalidProtocolBufferException { - BigQueryStatisticsRetriever retriever = - BigQueryStatisticsRetriever.newBuilder() - .setBigquery(BigQueryOptions.getDefaultInstance().getService()) - .setDatasetId("feast_test_20200202") - .setProjectId("aliz-development") - .build(); - - // FeatureSetSpec featureSetSpec = FeatureSetSpec.newBuilder() - // .setProject("metrics_test") - // .setName("customer_transactions") - // .setVersion(1) - // - // .addEntities(EntitySpec.newBuilder().setName("customer_id").setValueType(Enum.INT64)) - // - // .addFeatures(FeatureSpec.newBuilder().setName("total_transactions").setValueType(Enum.INT64)) - // - // .addFeatures(FeatureSpec.newBuilder().setName("daily_transactions").setValueType(Enum.FLOAT)) - // .build(); - - FeatureSetSpec.Builder featureSetSpec = FeatureSetSpec.newBuilder(); - String bigStatsJson = - "{\"project\":\"metrics_test\",\"maxAge\":\"345599s\",\"name\":\"big\",\"entities\":[{\"name\":\"driver\",\"valueType\":\"STRING\"}],\"features\":[{\"name\":\"ride_driver_id_num_completed\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_prop_completed\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_distance_completed\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_customer_distance_completed\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_distance_cancelled\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_customer_distance_cancelled\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_standing_completed_1\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_standing_completed_2\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_standing_completed_3\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_standing_cancelled_1\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_standing_cancelled_2\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_standing_cancelled_3\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_origin_completed_1\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_origin_completed_2\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_origin_completed_3\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_origin_cancelled_1\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_origin_cancelled_2\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_origin_cancelled_3\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_destination_completed_1\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_destination_completed_2\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_destination_completed_3\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_destination_cancelled_1\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_destination_cancelled_2\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_destination_cancelled_3\",\"valueType\":\"INT64\"},{\"name\":\"ride_driver_id_donut_count_completed\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_donut_count_cancelled\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_num_completed_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_prop_completed_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_distance_completed_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_customer_distance_completed_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_distance_cancelled_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_avg_customer_distance_cancelled_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_donut_count_completed_recent\",\"valueType\":\"FLOAT\"},{\"name\":\"ride_driver_id_donut_count_cancelled_recent\",\"valueType\":\"FLOAT\"}]}"; - JsonFormat.parser().merge(bigStatsJson, featureSetSpec); - featureSetSpec.setVersion(1); - FeatureSetStatistics featureStatistics = - retriever.getFeatureStatistics( - featureSetSpec.build(), - Arrays.asList( - "ride_driver_id_destination_cancelled_3", - "ride_driver_id_avg_customer_distance_completed_recent", - "ride_driver_id_avg_distance_completed_recent", - "ride_driver_id_origin_cancelled_2", - "ride_driver_id_donut_count_cancelled", - "ride_driver_id_destination_completed_3", - "ride_driver_id_prop_completed", - "ride_driver_id_origin_completed_2", - "ride_driver_id_standing_cancelled_1", - "ride_driver_id_avg_customer_distance_cancelled_recent", - "ride_driver_id_avg_customer_distance_cancelled", - "ride_driver_id_donut_count_completed_recent", - "ride_driver_id_standing_completed_1", - "ride_driver_id_avg_distance_cancelled_recent", - "ride_driver_id_destination_cancelled_1", - "ride_driver_id_standing_cancelled_2", - "ride_driver_id_origin_completed_3", - "ride_driver_id_standing_completed_2", - "ride_driver_id_donut_count_completed", - "ride_driver_id_origin_cancelled_1", - "ride_driver_id_origin_cancelled_3", - "ride_driver_id_origin_completed_1", - "ride_driver_id_num_completed", - "ride_driver_id_destination_cancelled_2", - "ride_driver_id_prop_completed_recent", - "ride_driver_id_standing_cancelled_3", - "ride_driver_id_avg_distance_cancelled", - "ride_driver_id_avg_customer_distance_completed", - "ride_driver_id_donut_count_cancelled_recent", - "ride_driver_id_destination_completed_2", - "ride_driver_id_avg_distance_completed", - "ride_driver_id_num_completed_recent", - "ride_driver_id_standing_completed_3", - "ride_driver_id_destination_completed_1"), - "dataset"); - } -} From 6d22f12ee2d5c6e89d6a219dd7c466adc81eeb5d Mon Sep 17 00:00:00 2001 From: zhilingc Date: Mon, 13 Apr 2020 15:35:08 +0800 Subject: [PATCH 04/12] Add example notebook --- .../core/dao/EntityStatisticsRepository.java | 31 - .../java/feast/core/grpc/CoreServiceImpl.java | 14 +- .../main/java/feast/core/model/Entity.java | 86 +-- .../feast/core/model/EntityReference.java | 70 ++ .../feast/core/model/EntityStatistics.java | 266 ------- .../main/java/feast/core/model/Feature.java | 51 +- .../{FieldId.java => FeatureReference.java} | 8 +- .../java/feast/core/model/FeatureSet.java | 74 +- .../feast/core/model/FeatureStatistics.java | 20 +- .../src/main/java/feast/core/model/Field.java | 75 -- .../java/feast/core/service/StatsService.java | 293 ++++---- .../feast/core/service/JobServiceTest.java | 10 +- .../feast/core/service/SpecServiceTest.java | 84 +-- .../feast/core/service/StatsServiceTest.java | 81 +- ...atistics with Feast, TFDV and Facets.ipynb | 706 ++++++++++++++++++ protos/feast/core/FeatureSet.proto | 40 - .../proto/v0/statistics.proto | 1 + sdk/python/feast/client.py | 9 +- .../api/statistics/StatisticsRetriever.java | 6 +- .../BigQueryStatisticsRetriever.java | 68 +- .../FeatureSetStatisticsQueryInfo.java | 40 +- .../statistics/FieldStatisticsQueryInfo.java | 9 +- .../{StatsUtil.java => StatsQueryResult.java} | 154 ++-- .../main/resources/templates/basic_stats.sql | 2 +- ...tilTest.java => StatsQueryResultTest.java} | 122 ++- tests/e2e/feature-validation.py | 152 +++- .../expected_output_basic_dataset.json | 310 -------- 27 files changed, 1556 insertions(+), 1226 deletions(-) delete mode 100644 core/src/main/java/feast/core/dao/EntityStatisticsRepository.java create mode 100644 core/src/main/java/feast/core/model/EntityReference.java delete mode 100644 core/src/main/java/feast/core/model/EntityStatistics.java rename core/src/main/java/feast/core/model/{FieldId.java => FeatureReference.java} (91%) delete mode 100644 core/src/main/java/feast/core/model/Field.java create mode 100644 examples/statistics/Historical Feature Statistics with Feast, TFDV and Facets.ipynb rename storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/{StatsUtil.java => StatsQueryResult.java} (65%) rename storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/{StatsUtilTest.java => StatsQueryResultTest.java} (65%) delete mode 100644 tests/e2e/statistics/expected_output_basic_dataset.json diff --git a/core/src/main/java/feast/core/dao/EntityStatisticsRepository.java b/core/src/main/java/feast/core/dao/EntityStatisticsRepository.java deleted file mode 100644 index a65daafb0bc..00000000000 --- a/core/src/main/java/feast/core/dao/EntityStatisticsRepository.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.dao; - -import feast.core.model.Entity; -import feast.core.model.EntityStatistics; -import java.util.Date; -import java.util.Optional; -import org.springframework.data.jpa.repository.JpaRepository; - -/** JPA repository supplying Statistics objects for entities keyed by id. */ -public interface EntityStatisticsRepository extends JpaRepository { - Optional findEntityStatisticsByEntityAndDatasetId( - Entity entity, String datasetId); - - Optional findEntityStatisticsByEntityAndDate(Entity entity, Date date); -} diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index 513fe23c79f..f3d93b9e9d1 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -30,7 +30,6 @@ import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; -import java.io.IOException; import java.util.List; import java.util.NoSuchElementException; import java.util.stream.Collectors; @@ -103,7 +102,18 @@ public void getFeatureStatistics( GetFeatureStatisticsResponse response = statsService.getFeatureStatistics(request); responseObserver.onNext(response); responseObserver.onCompleted(); - } catch (RetrievalException | IllegalArgumentException | IOException e) { + } catch (IllegalArgumentException e) { + log.error("Illegal arguments provided to GetFeatureStatistics method: ", e); + responseObserver.onError( + Status.INVALID_ARGUMENT + .withDescription(e.getMessage()) + .withCause(e) + .asRuntimeException()); + } catch (RetrievalException e) { + log.error("Unable to fetch feature set requested in GetFeatureStatistics method: ", e); + responseObserver.onError( + Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException()); + } catch (Exception e) { log.error("Exception has occurred in GetFeatureStatistics method: ", e); responseObserver.onError( Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); diff --git a/core/src/main/java/feast/core/model/Entity.java b/core/src/main/java/feast/core/model/Entity.java index 1b827aef6aa..b9d2a8827f2 100644 --- a/core/src/main/java/feast/core/model/Entity.java +++ b/core/src/main/java/feast/core/model/Entity.java @@ -19,6 +19,7 @@ import feast.core.FeatureSetProto.EntitySpec; import feast.types.ValueProto.ValueType; import java.util.Objects; +import javax.persistence.EmbeddedId; import lombok.Getter; import lombok.Setter; @@ -26,85 +27,26 @@ @Getter @Setter @javax.persistence.Entity -public class Entity extends Field { +public class Entity { + @EmbeddedId private EntityReference reference; + + private String type; public Entity() {} - public Entity(FieldId fieldId) { - this.setId(fieldId); + private Entity(String name, ValueType.Enum type) { + this.setReference(new EntityReference(name)); + this.setType(type.toString()); } - public Entity(String name, ValueType.Enum type) { - this.setId(new FieldId()); - this.setName(name); - this.setType(type.toString()); + public static Entity withRef(EntityReference entityRef) { + Entity entity = new Entity(); + entity.setReference(entityRef); + return entity; } public static Entity fromProto(EntitySpec entitySpec) { Entity entity = new Entity(entitySpec.getName(), entitySpec.getValueType()); - - switch (entitySpec.getPresenceConstraintsCase()) { - case PRESENCE: - entity.setPresence(entitySpec.getPresence().toByteArray()); - break; - case GROUP_PRESENCE: - entity.setGroupPresence(entitySpec.getGroupPresence().toByteArray()); - break; - case PRESENCECONSTRAINTS_NOT_SET: - break; - } - - switch (entitySpec.getShapeTypeCase()) { - case SHAPE: - entity.setShape(entitySpec.getShape().toByteArray()); - break; - case VALUE_COUNT: - entity.setValueCount(entitySpec.getValueCount().toByteArray()); - break; - case SHAPETYPE_NOT_SET: - break; - } - - switch (entitySpec.getDomainInfoCase()) { - case DOMAIN: - entity.setDomain(entitySpec.getDomain()); - break; - case INT_DOMAIN: - entity.setIntDomain(entitySpec.getIntDomain().toByteArray()); - break; - case FLOAT_DOMAIN: - entity.setFloatDomain(entitySpec.getFloatDomain().toByteArray()); - break; - case STRING_DOMAIN: - entity.setStringDomain(entitySpec.getStringDomain().toByteArray()); - break; - case BOOL_DOMAIN: - entity.setBoolDomain(entitySpec.getBoolDomain().toByteArray()); - break; - case STRUCT_DOMAIN: - entity.setStructDomain(entitySpec.getStructDomain().toByteArray()); - break; - case NATURAL_LANGUAGE_DOMAIN: - entity.setNaturalLanguageDomain(entitySpec.getNaturalLanguageDomain().toByteArray()); - break; - case IMAGE_DOMAIN: - entity.setImageDomain(entitySpec.getImageDomain().toByteArray()); - break; - case MID_DOMAIN: - entity.setMidDomain(entitySpec.getMidDomain().toByteArray()); - break; - case URL_DOMAIN: - entity.setUrlDomain(entitySpec.getUrlDomain().toByteArray()); - break; - case TIME_DOMAIN: - entity.setTimeDomain(entitySpec.getTimeDomain().toByteArray()); - break; - case TIME_OF_DAY_DOMAIN: - entity.setTimeOfDayDomain(entitySpec.getTimeOfDayDomain().toByteArray()); - break; - case DOMAININFO_NOT_SET: - break; - } return entity; } @@ -117,11 +59,11 @@ public boolean equals(Object o) { return false; } Entity feature = (Entity) o; - return getId().equals(feature.getId()) && getType().equals(feature.getType()); + return getReference().equals(feature.getReference()) && getType().equals(feature.getType()); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), getId(), getType()); + return Objects.hash(super.hashCode(), getReference(), getType()); } } diff --git a/core/src/main/java/feast/core/model/EntityReference.java b/core/src/main/java/feast/core/model/EntityReference.java new file mode 100644 index 00000000000..3b0966de808 --- /dev/null +++ b/core/src/main/java/feast/core/model/EntityReference.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.model; + +import java.io.Serializable; +import java.util.Objects; +import javax.persistence.Column; +import javax.persistence.Embeddable; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +@Embeddable +@AllArgsConstructor +@Getter +@Setter +public class EntityReference implements Serializable { + // Project the field belongs to + @Column(nullable = false) + private String project; + + // Feature set the field belongs to + @Column(name = "feature_set", nullable = false) + private String featureSet; + + // Version of the feature set this field belongs to + @Column(nullable = false) + private int version; + + // Name of the field + @Column(nullable = false) + private String name; + + EntityReference(String name) { + this.name = name; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + EntityReference fieldId = (EntityReference) o; + return Objects.equals(name, fieldId.getName()) + && Objects.equals(project, fieldId.getProject()) + && Objects.equals(featureSet, fieldId.getFeatureSet()); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), project, featureSet, name); + } +} diff --git a/core/src/main/java/feast/core/model/EntityStatistics.java b/core/src/main/java/feast/core/model/EntityStatistics.java deleted file mode 100644 index 2b46a5ae67e..00000000000 --- a/core/src/main/java/feast/core/model/EntityStatistics.java +++ /dev/null @@ -1,266 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import com.google.protobuf.InvalidProtocolBufferException; -import java.io.*; -import java.util.Date; -import java.util.List; -import javax.persistence.*; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import org.tensorflow.metadata.v0.*; - -@NoArgsConstructor -@Getter -@Setter -@javax.persistence.Entity -@Table( - name = "entity_statistics", - indexes = { - @Index( - name = "idx_entity_statistics_entity", - columnList = "project,feature_set,version,name"), - @Index(name = "idx_entity_statistics_dataset_id", columnList = "datasetId"), - @Index(name = "idx_entity_statistics_date", columnList = "date"), - }) -public class EntityStatistics { - @Id - @GeneratedValue(strategy = GenerationType.AUTO) - private int id; - - @ManyToOne - @JoinColumns({ - @JoinColumn(name = "project", referencedColumnName = "project"), - @JoinColumn(name = "feature_set", referencedColumnName = "feature_set"), - @JoinColumn(name = "version", referencedColumnName = "version"), - @JoinColumn(name = "name", referencedColumnName = "name") - }) - private Entity entity; - - // Only one of these fields should be populated. - private String datasetId; - private Date date; - - // General statistics - private String featureType; - private long count; - private long numMissing; - private long minNumValues; - private long maxNumValues; - private float avgNumValues; - private long totalNumValues; - private byte[] numValuesHistogram; - - // Numeric statistics - private double mean; - private double stdev; - private long zeroes; - private double min; - private double max; - private double median; - private byte[] numericValueHistogram; - private byte[] numericValueQuantiles; - - // String statistics - @Column(name = "n_unique") - private long unique; - - private float averageLength; - private byte[] rankHistogram; - private byte[] topValues; - - // Byte statistics - private float minBytes; - private float maxBytes; - private float avgBytes; - - // Instantiates a Statistics object from a tensorflow metadata FeatureNameStatistics object and a - // dataset ID. - public static EntityStatistics fromProto( - String project, - String featureSetName, - int version, - FeatureNameStatistics featureNameStatistics, - String datasetId) - throws IOException { - EntityStatistics featureStatistics = EntityStatistics.fromProto(featureNameStatistics); - Entity entity = new Entity(); - entity.setId( - new FieldId(project, featureSetName, version, featureNameStatistics.getPath().getStep(0))); - featureStatistics.setEntity(entity); - featureStatistics.setDatasetId(datasetId); - return featureStatistics; - } - - // Instantiates a Statistics object from a tensorflow metadata FeatureNameStatistics object and a - // date. - public static EntityStatistics fromProto( - String project, - String featureSetName, - int version, - FeatureNameStatistics featureNameStatistics, - Date date) - throws IOException { - EntityStatistics entityStatistics = EntityStatistics.fromProto(featureNameStatistics); - entityStatistics.setDate(date); - Entity entity = new Entity(); - entity.setId( - new FieldId(project, featureSetName, version, featureNameStatistics.getPath().getStep(0))); - entityStatistics.setEntity(entity); - return entityStatistics; - } - - public FeatureNameStatistics toProto() throws InvalidProtocolBufferException { - FeatureNameStatistics.Builder featureNameStatisticsBuilder = - FeatureNameStatistics.newBuilder() - .setType(FeatureNameStatistics.Type.valueOf(featureType)) - .setPath(Path.newBuilder().addStep(entity.getId().getName())); - CommonStatistics commonStatistics = - CommonStatistics.newBuilder() - .setNumNonMissing(count - numMissing) - .setNumMissing(numMissing) - .setMaxNumValues(maxNumValues) - .setMinNumValues(minNumValues) - .setTotNumValues(totalNumValues) - .setNumValuesHistogram(Histogram.parseFrom(numValuesHistogram)) - .build(); - - switch (featureNameStatisticsBuilder.getType()) { - case INT: - case FLOAT: - NumericStatistics numStats = - NumericStatistics.newBuilder() - .setCommonStats(commonStatistics) - .setMean(mean) - .setStdDev(stdev) - .setNumZeros(zeroes) - .setMin(min) - .setMax(max) - .setMedian(median) - .addHistograms(Histogram.parseFrom(numericValueHistogram)) - .addHistograms(Histogram.parseFrom(numericValueQuantiles)) - .build(); - featureNameStatisticsBuilder.setNumStats(numStats); - break; - case STRING: - StringStatistics.Builder stringStats = - StringStatistics.newBuilder() - .setCommonStats(commonStatistics) - .setUnique(unique) - .setAvgLength(averageLength); - if (rankHistogram == null) { - stringStats.setRankHistogram(RankHistogram.getDefaultInstance()); - } else { - stringStats.setRankHistogram(RankHistogram.parseFrom(rankHistogram)); - } - try (ByteArrayInputStream bis = new ByteArrayInputStream(topValues)) { - ObjectInputStream ois = new ObjectInputStream(bis); - List freqAndValueList = - (List) ois.readObject(); - stringStats.addAllTopValues(freqAndValueList); - } catch (IOException | ClassNotFoundException e) { - throw new InvalidProtocolBufferException( - "Failed to parse field: StringStatistics.TopValues. Check if the value is malformed."); - } - featureNameStatisticsBuilder.setStringStats(stringStats); - break; - case BYTES: - BytesStatistics bytesStats = - BytesStatistics.newBuilder() - .setCommonStats(commonStatistics) - .setAvgNumBytes(avgBytes) - .setMinNumBytes(minBytes) - .setMaxNumBytes(maxBytes) - .build(); - featureNameStatisticsBuilder.setBytesStats(bytesStats); - break; - case STRUCT: - StructStatistics structStats = - StructStatistics.newBuilder().setCommonStats(commonStatistics).build(); - featureNameStatisticsBuilder.setStructStats(structStats); - break; - } - return featureNameStatisticsBuilder.build(); - } - - private static EntityStatistics fromProto(FeatureNameStatistics featureNameStatistics) - throws IOException, IllegalArgumentException { - EntityStatistics featureStatistics = new EntityStatistics(); - featureStatistics.setFeatureType(featureNameStatistics.getType().toString()); - CommonStatistics commonStats; - switch (featureNameStatistics.getType()) { - case FLOAT: - case INT: - NumericStatistics numStats = featureNameStatistics.getNumStats(); - commonStats = numStats.getCommonStats(); - featureStatistics.setMean(numStats.getMean()); - featureStatistics.setStdev(numStats.getStdDev()); - featureStatistics.setZeroes(numStats.getNumZeros()); - featureStatistics.setMin(numStats.getMin()); - featureStatistics.setMax(numStats.getMax()); - featureStatistics.setMedian(numStats.getMedian()); - for (Histogram histogram : numStats.getHistogramsList()) { - switch (histogram.getType()) { - case STANDARD: - featureStatistics.setNumericValueHistogram(histogram.toByteArray()); - case QUANTILES: - featureStatistics.setNumericValueQuantiles(histogram.toByteArray()); - default: - // invalid type, dropping the values - } - } - break; - case STRING: - StringStatistics stringStats = featureNameStatistics.getStringStats(); - commonStats = stringStats.getCommonStats(); - featureStatistics.setUnique(stringStats.getUnique()); - featureStatistics.setAverageLength(stringStats.getAvgLength()); - featureStatistics.setRankHistogram(stringStats.getRankHistogram().toByteArray()); - try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) { - ObjectOutputStream oos = new ObjectOutputStream(bos); - oos.writeObject(stringStats.getTopValuesList()); - featureStatistics.setTopValues(bos.toByteArray()); - } - break; - case BYTES: - BytesStatistics bytesStats = featureNameStatistics.getBytesStats(); - commonStats = bytesStats.getCommonStats(); - featureStatistics.setUnique(bytesStats.getUnique()); - featureStatistics.setMaxBytes(bytesStats.getMaxNumBytes()); - featureStatistics.setMinBytes(bytesStats.getMinNumBytes()); - featureStatistics.setAvgBytes(bytesStats.getAvgNumBytes()); - break; - case STRUCT: - StructStatistics structStats = featureNameStatistics.getStructStats(); - commonStats = structStats.getCommonStats(); - break; - default: - throw new IllegalArgumentException("Feature statistics provided were of unknown type."); - } - featureStatistics.setCount(commonStats.getNumMissing() + commonStats.getNumNonMissing()); - featureStatistics.setNumMissing(commonStats.getNumMissing()); - featureStatistics.setMinNumValues(commonStats.getMinNumValues()); - featureStatistics.setMaxNumValues(commonStats.getMaxNumValues()); - featureStatistics.setAvgNumValues(commonStats.getAvgNumValues()); - featureStatistics.setTotalNumValues(commonStats.getTotNumValues()); - featureStatistics.setNumValuesHistogram(commonStats.getNumValuesHistogram().toByteArray()); - - return featureStatistics; - } -} diff --git a/core/src/main/java/feast/core/model/Feature.java b/core/src/main/java/feast/core/model/Feature.java index a2a840fc6cf..907e3446a1b 100644 --- a/core/src/main/java/feast/core/model/Feature.java +++ b/core/src/main/java/feast/core/model/Feature.java @@ -32,20 +32,51 @@ @Getter @Setter @Entity -public class Feature extends Field { +public class Feature { - public Feature() {} + @EmbeddedId private FeatureReference reference; - public Feature(FieldId fieldId) { - this.setId(fieldId); - } + // Type of the field + private String type; + + // Presence constraints (refer to proto feast.core.FeatureSet.FeatureSpec) + // Only one of them can be set. + private byte[] presence; + private byte[] groupPresence; + + // Shape type (refer to proto feast.core.FeatureSet.FeatureSpec) + // Only one of them can be set. + private byte[] shape; + private byte[] valueCount; + + // Domain info for the values (refer to proto feast.core.FeatureSet.FeatureSpec) + // Only one of them can be set. + private String domain; + private byte[] intDomain; + private byte[] floatDomain; + private byte[] stringDomain; + private byte[] boolDomain; + private byte[] structDomain; + private byte[] naturalLanguageDomain; + private byte[] imageDomain; + private byte[] midDomain; + private byte[] urlDomain; + private byte[] timeDomain; + private byte[] timeOfDayDomain; - public Feature(String name, ValueType.Enum type) { - this.setId(new FieldId()); - this.setName(name); + private Feature() {} + + private Feature(String name, ValueType.Enum type) { + this.setReference(new FeatureReference(name)); this.setType(type.toString()); } + public static Feature withReference(FeatureReference featureRef) { + Feature feature = new Feature(); + feature.setReference(featureRef); + return feature; + } + public static Feature fromProto(FeatureSpec featureSpec) { Feature feature = new Feature(featureSpec.getName(), featureSpec.getValueType()); @@ -123,7 +154,7 @@ public boolean equals(Object o) { return false; } Feature feature = (Feature) o; - return Objects.equals(getId(), feature.getId()) + return Objects.equals(getReference(), feature.getReference()) && Arrays.equals(getPresence(), feature.getPresence()) && Arrays.equals(getGroupPresence(), feature.getGroupPresence()) && Arrays.equals(getShape(), feature.getShape()) @@ -144,6 +175,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(super.hashCode(), getId(), getType()); + return Objects.hash(super.hashCode(), getReference(), getType()); } } diff --git a/core/src/main/java/feast/core/model/FieldId.java b/core/src/main/java/feast/core/model/FeatureReference.java similarity index 91% rename from core/src/main/java/feast/core/model/FieldId.java rename to core/src/main/java/feast/core/model/FeatureReference.java index 8f65f1f2091..b27c17f1b94 100644 --- a/core/src/main/java/feast/core/model/FieldId.java +++ b/core/src/main/java/feast/core/model/FeatureReference.java @@ -30,7 +30,7 @@ @AllArgsConstructor @Getter @Setter -public class FieldId implements Serializable { +public class FeatureReference implements Serializable { // Project the field belongs to @Column(nullable = false) private String project; @@ -47,6 +47,10 @@ public class FieldId implements Serializable { @Column(nullable = false) private String name; + FeatureReference(String name) { + this.name = name; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -55,7 +59,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - FieldId fieldId = (FieldId) o; + FeatureReference fieldId = (FeatureReference) o; return Objects.equals(name, fieldId.getName()) && Objects.equals(project, fieldId.getProject()) && Objects.equals(featureSet, fieldId.getFeatureSet()); diff --git a/core/src/main/java/feast/core/model/FeatureSet.java b/core/src/main/java/feast/core/model/FeatureSet.java index 6d9c3dec079..72aa9a9204b 100644 --- a/core/src/main/java/feast/core/model/FeatureSet.java +++ b/core/src/main/java/feast/core/model/FeatureSet.java @@ -159,9 +159,10 @@ public void addEntities(List entities) { } public void addEntity(Entity entity) { - entity.setProject(this.project.getName()); - entity.setFeatureSet(this.getName()); - entity.setVersion(this.getVersion()); + EntityReference entityReference = entity.getReference(); + entityReference.setProject(this.project.getName()); + entityReference.setFeatureSet(this.getName()); + entityReference.setVersion(this.getVersion()); entities.add(entity); } @@ -172,9 +173,10 @@ public void addFeatures(List features) { } public void addFeature(Feature feature) { - feature.setProject(this.project.getName()); - feature.setFeatureSet(this.getName()); - feature.setVersion(this.getVersion()); + FeatureReference featureReference = feature.getReference(); + featureReference.setProject(this.project.getName()); + featureReference.setFeatureSet(this.getName()); + featureReference.setVersion(this.getVersion()); features.add(feature); } @@ -212,58 +214,16 @@ public FeatureSetProto.FeatureSet toProto() throws InvalidProtocolBufferExceptio return FeatureSetProto.FeatureSet.newBuilder().setMeta(meta).setSpec(spec).build(); } - private void setEntitySpecFields(EntitySpec.Builder entitySpecBuilder, Entity entityField) - throws InvalidProtocolBufferException { + private void setEntitySpecFields(EntitySpec.Builder entitySpecBuilder, Entity entityField) { entitySpecBuilder - .setName(entityField.getId().getName()) + .setName(entityField.getReference().getName()) .setValueType(Enum.valueOf(entityField.getType())); - - if (entityField.getPresence() != null) { - entitySpecBuilder.setPresence(FeaturePresence.parseFrom(entityField.getPresence())); - } else if (entityField.getGroupPresence() != null) { - entitySpecBuilder.setGroupPresence( - FeaturePresenceWithinGroup.parseFrom(entityField.getGroupPresence())); - } - - if (entityField.getShape() != null) { - entitySpecBuilder.setShape(FixedShape.parseFrom(entityField.getShape())); - } else if (entityField.getValueCount() != null) { - entitySpecBuilder.setValueCount(ValueCount.parseFrom(entityField.getValueCount())); - } - - if (entityField.getDomain() != null) { - entitySpecBuilder.setDomain(entityField.getDomain()); - } else if (entityField.getIntDomain() != null) { - entitySpecBuilder.setIntDomain(IntDomain.parseFrom(entityField.getIntDomain())); - } else if (entityField.getFloatDomain() != null) { - entitySpecBuilder.setFloatDomain(FloatDomain.parseFrom(entityField.getFloatDomain())); - } else if (entityField.getStringDomain() != null) { - entitySpecBuilder.setStringDomain(StringDomain.parseFrom(entityField.getStringDomain())); - } else if (entityField.getBoolDomain() != null) { - entitySpecBuilder.setBoolDomain(BoolDomain.parseFrom(entityField.getBoolDomain())); - } else if (entityField.getStructDomain() != null) { - entitySpecBuilder.setStructDomain(StructDomain.parseFrom(entityField.getStructDomain())); - } else if (entityField.getNaturalLanguageDomain() != null) { - entitySpecBuilder.setNaturalLanguageDomain( - NaturalLanguageDomain.parseFrom(entityField.getNaturalLanguageDomain())); - } else if (entityField.getImageDomain() != null) { - entitySpecBuilder.setImageDomain(ImageDomain.parseFrom(entityField.getImageDomain())); - } else if (entityField.getMidDomain() != null) { - entitySpecBuilder.setIntDomain(IntDomain.parseFrom(entityField.getIntDomain())); - } else if (entityField.getUrlDomain() != null) { - entitySpecBuilder.setUrlDomain(URLDomain.parseFrom(entityField.getUrlDomain())); - } else if (entityField.getTimeDomain() != null) { - entitySpecBuilder.setTimeDomain(TimeDomain.parseFrom(entityField.getTimeDomain())); - } else if (entityField.getTimeOfDayDomain() != null) { - entitySpecBuilder.setTimeOfDayDomain( - TimeOfDayDomain.parseFrom(entityField.getTimeOfDayDomain())); - } } private void setFeatureSpecFields(FeatureSpec.Builder featureSpecBuilder, Feature featureField) throws InvalidProtocolBufferException { featureSpecBuilder - .setName(featureField.getId().getName()) + .setName(featureField.getReference().getName()) .setValueType(Enum.valueOf(featureField.getType())); if (featureField.getPresence() != null) { @@ -336,11 +296,11 @@ public boolean equalTo(FeatureSet other) { Map featuresMap = new HashMap<>(); for (Entity e : entities) { - entitiesMap.putIfAbsent(e.getId().getName(), e); + entitiesMap.putIfAbsent(e.getReference().getName(), e); } for (Feature f : features) { - featuresMap.putIfAbsent(f.getId().getName(), f); + featuresMap.putIfAbsent(f.getReference().getName(), f); } // Ensure map size is consistent with existing fields @@ -353,19 +313,19 @@ public boolean equalTo(FeatureSet other) { // Ensure the other entities and features exist in the field map for (Entity e : other.getEntities()) { - if (!entitiesMap.containsKey(e.getId().getName())) { + if (!entitiesMap.containsKey(e.getReference().getName())) { return false; } - if (!e.equals(entitiesMap.get(e.getId().getName()))) { + if (!e.equals(entitiesMap.get(e.getReference().getName()))) { return false; } } for (Feature f : other.getFeatures()) { - if (!featuresMap.containsKey(f.getId().getName())) { + if (!featuresMap.containsKey(f.getReference().getName())) { return false; } - if (!f.equals(featuresMap.get(f.getId().getName()))) { + if (!f.equals(featuresMap.get(f.getReference().getName()))) { return false; } } diff --git a/core/src/main/java/feast/core/model/FeatureStatistics.java b/core/src/main/java/feast/core/model/FeatureStatistics.java index 1cb9ebb46b2..a54dd35c059 100644 --- a/core/src/main/java/feast/core/model/FeatureStatistics.java +++ b/core/src/main/java/feast/core/model/FeatureStatistics.java @@ -93,7 +93,7 @@ public class FeatureStatistics { // Instantiates a Statistics object from a tensorflow metadata FeatureNameStatistics object and a // dataset ID. - public static FeatureStatistics fromProto( + public static FeatureStatistics createForDataset( String project, String featureSetName, int version, @@ -101,9 +101,10 @@ public static FeatureStatistics fromProto( String datasetId) throws IOException { FeatureStatistics featureStatistics = FeatureStatistics.fromProto(featureNameStatistics); - Feature feature = new Feature(); - feature.setId( - new FieldId(project, featureSetName, version, featureNameStatistics.getPath().getStep(0))); + Feature feature = + Feature.withReference( + new FeatureReference( + project, featureSetName, version, featureNameStatistics.getPath().getStep(0))); featureStatistics.setFeature(feature); featureStatistics.setDatasetId(datasetId); return featureStatistics; @@ -111,7 +112,7 @@ public static FeatureStatistics fromProto( // Instantiates a Statistics object from a tensorflow metadata FeatureNameStatistics object and a // date. - public static FeatureStatistics fromProto( + public static FeatureStatistics createForDate( String project, String featureSetName, int version, @@ -120,9 +121,10 @@ public static FeatureStatistics fromProto( throws IOException { FeatureStatistics featureStatistics = FeatureStatistics.fromProto(featureNameStatistics); featureStatistics.setDate(date); - Feature feature = new Feature(); - feature.setId( - new FieldId(project, featureSetName, version, featureNameStatistics.getPath().getStep(0))); + Feature feature = + Feature.withReference( + new FeatureReference( + project, featureSetName, version, featureNameStatistics.getPath().getStep(0))); featureStatistics.setFeature(feature); return featureStatistics; } @@ -131,7 +133,7 @@ public FeatureNameStatistics toProto() throws InvalidProtocolBufferException { FeatureNameStatistics.Builder featureNameStatisticsBuilder = FeatureNameStatistics.newBuilder() .setType(FeatureNameStatistics.Type.valueOf(featureType)) - .setPath(Path.newBuilder().addStep(feature.getId().getName())); + .setPath(Path.newBuilder().addStep(feature.getReference().getName())); CommonStatistics commonStatistics = CommonStatistics.newBuilder() .setNumNonMissing(count - numMissing) diff --git a/core/src/main/java/feast/core/model/Field.java b/core/src/main/java/feast/core/model/Field.java deleted file mode 100644 index 66dff6ec8f5..00000000000 --- a/core/src/main/java/feast/core/model/Field.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import javax.persistence.EmbeddedId; -import javax.persistence.MappedSuperclass; -import lombok.Getter; -import lombok.Setter; - -// A field in a feature set, which may or may not contain value constraints -// for validation purposes. -@Getter -@Setter -@MappedSuperclass -public abstract class Field { - @EmbeddedId private FieldId id; - - // Type of the field - private String type; - - // Presence constraints (refer to proto feast.core.FeatureSet.FeatureSpec) - // Only one of them can be set. - private byte[] presence; - private byte[] groupPresence; - - // Shape type (refer to proto feast.core.FeatureSet.FeatureSpec) - // Only one of them can be set. - private byte[] shape; - private byte[] valueCount; - - // Domain info for the values (refer to proto feast.core.FeatureSet.FeatureSpec) - // Only one of them can be set. - private String domain; - private byte[] intDomain; - private byte[] floatDomain; - private byte[] stringDomain; - private byte[] boolDomain; - private byte[] structDomain; - private byte[] naturalLanguageDomain; - private byte[] imageDomain; - private byte[] midDomain; - private byte[] urlDomain; - private byte[] timeDomain; - private byte[] timeOfDayDomain; - - public void setName(String name) { - this.id.setName(name); - } - - public void setProject(String project) { - this.id.setProject(project); - } - - public void setVersion(int version) { - this.id.setVersion(version); - } - - public void setFeatureSet(String featureSet) { - this.id.setFeatureSet(featureSet); - } -} diff --git a/core/src/main/java/feast/core/service/StatsService.java b/core/src/main/java/feast/core/service/StatsService.java index ad2ae912c73..3331482bd35 100644 --- a/core/src/main/java/feast/core/service/StatsService.java +++ b/core/src/main/java/feast/core/service/StatsService.java @@ -18,24 +18,21 @@ import static java.lang.Math.*; -import com.google.cloud.bigquery.BigQueryOptions; import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Timestamp; -import feast.core.CoreServiceProto.GetFeatureSetRequest; -import feast.core.CoreServiceProto.GetFeatureStatisticsRequest; -import feast.core.CoreServiceProto.GetFeatureStatisticsResponse; -import feast.core.FeatureSetProto.EntitySpec; +import feast.core.CoreServiceProto.*; +import feast.core.CoreServiceProto.ListStoresRequest.Filter; import feast.core.FeatureSetProto.FeatureSet; import feast.core.FeatureSetProto.FeatureSetSpec; import feast.core.FeatureSetProto.FeatureSpec; import feast.core.StoreProto.Store; import feast.core.StoreProto.Store.StoreType; -import feast.core.dao.EntityStatisticsRepository; import feast.core.dao.FeatureStatisticsRepository; -import feast.core.dao.StoreRepository; -import feast.core.model.*; +import feast.core.exception.RetrievalException; import feast.core.model.Feature; +import feast.core.model.FeatureReference; +import feast.core.model.FeatureStatistics; import feast.storage.api.statistics.FeatureSetStatistics; import feast.storage.api.statistics.StatisticsRetriever; import feast.storage.connectors.bigquery.statistics.BigQueryStatisticsRetriever; @@ -55,20 +52,13 @@ @Service public class StatsService { - private StoreRepository storeRepository; private SpecService specService; private FeatureStatisticsRepository featureStatisticsRepository; - private EntityStatisticsRepository entityStatisticsRepository; @Autowired public StatsService( - StoreRepository storeRepository, - SpecService specService, - EntityStatisticsRepository entityStatisticsRepository, - FeatureStatisticsRepository featureStatisticsRepository) { - this.storeRepository = storeRepository; + SpecService specService, FeatureStatisticsRepository featureStatisticsRepository) { this.specService = specService; - this.entityStatisticsRepository = entityStatisticsRepository; this.featureStatisticsRepository = featureStatisticsRepository; } @@ -95,8 +85,24 @@ public StatsService( @Transactional public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsRequest request) throws IOException { + + // Validate the request + validateRequest(request); + + // Get the stats retriever for the store requested StatisticsRetriever statisticsRetriever = getStatisticsRetriever(request.getStore()); - FeatureSetSpec featureSetSpec = getFeatureSetSpec(request.getFeatureSetId()); + + // 1. Retrieve the feature set spec from the db + FeatureSetSpec featureSetSpec; + try { + featureSetSpec = getFeatureSetSpec(request.getFeatureSetId()); + } catch (IllegalArgumentException | RetrievalException e) { + throw new RetrievalException( + String.format("Unable to find feature set %s", request.getFeatureSetId()), e); + } + + // 2. Filter out the features requested by the user. If none are provided, + // use all features in the feature set. List features = request.getFeatureIdsList(); if (features.size() == 0) { features = @@ -104,20 +110,20 @@ public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsReq .map(FeatureSpec::getName) .collect(Collectors.toList()); } - List entities = - featureSetSpec.getEntitiesList().stream() - .map(EntitySpec::getName) - .collect(Collectors.toList()); + + // 3. Retrieve the statistics from the StatsRetriever. List> featureNameStatisticsList = new ArrayList<>(); if (request.getDatasetIdsCount() == 0) { - // retrieve by date - long timestamp = request.getStartDate().getSeconds(); - while (timestamp < request.getEndDate().getSeconds()) { + Timestamp endDate = request.getEndDate(); + Timestamp startDate = request.getStartDate(); + // If no dataset provided, retrieve by date + + long timestamp = startDate.getSeconds(); + while (timestamp < endDate.getSeconds()) { List featureNameStatistics = getFeatureNameStatisticsByDate( statisticsRetriever, featureSetSpec, - entities, features, timestamp, request.getForceRefresh()); @@ -125,19 +131,20 @@ public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsReq timestamp += 86400; // advance by a day } } else { - // retrieve by dataset + // else, retrieve by dataset for (String datasetId : request.getDatasetIdsList()) { List featureNameStatistics = getFeatureNameStatisticsByDataset( statisticsRetriever, featureSetSpec, - entities, features, datasetId, request.getForceRefresh()); featureNameStatisticsList.add(featureNameStatistics); } } + + // Merge statistics values across days/datasets List featureNameStatistics = mergeStatistics(featureNameStatisticsList); long totalCount = getTotalCount(featureNameStatistics.get(0)); return GetFeatureStatisticsResponse.newBuilder() @@ -150,25 +157,40 @@ public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsReq .build(); } + /** + * Get {@link FeatureNameStatistics} by dataset id. + * + * @param statisticsRetriever {@link StatisticsRetriever} corresponding to the store to get the + * data from. + * @param featureSetSpec {@link FeatureSetSpec} of the feature set requested + * @param features features to retrieve + * @param datasetId dataset id to subset the data by + * @param forceRefresh whether to override the values in the cache + * @return {@link FeatureNameStatistics} for the data within the dataset id provided + * @throws IOException + */ private List getFeatureNameStatisticsByDataset( StatisticsRetriever statisticsRetriever, FeatureSetSpec featureSetSpec, - List entities, List features, String datasetId, boolean forceRefresh) throws IOException { List featureNameStatistics = new ArrayList<>(); List featuresMissingStats = new ArrayList<>(); - List entitiesMissingStats = new ArrayList<>(); + + // For each feature requested, check if statistics already exist in the cache + // If not refreshing data in the cache, retrieve the cached data and add it to the + // list of FeatureNameStatistics for this dataset. + // Else, add to the list of features we still need to retrieve statistics for. for (String featureName : features) { - Feature feature = - new Feature( - new FieldId( - featureSetSpec.getProject(), - featureSetSpec.getName(), - featureSetSpec.getVersion(), - featureName)); + FeatureReference featureReference = + new FeatureReference( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + featureName); + Feature feature = Feature.withReference(featureReference); Optional cachedFeatureStatistics = Optional.empty(); if (!forceRefresh) { cachedFeatureStatistics = @@ -181,58 +203,48 @@ private List getFeatureNameStatisticsByDataset( featuresMissingStats.add(featureName); } } - for (String entityName : entities) { - Entity entity = - new Entity( - new FieldId( - featureSetSpec.getProject(), - featureSetSpec.getName(), - featureSetSpec.getVersion(), - entityName)); - Optional cachedEntityStatistics = Optional.empty(); - if (!forceRefresh) { - cachedEntityStatistics = - entityStatisticsRepository.findEntityStatisticsByEntityAndDatasetId(entity, datasetId); - } - if (cachedEntityStatistics.isPresent()) { - featureNameStatistics.add(cachedEntityStatistics.get().toProto()); - } else { - entitiesMissingStats.add(entityName); - } - } - if (featuresMissingStats.size() > 0 || entitiesMissingStats.size() > 0) { + + // Retrieve the balance of statistics after checking the cache, and add it to the + // list of FeatureNameStatistics. + if (featuresMissingStats.size() > 0) { FeatureSetStatistics featureSetStatistics = - statisticsRetriever.getFeatureStatistics( - featureSetSpec, entitiesMissingStats, featuresMissingStats, datasetId); + statisticsRetriever.getFeatureStatistics(featureSetSpec, featuresMissingStats, datasetId); + + // Persist the newly retrieved statistics in the cache. for (FeatureNameStatistics stat : featureSetStatistics.getFeatureNameStatistics()) { - String name = stat.getPath().getStep(0); - if (features.contains(name)) { - featureStatisticsRepository.save( - FeatureStatistics.fromProto( - featureSetSpec.getProject(), - featureSetSpec.getName(), - featureSetSpec.getVersion(), - stat, - datasetId)); - } else if (entities.contains(name)) { - entityStatisticsRepository.save( - EntityStatistics.fromProto( - featureSetSpec.getProject(), - featureSetSpec.getName(), - featureSetSpec.getVersion(), - stat, - datasetId)); - } + FeatureStatistics featureStatistics = + FeatureStatistics.createForDataset( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + stat, + datasetId); + Optional existingRecord = + featureStatisticsRepository.findFeatureStatisticsByFeatureAndDatasetId( + featureStatistics.getFeature(), datasetId); + existingRecord.ifPresent(statistics -> featureStatistics.setId(statistics.getId())); + featureStatisticsRepository.save(featureStatistics); } featureNameStatistics.addAll(featureSetStatistics.getFeatureNameStatistics()); } return featureNameStatistics; } + /** + * Get {@link FeatureNameStatistics} by date. + * + * @param statisticsRetriever {@link StatisticsRetriever} corresponding to the store to get the + * data from. + * @param featureSetSpec {@link FeatureSetSpec} of the feature set requested + * @param features features to retrieve + * @param timestamp timestamp of the date to subset the data + * @param forceRefresh whether to override the values in the cache + * @return {@link FeatureNameStatistics} for the data within the dataset id provided + * @throws IOException + */ private List getFeatureNameStatisticsByDate( StatisticsRetriever statisticsRetriever, FeatureSetSpec featureSetSpec, - List entities, List features, long timestamp, boolean forceRefresh) @@ -240,15 +252,19 @@ private List getFeatureNameStatisticsByDate( Date date = Date.from(Instant.ofEpochSecond(timestamp)); List featureNameStatistics = new ArrayList<>(); List featuresMissingStats = new ArrayList<>(); - List entitiesMissingStats = new ArrayList<>(); + + // For each feature requested, check if statistics already exist in the cache + // If not refreshing data in the cache, retrieve the cached data and add it to the + // list of FeatureNameStatistics for this date. + // Else, add to the list of features we still need to retrieve statistics for. for (String featureName : features) { - Feature feature = - new Feature( - new FieldId( - featureSetSpec.getProject(), - featureSetSpec.getName(), - featureSetSpec.getVersion(), - featureName)); + FeatureReference featureReference = + new FeatureReference( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + featureName); + Feature feature = Feature.withReference(featureReference); Optional cachedFeatureStatistics = Optional.empty(); if (!forceRefresh) { cachedFeatureStatistics = @@ -260,72 +276,57 @@ private List getFeatureNameStatisticsByDate( featuresMissingStats.add(featureName); } } - for (String entityName : entities) { - Entity entity = - new Entity( - new FieldId( - featureSetSpec.getProject(), - featureSetSpec.getName(), - featureSetSpec.getVersion(), - entityName)); - Optional cachedEntityStatistics = Optional.empty(); - if (!forceRefresh) { - cachedEntityStatistics = - entityStatisticsRepository.findEntityStatisticsByEntityAndDate(entity, date); - } - if (cachedEntityStatistics.isPresent()) { - featureNameStatistics.add(cachedEntityStatistics.get().toProto()); - } else { - entitiesMissingStats.add(entityName); - } - } + + // Retrieve the balance of statistics after checking the cache, and add it to the + // list of FeatureNameStatistics. if (featuresMissingStats.size() > 0) { FeatureSetStatistics featureSetStatistics = statisticsRetriever.getFeatureStatistics( featureSetSpec, featuresMissingStats, - entitiesMissingStats, Timestamp.newBuilder().setSeconds(timestamp).build()); + + // Persist the newly retrieved statistics in the cache. for (FeatureNameStatistics stat : featureSetStatistics.getFeatureNameStatistics()) { - String name = stat.getPath().getStep(0); - if (features.contains(name)) { - featureStatisticsRepository.save( - FeatureStatistics.fromProto( - featureSetSpec.getProject(), - featureSetSpec.getName(), - featureSetSpec.getVersion(), - stat, - date)); - } else if (entities.contains(name)) { - entityStatisticsRepository.save( - EntityStatistics.fromProto( - featureSetSpec.getProject(), - featureSetSpec.getName(), - featureSetSpec.getVersion(), - stat, - date)); - } + FeatureStatistics featureStatistics = + FeatureStatistics.createForDate( + featureSetSpec.getProject(), + featureSetSpec.getName(), + featureSetSpec.getVersion(), + stat, + date); + Optional existingRecord = + featureStatisticsRepository.findFeatureStatisticsByFeatureAndDate( + featureStatistics.getFeature(), date); + existingRecord.ifPresent(statistics -> featureStatistics.setId(statistics.getId())); + featureStatisticsRepository.save(featureStatistics); } featureNameStatistics.addAll(featureSetStatistics.getFeatureNameStatistics()); } return featureNameStatistics; } - private StatisticsRetriever getStatisticsRetriever(String storeName) - throws InvalidProtocolBufferException { - Store store = storeRepository.getOne(storeName).toProto(); + /** + * Get the {@link StatisticsRetriever} corresponding to the store name provided. + * + * @param storeName name of the store to retrieve statistics from + * @return {@link StatisticsRetriever} + */ + private StatisticsRetriever getStatisticsRetriever(String storeName) { + ListStoresResponse listStoresResponse = + specService.listStores(Filter.newBuilder().setName(storeName).build()); + Store store = listStoresResponse.getStore(0); if (store.getType() != StoreType.BIGQUERY) { - throw new IllegalArgumentException("Batch statistics are only supported for BigQuery stores"); + throw new IllegalArgumentException( + String.format( + "Invalid store %s with type %s specified. Batch statistics are only supported for BigQuery stores", + store.getName(), store.getType())); } - return BigQueryStatisticsRetriever.newBuilder() - .setProjectId(store.getBigqueryConfig().getProjectId()) - .setDatasetId(store.getBigqueryConfig().getDatasetId()) - .setBigquery(BigQueryOptions.getDefaultInstance().getService()) - .build(); + return BigQueryStatisticsRetriever.create(store.getBigqueryConfig()); } private FeatureSetSpec getFeatureSetSpec(String featureSetId) - throws InvalidProtocolBufferException { + throws InvalidProtocolBufferException, IllegalArgumentException, RetrievalException { String[] split = featureSetId.split("/"); String project = split[0]; split = split[1].split(":"); @@ -341,6 +342,13 @@ private FeatureSetSpec getFeatureSetSpec(String featureSetId) return featureSet.getSpec(); } + /** + * Merge feature statistics by name. This method is used to merge statistics retrieved over + * multiple days or datasets. + * + * @param featureNameStatistics {@link FeatureNameStatistics} retrieved from the store + * @return Merged list of {@link FeatureNameStatistics} by name + */ @VisibleForTesting public List mergeStatistics( List> featureNameStatistics) { @@ -410,7 +418,7 @@ private FeatureNameStatistics mergeStructStats( StructStatistics.newBuilder() .setCommonStats( CommonStatistics.newBuilder() - .setTotNumValues(totalCount) + .setTotNumValues(totalNumValues) .setNumNonMissing(totalCount) .setAvgNumValues((float) totalNumValues / totalCount) .setMaxNumValues(maxNumValues) @@ -571,4 +579,21 @@ private long getTotalCount(FeatureNameStatistics featureNameStatistics) { } return commonStats.getNumNonMissing() + commonStats.getNumMissing(); } + + private void validateRequest(GetFeatureStatisticsRequest request) { + if (request.getDatasetIdsCount() == 0) { + Timestamp startDate = request.getStartDate(); + Timestamp endDate = request.getEndDate(); + if (!request.hasStartDate() || !request.hasEndDate()) { + throw new IllegalArgumentException( + "Invalid request. Either provide dataset ids to retrieve statistics over, or a start date and end date."); + } + if (endDate.getSeconds() < startDate.getSeconds()) { + throw new IllegalArgumentException( + String.format( + "Invalid request. Start timestamp %d is greater than the end timestamp %d", + startDate.getSeconds(), endDate.getSeconds())); + } + } + } } diff --git a/core/src/test/java/feast/core/service/JobServiceTest.java b/core/src/test/java/feast/core/service/JobServiceTest.java index d21ad8e360d..bc3c9f8ba02 100644 --- a/core/src/test/java/feast/core/service/JobServiceTest.java +++ b/core/src/test/java/feast/core/service/JobServiceTest.java @@ -34,7 +34,9 @@ import feast.core.CoreServiceProto.RestartIngestionJobResponse; import feast.core.CoreServiceProto.StopIngestionJobRequest; import feast.core.CoreServiceProto.StopIngestionJobResponse; +import feast.core.FeatureSetProto.EntitySpec; import feast.core.FeatureSetProto.FeatureSetStatus; +import feast.core.FeatureSetProto.FeatureSpec; import feast.core.FeatureSetReferenceProto.FeatureSetReference; import feast.core.IngestionJobProto.IngestionJob; import feast.core.SourceProto.KafkaSourceConfig; @@ -159,8 +161,12 @@ public void setupJobManager() { // dummy model constructorss private FeatureSet newDummyFeatureSet(String name, int version, String project) { - Feature feature = new Feature(name + "_feature", Enum.INT64); - Entity entity = new Entity(name + "_entity", Enum.STRING); + FeatureSpec featureSpec = + FeatureSpec.newBuilder().setName(name + "_feature").setValueType(Enum.INT64).build(); + Feature feature = Feature.fromProto(featureSpec); + EntitySpec entitySpec = + EntitySpec.newBuilder().setName(name + "_entity").setValueType(Enum.STRING).build(); + Entity entity = Entity.fromProto(entitySpec); FeatureSet fs = new FeatureSet( diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index cd0e2681b3a..2aa7563cfaa 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -52,6 +52,7 @@ import feast.core.dao.StoreRepository; import feast.core.exception.RetrievalException; import feast.core.model.*; +import feast.types.ValueProto.ValueType; import feast.types.ValueProto.ValueType.Enum; import java.sql.Date; import java.time.Instant; @@ -120,9 +121,9 @@ public void setUp() { FeatureSet featureSet1v3 = newDummyFeatureSet("f1", 3, "project1"); FeatureSet featureSet2v1 = newDummyFeatureSet("f2", 1, "project1"); - Feature f3f1 = new Feature("f3f1", Enum.INT64); - Feature f3f2 = new Feature("f3f2", Enum.INT64); - Entity f3e1 = new Entity("f3e1", Enum.STRING); + Feature f3f1 = newFeature("f3f1", Enum.INT64); + Feature f3f2 = newFeature("f3f2", Enum.INT64); + Entity f3e1 = newEntity("f3e1", Enum.STRING); FeatureSet featureSet3v1 = new FeatureSet( "f3", @@ -486,9 +487,9 @@ public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists() public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered() throws InvalidProtocolBufferException { - Feature f3f1 = new Feature("f3f1", Enum.INT64); - Feature f3f2 = new Feature("f3f2", Enum.INT64); - Entity f3e1 = new Entity("f3e1", Enum.STRING); + Feature f3f1 = newFeature("f3f1", Enum.INT64); + Feature f3f2 = newFeature("f3f2", Enum.INT64); + Entity f3e1 = newEntity("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = (new FeatureSet( "f3", @@ -519,46 +520,11 @@ public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered() public void applyFeatureSetShouldAcceptPresenceShapeAndDomainConstraints() throws InvalidProtocolBufferException { List entitySpecs = new ArrayList<>(); - entitySpecs.add( - EntitySpec.newBuilder() - .setName("entity1") - .setValueType(Enum.INT64) - .setPresence(FeaturePresence.getDefaultInstance()) - .setShape(FixedShape.getDefaultInstance()) - .setDomain("mydomain") - .build()); - entitySpecs.add( - EntitySpec.newBuilder() - .setName("entity2") - .setValueType(Enum.INT64) - .setGroupPresence(FeaturePresenceWithinGroup.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setIntDomain(IntDomain.getDefaultInstance()) - .build()); - entitySpecs.add( - EntitySpec.newBuilder() - .setName("entity3") - .setValueType(Enum.FLOAT) - .setPresence(FeaturePresence.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setFloatDomain(FloatDomain.getDefaultInstance()) - .build()); - entitySpecs.add( - EntitySpec.newBuilder() - .setName("entity4") - .setValueType(Enum.STRING) - .setPresence(FeaturePresence.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setStringDomain(StringDomain.getDefaultInstance()) - .build()); - entitySpecs.add( - EntitySpec.newBuilder() - .setName("entity5") - .setValueType(Enum.BOOL) - .setPresence(FeaturePresence.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setBoolDomain(BoolDomain.getDefaultInstance()) - .build()); + entitySpecs.add(EntitySpec.newBuilder().setName("entity1").setValueType(Enum.INT64).build()); + entitySpecs.add(EntitySpec.newBuilder().setName("entity2").setValueType(Enum.INT64).build()); + entitySpecs.add(EntitySpec.newBuilder().setName("entity3").setValueType(Enum.FLOAT).build()); + entitySpecs.add(EntitySpec.newBuilder().setName("entity4").setValueType(Enum.STRING).build()); + entitySpecs.add(EntitySpec.newBuilder().setName("entity5").setValueType(Enum.BOOL).build()); List featureSpecs = new ArrayList<>(); featureSpecs.add( @@ -709,9 +675,9 @@ public void applyFeatureSetShouldUpdateFeatureSetWhenConstraintsAreUpdated() @Test public void applyFeatureSetShouldCreateProjectWhenNotAlreadyExists() throws InvalidProtocolBufferException { - Feature f3f1 = new Feature("f3f1", Enum.INT64); - Feature f3f2 = new Feature("f3f2", Enum.INT64); - Entity f3e1 = new Entity("f3e1", Enum.STRING); + Feature f3f1 = newFeature("f3f1", Enum.INT64); + Feature f3f2 = newFeature("f3f2", Enum.INT64); + Entity f3e1 = newEntity("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = (new FeatureSet( "f3", @@ -735,9 +701,9 @@ public void applyFeatureSetShouldCreateProjectWhenNotAlreadyExists() @Test public void applyFeatureSetShouldFailWhenProjectIsArchived() throws InvalidProtocolBufferException { - Feature f3f1 = new Feature("f3f1", Enum.INT64); - Feature f3f2 = new Feature("f3f2", Enum.INT64); - Entity f3e1 = new Entity("f3e1", Enum.STRING); + Feature f3f1 = newFeature("f3f1", Enum.INT64); + Feature f3f2 = newFeature("f3f2", Enum.INT64); + Entity f3e1 = newEntity("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = (new FeatureSet( "f3", @@ -802,8 +768,8 @@ public void shouldFailIfGetFeatureSetWithoutProject() throws InvalidProtocolBuff } private FeatureSet newDummyFeatureSet(String name, int version, String project) { - Feature feature = new Feature("feature", Enum.INT64); - Entity entity = new Entity("entity", Enum.STRING); + Feature feature = newFeature("feature", Enum.INT64); + Entity entity = newEntity("entity", Enum.STRING); FeatureSet fs = new FeatureSet( @@ -828,4 +794,14 @@ private Store newDummyStore(String name) { store.setConfig(RedisConfig.newBuilder().setPort(6379).build().toByteArray()); return store; } + + private Feature newFeature(String name, ValueType.Enum type) { + FeatureSpec spec = FeatureSpec.newBuilder().setName(name).setValueType(type).build(); + return Feature.fromProto(spec); + } + + private Entity newEntity(String name, ValueType.Enum type) { + EntitySpec spec = EntitySpec.newBuilder().setName(name).setValueType(type).build(); + return Entity.fromProto(spec); + } } diff --git a/core/src/test/java/feast/core/service/StatsServiceTest.java b/core/src/test/java/feast/core/service/StatsServiceTest.java index 764af9be0c9..126aa518a7f 100644 --- a/core/src/test/java/feast/core/service/StatsServiceTest.java +++ b/core/src/test/java/feast/core/service/StatsServiceTest.java @@ -18,13 +18,22 @@ import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.*; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; -import feast.core.dao.EntityStatisticsRepository; +import com.google.protobuf.Timestamp; +import feast.core.CoreServiceProto.GetFeatureStatisticsRequest; +import feast.core.CoreServiceProto.ListStoresRequest; +import feast.core.CoreServiceProto.ListStoresResponse; +import feast.core.StoreProto.Store; +import feast.core.StoreProto.Store.StoreType; import feast.core.dao.FeatureStatisticsRepository; -import feast.core.dao.StoreRepository; +import java.io.IOException; import java.util.Arrays; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.mockito.Mock; import org.tensorflow.metadata.v0.*; import org.tensorflow.metadata.v0.FeatureNameStatistics.Type; @@ -32,16 +41,74 @@ public class StatsServiceTest { private StatsService statsService; - @Mock private StoreRepository storeRepository; @Mock private FeatureStatisticsRepository featureStatisticsRepository; - @Mock private EntityStatisticsRepository entityStatisticsRepository; @Mock private SpecService specService; + @Rule public final ExpectedException expectedException = ExpectedException.none(); + @Before public void setUp() { - statsService = - new StatsService( - storeRepository, specService, entityStatisticsRepository, featureStatisticsRepository); + initMocks(this); + statsService = new StatsService(specService, featureStatisticsRepository); + } + + @Test + public void shouldThrowExceptionIfNeitherDatesNorDatasetsProvided() throws IOException { + GetFeatureStatisticsRequest request = GetFeatureStatisticsRequest.newBuilder().build(); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage( + "Invalid request. Either provide dataset ids to retrieve statistics over, or a start date and end date."); + statsService.getFeatureStatistics(request); + } + + @Test + public void shouldThrowExceptionIfInvalidDatesProvided() throws IOException { + GetFeatureStatisticsRequest request = + GetFeatureStatisticsRequest.newBuilder() + .setStartDate(Timestamp.newBuilder().setSeconds(1)) + .setEndDate(Timestamp.newBuilder().setSeconds(0)) + .build(); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage( + "Invalid request. Start timestamp 1 is greater than the end timestamp 0"); + statsService.getFeatureStatistics(request); + } + + @Test + public void shouldThrowExceptionIfInvalidStoreProvided() throws IOException { + GetFeatureStatisticsRequest request = + GetFeatureStatisticsRequest.newBuilder() + .setStartDate(Timestamp.newBuilder().setSeconds(0)) + .setEndDate(Timestamp.newBuilder().setSeconds(1)) + .setStore("redis") + .build(); + + when(specService.listStores(ListStoresRequest.Filter.newBuilder().setName("redis").build())) + .thenReturn( + ListStoresResponse.newBuilder() + .addStore(Store.newBuilder().setName("redis").setType(StoreType.REDIS).build()) + .build()); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage( + "Invalid store redis with type REDIS specified. Batch statistics are only supported for BigQuery stores"); + statsService.getFeatureStatistics(request); + } + + @Test + public void shouldThrowExceptionIfFeatureSetNotFound() throws IOException { + GetFeatureStatisticsRequest request = + GetFeatureStatisticsRequest.newBuilder() + .setStartDate(Timestamp.newBuilder().setSeconds(1)) + .setEndDate(Timestamp.newBuilder().setSeconds(0)) + .build(); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage( + "Invalid request. Start timestamp 1 is greater than the end timestamp 0"); + statsService.getFeatureStatistics(request); } @Test diff --git a/examples/statistics/Historical Feature Statistics with Feast, TFDV and Facets.ipynb b/examples/statistics/Historical Feature Statistics with Feast, TFDV and Facets.ipynb new file mode 100644 index 00000000000..2ee48e1b1d5 --- /dev/null +++ b/examples/statistics/Historical Feature Statistics with Feast, TFDV and Facets.ipynb @@ -0,0 +1,706 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Historical Feature Statistics with Feast, TFDV and Facets\n", + "\n", + "This tutorial covers how Feast can be used in conjunction with TFDV and Facets to retrieve statistics about feature datasets. \n", + "\n", + "The notebook showcases how Feast's integration with TFDV allows users to:\n", + "\n", + "1. Define TFX feature schemas and persist these properties in the Feature Store\n", + "2. Validate new data against the defined schema\n", + "3. Validate data already in Feast against the defined schema\n", + "\n", + "**Prerequisites**:\n", + "\n", + "- Feast running with at least 1 BigQuery warehouse store" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "setting project to statistics...\n", + "project already exists, skipping.\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "import pytest\n", + "import pytz\n", + "import uuid\n", + "import time\n", + "from datetime import datetime, timedelta\n", + "\n", + "from feast.client import Client\n", + "from feast.entity import Entity\n", + "from feast.feature import Feature\n", + "from feast.feature_set import FeatureSet\n", + "from feast.type_map import ValueType\n", + "from google.protobuf import json_format\n", + "from google.protobuf.duration_pb2 import Duration\n", + "from tensorflow_metadata.proto.v0 import statistics_pb2\n", + "import tensorflow_data_validation as tfdv\n", + "\n", + "PROJECT_NAME = \"statistics\"\n", + "IRIS_DATASET = \"http://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data\"\n", + "BIGQUERY_STORE_NAME = \"serving\"\n", + "client = Client(core_url=\"localhost:6565\")\n", + "print(f\"setting project to {PROJECT_NAME}...\")\n", + "try:\n", + " client.create_project(PROJECT_NAME)\n", + "except:\n", + " print(\"project already exists, skipping.\")\n", + "client.set_project(PROJECT_NAME)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this example, we are using the iris dataset. More information about this dataset can be found [here](http://archive.ics.uci.edu/ml/datasets/iris)." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "

\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
sepal_lengthsepal_widthpetal_lengthpetal_widthclassdatetime
05.13.51.40.2Iris-setosa2020-04-12 07:22:07.065951+00:00
14.93.01.40.2Iris-setosa2020-04-12 07:22:07.065951+00:00
24.73.21.30.2Iris-setosa2020-04-12 07:22:07.065951+00:00
34.63.11.50.2Iris-setosa2020-04-12 07:22:07.065951+00:00
45.03.61.40.2Iris-setosa2020-04-12 07:22:07.065951+00:00
\n", + "
" + ], + "text/plain": [ + " sepal_length sepal_width petal_length petal_width class \\\n", + "0 5.1 3.5 1.4 0.2 Iris-setosa \n", + "1 4.9 3.0 1.4 0.2 Iris-setosa \n", + "2 4.7 3.2 1.3 0.2 Iris-setosa \n", + "3 4.6 3.1 1.5 0.2 Iris-setosa \n", + "4 5.0 3.6 1.4 0.2 Iris-setosa \n", + "\n", + " datetime \n", + "0 2020-04-12 07:22:07.065951+00:00 \n", + "1 2020-04-12 07:22:07.065951+00:00 \n", + "2 2020-04-12 07:22:07.065951+00:00 \n", + "3 2020-04-12 07:22:07.065951+00:00 \n", + "4 2020-04-12 07:22:07.065951+00:00 " + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "iris_feature_names = [\"sepal_length\",\"sepal_width\",\"petal_length\",\"petal_width\"]\n", + "df = pd.read_csv(IRIS_DATASET, names=iris_feature_names + [\"class\"])\n", + "\n", + "# Add datetime to satisfy Feast\n", + "current_datetime = datetime.utcnow().replace(tzinfo=pytz.utc)\n", + "df['datetime'] = current_datetime - timedelta(days=1)\n", + "\n", + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TFDV schema as part of the feature set definition\n", + "\n", + "An integral part of TFDV is the feature [schemas](https://github.com/tensorflow/metadata/blob/master/tensorflow_metadata/proto/v0/schema.proto) that describe the expected properties of the data in a dataset, such as:\n", + "- expected feature presence\n", + "- type\n", + "- expected domains of features\n", + "\n", + "These schemas, which can be [manually defined or generated by TFDV](https://www.tensorflow.org/tfx/data_validation/get_started#inferring_a_schema_over_the_data), can be then used to extend the definition of features within the feature set. As part of the spec, the schema is persisted within Feast, and is used for both in-flight data validation, as well as offline integration with TFDV.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:Ignoring feature datetime of type datetime64[ns, UTC]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Entity class(ValueType.STRING) manually updated (replacing an existing field).\n", + "Feature sepal_length (ValueType.DOUBLE) added from dataframe.\n", + "Feature sepal_width (ValueType.DOUBLE) added from dataframe.\n", + "Feature petal_length (ValueType.DOUBLE) added from dataframe.\n", + "Feature petal_width (ValueType.DOUBLE) added from dataframe.\n", + "\n", + "{\n", + " \"spec\": {\n", + " \"name\": \"iris\",\n", + " \"entities\": [\n", + " {\n", + " \"name\": \"class\",\n", + " \"valueType\": \"STRING\",\n", + " \"presence\": {\n", + " \"minFraction\": 1.0,\n", + " \"minCount\": \"1\"\n", + " },\n", + " \"shape\": {\n", + " \"dim\": [\n", + " {\n", + " \"size\": \"1\"\n", + " }\n", + " ]\n", + " },\n", + " \"stringDomain\": {\n", + " \"name\": \"class\",\n", + " \"value\": [\n", + " \"Iris-setosa\",\n", + " \"Iris-versicolor\",\n", + " \"Iris-virginica\"\n", + " ]\n", + " }\n", + " }\n", + " ],\n", + " \"features\": [\n", + " {\n", + " \"name\": \"sepal_length\",\n", + " \"valueType\": \"DOUBLE\",\n", + " \"presence\": {\n", + " \"minFraction\": 1.0,\n", + " \"minCount\": \"1\"\n", + " },\n", + " \"shape\": {\n", + " \"dim\": [\n", + " {\n", + " \"size\": \"1\"\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"sepal_width\",\n", + " \"valueType\": \"DOUBLE\",\n", + " \"presence\": {\n", + " \"minFraction\": 1.0,\n", + " \"minCount\": \"1\"\n", + " },\n", + " \"shape\": {\n", + " \"dim\": [\n", + " {\n", + " \"size\": \"1\"\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"petal_length\",\n", + " \"valueType\": \"DOUBLE\",\n", + " \"presence\": {\n", + " \"minFraction\": 1.0,\n", + " \"minCount\": \"1\"\n", + " },\n", + " \"shape\": {\n", + " \"dim\": [\n", + " {\n", + " \"size\": \"1\"\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"petal_width\",\n", + " \"valueType\": \"DOUBLE\",\n", + " \"presence\": {\n", + " \"minFraction\": 1.0,\n", + " \"minCount\": \"1\"\n", + " },\n", + " \"shape\": {\n", + " \"dim\": [\n", + " {\n", + " \"size\": \"1\"\n", + " }\n", + " ]\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"meta\": {}\n", + "}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/zhiling/.pyenv/versions/3.7.2/envs/feast-0.2-dev/lib/python3.7/site-packages/tensorflow_data_validation/arrow/arrow_util.py:236: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", + " types.FeaturePath([column_name]), column.data.chunk(0), weights):\n" + ] + } + ], + "source": [ + "# Infer a schema over the iris dataset. These values can be tweaked as necessary.\n", + "stats = tfdv.generate_statistics_from_dataframe(df)\n", + "schema = tfdv.infer_schema(statistics=stats)\n", + "\n", + "# Create a new FeatureSet or retrieve an existing FeatureSet in Feast\n", + "feature_set = FeatureSet(name=\"iris\")\n", + "feature_set.infer_fields_from_df(df[['datetime'] + iris_feature_names], \n", + " entities=[Entity(name=\"class\", dtype=ValueType.STRING)])\n", + "\n", + "# Update the entities and features with constraints defined in the schema\n", + "feature_set.import_tfx_schema(schema)\n", + "print(feature_set)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Computing statistics over an ingested dataset\n", + "\n", + "Feast is able to compute statistics for any data that has been ingested into the system. Statistics can be computed over either discrete datasets using *dataset_ids* or periods of time using a specified time range.\n", + "\n", + "These statistics are computed at a historical store (caveat: only BQ is supported at the moment). The feature statistics returned in the form of TFX's `DatasetFeatureStatisticsList`, which can then be directly fed back into TFDV methods to either visualise the data statistics, or validate the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Feature set updated/created: \"iris:1\"\n", + "Waiting for feature set to be ready for ingestion...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 150/150 [00:01<00:00, 142.05rows/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ingestion complete!\n", + "\n", + "Ingestion statistics:\n", + "Success: 150/150\n", + "Removing temporary file(s)...\n", + "dataset id: a45b5760-76c9-3cfe-8479-2eb6020a73e3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# Apply the featureset\n", + "client.apply(feature_set)\n", + "\n", + "# When a dataset is ingested into Feast, a unique dataset id referencing the ingested dataset is returned. \n", + "dataset_id = client.ingest(feature_set, df)\n", + "print(\"dataset id: \" + dataset_id)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Get statistics from Feast for the ingested dataset.\n", + "# The statistics are calculated over the data in the store specified.\n", + "stats = client.get_statistics(\n", + " feature_set_id=f'{PROJECT_NAME}/iris:1', \n", + " store=BIGQUERY_STORE_NAME, \n", + " features=iris_feature_names, \n", + " dataset_ids=[dataset_id])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Visualising statistics with facets\n", + "\n", + "Since Feast outputs statistics in a format compatible with the TFDV API, the stats object can be directly passed to `tfdv.visualize_statistics()` to visualise, in-line, the output statistics on [Facets](https://pair-code.github.io/facets/), allowing for easy and interactive exploration of the shape and distribution of the data inside Feast." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "tfdv.visualize_statistics(stats)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Validating correctness of subsequent datasets \n", + "\n", + "While it is useful to explore dataset statistics using facets, since we have already defined a schema that specifies a dataset's bounds of correctness, we can leverage TFDV's `validate_statistics` to validate if subsequent datasets are problematic or not. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It is possible to validate correctness of a new dataset prior to ingestion by retrieving the schema from the feature set, and comparing computed statistics against that schema. \n", + "\n", + "This can be useful if we want to avoid ingesting problematic data into Feast." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:Ignoring feature datetime of type datetime64[ns, UTC]\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Anomaly short descriptionAnomaly long description
Feature name
'class'Unexpected string valuesExamples contain values missing from the schema: Iris-nonsensica (~33%).
\n", + "
" + ], + "text/plain": [ + " Anomaly short description \\\n", + "Feature name \n", + "'class' Unexpected string values \n", + "\n", + " Anomaly long description \n", + "Feature name \n", + "'class' Examples contain values missing from the schema: Iris-nonsensica (~33%). " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Ingest a new dataset with obviously incorrect data\n", + "df_2 = pd.DataFrame(\n", + " {\n", + " \"datetime\": current_datetime,\n", + " \"class\": [\"Iris-setosa\", \"Iris-virginica\", \"Iris-nonsensica\"],\n", + " \"sepal_length\": [4.3, 6.9, 12],\n", + " \"sepal_width\": [3.0, 2.8, 1.1],\n", + " \"petal_length\": [1.2, 4.9, 2.2],\n", + " \"petal_width\": [0.1, 1.8, 0]\n", + " }\n", + ")\n", + "\n", + "# Validate correctness\n", + "stats_2 = tfdv.generate_statistics_from_dataframe(df_2)\n", + "anomalies = tfdv.validate_statistics(statistics=stats_2, schema=feature_set.export_tfx_schema())\n", + "tfdv.display_anomalies(anomalies)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Alternatively, the data can be ingested into Feast, and the statistics computed at the store. This has the benefit of offloading statistics computation for large datasets to Feast." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/3 [00:00\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Anomaly short descriptionAnomaly long description
Feature name
'class'Unexpected string valuesExamples contain values missing from the schema: Iris-nonsensica (~33%).
\n", + "" + ], + "text/plain": [ + " Anomaly short description \\\n", + "Feature name \n", + "'class' Unexpected string values \n", + "\n", + " Anomaly long description \n", + "Feature name \n", + "'class' Examples contain values missing from the schema: Iris-nonsensica (~33%). " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Ingest the data into Feast\n", + "dataset_id_2 = client.ingest(feature_set, df_2)\n", + "time.sleep(10) # Sleep is not necessary if not using DirectRunner\n", + "\n", + "# Compute statistics over the new dataset\n", + "stats_2 = client.get_statistics(\n", + " feature_set_id=f'{PROJECT_NAME}/iris:1', \n", + " store=BIGQUERY_STORE_NAME, \n", + " features=iris_feature_names, \n", + " dataset_ids=[dataset_id_2])\n", + "\n", + "# Detect anomalies in the dataset\n", + "anomalies = tfdv.validate_statistics(statistics=stats_2, schema=feature_set.export_tfx_schema())\n", + "tfdv.display_anomalies(anomalies)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/protos/feast/core/FeatureSet.proto b/protos/feast/core/FeatureSet.proto index 429d99c8547..40eb468a47e 100644 --- a/protos/feast/core/FeatureSet.proto +++ b/protos/feast/core/FeatureSet.proto @@ -68,46 +68,6 @@ message EntitySpec { // Value type of the feature. feast.types.ValueType.Enum value_type = 2; - - // presence_constraints, shape_type and domain_info are referenced from: - // https://github.com/tensorflow/metadata/blob/36f65d1268cbc92cdbcf812ee03dcf47fb53b91e/tensorflow_metadata/proto/v0/schema.proto#L107 - - oneof presence_constraints { - // Constraints on the presence of this feature in the examples. - tensorflow.metadata.v0.FeaturePresence presence = 3; - // Only used in the context of a "group" context, e.g., inside a sequence. - tensorflow.metadata.v0.FeaturePresenceWithinGroup group_presence = 4; - } - - // The shape of the feature which governs the number of values that appear in - // each example. - oneof shape_type { - // The feature has a fixed shape corresponding to a multi-dimensional - // tensor. - tensorflow.metadata.v0.FixedShape shape = 5; - // The feature doesn't have a well defined shape. All we know are limits on - // the minimum and maximum number of values. - tensorflow.metadata.v0.ValueCount value_count = 6; - } - - // Domain for the values of the feature. - oneof domain_info { - // Reference to a domain defined at the schema level. - string domain = 7; - // Inline definitions of domains. - tensorflow.metadata.v0.IntDomain int_domain = 8; - tensorflow.metadata.v0.FloatDomain float_domain = 9; - tensorflow.metadata.v0.StringDomain string_domain = 10; - tensorflow.metadata.v0.BoolDomain bool_domain = 11; - tensorflow.metadata.v0.StructDomain struct_domain = 12; - // Supported semantic domains. - tensorflow.metadata.v0.NaturalLanguageDomain natural_language_domain = 13; - tensorflow.metadata.v0.ImageDomain image_domain = 14; - tensorflow.metadata.v0.MIDDomain mid_domain = 15; - tensorflow.metadata.v0.URLDomain url_domain = 16; - tensorflow.metadata.v0.TimeDomain time_domain = 17; - tensorflow.metadata.v0.TimeOfDayDomain time_of_day_domain = 18; - } } message FeatureSpec { diff --git a/protos/tensorflow_metadata/proto/v0/statistics.proto b/protos/tensorflow_metadata/proto/v0/statistics.proto index 6258d06dfcf..b07f5e5fc29 100644 --- a/protos/tensorflow_metadata/proto/v0/statistics.proto +++ b/protos/tensorflow_metadata/proto/v0/statistics.proto @@ -23,6 +23,7 @@ package tensorflow.metadata.v0; option java_package = "org.tensorflow.metadata.v0"; option java_multiple_files = true; +option go_package = "github.com/gojek/feast/sdk/go/protos/tensorflow_metadata/proto/v0"; import "tensorflow_metadata/proto/v0/path.proto"; diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 1c4de9edd03..8e11bc94a2d 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -51,7 +51,6 @@ GetFeatureSetRequest, GetFeatureSetResponse, GetFeatureStatisticsRequest, - GetFeatureStatisticsResponse, ListFeatureSetsRequest, ListFeatureSetsResponse, ListIngestionJobsRequest, @@ -665,7 +664,6 @@ def get_online_features( return response - def list_ingest_jobs( self, job_id: str = None, @@ -870,7 +868,6 @@ def ingest( return dataset_id - def get_statistics( self, feature_set_id: str, @@ -921,7 +918,10 @@ def get_statistics( ) request = GetFeatureStatisticsRequest( - feature_set_id=feature_set_id, feature_ids=features, store=store, force_refresh=force_refresh + feature_set_id=feature_set_id, + feature_ids=features, + store=store, + force_refresh=force_refresh, ) if dataset_ids is not None: request.dataset_ids.extend(dataset_ids) @@ -938,7 +938,6 @@ def get_statistics( return self._core_service_stub.GetFeatureStatistics( request ).dataset_feature_statistics_list - return None def _build_feature_references( diff --git a/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java b/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java index 251108304d6..c527b3c80cd 100644 --- a/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java +++ b/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java @@ -26,23 +26,21 @@ public interface StatisticsRetriever { * Get feature set statistics for a single feature set, for a single dataset id. * * @param featureSetSpec feature set spec of the requested feature set - * @param entities subset of entities to retrieve. * @param features subset of features to retrieve. * @param dataset dataset id to filter the data by * @return {@link FeatureSetStatistics} containing statistics for the requested features. */ FeatureSetStatistics getFeatureStatistics( - FeatureSetSpec featureSetSpec, List entities, List features, String dataset); + FeatureSetSpec featureSetSpec, List features, String dataset); /** * Get feature set statistics for a single feature set, for a single day. * * @param featureSetSpec feature set spec of the requested feature set - * @param entities subset of entities to retrieve. * @param features subset of features to retrieve. * @param date date to filter the data by * @return {@link FeatureSetStatistics} containing statistics for the requested features. */ FeatureSetStatistics getFeatureStatistics( - FeatureSetSpec featureSetSpec, List entities, List features, Timestamp date); + FeatureSetSpec featureSetSpec, List features, Timestamp date); } diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/BigQueryStatisticsRetriever.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/BigQueryStatisticsRetriever.java index 8509a7e3c90..59a1f546429 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/BigQueryStatisticsRetriever.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/BigQueryStatisticsRetriever.java @@ -16,18 +16,13 @@ */ package feast.storage.connectors.bigquery.statistics; -import static feast.storage.connectors.bigquery.statistics.StatsUtil.toFeatureNameStatistics; - import com.google.auto.value.AutoValue; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.FieldValueList; -import com.google.cloud.bigquery.QueryJobConfiguration; -import com.google.cloud.bigquery.TableResult; +import com.google.cloud.bigquery.*; import com.google.common.collect.Streams; import com.google.protobuf.Timestamp; -import feast.core.FeatureSetProto.EntitySpec; import feast.core.FeatureSetProto.FeatureSetSpec; import feast.core.FeatureSetProto.FeatureSpec; +import feast.core.StoreProto.Store.BigQueryConfig; import feast.storage.api.statistics.FeatureSetStatistics; import feast.storage.api.statistics.StatisticsRetriever; import java.io.IOException; @@ -45,12 +40,22 @@ public abstract class BigQueryStatisticsRetriever implements StatisticsRetriever public abstract BigQuery bigquery(); - public static Builder newBuilder() { + public static BigQueryStatisticsRetriever create(BigQueryConfig config) { + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + return newBuilder() + .setBigquery(bigquery) + .setDatasetId(config.getDatasetId()) + .setProjectId(config.getProjectId()) + .build(); + } + + static Builder newBuilder() { return new AutoValue_BigQueryStatisticsRetriever.Builder(); } @AutoValue.Builder - public abstract static class Builder { + abstract static class Builder { + public abstract Builder setProjectId(String projectId); public abstract Builder setDatasetId(String datasetId); @@ -62,52 +67,41 @@ public abstract static class Builder { @Override public FeatureSetStatistics getFeatureStatistics( - FeatureSetSpec featureSetSpec, List entities, List features, String dataset) { + FeatureSetSpec featureSetSpec, List features, String dataset) { FeatureSetStatisticsQueryInfo featureSetStatisticsQueryInfo = new FeatureSetStatisticsQueryInfo( featureSetSpec.getProject(), featureSetSpec.getName(), featureSetSpec.getVersion(), dataset); - return getFeatureSetStatistics( - featureSetSpec, entities, features, featureSetStatisticsQueryInfo); + return getFeatureSetStatistics(featureSetSpec, features, featureSetStatisticsQueryInfo); } @Override public FeatureSetStatistics getFeatureStatistics( - FeatureSetSpec featureSetSpec, List entities, List features, Timestamp date) { + FeatureSetSpec featureSetSpec, List features, Timestamp date) { FeatureSetStatisticsQueryInfo featureSetStatisticsQueryInfo = new FeatureSetStatisticsQueryInfo( featureSetSpec.getProject(), featureSetSpec.getName(), featureSetSpec.getVersion(), date); - return getFeatureSetStatistics( - featureSetSpec, entities, features, featureSetStatisticsQueryInfo); + return getFeatureSetStatistics(featureSetSpec, features, featureSetStatisticsQueryInfo); } private FeatureSetStatistics getFeatureSetStatistics( FeatureSetSpec featureSetSpec, - List entities, List features, FeatureSetStatisticsQueryInfo featureSetStatisticsQueryInfo) { List featuresList = featureSetSpec.getFeaturesList(); - List entitiesList = featureSetSpec.getEntitiesList(); - FeatureSetSpec.Builder featureSetSpecBuilder = - featureSetSpec.toBuilder().clearFeatures().clearEntities(); + FeatureSetSpec.Builder featureSetSpecBuilder = featureSetSpec.toBuilder().clearFeatures(); for (FeatureSpec featureSpec : featuresList) { if (features.contains(featureSpec.getName())) { featureSetStatisticsQueryInfo.addFeature(featureSpec); featureSetSpecBuilder.addFeatures(featureSpec); } } - for (EntitySpec entitySpec : entitiesList) { - if (entities.contains(entitySpec.getName())) { - featureSetStatisticsQueryInfo.addEntity(entitySpec); - featureSetSpecBuilder.addEntities(entitySpec); - } - } featureSetSpec = featureSetSpecBuilder.build(); try { @@ -131,8 +125,7 @@ private FeatureSetStatistics getFeatureSetStatistics( Map histValues = getTableResultByFeatureName(hist); int totalCountIndex = basicStats.getSchema().getFields().getIndex("total_count"); - String ref = (features.size() > 0) ? features.get(0) : entities.get(0); - ; + String ref = features.get(0); FeatureSetStatistics.Builder featureSetStatisticsBuilder = FeatureSetStatistics.newBuilder() .setNumExamples(basicStatsValues.get(ref).get(totalCountIndex).getLongValue()); @@ -140,22 +133,11 @@ private FeatureSetStatistics getFeatureSetStatistics( // Convert BQ rows to FeatureNameStatistics for (FeatureSpec featureSpec : featureSetSpec.getFeaturesList()) { FeatureNameStatistics featureNameStatistics = - toFeatureNameStatistics( - featureSpec.getValueType(), - basicStats.getSchema(), - basicStatsValues.get(featureSpec.getName()), - hist.getSchema(), - histValues.get(featureSpec.getName())); - featureSetStatisticsBuilder.addFeatureNameStatistics(featureNameStatistics); - } - for (EntitySpec entitySpec : featureSetSpec.getEntitiesList()) { - FeatureNameStatistics featureNameStatistics = - toFeatureNameStatistics( - entitySpec.getValueType(), - basicStats.getSchema(), - basicStatsValues.get(entitySpec.getName()), - hist.getSchema(), - histValues.get(entitySpec.getName())); + StatsQueryResult.create() + .withBasicStatsResults( + basicStats.getSchema(), basicStatsValues.get(featureSpec.getName())) + .withHistResults(hist.getSchema(), histValues.get(featureSpec.getName())) + .toFeatureNameStatistics(featureSpec.getValueType()); featureSetStatisticsBuilder.addFeatureNameStatistics(featureNameStatistics); } return featureSetStatisticsBuilder.build(); diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java index e6f0db7f062..686b4f549d9 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java @@ -31,32 +31,32 @@ * queries. */ public class FeatureSetStatisticsQueryInfo { + // Feast project name private final String project; + + // Feature set name private final String name; + + // Version of the feature set private final int version; + + // Dataset ID to retrieve statistics over private String datasetId = ""; + + // Date to retrieve statistics over private String date = ""; - private final List features; - public FeatureSetStatisticsQueryInfo( - String project, - String name, - int version, - String datasetId, - String date, - List features) { - this.project = project; - this.name = name; - this.version = version; - this.datasetId = datasetId; - this.date = date; - this.features = features; - } + // List of entity names in this feature set + private final List entityNames; + + // List of fields to get stats for + private final List features; public FeatureSetStatisticsQueryInfo(String project, String name, int version, String datasetId) { this.project = project; this.name = name; this.version = version; + this.entityNames = new ArrayList<>(); this.features = new ArrayList<>(); this.datasetId = datasetId; } @@ -65,6 +65,7 @@ public FeatureSetStatisticsQueryInfo(String project, String name, int version, T this.project = project; this.name = name; this.version = version; + this.entityNames = new ArrayList<>(); this.features = new ArrayList<>(); DateTime dateTime = new DateTime(date.getSeconds() * 1000, DateTimeZone.UTC); DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd"); @@ -76,6 +77,7 @@ public void addFeature(FeatureSpec featureSpec) { } public void addEntity(EntitySpec entitySpec) { + this.entityNames.add(entitySpec.getName()); this.features.add(FieldStatisticsQueryInfo.fromProto(entitySpec)); } @@ -95,6 +97,14 @@ public String getDatasetId() { return datasetId; } + public String getDate() { + return date; + } + + public List getEntityNames() { + return entityNames; + } + public List getFeatures() { return features; } diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FieldStatisticsQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FieldStatisticsQueryInfo.java index 4f1e085ff24..c73b21922b1 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FieldStatisticsQueryInfo.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FieldStatisticsQueryInfo.java @@ -24,7 +24,10 @@ * Value class for Features containing information necessary to template stats-retrieving queries. */ public class FieldStatisticsQueryInfo { + // Name of the field private final String name; + + // Type of the field private final String type; private FieldStatisticsQueryInfo(String name, String type) { @@ -54,7 +57,8 @@ public static FieldStatisticsQueryInfo fromProto(FeatureSpec featureSpec) { case STRING_LIST: return new FieldStatisticsQueryInfo(featureSpec.getName(), "LIST"); default: - throw new IllegalArgumentException("Invalid feature type provided"); + throw new IllegalArgumentException( + String.format("Invalid feature type provided: %s", valueType)); } } @@ -80,7 +84,8 @@ public static FieldStatisticsQueryInfo fromProto(EntitySpec entitySpec) { case STRING_LIST: return new FieldStatisticsQueryInfo(entitySpec.getName(), "LIST"); default: - throw new IllegalArgumentException("Invalid entity type provided"); + throw new IllegalArgumentException( + String.format("Invalid entity type provided: %s", valueType)); } } diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsUtil.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryResult.java similarity index 65% rename from storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsUtil.java rename to storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryResult.java index 012959f8951..7ed9cc797a0 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsUtil.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryResult.java @@ -16,80 +16,137 @@ */ package feast.storage.connectors.bigquery.statistics; -import com.google.cloud.bigquery.FieldList; -import com.google.cloud.bigquery.FieldValue; -import com.google.cloud.bigquery.FieldValueList; +import com.google.auto.value.AutoValue; +import com.google.cloud.bigquery.*; import com.google.cloud.bigquery.Schema; import com.google.common.collect.ComparisonChain; import com.google.common.collect.Ordering; import feast.types.ValueProto.ValueType; -import feast.types.ValueProto.ValueType.Enum; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import javax.annotation.Nullable; import org.tensorflow.metadata.v0.*; -import org.tensorflow.metadata.v0.FeatureNameStatistics.Builder; -import org.tensorflow.metadata.v0.FeatureNameStatistics.Type; import org.tensorflow.metadata.v0.Histogram.Bucket; import org.tensorflow.metadata.v0.Histogram.HistogramType; import org.tensorflow.metadata.v0.StringStatistics.FreqAndValue; -public class StatsUtil { - private static final Map TFDV_TYPE_MAP = new HashMap<>(); +@AutoValue +public abstract class StatsQueryResult { + // Map converting Feast type to TFDV type + private static final Map TFDV_TYPE_MAP = + new HashMap<>(); static { - TFDV_TYPE_MAP.put(Enum.INT64, Type.INT); - TFDV_TYPE_MAP.put(Enum.INT32, Type.INT); - TFDV_TYPE_MAP.put(Enum.BOOL, Type.INT); - TFDV_TYPE_MAP.put(Enum.FLOAT, Type.FLOAT); - TFDV_TYPE_MAP.put(Enum.DOUBLE, Type.FLOAT); - TFDV_TYPE_MAP.put(Enum.STRING, Type.STRING); - TFDV_TYPE_MAP.put(Enum.BYTES, Type.BYTES); - TFDV_TYPE_MAP.put(Enum.BYTES_LIST, Type.STRUCT); - TFDV_TYPE_MAP.put(Enum.STRING_LIST, Type.STRUCT); - TFDV_TYPE_MAP.put(Enum.INT32_LIST, Type.STRUCT); - TFDV_TYPE_MAP.put(Enum.INT64_LIST, Type.STRUCT); - TFDV_TYPE_MAP.put(Enum.BOOL_LIST, Type.STRUCT); - TFDV_TYPE_MAP.put(Enum.FLOAT_LIST, Type.STRUCT); - TFDV_TYPE_MAP.put(Enum.DOUBLE_LIST, Type.STRUCT); + TFDV_TYPE_MAP.put(ValueType.Enum.INT64, FeatureNameStatistics.Type.INT); + TFDV_TYPE_MAP.put(ValueType.Enum.INT32, FeatureNameStatistics.Type.INT); + TFDV_TYPE_MAP.put(ValueType.Enum.BOOL, FeatureNameStatistics.Type.INT); + TFDV_TYPE_MAP.put(ValueType.Enum.FLOAT, FeatureNameStatistics.Type.FLOAT); + TFDV_TYPE_MAP.put(ValueType.Enum.DOUBLE, FeatureNameStatistics.Type.FLOAT); + TFDV_TYPE_MAP.put(ValueType.Enum.STRING, FeatureNameStatistics.Type.STRING); + TFDV_TYPE_MAP.put(ValueType.Enum.BYTES, FeatureNameStatistics.Type.BYTES); + TFDV_TYPE_MAP.put(ValueType.Enum.BYTES_LIST, FeatureNameStatistics.Type.STRUCT); + TFDV_TYPE_MAP.put(ValueType.Enum.STRING_LIST, FeatureNameStatistics.Type.STRUCT); + TFDV_TYPE_MAP.put(ValueType.Enum.INT32_LIST, FeatureNameStatistics.Type.STRUCT); + TFDV_TYPE_MAP.put(ValueType.Enum.INT64_LIST, FeatureNameStatistics.Type.STRUCT); + TFDV_TYPE_MAP.put(ValueType.Enum.BOOL_LIST, FeatureNameStatistics.Type.STRUCT); + TFDV_TYPE_MAP.put(ValueType.Enum.FLOAT_LIST, FeatureNameStatistics.Type.STRUCT); + TFDV_TYPE_MAP.put(ValueType.Enum.DOUBLE_LIST, FeatureNameStatistics.Type.STRUCT); } + // Schema of the table returned by the basic stats retrieval query + @Nullable + abstract Schema basicStatsSchema(); + + // Table values returned by the basic stats retrieval query + @Nullable + abstract FieldValueList basicStatsFieldValues(); + + // Schema of the table returned by the histogram retrieval query + @Nullable + abstract Schema histSchema(); + + // Table values returned by the histogram retrieval query + @Nullable + abstract FieldValueList histFieldValues(); + + public static StatsQueryResult create() { + return StatsQueryResult.newBuilder().build(); + } + + private static StatsQueryResult.Builder newBuilder() { + return new AutoValue_StatsQueryResult.Builder(); + } + + abstract Builder toBuilder(); + /** - * Convert BQ-retrieved statistics to the corresponding TFDV {@link FeatureNameStatistics} - * specific to the feature type. + * Add basic stats query results to the StatsQueryResult. * - * @param valueType {@link ValueType.Enum} denoting the value type of the feature * @param basicStatsSchema BigQuery {@link Schema} of the retrieved statistics row for the * non-histogram statistics. Used to retrieve the column names corresponding to each value in * the row. - * @param basicStatsValues BigQuery {@link FieldValueList} containing a single row of + * @param basicStatsFieldValues BigQuery {@link FieldValueList} containing a single row of * non-histogram statistics retrieved from BigQuery + * @return {@link StatsQueryResult} + */ + public StatsQueryResult withBasicStatsResults( + Schema basicStatsSchema, FieldValueList basicStatsFieldValues) { + return toBuilder() + .setBasicStatsSchema(basicStatsSchema) + .setBasicStatsFieldValues(basicStatsFieldValues) + .build(); + } + + /** + * Add histogram stats query results to the StatsQueryResult. + * * @param histSchema BigQuery {@link Schema} of the retrieved statistics row for the histogram * statistics. Used to retrieve the column names corresponding to each value in the row. - * @param histValues BigQuery {@link FieldValueList} containing a single row of histogram + * @param histFieldValues BigQuery {@link FieldValueList} containing a single row of histogram * statistics retrieved from BigQuery + * @return {@link StatsQueryResult} + */ + public StatsQueryResult withHistResults(Schema histSchema, FieldValueList histFieldValues) { + return toBuilder().setHistSchema(histSchema).setHistFieldValues(histFieldValues).build(); + } + + @AutoValue.Builder + abstract static class Builder { + abstract Builder setBasicStatsSchema(Schema basicStatsSchema); + + abstract Builder setBasicStatsFieldValues(FieldValueList basicStatsFieldValues); + + abstract Builder setHistSchema(Schema histSchema); + + abstract Builder setHistFieldValues(FieldValueList histFieldValues); + + public abstract StatsQueryResult build(); + } + + /** + * Convert BQ-retrieved statistics to the corresponding TFDV {@link FeatureNameStatistics} + * specific to the feature type. + * + * @param valueType {@link ValueType.Enum} denoting the value type of the feature * @return {@link FeatureNameStatistics} */ - public static FeatureNameStatistics toFeatureNameStatistics( - ValueType.Enum valueType, - Schema basicStatsSchema, - FieldValueList basicStatsValues, - Schema histSchema, - FieldValueList histValues) { + public FeatureNameStatistics toFeatureNameStatistics(ValueType.Enum valueType) { Map valuesMap = new HashMap<>(); - FieldList basicStatsfields = basicStatsSchema.getFields(); - for (int i = 0; i < basicStatsSchema.getFields().size(); i++) { - valuesMap.put(basicStatsfields.get(i).getName(), basicStatsValues.get(i)); + // Convert the table values to a map of field name : table value for easy retrieval + FieldList basicStatsfields = basicStatsSchema().getFields(); + for (int i = 0; i < basicStatsSchema().getFields().size(); i++) { + valuesMap.put(basicStatsfields.get(i).getName(), basicStatsFieldValues().get(i)); } - FieldList histFields = histSchema.getFields(); - for (int i = 0; i < histSchema.getFields().size(); i++) { - valuesMap.put(histFields.get(i).getName(), histValues.get(i)); + FieldList histFields = histSchema().getFields(); + for (int i = 0; i < histSchema().getFields().size(); i++) { + valuesMap.put(histFields.get(i).getName(), histFieldValues().get(i)); } - Builder featureNameStatisticsBuilder = + FeatureNameStatistics.Builder featureNameStatisticsBuilder = FeatureNameStatistics.newBuilder() .setPath(Path.newBuilder().addStep(valuesMap.get("feature_name").getStringValue())) .setType(TFDV_TYPE_MAP.get(valueType)); @@ -128,7 +185,7 @@ public static FeatureNameStatistics toFeatureNameStatistics( return featureNameStatisticsBuilder.build(); } - private static BytesStatistics getBytesStatistics(Map valuesMap) { + private BytesStatistics getBytesStatistics(Map valuesMap) { if (valuesMap.get("total_count").getLongValue() == 0) { return BytesStatistics.getDefaultInstance(); } @@ -141,7 +198,7 @@ private static BytesStatistics getBytesStatistics(Map values .setMinNumValues(1) .setMaxNumValues(1) .setAvgNumValues(1) - .setTotNumValues(valuesMap.get("total_count").getLongValue())) + .setTotNumValues(valuesMap.get("feature_count").getLongValue())) .setUnique(valuesMap.get("unique").getLongValue()) .setMaxNumBytes((float) valuesMap.get("max").getDoubleValue()) .setMinNumBytes((float) valuesMap.get("min").getDoubleValue()) @@ -149,7 +206,7 @@ private static BytesStatistics getBytesStatistics(Map values .build(); } - private static StringStatistics getStringStatistics(Map valuesMap) { + private StringStatistics getStringStatistics(Map valuesMap) { if (valuesMap.get("total_count").getLongValue() == 0) { return StringStatistics.getDefaultInstance(); } @@ -186,6 +243,7 @@ private static StringStatistics getStringStatistics(Map valu return StringStatistics.newBuilder() .setUnique(valuesMap.get("unique").getLongValue()) + .setAvgLength((long) valuesMap.get("mean").getDoubleValue()) .setCommonStats( CommonStatistics.newBuilder() .setNumMissing(valuesMap.get("missing_count").getLongValue()) @@ -193,13 +251,13 @@ private static StringStatistics getStringStatistics(Map valu .setMinNumValues(1) .setMaxNumValues(1) .setAvgNumValues(1) - .setTotNumValues(valuesMap.get("total_count").getLongValue())) + .setTotNumValues(valuesMap.get("feature_count").getLongValue())) .setRankHistogram(rankHistogram) .addAllTopValues(topCount) .build(); } - private static NumericStatistics getNumericStatistics(Map valuesMap) { + private NumericStatistics getNumericStatistics(Map valuesMap) { if (valuesMap.get("total_count").getLongValue() == 0) { return NumericStatistics.getDefaultInstance(); } @@ -253,7 +311,7 @@ private static NumericStatistics getNumericStatistics(Map va .build(); } - private static StructStatistics getStructStatistics(Map valuesMap) { + private StructStatistics getStructStatistics(Map valuesMap) { if (valuesMap.get("total_count").getLongValue() == 0) { return StructStatistics.getDefaultInstance(); } @@ -266,7 +324,9 @@ private static StructStatistics getStructStatistics(Map valu .setMinNumValues(valuesMap.get("min").getLongValue()) .setMaxNumValues(valuesMap.get("max").getLongValue()) .setAvgNumValues(valuesMap.get("mean").getLongValue()) - .setTotNumValues(valuesMap.get("total_count").getLongValue())) + .setTotNumValues( + valuesMap.get("feature_count").getLongValue() + * valuesMap.get("mean").getLongValue())) .build(); } } diff --git a/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql b/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql index c4c51ef0636..d52da5d6a65 100644 --- a/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql +++ b/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql @@ -33,7 +33,7 @@ SELECT null as unique {% elseif feature.type equals "CATEGORICAL" %} -- mean - null as mean, + AVG(LENGTH({{ feature.name }})) as mean, -- stdev null as stdev, -- zeroes diff --git a/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsUtilTest.java b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsQueryResultTest.java similarity index 65% rename from storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsUtilTest.java rename to storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsQueryResultTest.java index 813dcc7c704..6c49153c13b 100644 --- a/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsUtilTest.java +++ b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsQueryResultTest.java @@ -16,9 +16,8 @@ */ package feast.storage.connectors.bigquery.statistics; -import static feast.storage.connectors.bigquery.statistics.StatsUtil.toFeatureNameStatistics; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertThat; import com.google.cloud.bigquery.FieldValue; import com.google.cloud.bigquery.FieldValue.Attribute; @@ -29,12 +28,11 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import feast.core.FeatureSetProto.FeatureSpec; -import feast.types.ValueProto.ValueType.Enum; +import feast.types.ValueProto.ValueType; import org.junit.Test; import org.tensorflow.metadata.v0.FeatureNameStatistics; -public class StatsUtilTest { - +public class StatsQueryResultTest { private Schema basicStatsSchema = Schema.of( com.google.cloud.bigquery.Field.of("feature_name", LegacySQLTypeName.STRING), @@ -51,7 +49,7 @@ public class StatsUtilTest { private Schema histStatsSchema = Schema.of( - com.google.cloud.bigquery.Field.of("feature", LegacySQLTypeName.STRING), + com.google.cloud.bigquery.Field.of("field", LegacySQLTypeName.STRING), com.google.cloud.bigquery.Field.of( "num_hist", LegacySQLTypeName.RECORD, @@ -121,15 +119,14 @@ public void voidShouldConvertNumericStatsToFeatureNameStatistics() FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())))); FeatureSpec featureSpec = - FeatureSpec.newBuilder().setName("floats").setValueType(Enum.DOUBLE).build(); + FeatureSpec.newBuilder().setName("floats").setValueType(ValueType.Enum.DOUBLE).build(); FeatureNameStatistics actual = - toFeatureNameStatistics( - featureSpec.getValueType(), - basicStatsSchema, - numericFieldValueList, - histStatsSchema, - numericHistFieldValueList); + StatsQueryResult.create() + .withBasicStatsResults(basicStatsSchema, numericFieldValueList) + .withHistResults(histStatsSchema, numericHistFieldValueList) + .toFeatureNameStatistics(featureSpec.getValueType()); + String expectedJson = "{\"type\":\"FLOAT\",\"numStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"1\",\"maxNumValues\":\"1\",\"avgNumValues\":1,\"totNumValues\":\"20\"},\"mean\":1,\"stdDev\":6,\"min\":-8.5,\"median\":0.5,\"max\":10.5,\"histograms\":[{\"buckets\":[{\"lowValue\":-8.5,\"highValue\":-7.5,\"sampleCount\":2},{\"lowValue\":-7.5,\"highValue\":-5.5,\"sampleCount\":2},{\"lowValue\":-5.5,\"highValue\":-3.5,\"sampleCount\":2},{\"lowValue\":-3.5,\"highValue\":-1.5,\"sampleCount\":2},{\"lowValue\":-1.5,\"highValue\":0.5,\"sampleCount\":2},{\"lowValue\":0.5,\"highValue\":2.5,\"sampleCount\":2},{\"lowValue\":2.5,\"highValue\":4.5,\"sampleCount\":2},{\"lowValue\":4.5,\"highValue\":6.5,\"sampleCount\":2},{\"lowValue\":6.5,\"highValue\":8.5,\"sampleCount\":2},{\"lowValue\":8.5,\"highValue\":10.5,\"sampleCount\":2}],\"type\":\"QUANTILES\"},{\"buckets\":[{\"lowValue\":1,\"highValue\":2,\"sampleCount\":1},{\"lowValue\":2,\"highValue\":3,\"sampleCount\":2}]}]},\"path\":{\"step\":[\"floats\"]}}"; FeatureNameStatistics.Builder expected = FeatureNameStatistics.newBuilder(); @@ -147,7 +144,7 @@ public void voidShouldConvertStringStatsToFeatureNameStatistics() FieldValue.of(Attribute.PRIMITIVE, "20"), FieldValue.of(Attribute.PRIMITIVE, "20"), FieldValue.of(Attribute.PRIMITIVE, "0"), - FieldValue.of(Attribute.PRIMITIVE, null), + FieldValue.of(Attribute.PRIMITIVE, "1"), FieldValue.of(Attribute.PRIMITIVE, null), FieldValue.of(Attribute.PRIMITIVE, null), FieldValue.of(Attribute.PRIMITIVE, null), @@ -178,17 +175,98 @@ public void voidShouldConvertStringStatsToFeatureNameStatistics() FieldValue.of(Attribute.PRIMITIVE, "2"))))))))); FeatureSpec featureSpec = - FeatureSpec.newBuilder().setName("strings").setValueType(Enum.STRING).build(); + FeatureSpec.newBuilder().setName("strings").setValueType(ValueType.Enum.STRING).build(); FeatureNameStatistics actual = - toFeatureNameStatistics( - featureSpec.getValueType(), - basicStatsSchema, - stringFieldValueList, - histStatsSchema, - stringHistFieldValueList); + StatsQueryResult.create() + .withBasicStatsResults(basicStatsSchema, stringFieldValueList) + .withHistResults(histStatsSchema, stringHistFieldValueList) + .toFeatureNameStatistics(featureSpec.getValueType()); + + String expectedJson = + "{\"type\":\"STRING\",\"stringStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"1\",\"maxNumValues\":\"1\",\"avgNumValues\":1,\"totNumValues\":\"20\"},\"unique\":\"2\",\"topValues\":[{\"value\":\"b\",\"frequency\":2},{\"value\":\"a\",\"frequency\":1}],\"avgLength\":1,\"rankHistogram\":{\"buckets\":[{\"label\":\"a\",\"sampleCount\":1},{\"label\":\"b\",\"sampleCount\":2}]}},\"path\":{\"step\":[\"strings\"]}}"; + FeatureNameStatistics.Builder expected = FeatureNameStatistics.newBuilder(); + JsonFormat.parser().merge(expectedJson, expected); + assertThat(actual, equalTo(expected.build())); + } + + @Test + public void voidShouldConvertBytesStatsToFeatureNameStatistics() + throws InvalidProtocolBufferException { + FieldValueList stringFieldValueList = + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "bytes"), + FieldValue.of(Attribute.PRIMITIVE, "20"), + FieldValue.of(Attribute.PRIMITIVE, "20"), + FieldValue.of(Attribute.PRIMITIVE, "0"), + FieldValue.of(Attribute.PRIMITIVE, "5"), + FieldValue.of(Attribute.PRIMITIVE, null), + FieldValue.of(Attribute.PRIMITIVE, null), + FieldValue.of(Attribute.PRIMITIVE, "3"), + FieldValue.of(Attribute.PRIMITIVE, "7"), + FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())), + FieldValue.of(Attribute.PRIMITIVE, "2"))); + + FieldValueList stringHistFieldValueList = + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "bytes"), + FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())), + FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())))); + + FeatureSpec featureSpec = + FeatureSpec.newBuilder().setName("bytes").setValueType(ValueType.Enum.BYTES).build(); + + FeatureNameStatistics actual = + StatsQueryResult.create() + .withBasicStatsResults(basicStatsSchema, stringFieldValueList) + .withHistResults(histStatsSchema, stringHistFieldValueList) + .toFeatureNameStatistics(featureSpec.getValueType()); + + String expectedJson = + "{\"type\":\"BYTES\",\"bytesStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"1\",\"maxNumValues\":\"1\",\"avgNumValues\":1,\"totNumValues\":\"20\"},\"unique\":\"2\",\"avgNumBytes\":5,\"minNumBytes\":3,\"maxNumBytes\":7},\"path\":{\"step\":[\"bytes\"]}}"; + FeatureNameStatistics.Builder expected = FeatureNameStatistics.newBuilder(); + JsonFormat.parser().merge(expectedJson, expected); + assertThat(actual, equalTo(expected.build())); + } + + @Test + public void voidShouldConvertStructStatsToFeatureNameStatistics() + throws InvalidProtocolBufferException { + FieldValueList stringFieldValueList = + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "list"), + FieldValue.of(Attribute.PRIMITIVE, "20"), + FieldValue.of(Attribute.PRIMITIVE, "20"), + FieldValue.of(Attribute.PRIMITIVE, "0"), + FieldValue.of(Attribute.PRIMITIVE, "5"), + FieldValue.of(Attribute.PRIMITIVE, null), + FieldValue.of(Attribute.PRIMITIVE, null), + FieldValue.of(Attribute.PRIMITIVE, "3"), + FieldValue.of(Attribute.PRIMITIVE, "7"), + FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())), + FieldValue.of(Attribute.PRIMITIVE, null))); + + FieldValueList stringHistFieldValueList = + FieldValueList.of( + Lists.newArrayList( + FieldValue.of(Attribute.PRIMITIVE, "list"), + FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())), + FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())))); + + FeatureSpec featureSpec = + FeatureSpec.newBuilder().setName("list").setValueType(ValueType.Enum.STRING_LIST).build(); + + FeatureNameStatistics actual = + StatsQueryResult.create() + .withBasicStatsResults(basicStatsSchema, stringFieldValueList) + .withHistResults(histStatsSchema, stringHistFieldValueList) + .toFeatureNameStatistics(featureSpec.getValueType()); + String expectedJson = - "{\"type\":\"STRING\",\"stringStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"1\",\"maxNumValues\":\"1\",\"avgNumValues\":1,\"totNumValues\":\"20\"},\"unique\":\"2\",\"topValues\":[{\"value\":\"b\",\"frequency\":2},{\"value\":\"a\",\"frequency\":1}],\"rankHistogram\":{\"buckets\":[{\"label\":\"a\",\"sampleCount\":1},{\"label\":\"b\",\"sampleCount\":2}]}},\"path\":{\"step\":[\"strings\"]}}"; + "{\"type\":\"STRUCT\",\"structStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"3\",\"maxNumValues\":\"7\",\"avgNumValues\":5,\"totNumValues\":\"100\"}},\"path\":{\"step\":[\"list\"]}}"; FeatureNameStatistics.Builder expected = FeatureNameStatistics.newBuilder(); JsonFormat.parser().merge(expectedJson, expected); assertThat(actual, equalTo(expected.build())); diff --git a/tests/e2e/feature-validation.py b/tests/e2e/feature-validation.py index bac2aef5d93..97515b00688 100644 --- a/tests/e2e/feature-validation.py +++ b/tests/e2e/feature-validation.py @@ -2,17 +2,18 @@ import pytest import pytz import uuid -import time -from datetime import datetime +from datetime import datetime, timedelta from feast.client import Client from feast.entity import Entity from feast.feature import Feature from feast.feature_set import FeatureSet from feast.type_map import ValueType -from google.protobuf import json_format from google.protobuf.duration_pb2 import Duration -from tensorflow_metadata.proto.v0 import statistics_pb2 +import tensorflow_data_validation as tfdv +from deepdiff import DeepDiff +from google.protobuf.json_format import MessageToDict + pd.set_option("display.max_columns", None) @@ -38,7 +39,7 @@ def client(core_url, allow_dirty): @pytest.fixture(scope="module") -def dataset_basic(client): +def feature_validation_feature_set(client): fv_fs = FeatureSet( "feature_validation", features=[ @@ -50,7 +51,11 @@ def dataset_basic(client): max_age=Duration(seconds=100), ) client.apply(fv_fs) - time.sleep(20) + return fv_fs + + +@pytest.fixture(scope="module") +def dataset_basic(client, feature_validation_feature_set): N_ROWS = 20 @@ -65,35 +70,150 @@ def dataset_basic(client): } ) - expected_stats = statistics_pb2.DatasetFeatureStatisticsList() - with open("statistics/expected_output_basic_dataset.json", "r") as fo: - contents = fo.read() - json_format.Parse(contents, expected_stats) + expected_stats = tfdv.generate_statistics_from_dataframe( + df[["entity_id", "strings", "ints", "floats"]] + ) + clear_unsupported_fields(expected_stats) return { - "id": client.ingest(fv_fs, df), - "date": datetime.datetime( + "id": client.ingest(feature_validation_feature_set, df), + "date": datetime( time_offset.year, time_offset.month, time_offset.day ).replace(tzinfo=pytz.utc), "stats": expected_stats, } +@pytest.fixture(scope="module") +def dataset_agg(client, feature_validation_feature_set): + time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) + start_date = time_offset - timedelta(days=10) + end_date = time_offset - timedelta(days=8) + df1 = pd.DataFrame( + { + "datetime": [start_date] * 5, + "entity_id": [i for i in range(5)], + "strings": ["a", "b", "b", "b", "a"], + "ints": [4, 3, 2, 6, 3], + "floats": [2.1, 5.2, 4.3, 0.6, 0.1], + } + ) + dataset_id_1 = client.ingest(feature_validation_feature_set, df1) + df2 = pd.DataFrame( + { + "datetime": [start_date + timedelta(days=1)] * 3, + "entity_id": [i for i in range(3)], + "strings": ["a", "b", "c"], + "ints": [2, 6, 7], + "floats": [1.6, 2.4, 2], + } + ) + dataset_id_2 = client.ingest(feature_validation_feature_set, df2) + + combined_df = pd.concat([df1, df2])[["entity_id", "strings", "ints", "floats"]] + expected_stats = tfdv.generate_statistics_from_dataframe(combined_df) + clear_unsupported_agg_fields(expected_stats) + + # Temporary until TFDV fixes their std dev computation + for feature in expected_stats.datasets[0].features: + if feature.HasField("num_stats"): + name = feature.path.step[0] + std = combined_df[name].std() + feature.num_stats.std_dev = std + + return { + "ids": [dataset_id_1, dataset_id_2], + "start_date": datetime( + start_date.year, start_date.month, start_date.day + ).replace(tzinfo=pytz.utc), + "end_date": datetime( + end_date.year, end_date.month, end_date.day + ).replace(tzinfo=pytz.utc), + "stats": expected_stats, + } + + def test_basic_retrieval_by_single_dataset(client, dataset_basic): stats = client.get_statistics( - feature_refs=["strings", "ints", "floats"], + features=["strings", "ints", "floats"], store="bigquery", dataset_ids=[dataset_basic["id"]], ) - assert stats == dataset_basic["stats"] + assert_stats_equal(dataset_basic["stats"], stats) def test_basic_by_date(client, dataset_basic): stats = client.get_statistics( - feature_refs=["strings", "ints", "floats"], + features=["strings", "ints", "floats"], store="bigquery", start_date=dataset_basic["date"], end_date=dataset_basic["date"], ) - assert stats == dataset_basic["stats"] \ No newline at end of file + assert_stats_equal(dataset_basic["stats"], stats) + + +def test_agg_over_datasets(client, dataset_agg): + stats = client.get_statistics( + features=["strings", "ints", "floats"], + store="bigquery", + dataset_ids=[dataset_basic["ids"]], + ) + assert_stats_equal(dataset_basic["stats"], stats) + + +def test_agg_over_dates(client, dataset_agg): + stats = client.get_statistics( + features=["strings", "ints", "floats"], + store="bigquery", + start_date=dataset_basic["start_date"], + end_date=dataset_basic["end_date"], + ) + assert_stats_equal(dataset_basic["stats"], stats) + + +def clear_unsupported_fields(datasets): + dataset = datasets.datasets[0] + for feature in dataset.features: + if feature.HasField("num_stats"): + feature.num_stats.common_stats.ClearField("num_values_histogram") + elif feature.HasField("string_stats"): + feature.string_stats.common_stats.ClearField("num_values_histogram") + for bucket in feature.string_stats.rank_histogram.buckets: + bucket.ClearField("low_rank") + bucket.ClearField("high_rank") + elif feature.HasField("struct_stats"): + feature.string_stats.struct_stats.ClearField("num_values_histogram") + elif feature.HasField("bytes_stats"): + feature.string_stats.bytes_stats.ClearField("num_values_histogram") + + +def clear_unsupported_agg_fields(datasets): + dataset = datasets.datasets[0] + for feature in dataset.features: + if feature.HasField("num_stats"): + feature.num_stats.common_stats.ClearField("num_values_histogram") + feature.num_stats.ClearField("histograms") + elif feature.HasField("string_stats"): + feature.string_stats.common_stats.ClearField("num_values_histogram") + feature.string_stats.ClearField("histograms") + feature.string_stats.ClearField("rank_histogram") + feature.string_stats.ClearField("top_values") + feature.string_stats.ClearField("unique") + elif feature.HasField("struct_stats"): + feature.string_stats.struct_stats.ClearField("num_values_histogram") + elif feature.HasField("bytes_stats"): + feature.string_stats.bytes_stats.ClearField("num_values_histogram") + + +def assert_stats_equal(left, right): + left_stats = MessageToDict(left)["datasets"][0] + right_stats = MessageToDict(right)["datasets"][0] + assert ( + left_stats["numExamples"] == right_stats["numExamples"] + ), f"Number of examples do not match. Expected {left_stats['numExamples']}, got {right_stats['numExamples']}" + + left_features = sorted(left_stats["features"], key=lambda k: k["path"]["step"][0]) + right_features = sorted(right_stats["features"], key=lambda k: k["path"]["step"][0]) + diff = DeepDiff(left_features, right_features) + assert len(diff) == 0, f"Statistics do not match: \n{diff}" diff --git a/tests/e2e/statistics/expected_output_basic_dataset.json b/tests/e2e/statistics/expected_output_basic_dataset.json deleted file mode 100644 index 0cf4a9fbab2..00000000000 --- a/tests/e2e/statistics/expected_output_basic_dataset.json +++ /dev/null @@ -1,310 +0,0 @@ -{ - "datasets": [ - { - "type": "FLOAT", - "numStats": { - "commonStats": { - "numNonMissing": "20", - "minNumValues": "1", - "maxNumValues": "1", - "avgNumValues": 1.0, - "totNumValues": "20" - }, - "mean": 1.0, - "stdDev": 5.766281297335398, - "min": -8.5, - "median": 1.5, - "max": 10.5, - "histograms": [ - { - "buckets": [ - { - "lowValue": -8.5, - "highValue": -6.6, - "sampleCount": 1.998 - }, - { - "lowValue": -6.6, - "highValue": -4.7, - "sampleCount": 1.998 - }, - { - "lowValue": -4.7, - "highValue": -2.8000000000000007, - "sampleCount": 1.998 - }, - { - "lowValue": -2.8000000000000007, - "highValue": -0.9000000000000004, - "sampleCount": 1.998 - }, - { - "lowValue": -0.9000000000000004, - "highValue": 1.0, - "sampleCount": 1.998 - }, - { - "lowValue": 1.0, - "highValue": 2.8999999999999986, - "sampleCount": 1.998 - }, - { - "lowValue": 2.8999999999999986, - "highValue": 4.799999999999999, - "sampleCount": 1.998 - }, - { - "lowValue": 4.799999999999999, - "highValue": 6.699999999999999, - "sampleCount": 1.998 - }, - { - "lowValue": 6.699999999999999, - "highValue": 8.599999999999998, - "sampleCount": 1.998 - }, - { - "lowValue": 8.599999999999998, - "highValue": 10.5, - "sampleCount": 2.0180000000000002 - } - ] - }, - { - "buckets": [ - { - "lowValue": -8.5, - "highValue": -6.5, - "sampleCount": 2.0 - }, - { - "lowValue": -6.5, - "highValue": -4.5, - "sampleCount": 2.0 - }, - { - "lowValue": -4.5, - "highValue": -2.5, - "sampleCount": 2.0 - }, - { - "lowValue": -2.5, - "highValue": -0.5, - "sampleCount": 2.0 - }, - { - "lowValue": -0.5, - "highValue": 1.5, - "sampleCount": 2.0 - }, - { - "lowValue": 1.5, - "highValue": 3.5, - "sampleCount": 2.0 - }, - { - "lowValue": 3.5, - "highValue": 5.5, - "sampleCount": 2.0 - }, - { - "lowValue": 5.5, - "highValue": 7.5, - "sampleCount": 2.0 - }, - { - "lowValue": 7.5, - "highValue": 9.5, - "sampleCount": 2.0 - }, - { - "lowValue": 9.5, - "highValue": 10.5, - "sampleCount": 2.0 - } - ], - "type": "QUANTILES" - } - ] - }, - "path": { - "step": [ - "floats" - ] - } - }, - { - "numStats": { - "commonStats": { - "numNonMissing": "20", - "minNumValues": "1", - "maxNumValues": "1", - "avgNumValues": 1.0, - "totNumValues": "20" - }, - "mean": 9.5, - "stdDev": 5.766281297335398, - "numZeros": "1", - "median": 10.0, - "max": 19.0, - "histograms": [ - { - "buckets": [ - { - "highValue": 1.9, - "sampleCount": 1.998 - }, - { - "lowValue": 1.9, - "highValue": 3.8, - "sampleCount": 1.998 - }, - { - "lowValue": 3.8, - "highValue": 5.699999999999999, - "sampleCount": 1.998 - }, - { - "lowValue": 5.699999999999999, - "highValue": 7.6, - "sampleCount": 1.998 - }, - { - "lowValue": 7.6, - "highValue": 9.5, - "sampleCount": 1.998 - }, - { - "lowValue": 9.5, - "highValue": 11.399999999999999, - "sampleCount": 1.998 - }, - { - "lowValue": 11.399999999999999, - "highValue": 13.299999999999999, - "sampleCount": 1.998 - }, - { - "lowValue": 13.299999999999999, - "highValue": 15.2, - "sampleCount": 1.998 - }, - { - "lowValue": 15.2, - "highValue": 17.099999999999998, - "sampleCount": 1.998 - }, - { - "lowValue": 17.099999999999998, - "highValue": 19.0, - "sampleCount": 2.0180000000000002 - } - ] - }, - { - "buckets": [ - { - "highValue": 2.0, - "sampleCount": 2.0 - }, - { - "lowValue": 2.0, - "highValue": 4.0, - "sampleCount": 2.0 - }, - { - "lowValue": 4.0, - "highValue": 6.0, - "sampleCount": 2.0 - }, - { - "lowValue": 6.0, - "highValue": 8.0, - "sampleCount": 2.0 - }, - { - "lowValue": 8.0, - "highValue": 10.0, - "sampleCount": 2.0 - }, - { - "lowValue": 10.0, - "highValue": 12.0, - "sampleCount": 2.0 - }, - { - "lowValue": 12.0, - "highValue": 14.0, - "sampleCount": 2.0 - }, - { - "lowValue": 14.0, - "highValue": 16.0, - "sampleCount": 2.0 - }, - { - "lowValue": 16.0, - "highValue": 18.0, - "sampleCount": 2.0 - }, - { - "lowValue": 18.0, - "highValue": 19.0, - "sampleCount": 2.0 - } - ], - "type": "QUANTILES" - } - ] - }, - "path": { - "step": [ - "ints" - ] - } - }, - { - "type": "STRING", - "stringStats": { - "commonStats": { - "numNonMissing": "20", - "minNumValues": "1", - "maxNumValues": "1", - "avgNumValues": 1.0, - "totNumValues": "20" - }, - "unique": "2", - "topValues": [ - { - "value": "b", - "frequency": 10.0 - }, - { - "value": "a", - "frequency": 10.0 - } - ], - "avgLength": 1.0, - "rankHistogram": { - "buckets": [ - { - "label": "b", - "sampleCount": 10.0 - }, - { - "label": "a", - "sampleCount": 10.0 - } - ] - } - }, - "path": { - "step": [ - "strings" - ] - } - } - ] - } - ] -} \ No newline at end of file From f7f80b86e6cec1c39c6aaee1c6e95c64c179a3bf Mon Sep 17 00:00:00 2001 From: zhilingc Date: Wed, 22 Apr 2020 16:51:22 +0800 Subject: [PATCH 05/12] Minor fixes, add force refresh end to end test Apply python lint --- .../feast/core/model/EntityReference.java | 2 + sdk/python/feast/client.py | 6 +- .../FeatureSetStatisticsQueryInfo.java | 8 +-- ...o.java => FeatureStatisticsQueryInfo.java} | 24 +++---- .../main/resources/templates/basic_stats.sql | 2 +- .../main/resources/templates/hist_stats.sql | 2 +- tests/e2e/feature-validation.py | 66 ++++++++++++++----- 7 files changed, 73 insertions(+), 37 deletions(-) rename storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/{FieldStatisticsQueryInfo.java => FeatureStatisticsQueryInfo.java} (70%) diff --git a/core/src/main/java/feast/core/model/EntityReference.java b/core/src/main/java/feast/core/model/EntityReference.java index 3b0966de808..88d808df7fc 100644 --- a/core/src/main/java/feast/core/model/EntityReference.java +++ b/core/src/main/java/feast/core/model/EntityReference.java @@ -22,9 +22,11 @@ import javax.persistence.Embeddable; import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; @Embeddable +@NoArgsConstructor @AllArgsConstructor @Getter @Setter diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 8e11bc94a2d..2611ef6da75 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -928,12 +928,10 @@ def get_statistics( else: if start_date is not None: request.start_date.CopyFrom( - Timestamp(seconds=int(start_date.strftime("%s"))) + Timestamp(seconds=int(start_date.timestamp())) ) if end_date is not None: - request.end_date.CopyFrom( - Timestamp(seconds=int(end_date.strftime("%s"))) - ) + request.end_date.CopyFrom(Timestamp(seconds=int(end_date.timestamp()))) return self._core_service_stub.GetFeatureStatistics( request diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java index 686b4f549d9..5bac2bd7771 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java @@ -50,7 +50,7 @@ public class FeatureSetStatisticsQueryInfo { private final List entityNames; // List of fields to get stats for - private final List features; + private final List features; public FeatureSetStatisticsQueryInfo(String project, String name, int version, String datasetId) { this.project = project; @@ -73,12 +73,12 @@ public FeatureSetStatisticsQueryInfo(String project, String name, int version, T } public void addFeature(FeatureSpec featureSpec) { - this.features.add(FieldStatisticsQueryInfo.fromProto(featureSpec)); + this.features.add(FeatureStatisticsQueryInfo.fromProto(featureSpec)); } public void addEntity(EntitySpec entitySpec) { this.entityNames.add(entitySpec.getName()); - this.features.add(FieldStatisticsQueryInfo.fromProto(entitySpec)); + this.features.add(FeatureStatisticsQueryInfo.fromProto(entitySpec)); } public String getProject() { @@ -105,7 +105,7 @@ public List getEntityNames() { return entityNames; } - public List getFeatures() { + public List getFeatures() { return features; } } diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FieldStatisticsQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureStatisticsQueryInfo.java similarity index 70% rename from storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FieldStatisticsQueryInfo.java rename to storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureStatisticsQueryInfo.java index c73b21922b1..6461f8350b8 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FieldStatisticsQueryInfo.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureStatisticsQueryInfo.java @@ -23,19 +23,19 @@ /** * Value class for Features containing information necessary to template stats-retrieving queries. */ -public class FieldStatisticsQueryInfo { +public class FeatureStatisticsQueryInfo { // Name of the field private final String name; // Type of the field private final String type; - private FieldStatisticsQueryInfo(String name, String type) { + private FeatureStatisticsQueryInfo(String name, String type) { this.name = name; this.type = type; } - public static FieldStatisticsQueryInfo fromProto(FeatureSpec featureSpec) { + public static FeatureStatisticsQueryInfo fromProto(FeatureSpec featureSpec) { Enum valueType = featureSpec.getValueType(); switch (valueType) { case FLOAT: @@ -43,11 +43,11 @@ public static FieldStatisticsQueryInfo fromProto(FeatureSpec featureSpec) { case INT32: case INT64: case BOOL: - return new FieldStatisticsQueryInfo(featureSpec.getName(), "NUMERIC"); + return new FeatureStatisticsQueryInfo(featureSpec.getName(), "NUMERIC"); case STRING: - return new FieldStatisticsQueryInfo(featureSpec.getName(), "CATEGORICAL"); + return new FeatureStatisticsQueryInfo(featureSpec.getName(), "CATEGORICAL"); case BYTES: - return new FieldStatisticsQueryInfo(featureSpec.getName(), "BYTES"); + return new FeatureStatisticsQueryInfo(featureSpec.getName(), "BYTES"); case BYTES_LIST: case BOOL_LIST: case FLOAT_LIST: @@ -55,14 +55,14 @@ public static FieldStatisticsQueryInfo fromProto(FeatureSpec featureSpec) { case INT64_LIST: case DOUBLE_LIST: case STRING_LIST: - return new FieldStatisticsQueryInfo(featureSpec.getName(), "LIST"); + return new FeatureStatisticsQueryInfo(featureSpec.getName(), "LIST"); default: throw new IllegalArgumentException( String.format("Invalid feature type provided: %s", valueType)); } } - public static FieldStatisticsQueryInfo fromProto(EntitySpec entitySpec) { + public static FeatureStatisticsQueryInfo fromProto(EntitySpec entitySpec) { Enum valueType = entitySpec.getValueType(); switch (valueType) { case FLOAT: @@ -70,11 +70,11 @@ public static FieldStatisticsQueryInfo fromProto(EntitySpec entitySpec) { case INT32: case INT64: case BOOL: - return new FieldStatisticsQueryInfo(entitySpec.getName(), "NUMERIC"); + return new FeatureStatisticsQueryInfo(entitySpec.getName(), "NUMERIC"); case STRING: - return new FieldStatisticsQueryInfo(entitySpec.getName(), "CATEGORICAL"); + return new FeatureStatisticsQueryInfo(entitySpec.getName(), "CATEGORICAL"); case BYTES: - return new FieldStatisticsQueryInfo(entitySpec.getName(), "BYTES"); + return new FeatureStatisticsQueryInfo(entitySpec.getName(), "BYTES"); case BYTES_LIST: case BOOL_LIST: case FLOAT_LIST: @@ -82,7 +82,7 @@ public static FieldStatisticsQueryInfo fromProto(EntitySpec entitySpec) { case INT64_LIST: case DOUBLE_LIST: case STRING_LIST: - return new FieldStatisticsQueryInfo(entitySpec.getName(), "LIST"); + return new FeatureStatisticsQueryInfo(entitySpec.getName(), "LIST"); default: throw new IllegalArgumentException( String.format("Invalid entity type provided: %s", valueType)); diff --git a/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql b/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql index d52da5d6a65..331d9a108ef 100644 --- a/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql +++ b/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql @@ -1,7 +1,7 @@ WITH subset AS ( SELECT * FROM `{{ projectId }}.{{ datasetId }}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}` {% if featureSet.datasetId == "" %} -WHERE event_timestamp >= '{{ featureSet.date }} 00:00:00 UTC' AND event_timestamp < DATETIME_ADD('{{ featureSet.date }} 00:00:00 UTC', INTERVAL 1 DAY) +WHERE DATE(event_timestamp) = '{{ featureSet.date }}' {% else %} WHERE dataset_id='{{ featureSet.datasetId }}' {% endif %} diff --git a/storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql b/storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql index d611cb3c36c..4908a9124c9 100644 --- a/storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql +++ b/storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql @@ -1,7 +1,7 @@ WITH subset AS ( SELECT * FROM `{{ projectId }}.{{ datasetId }}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}` {% if featureSet.datasetId == "" %} -WHERE event_timestamp >= '{{ featureSet.date }} 00:00:00 UTC' AND event_timestamp < DATETIME_ADD('{{ featureSet.date }} 00:00:00 UTC', INTERVAL 1 DAY) +WHERE DATE(event_timestamp) = '{{ featureSet.date }}' {% else %} WHERE dataset_id='{{ featureSet.datasetId }}' {% endif %} diff --git a/tests/e2e/feature-validation.py b/tests/e2e/feature-validation.py index 97515b00688..90f0a717329 100644 --- a/tests/e2e/feature-validation.py +++ b/tests/e2e/feature-validation.py @@ -18,6 +18,7 @@ pd.set_option("display.max_columns", None) PROJECT_NAME = "batch_" + uuid.uuid4().hex.upper()[0:6] +STORE_NAME = "historical" @pytest.fixture(scope="module") @@ -76,10 +77,11 @@ def dataset_basic(client, feature_validation_feature_set): clear_unsupported_fields(expected_stats) return { + "df": df, "id": client.ingest(feature_validation_feature_set, df), - "date": datetime( - time_offset.year, time_offset.month, time_offset.day - ).replace(tzinfo=pytz.utc), + "date": datetime(time_offset.year, time_offset.month, time_offset.day).replace( + tzinfo=pytz.utc + ), "stats": expected_stats, } @@ -88,7 +90,7 @@ def dataset_basic(client, feature_validation_feature_set): def dataset_agg(client, feature_validation_feature_set): time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) start_date = time_offset - timedelta(days=10) - end_date = time_offset - timedelta(days=8) + end_date = time_offset - timedelta(days=7) df1 = pd.DataFrame( { "datetime": [start_date] * 5, @@ -126,17 +128,18 @@ def dataset_agg(client, feature_validation_feature_set): "start_date": datetime( start_date.year, start_date.month, start_date.day ).replace(tzinfo=pytz.utc), - "end_date": datetime( - end_date.year, end_date.month, end_date.day - ).replace(tzinfo=pytz.utc), + "end_date": datetime(end_date.year, end_date.month, end_date.day).replace( + tzinfo=pytz.utc + ), "stats": expected_stats, } def test_basic_retrieval_by_single_dataset(client, dataset_basic): stats = client.get_statistics( + f"{PROJECT_NAME}/feature_validation:1", features=["strings", "ints", "floats"], - store="bigquery", + store=STORE_NAME, dataset_ids=[dataset_basic["id"]], ) @@ -145,18 +148,20 @@ def test_basic_retrieval_by_single_dataset(client, dataset_basic): def test_basic_by_date(client, dataset_basic): stats = client.get_statistics( + f"{PROJECT_NAME}/feature_validation:1", features=["strings", "ints", "floats"], - store="bigquery", + store=STORE_NAME, start_date=dataset_basic["date"], - end_date=dataset_basic["date"], + end_date=dataset_basic["date"] + timedelta(days=1), ) assert_stats_equal(dataset_basic["stats"], stats) def test_agg_over_datasets(client, dataset_agg): stats = client.get_statistics( + f"{PROJECT_NAME}/feature_validation:1", features=["strings", "ints", "floats"], - store="bigquery", + store=STORE_NAME, dataset_ids=[dataset_basic["ids"]], ) assert_stats_equal(dataset_basic["stats"], stats) @@ -164,12 +169,43 @@ def test_agg_over_datasets(client, dataset_agg): def test_agg_over_dates(client, dataset_agg): stats = client.get_statistics( + f"{PROJECT_NAME}/feature_validation:1", features=["strings", "ints", "floats"], - store="bigquery", - start_date=dataset_basic["start_date"], - end_date=dataset_basic["end_date"], + store=STORE_NAME, + start_date=dataset_agg["start_date"], + end_date=dataset_agg["end_date"], ) - assert_stats_equal(dataset_basic["stats"], stats) + assert_stats_equal(dataset_agg["stats"], stats) + + +def test_force_refresh(client, dataset_basic, feature_validation_feature_set): + df = dataset_basic["df"] + + df2 = pd.DataFrame( + { + "datetime": [df.iloc[0].datetime], + "entity_id": [10], + "strings": ["c"], + "ints": [2], + "floats": [1.3], + } + ) + client.ingest(feature_validation_feature_set, df2) + + actual_stats = client.get_statistics( + f"{PROJECT_NAME}/feature_validation:1", + features=["strings", "ints", "floats"], + store="historical", + start_date=dataset_basic["date"], + end_date=dataset_basic["date"] + timedelta(days=1), + force_refresh=True, + ) + + combined_df = pd.concat([df, df2]) + expected_stats = tfdv.generate_statistics_from_dataframe(combined_df) + clear_unsupported_fields(expected_stats) + + assert_stats_equal(expected_stats, actual_stats) def clear_unsupported_fields(datasets): From 7a3a39412cf11b937b57a74c95bc83ec82b39e4f Mon Sep 17 00:00:00 2001 From: zhilingc Date: Wed, 22 Apr 2020 22:10:05 +0800 Subject: [PATCH 06/12] Remove tfx schema from entities Apply black Rename feature_ids to features --- .../java/feast/core/service/StatsService.java | 2 +- protos/feast/core/CoreService.proto | 2 +- sdk/python/feast/client.py | 2 +- sdk/python/feast/entity.py | 24 +------------------ sdk/python/feast/feature_set.py | 2 ++ .../bikeshare_feature_set.yaml | 9 ------- .../tensorflow_metadata/bikeshare_schema.json | 19 --------------- sdk/python/tests/test_feature_set.py | 3 --- 8 files changed, 6 insertions(+), 57 deletions(-) diff --git a/core/src/main/java/feast/core/service/StatsService.java b/core/src/main/java/feast/core/service/StatsService.java index 3331482bd35..af34a1ccb3c 100644 --- a/core/src/main/java/feast/core/service/StatsService.java +++ b/core/src/main/java/feast/core/service/StatsService.java @@ -103,7 +103,7 @@ public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsReq // 2. Filter out the features requested by the user. If none are provided, // use all features in the feature set. - List features = request.getFeatureIdsList(); + List features = request.getFeaturesList(); if (features.size() == 0) { features = featureSetSpec.getFeaturesList().stream() diff --git a/protos/feast/core/CoreService.proto b/protos/feast/core/CoreService.proto index 08a2af4ce68..fafefa07d9c 100644 --- a/protos/feast/core/CoreService.proto +++ b/protos/feast/core/CoreService.proto @@ -289,7 +289,7 @@ message GetFeatureStatisticsRequest { // Optional filter which filters returned statistics by selected features. These // features must be present in the data that is being processed. - repeated string feature_ids = 2; + repeated string features = 2; // Optional filter to select store over which the statistics will retrieved. // Only historical stores are allowed. diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 2611ef6da75..74560e572c7 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -919,7 +919,7 @@ def get_statistics( request = GetFeatureStatisticsRequest( feature_set_id=feature_set_id, - feature_ids=features, + features=features, store=store, force_refresh=force_refresh, ) diff --git a/sdk/python/feast/entity.py b/sdk/python/feast/entity.py index 9c5a027b974..012d01631af 100644 --- a/sdk/python/feast/entity.py +++ b/sdk/python/feast/entity.py @@ -29,26 +29,7 @@ def to_proto(self) -> EntityProto: Returns EntitySpec object """ value_type = ValueTypeProto.ValueType.Enum.Value(self.dtype.name) - return EntityProto( - name=self.name, - value_type=value_type, - presence=self.presence, - group_presence=self.group_presence, - shape=self.shape, - value_count=self.value_count, - domain=self.domain, - int_domain=self.int_domain, - float_domain=self.float_domain, - string_domain=self.string_domain, - bool_domain=self.bool_domain, - struct_domain=self.struct_domain, - natural_language_domain=self.natural_language_domain, - image_domain=self.image_domain, - mid_domain=self.mid_domain, - url_domain=self.url_domain, - time_domain=self.time_domain, - time_of_day_domain=self.time_of_day_domain, - ) + return EntityProto(name=self.name, value_type=value_type,) @classmethod def from_proto(cls, entity_proto: EntityProto): @@ -62,7 +43,4 @@ def from_proto(cls, entity_proto: EntityProto): Entity object """ entity = cls(name=entity_proto.name, dtype=ValueType(entity_proto.value_type)) - entity.update_presence_constraints(entity_proto) - entity.update_shape_type(entity_proto) - entity.update_domain_info(entity_proto) return entity diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py index 760e947318f..ace7f165de1 100644 --- a/sdk/python/feast/feature_set.py +++ b/sdk/python/feast/feature_set.py @@ -716,6 +716,8 @@ def export_tfx_schema(self) -> schema_pb2.Schema: ] for _, field in self._fields.items(): + if isinstance(field, Entity): + continue feature = schema_pb2.Feature() for attr in attributes_to_copy_from_field_to_feature: if getattr(field, attr) is None: diff --git a/sdk/python/tests/data/tensorflow_metadata/bikeshare_feature_set.yaml b/sdk/python/tests/data/tensorflow_metadata/bikeshare_feature_set.yaml index daa0a35f0ab..48c595712cb 100644 --- a/sdk/python/tests/data/tensorflow_metadata/bikeshare_feature_set.yaml +++ b/sdk/python/tests/data/tensorflow_metadata/bikeshare_feature_set.yaml @@ -3,15 +3,6 @@ spec: entities: - name: station_id valueType: INT64 - intDomain: - min: 1 - max: 5000 - presence: - minFraction: 1.0 - minCount: 1 - shape: - dim: - - size: 1 features: - name: location valueType: STRING diff --git a/sdk/python/tests/data/tensorflow_metadata/bikeshare_schema.json b/sdk/python/tests/data/tensorflow_metadata/bikeshare_schema.json index e7a886053c1..fa9f97cca0d 100644 --- a/sdk/python/tests/data/tensorflow_metadata/bikeshare_schema.json +++ b/sdk/python/tests/data/tensorflow_metadata/bikeshare_schema.json @@ -85,25 +85,6 @@ } ] } - }, - { - "name": "station_id", - "type": "INT", - "presence": { - "minFraction": 1.0, - "minCount": "1" - }, - "int_domain": { - "min": 1, - "max": 5000 - }, - "shape": { - "dim": [ - { - "size": "1" - } - ] - } } ], "stringDomain": [ diff --git a/sdk/python/tests/test_feature_set.py b/sdk/python/tests/test_feature_set.py index 0a7d1ebabea..a2cc12fe113 100644 --- a/sdk/python/tests/test_feature_set.py +++ b/sdk/python/tests/test_feature_set.py @@ -210,9 +210,6 @@ def test_import_tfx_schema(self): feature_set.import_tfx_schema(test_input_schema) # After update - for entity in feature_set.entities: - assert entity.presence is not None - assert entity.shape is not None for feature in feature_set.features: assert feature.presence is not None assert feature.shape is not None From 0dc1f07a2fd16cebb4674467caef75002a325b1e Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 23 Apr 2020 18:40:35 +0800 Subject: [PATCH 07/12] Group tests by serving type --- infra/scripts/test-end-to-end-batch.sh | 2 +- infra/scripts/test-end-to-end.sh | 2 +- tests/e2e/{ => bq}/bq-batch-retrieval.py | 154 ++++++++++------- .../feature-stats.py} | 74 +++++--- tests/e2e/conftest.py | 4 +- .../all_types_parquet/all_types_parquet.yaml | 0 .../{ => redis}/basic-ingest-redis-serving.py | 158 +++++++++--------- .../e2e/{ => redis}/basic/cust_trans_fs.yaml | 0 tests/e2e/{ => redis}/basic/data.csv | 0 .../large_volume/cust_trans_large_fs.yaml | 0 tests/e2e/requirements.txt | 2 + 11 files changed, 230 insertions(+), 166 deletions(-) rename tests/e2e/{ => bq}/bq-batch-retrieval.py (70%) rename tests/e2e/{feature-validation.py => bq/feature-stats.py} (77%) rename tests/e2e/{ => redis}/all_types_parquet/all_types_parquet.yaml (100%) rename tests/e2e/{ => redis}/basic-ingest-redis-serving.py (84%) rename tests/e2e/{ => redis}/basic/cust_trans_fs.yaml (100%) rename tests/e2e/{ => redis}/basic/data.csv (100%) rename tests/e2e/{ => redis}/large_volume/cust_trans_large_fs.yaml (100%) diff --git a/infra/scripts/test-end-to-end-batch.sh b/infra/scripts/test-end-to-end-batch.sh index 81a314315b7..9dcd74877f3 100755 --- a/infra/scripts/test-end-to-end-batch.sh +++ b/infra/scripts/test-end-to-end-batch.sh @@ -251,7 +251,7 @@ ORIGINAL_DIR=$(pwd) cd tests/e2e set +e -pytest bq-batch-retrieval.py feature-validation.py --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml +pytest bq/* --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml TEST_EXIT_CODE=$? if [[ ${TEST_EXIT_CODE} != 0 ]]; then diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index c33dadc5413..f5943f2518d 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -216,7 +216,7 @@ ORIGINAL_DIR=$(pwd) cd tests/e2e set +e -pytest basic-ingest-redis-serving.py --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml +pytest redis/* --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml TEST_EXIT_CODE=$? if [[ ${TEST_EXIT_CODE} != 0 ]]; then diff --git a/tests/e2e/bq-batch-retrieval.py b/tests/e2e/bq/bq-batch-retrieval.py similarity index 70% rename from tests/e2e/bq-batch-retrieval.py rename to tests/e2e/bq/bq-batch-retrieval.py index 0cf05e77e1d..fc37f3faf86 100644 --- a/tests/e2e/bq-batch-retrieval.py +++ b/tests/e2e/bq/bq-batch-retrieval.py @@ -18,9 +18,9 @@ from google.protobuf.duration_pb2 import Duration from pandavro import to_avro -pd.set_option('display.max_columns', None) +pd.set_option("display.max_columns", None) -PROJECT_NAME = 'batch_' + uuid.uuid4().hex.upper()[0:6] +PROJECT_NAME = "batch_" + uuid.uuid4().hex.upper()[0:6] @pytest.fixture(scope="module") @@ -54,60 +54,63 @@ def client(core_url, serving_url, allow_dirty): if not allow_dirty: feature_sets = client.list_feature_sets() if len(feature_sets) > 0: - raise Exception("Feast cannot have existing feature sets registered. Exiting tests.") + raise Exception( + "Feast cannot have existing feature sets registered. Exiting tests." + ) return client + @pytest.mark.first -def test_apply_all_featuresets(client): +def test_batch_apply_all_featuresets(client): client.set_project(PROJECT_NAME) file_fs1 = FeatureSet( - "file_feature_set", - features=[Feature("feature_value1", ValueType.STRING)], - entities=[Entity("entity_id", ValueType.INT64)], - max_age=Duration(seconds=100), - ) + "file_feature_set", + features=[Feature("feature_value1", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) client.apply(file_fs1) gcs_fs1 = FeatureSet( - "gcs_feature_set", - features=[Feature("feature_value2", ValueType.STRING)], - entities=[Entity("entity_id", ValueType.INT64)], - max_age=Duration(seconds=100), - ) + "gcs_feature_set", + features=[Feature("feature_value2", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) client.apply(gcs_fs1) proc_time_fs = FeatureSet( - "processing_time", - features=[Feature("feature_value3", ValueType.STRING)], - entities=[Entity("entity_id", ValueType.INT64)], - max_age=Duration(seconds=100), - ) + "processing_time", + features=[Feature("feature_value3", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) client.apply(proc_time_fs) add_cols_fs = FeatureSet( - "additional_columns", - features=[Feature("feature_value4", ValueType.STRING)], - entities=[Entity("entity_id", ValueType.INT64)], - max_age=Duration(seconds=100), - ) + "additional_columns", + features=[Feature("feature_value4", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) client.apply(add_cols_fs) historical_fs = FeatureSet( - "historical", - features=[Feature("feature_value5", ValueType.STRING)], - entities=[Entity("entity_id", ValueType.INT64)], - max_age=Duration(seconds=100), - ) + "historical", + features=[Feature("feature_value5", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) client.apply(historical_fs) fs1 = FeatureSet( - "feature_set_1", - features=[Feature("feature_value6", ValueType.STRING)], - entities=[Entity("entity_id", ValueType.INT64)], - max_age=Duration(seconds=100), - ) + "feature_set_1", + features=[Feature("feature_value6", ValueType.STRING)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=100), + ) fs2 = FeatureSet( "feature_set_2", @@ -127,7 +130,7 @@ def test_apply_all_featuresets(client): client.apply(no_max_age_fs) -def test_get_batch_features_with_file(client): +def test_batch_get_batch_features_with_file(client): file_fs1 = client.get_feature_set(name="file_feature_set", version=1) N_ROWS = 10 @@ -144,20 +147,26 @@ def test_get_batch_features_with_file(client): # Rename column (datetime -> event_timestamp) features_1_df = features_1_df.rename(columns={"datetime": "event_timestamp"}) - to_avro(df=features_1_df[["event_timestamp", "entity_id"]], file_path_or_buffer="file_feature_set.avro") + to_avro( + df=features_1_df[["event_timestamp", "entity_id"]], + file_path_or_buffer="file_feature_set.avro", + ) time.sleep(15) feature_retrieval_job = client.get_batch_features( - entity_rows="file://file_feature_set.avro", feature_refs=[f"{PROJECT_NAME}/feature_value1:1"] + entity_rows="file://file_feature_set.avro", + feature_refs=[f"{PROJECT_NAME}/feature_value1:1"], ) output = feature_retrieval_job.to_dataframe() print(output.head()) - assert output["entity_id"].to_list() == [int(i) for i in output["feature_value1"].to_list()] + assert output["entity_id"].to_list() == [ + int(i) for i in output["feature_value1"].to_list() + ] -def test_get_batch_features_with_gs_path(client, gcs_path): +def test_batch_get_batch_features_with_gs_path(client, gcs_path): gcs_fs1 = client.get_feature_set(name="gcs_feature_set", version=1) N_ROWS = 10 @@ -176,7 +185,10 @@ def test_get_batch_features_with_gs_path(client, gcs_path): # Output file to local file_name = "gcs_feature_set.avro" - to_avro(df=features_1_df[["event_timestamp", "entity_id"]], file_path_or_buffer=file_name) + to_avro( + df=features_1_df[["event_timestamp", "entity_id"]], + file_path_or_buffer=file_name, + ) uri = urlparse(gcs_path) bucket = uri.hostname @@ -192,16 +204,18 @@ def test_get_batch_features_with_gs_path(client, gcs_path): time.sleep(15) feature_retrieval_job = client.get_batch_features( entity_rows=f"{gcs_path}{ts}/*", - feature_refs=[f"{PROJECT_NAME}/feature_value2:1"] + feature_refs=[f"{PROJECT_NAME}/feature_value2:1"], ) output = feature_retrieval_job.to_dataframe() print(output.head()) - assert output["entity_id"].to_list() == [int(i) for i in output["feature_value2"].to_list()] + assert output["entity_id"].to_list() == [ + int(i) for i in output["feature_value2"].to_list() + ] -def test_order_by_creation_time(client): +def test_batch_order_by_creation_time(client): proc_time_fs = client.get_feature_set(name="processing_time", version=1) time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) @@ -224,7 +238,8 @@ def test_order_by_creation_time(client): time.sleep(15) client.ingest(proc_time_fs, correct_df) feature_retrieval_job = client.get_batch_features( - entity_rows=incorrect_df[["datetime", "entity_id"]], feature_refs=[f"{PROJECT_NAME}/feature_value3:1"] + entity_rows=incorrect_df[["datetime", "entity_id"]], + feature_refs=[f"{PROJECT_NAME}/feature_value3:1"], ) output = feature_retrieval_job.to_dataframe() print(output.head()) @@ -232,13 +247,17 @@ def test_order_by_creation_time(client): assert output["feature_value3"].to_list() == ["CORRECT"] * N_ROWS -def test_additional_columns_in_entity_table(client): +def test_batch_additional_columns_in_entity_table(client): add_cols_fs = client.get_feature_set(name="additional_columns", version=1) N_ROWS = 10 time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) features_df = pd.DataFrame( - {"datetime": [time_offset] * N_ROWS, "entity_id": [i for i in range(N_ROWS)], "feature_value4": ["abc"] * N_ROWS} + { + "datetime": [time_offset] * N_ROWS, + "entity_id": [i for i in range(N_ROWS)], + "feature_value4": ["abc"] * N_ROWS, + } ) client.ingest(add_cols_fs, features_df) @@ -258,12 +277,17 @@ def test_additional_columns_in_entity_table(client): output = feature_retrieval_job.to_dataframe().sort_values(by=["entity_id"]) print(output.head(10)) - assert np.allclose(output["additional_float_col"], entity_df["additional_float_col"]) - assert output["additional_string_col"].to_list() == entity_df["additional_string_col"].to_list() + assert np.allclose( + output["additional_float_col"], entity_df["additional_float_col"] + ) + assert ( + output["additional_string_col"].to_list() + == entity_df["additional_string_col"].to_list() + ) assert output["feature_value4"].to_list() == features_df["feature_value4"].to_list() -def test_point_in_time_correctness_join(client): +def test_batch_point_in_time_correctness_join(client): historical_fs = client.get_feature_set(name="historical", version=1) time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) @@ -281,20 +305,25 @@ def test_point_in_time_correctness_join(client): } ) entity_df = pd.DataFrame( - {"datetime": [time_offset - timedelta(seconds=10)] * N_EXAMPLES, "entity_id": [i for i in range(N_EXAMPLES)]} + { + "datetime": [time_offset - timedelta(seconds=10)] * N_EXAMPLES, + "entity_id": [i for i in range(N_EXAMPLES)], + } ) client.ingest(historical_fs, historical_df) time.sleep(15) - feature_retrieval_job = client.get_batch_features(entity_rows=entity_df, feature_refs=[f"{PROJECT_NAME}/feature_value5"]) + feature_retrieval_job = client.get_batch_features( + entity_rows=entity_df, feature_refs=[f"{PROJECT_NAME}/feature_value5"] + ) output = feature_retrieval_job.to_dataframe() print(output.head()) assert output["feature_value5"].to_list() == ["CORRECT"] * N_EXAMPLES -def test_multiple_featureset_joins(client): +def test_batch_multiple_featureset_joins(client): fs1 = client.get_feature_set(name="feature_set_1", version=1) fs2 = client.get_feature_set(name="feature_set_2", version=1) @@ -328,16 +357,24 @@ def test_multiple_featureset_joins(client): time.sleep(15) feature_retrieval_job = client.get_batch_features( - entity_rows=entity_df, feature_refs=[f"{PROJECT_NAME}/feature_value6:1", f"{PROJECT_NAME}/other_feature_value7:1"] + entity_rows=entity_df, + feature_refs=[ + f"{PROJECT_NAME}/feature_value6:1", + f"{PROJECT_NAME}/other_feature_value7:1", + ], ) output = feature_retrieval_job.to_dataframe() print(output.head()) - assert output["entity_id"].to_list() == [int(i) for i in output["feature_value6"].to_list()] - assert output["other_entity_id"].to_list() == output["other_feature_value7"].to_list() + assert output["entity_id"].to_list() == [ + int(i) for i in output["feature_value6"].to_list() + ] + assert ( + output["other_entity_id"].to_list() == output["other_feature_value7"].to_list() + ) -def test_no_max_age(client): +def test_batch_no_max_age(client): no_max_age_fs = client.get_feature_set(name="no_max_age", version=1) time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) @@ -353,10 +390,11 @@ def test_no_max_age(client): time.sleep(15) feature_retrieval_job = client.get_batch_features( - entity_rows=features_8_df[["datetime", "entity_id"]], feature_refs=[f"{PROJECT_NAME}/feature_value8:1"] + entity_rows=features_8_df[["datetime", "entity_id"]], + feature_refs=[f"{PROJECT_NAME}/feature_value8:1"], ) output = feature_retrieval_job.to_dataframe() print(output.head()) - assert output["entity_id"].to_list() == output["feature_value8"].to_list() \ No newline at end of file + assert output["entity_id"].to_list() == output["feature_value8"].to_list() diff --git a/tests/e2e/feature-validation.py b/tests/e2e/bq/feature-stats.py similarity index 77% rename from tests/e2e/feature-validation.py rename to tests/e2e/bq/feature-stats.py index 90f0a717329..298efa3fcce 100644 --- a/tests/e2e/feature-validation.py +++ b/tests/e2e/bq/feature-stats.py @@ -21,6 +21,26 @@ STORE_NAME = "historical" +@pytest.fixture(scope="module") +def core_url(pytestconfig): + return pytestconfig.getoption("core_url") + + +@pytest.fixture(scope="module") +def serving_url(pytestconfig): + return pytestconfig.getoption("serving_url") + + +@pytest.fixture(scope="module") +def allow_dirty(pytestconfig): + return True if pytestconfig.getoption("allow_dirty").lower() == "true" else False + + +@pytest.fixture(scope="module") +def gcs_path(pytestconfig): + return pytestconfig.getoption("gcs_path") + + @pytest.fixture(scope="module") def client(core_url, allow_dirty): # Get client for core and serving @@ -40,9 +60,9 @@ def client(core_url, allow_dirty): @pytest.fixture(scope="module") -def feature_validation_feature_set(client): +def feature_stats_feature_set(client): fv_fs = FeatureSet( - "feature_validation", + "feature_stats", features=[ Feature("strings", ValueType.STRING), Feature("ints", ValueType.INT64), @@ -56,7 +76,7 @@ def feature_validation_feature_set(client): @pytest.fixture(scope="module") -def dataset_basic(client, feature_validation_feature_set): +def feature_stats_feature_stats_dataset_basic(client, feature_stats_feature_set): N_ROWS = 20 @@ -78,7 +98,7 @@ def dataset_basic(client, feature_validation_feature_set): return { "df": df, - "id": client.ingest(feature_validation_feature_set, df), + "id": client.ingest(feature_stats_feature_set, df), "date": datetime(time_offset.year, time_offset.month, time_offset.day).replace( tzinfo=pytz.utc ), @@ -87,7 +107,7 @@ def dataset_basic(client, feature_validation_feature_set): @pytest.fixture(scope="module") -def dataset_agg(client, feature_validation_feature_set): +def feature_stats_feature_stats_dataset_agg(client, feature_stats_feature_set): time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) start_date = time_offset - timedelta(days=10) end_date = time_offset - timedelta(days=7) @@ -100,7 +120,7 @@ def dataset_agg(client, feature_validation_feature_set): "floats": [2.1, 5.2, 4.3, 0.6, 0.1], } ) - dataset_id_1 = client.ingest(feature_validation_feature_set, df1) + dataset_id_1 = client.ingest(feature_stats_feature_set, df1) df2 = pd.DataFrame( { "datetime": [start_date + timedelta(days=1)] * 3, @@ -110,7 +130,7 @@ def dataset_agg(client, feature_validation_feature_set): "floats": [1.6, 2.4, 2], } ) - dataset_id_2 = client.ingest(feature_validation_feature_set, df2) + dataset_id_2 = client.ingest(feature_stats_feature_set, df2) combined_df = pd.concat([df1, df2])[["entity_id", "strings", "ints", "floats"]] expected_stats = tfdv.generate_statistics_from_dataframe(combined_df) @@ -135,51 +155,53 @@ def dataset_agg(client, feature_validation_feature_set): } -def test_basic_retrieval_by_single_dataset(client, dataset_basic): +def test_feature_stats_retrieval_by_single_dataset(client, feature_stats_dataset_basic): stats = client.get_statistics( f"{PROJECT_NAME}/feature_validation:1", features=["strings", "ints", "floats"], store=STORE_NAME, - dataset_ids=[dataset_basic["id"]], + dataset_ids=[feature_stats_dataset_basic["id"]], ) - assert_stats_equal(dataset_basic["stats"], stats) + assert_stats_equal(feature_stats_dataset_basic["stats"], stats) -def test_basic_by_date(client, dataset_basic): +def test_feature_stats_by_date(client, feature_stats_dataset_basic): stats = client.get_statistics( f"{PROJECT_NAME}/feature_validation:1", features=["strings", "ints", "floats"], store=STORE_NAME, - start_date=dataset_basic["date"], - end_date=dataset_basic["date"] + timedelta(days=1), + start_date=feature_stats_dataset_basic["date"], + end_date=feature_stats_dataset_basic["date"] + timedelta(days=1), ) - assert_stats_equal(dataset_basic["stats"], stats) + assert_stats_equal(feature_stats_dataset_basic["stats"], stats) -def test_agg_over_datasets(client, dataset_agg): +def test_feature_stats_agg_over_datasets(client, feature_stats_dataset_agg): stats = client.get_statistics( f"{PROJECT_NAME}/feature_validation:1", features=["strings", "ints", "floats"], store=STORE_NAME, - dataset_ids=[dataset_basic["ids"]], + dataset_ids=[feature_stats_dataset_basic["ids"]], ) - assert_stats_equal(dataset_basic["stats"], stats) + assert_stats_equal(feature_stats_dataset_basic["stats"], stats) -def test_agg_over_dates(client, dataset_agg): +def test_feature_stats_agg_over_dates(client, feature_stats_dataset_agg): stats = client.get_statistics( f"{PROJECT_NAME}/feature_validation:1", features=["strings", "ints", "floats"], store=STORE_NAME, - start_date=dataset_agg["start_date"], - end_date=dataset_agg["end_date"], + start_date=feature_stats_dataset_agg["start_date"], + end_date=feature_stats_dataset_agg["end_date"], ) - assert_stats_equal(dataset_agg["stats"], stats) + assert_stats_equal(feature_stats_dataset_agg["stats"], stats) -def test_force_refresh(client, dataset_basic, feature_validation_feature_set): - df = dataset_basic["df"] +def test_feature_stats_force_refresh( + client, feature_stats_dataset_basic, feature_stats_feature_set +): + df = feature_stats_dataset_basic["df"] df2 = pd.DataFrame( { @@ -190,14 +212,14 @@ def test_force_refresh(client, dataset_basic, feature_validation_feature_set): "floats": [1.3], } ) - client.ingest(feature_validation_feature_set, df2) + client.ingest(feature_stats_feature_set, df2) actual_stats = client.get_statistics( f"{PROJECT_NAME}/feature_validation:1", features=["strings", "ints", "floats"], store="historical", - start_date=dataset_basic["date"], - end_date=dataset_basic["date"] + timedelta(days=1), + start_date=feature_stats_dataset_basic["date"], + end_date=feature_stats_dataset_basic["date"] + timedelta(days=1), force_refresh=True, ) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 8ea472b6620..c1c3726844f 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -2,4 +2,6 @@ def pytest_addoption(parser): parser.addoption("--core_url", action="store", default="localhost:6565") parser.addoption("--serving_url", action="store", default="localhost:6566") parser.addoption("--allow_dirty", action="store", default="False") - parser.addoption("--gcs_path", action="store", default="gs://feast-templocation-kf-feast/") + parser.addoption( + "--gcs_path", action="store", default="gs://feast-templocation-kf-feast/" + ) diff --git a/tests/e2e/all_types_parquet/all_types_parquet.yaml b/tests/e2e/redis/all_types_parquet/all_types_parquet.yaml similarity index 100% rename from tests/e2e/all_types_parquet/all_types_parquet.yaml rename to tests/e2e/redis/all_types_parquet/all_types_parquet.yaml diff --git a/tests/e2e/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py similarity index 84% rename from tests/e2e/basic-ingest-redis-serving.py rename to tests/e2e/redis/basic-ingest-redis-serving.py index 8e40794344e..92a7ffb3a8f 100644 --- a/tests/e2e/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -24,25 +24,25 @@ import uuid FLOAT_TOLERANCE = 0.00001 -PROJECT_NAME = 'basic_' + uuid.uuid4().hex.upper()[0:6] +PROJECT_NAME = "basic_" + uuid.uuid4().hex.upper()[0:6] -@pytest.fixture(scope='module') + +@pytest.fixture(scope="module") def core_url(pytestconfig): return pytestconfig.getoption("core_url") -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def serving_url(pytestconfig): return pytestconfig.getoption("serving_url") -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def allow_dirty(pytestconfig): - return True if pytestconfig.getoption( - "allow_dirty").lower() == "true" else False + return True if pytestconfig.getoption("allow_dirty").lower() == "true" else False -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def client(core_url, serving_url, allow_dirty): # Get client for core and serving client = Client(core_url=core_url, serving_url=serving_url) @@ -60,13 +60,12 @@ def client(core_url, serving_url, allow_dirty): return client -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def basic_dataframe(): offset = random.randint(1000, 100000) # ensure a unique key space is used return pd.DataFrame( { - "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in - range(5)], + "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in range(5)], "customer_id": [offset + inc for inc in range(5)], "daily_transactions": [np.random.rand() for _ in range(5)], "total_transactions": [512 for _ in range(5)], @@ -109,6 +108,7 @@ def test_basic_ingest_success(client, basic_dataframe): client.ingest(cust_trans_fs, basic_dataframe) time.sleep(5) + @pytest.mark.timeout(45) @pytest.mark.run(order=12) def test_basic_retrieve_online_success(client, basic_dataframe): @@ -128,10 +128,7 @@ def test_basic_retrieve_online_success(client, basic_dataframe): } ) ], - feature_refs=[ - "daily_transactions", - "total_transactions", - ], + feature_refs=["daily_transactions", "total_transactions",], ) # type: GetOnlineFeaturesResponse if response is None: @@ -139,11 +136,10 @@ def test_basic_retrieve_online_success(client, basic_dataframe): returned_daily_transactions = float( response.field_values[0] - .fields[PROJECT_NAME + "/daily_transactions"] - .float_val + .fields[PROJECT_NAME + "/daily_transactions"] + .float_val ) - sent_daily_transactions = float( - basic_dataframe.iloc[0]["daily_transactions"]) + sent_daily_transactions = float(basic_dataframe.iloc[0]["daily_transactions"]) if math.isclose( sent_daily_transactions, @@ -152,15 +148,19 @@ def test_basic_retrieve_online_success(client, basic_dataframe): ): break + @pytest.mark.timeout(300) @pytest.mark.run(order=19) def test_basic_ingest_jobs(client, basic_dataframe): # list ingestion jobs given featureset cust_trans_fs = client.get_feature_set(name="customer_transactions") ingest_jobs = client.list_ingest_jobs( - feature_set_ref=FeatureSetRef.from_feature_set(cust_trans_fs)) + feature_set_ref=FeatureSetRef.from_feature_set(cust_trans_fs) + ) # filter ingestion jobs to only those that are running - ingest_jobs = [job for job in ingest_jobs if job.status == IngestionJobStatus.RUNNING] + ingest_jobs = [ + job for job in ingest_jobs if job.status == IngestionJobStatus.RUNNING + ] assert len(ingest_jobs) >= 1 for ingest_job in ingest_jobs: @@ -175,18 +175,16 @@ def test_basic_ingest_jobs(client, basic_dataframe): assert ingest_job.status == IngestionJobStatus.ABORTED -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def all_types_dataframe(): return pd.DataFrame( { - "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in - range(3)], + "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in range(3)], "user_id": [1001, 1002, 1003], "int32_feature": [np.int32(1), np.int32(2), np.int32(3)], "int64_feature": [np.int64(1), np.int64(2), np.int64(3)], "float_feature": [np.float(0.1), np.float(0.2), np.float(0.3)], - "double_feature": [np.float64(0.1), np.float64(0.2), - np.float64(0.3)], + "double_feature": [np.float64(0.1), np.float64(0.2), np.float64(0.3)], "string_feature": ["one", "two", "three"], "bytes_feature": [b"one", b"two", b"three"], "bool_feature": [True, False, False], @@ -248,8 +246,7 @@ def test_all_types_register_feature_set_success(client): Feature(name="float_list_feature", dtype=ValueType.FLOAT_LIST), Feature(name="int64_list_feature", dtype=ValueType.INT64_LIST), Feature(name="int32_list_feature", dtype=ValueType.INT32_LIST), - Feature(name="string_list_feature", - dtype=ValueType.STRING_LIST), + Feature(name="string_list_feature", dtype=ValueType.STRING_LIST), Feature(name="bytes_list_feature", dtype=ValueType.BYTES_LIST), ], max_age=Duration(seconds=3600), @@ -295,8 +292,11 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( - fields={"user_id": Value( - int64_val=all_types_dataframe.iloc[0]["user_id"])} + fields={ + "user_id": Value( + int64_val=all_types_dataframe.iloc[0]["user_id"] + ) + } ) ], feature_refs=[ @@ -319,11 +319,10 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): if response is None: continue - returned_float_list = ( response.field_values[0] - .fields[PROJECT_NAME+"/float_list_feature"] - .float_list_val.val + .fields[PROJECT_NAME + "/float_list_feature"] + .float_list_val.val ) sent_float_list = all_types_dataframe.iloc[0]["float_list_feature"] @@ -333,15 +332,19 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): ): break + @pytest.mark.timeout(300) @pytest.mark.run(order=29) def test_all_types_ingest_jobs(client, all_types_dataframe): # list ingestion jobs given featureset all_types_fs = client.get_feature_set(name="all_types") ingest_jobs = client.list_ingest_jobs( - feature_set_ref=FeatureSetRef.from_feature_set(all_types_fs)) + feature_set_ref=FeatureSetRef.from_feature_set(all_types_fs) + ) # filter ingestion jobs to only those that are running - ingest_jobs = [job for job in ingest_jobs if job.status == IngestionJobStatus.RUNNING] + ingest_jobs = [ + job for job in ingest_jobs if job.status == IngestionJobStatus.RUNNING + ] assert len(ingest_jobs) >= 1 for ingest_job in ingest_jobs: @@ -355,15 +358,15 @@ def test_all_types_ingest_jobs(client, all_types_dataframe): ingest_job.wait(IngestionJobStatus.ABORTED) assert ingest_job.status == IngestionJobStatus.ABORTED -@pytest.fixture(scope='module') + +@pytest.fixture(scope="module") def large_volume_dataframe(): ROW_COUNT = 100000 offset = random.randint(1000000, 10000000) # ensure a unique key space customer_data = pd.DataFrame( { "datetime": [ - datetime.utcnow().replace(tzinfo=pytz.utc) for _ in - range(ROW_COUNT) + datetime.utcnow().replace(tzinfo=pytz.utc) for _ in range(ROW_COUNT) ], "customer_id": [offset + inc for inc in range(ROW_COUNT)], "daily_transactions_large": [np.random.rand() for _ in range(ROW_COUNT)], @@ -377,7 +380,8 @@ def large_volume_dataframe(): @pytest.mark.run(order=30) def test_large_volume_register_feature_set_success(client): cust_trans_fs_expected = FeatureSet.from_yaml( - "large_volume/cust_trans_large_fs.yaml") + "large_volume/cust_trans_large_fs.yaml" + ) # Register feature set client.apply(cust_trans_fs_expected) @@ -385,8 +389,7 @@ def test_large_volume_register_feature_set_success(client): # Feast Core needs some time to fully commit the FeatureSet applied # when there is no existing job yet for the Featureset time.sleep(10) - cust_trans_fs_actual = client.get_feature_set( - name="customer_transactions_large") + cust_trans_fs_actual = client.get_feature_set(name="customer_transactions_large") assert cust_trans_fs_actual == cust_trans_fs_expected @@ -421,16 +424,12 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): GetOnlineFeaturesRequest.EntityRow( fields={ "customer_id": Value( - int64_val=large_volume_dataframe.iloc[0][ - "customer_id"] + int64_val=large_volume_dataframe.iloc[0]["customer_id"] ) } ) ], - feature_refs=[ - "daily_transactions_large", - "total_transactions_large", - ], + feature_refs=["daily_transactions_large", "total_transactions_large",], ) # type: GetOnlineFeaturesResponse if response is None: @@ -438,11 +437,12 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): returned_daily_transactions = float( response.field_values[0] - .fields[PROJECT_NAME + "/daily_transactions_large"] - .float_val + .fields[PROJECT_NAME + "/daily_transactions_large"] + .float_val ) sent_daily_transactions = float( - large_volume_dataframe.iloc[0]["daily_transactions_large"]) + large_volume_dataframe.iloc[0]["daily_transactions_large"] + ) if math.isclose( sent_daily_transactions, @@ -452,49 +452,47 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): break -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def all_types_parquet_file(): COUNT = 20000 df = pd.DataFrame( { "datetime": [datetime.utcnow() for _ in range(COUNT)], - "customer_id": [np.int32(random.randint(0, 10000)) for _ in - range(COUNT)], - "int32_feature_parquet": [np.int32(random.randint(0, 10000)) for _ in - range(COUNT)], - "int64_feature_parquet": [np.int64(random.randint(0, 10000)) for _ in - range(COUNT)], + "customer_id": [np.int32(random.randint(0, 10000)) for _ in range(COUNT)], + "int32_feature_parquet": [ + np.int32(random.randint(0, 10000)) for _ in range(COUNT) + ], + "int64_feature_parquet": [ + np.int64(random.randint(0, 10000)) for _ in range(COUNT) + ], "float_feature_parquet": [np.float(random.random()) for _ in range(COUNT)], - "double_feature_parquet": [np.float64(random.random()) for _ in - range(COUNT)], - "string_feature_parquet": ["one" + str(random.random()) for _ in - range(COUNT)], + "double_feature_parquet": [ + np.float64(random.random()) for _ in range(COUNT) + ], + "string_feature_parquet": [ + "one" + str(random.random()) for _ in range(COUNT) + ], "bytes_feature_parquet": [b"one" for _ in range(COUNT)], "int32_list_feature_parquet": [ np.array([1, 2, 3, random.randint(0, 10000)], dtype=np.int32) - for _ - in range(COUNT) + for _ in range(COUNT) ], "int64_list_feature_parquet": [ np.array([1, random.randint(0, 10000), 3, 4], dtype=np.int64) - for _ - in range(COUNT) + for _ in range(COUNT) ], "float_list_feature_parquet": [ - np.array([1.1, 1.2, 1.3, random.random()], dtype=np.float32) for - _ - in range(COUNT) + np.array([1.1, 1.2, 1.3, random.random()], dtype=np.float32) + for _ in range(COUNT) ], "double_list_feature_parquet": [ - np.array([1.1, 1.2, 1.3, random.random()], dtype=np.float64) for - _ - in range(COUNT) + np.array([1.1, 1.2, 1.3, random.random()], dtype=np.float64) + for _ in range(COUNT) ], "string_list_feature_parquet": [ - np.array(["one", "two" + str(random.random()), "three"]) for _ - in - range(COUNT) + np.array(["one", "two" + str(random.random()), "three"]) + for _ in range(COUNT) ], "bytes_list_feature_parquet": [ np.array([b"one", b"two", b"three"]) for _ in range(COUNT) @@ -505,16 +503,18 @@ def all_types_parquet_file(): # TODO: Boolean list is not being tested. # https://github.com/gojek/feast/issues/341 - file_path = os.path.join(tempfile.mkdtemp(), 'all_types.parquet') + file_path = os.path.join(tempfile.mkdtemp(), "all_types.parquet") df.to_parquet(file_path, allow_truncated_timestamps=True) return file_path + @pytest.mark.timeout(300) @pytest.mark.run(order=40) def test_all_types_parquet_register_feature_set_success(client): # Load feature set from file all_types_parquet_expected = FeatureSet.from_yaml( - "all_types_parquet/all_types_parquet.yaml") + "all_types_parquet/all_types_parquet.yaml" + ) # Register feature set client.apply(all_types_parquet_expected) @@ -538,11 +538,11 @@ def test_all_types_parquet_register_feature_set_success(client): @pytest.mark.timeout(600) @pytest.mark.run(order=41) -def test_all_types_infer_register_ingest_file_success(client, - all_types_parquet_file): +def test_all_types_infer_register_ingest_file_success(client, all_types_parquet_file): # Get feature set all_types_fs = client.get_feature_set(name="all_types_parquet") # Ingest user embedding data - client.ingest(feature_set=all_types_fs, source=all_types_parquet_file, - force_update=True) + client.ingest( + feature_set=all_types_fs, source=all_types_parquet_file, force_update=True + ) diff --git a/tests/e2e/basic/cust_trans_fs.yaml b/tests/e2e/redis/basic/cust_trans_fs.yaml similarity index 100% rename from tests/e2e/basic/cust_trans_fs.yaml rename to tests/e2e/redis/basic/cust_trans_fs.yaml diff --git a/tests/e2e/basic/data.csv b/tests/e2e/redis/basic/data.csv similarity index 100% rename from tests/e2e/basic/data.csv rename to tests/e2e/redis/basic/data.csv diff --git a/tests/e2e/large_volume/cust_trans_large_fs.yaml b/tests/e2e/redis/large_volume/cust_trans_large_fs.yaml similarity index 100% rename from tests/e2e/large_volume/cust_trans_large_fs.yaml rename to tests/e2e/redis/large_volume/cust_trans_large_fs.yaml diff --git a/tests/e2e/requirements.txt b/tests/e2e/requirements.txt index 0ba345a000f..82a65ff971e 100644 --- a/tests/e2e/requirements.txt +++ b/tests/e2e/requirements.txt @@ -7,3 +7,5 @@ pytest-benchmark==3.2.2 pytest-mock==1.10.4 pytest-timeout==1.3.3 pytest-ordering==0.6.* +tensorflow-data-validation==0.21.2 +deepdiff==4.3.2 \ No newline at end of file From e221414bd12a35a5d095f2ed9a5615bd83ca22fd Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 23 Apr 2020 20:21:45 +0800 Subject: [PATCH 08/12] Remove stutter in fixture --- tests/e2e/bq/feature-stats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/bq/feature-stats.py b/tests/e2e/bq/feature-stats.py index 298efa3fcce..0dce7c07256 100644 --- a/tests/e2e/bq/feature-stats.py +++ b/tests/e2e/bq/feature-stats.py @@ -76,7 +76,7 @@ def feature_stats_feature_set(client): @pytest.fixture(scope="module") -def feature_stats_feature_stats_dataset_basic(client, feature_stats_feature_set): +def feature_stats_dataset_basic(client, feature_stats_feature_set): N_ROWS = 20 @@ -107,7 +107,7 @@ def feature_stats_feature_stats_dataset_basic(client, feature_stats_feature_set) @pytest.fixture(scope="module") -def feature_stats_feature_stats_dataset_agg(client, feature_stats_feature_set): +def feature_stats_dataset_agg(client, feature_stats_feature_set): time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) start_date = time_offset - timedelta(days=10) end_date = time_offset - timedelta(days=7) From ba0b535c18a70aaf53cbbf2e67c7e7e07971a1c5 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 23 Apr 2020 22:01:44 +0800 Subject: [PATCH 09/12] Add tfx-bsl to dependencies --- tests/e2e/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e/requirements.txt b/tests/e2e/requirements.txt index 82a65ff971e..1183a68587b 100644 --- a/tests/e2e/requirements.txt +++ b/tests/e2e/requirements.txt @@ -8,4 +8,5 @@ pytest-mock==1.10.4 pytest-timeout==1.3.3 pytest-ordering==0.6.* tensorflow-data-validation==0.21.2 -deepdiff==4.3.2 \ No newline at end of file +deepdiff==4.3.2 +tfx-bsl==0.21.2 \ No newline at end of file From 91185540fb8cebb611f27ca2fe9b2ad5cdb6eb53 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Fri, 24 Apr 2020 12:10:00 +0800 Subject: [PATCH 10/12] Remove tfx-bsl from dependencies --- sdk/python/setup.py | 2 +- tests/e2e/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 9d8a3786505..10c4c29a8a6 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -42,7 +42,7 @@ "tabulate==0.8.*", "toml==0.10.*", "tqdm==4.*", - "pyarrow>=0.15.1", + "pyarrow<0.16.0,>=0.15.1", "numpy", "google", "confluent_kafka", diff --git a/tests/e2e/requirements.txt b/tests/e2e/requirements.txt index 1183a68587b..5a21a6a0cea 100644 --- a/tests/e2e/requirements.txt +++ b/tests/e2e/requirements.txt @@ -9,4 +9,4 @@ pytest-timeout==1.3.3 pytest-ordering==0.6.* tensorflow-data-validation==0.21.2 deepdiff==4.3.2 -tfx-bsl==0.21.2 \ No newline at end of file +tensorflow==2.1.0 \ No newline at end of file From 7a7a5b51eb7c1fc6f69f42f3a87d73e6eb9df15f Mon Sep 17 00:00:00 2001 From: zhilingc Date: Fri, 24 Apr 2020 13:08:54 +0800 Subject: [PATCH 11/12] Fix tests --- .../java/feast/core/service/StatsService.java | 57 +++++++++++++++++- .../bigquery/statistics/StatsQueryResult.java | 2 +- .../statistics/StatsQueryResultTest.java | 4 +- tests/e2e/bq/feature-stats.py | 58 ++++++++++++++----- tests/e2e/redis/basic-ingest-redis-serving.py | 7 ++- 5 files changed, 104 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/feast/core/service/StatsService.java b/core/src/main/java/feast/core/service/StatsService.java index af34a1ccb3c..0f1fea1ac37 100644 --- a/core/src/main/java/feast/core/service/StatsService.java +++ b/core/src/main/java/feast/core/service/StatsService.java @@ -41,6 +41,10 @@ import java.util.*; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -130,6 +134,15 @@ public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsReq featureNameStatisticsList.add(featureNameStatistics); timestamp += 86400; // advance by a day } + if (featureNameStatisticsList.size() == 0) { + DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd"); + DateTime startDateTime = new DateTime(startDate.getSeconds() * 1000, DateTimeZone.UTC); + DateTime endDateTime = new DateTime(endDate.getSeconds() * 1000, DateTimeZone.UTC); + throw new RetrievalException( + String.format( + "Unable to find any data over provided dates [%s, %s)", + fmt.print(startDateTime), fmt.print(endDateTime))); + } } else { // else, retrieve by dataset for (String datasetId : request.getDatasetIdsList()) { @@ -141,6 +154,12 @@ public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsReq datasetId, request.getForceRefresh()); featureNameStatisticsList.add(featureNameStatistics); + if (featureNameStatisticsList.size() == 0) { + throw new RetrievalException( + String.format( + "Unable to find any data over provided datasets %s", + request.getDatasetIdsList())); + } } } @@ -212,6 +231,9 @@ private List getFeatureNameStatisticsByDataset( // Persist the newly retrieved statistics in the cache. for (FeatureNameStatistics stat : featureSetStatistics.getFeatureNameStatistics()) { + if (isEmpty(stat)) { + continue; + } FeatureStatistics featureStatistics = FeatureStatistics.createForDataset( featureSetSpec.getProject(), @@ -224,8 +246,8 @@ private List getFeatureNameStatisticsByDataset( featureStatistics.getFeature(), datasetId); existingRecord.ifPresent(statistics -> featureStatistics.setId(statistics.getId())); featureStatisticsRepository.save(featureStatistics); + featureNameStatistics.add(stat); } - featureNameStatistics.addAll(featureSetStatistics.getFeatureNameStatistics()); } return featureNameStatistics; } @@ -288,6 +310,9 @@ private List getFeatureNameStatisticsByDate( // Persist the newly retrieved statistics in the cache. for (FeatureNameStatistics stat : featureSetStatistics.getFeatureNameStatistics()) { + if (isEmpty(stat)) { + continue; + } FeatureStatistics featureStatistics = FeatureStatistics.createForDate( featureSetSpec.getProject(), @@ -300,8 +325,8 @@ private List getFeatureNameStatisticsByDate( featureStatistics.getFeature(), date); existingRecord.ifPresent(statistics -> featureStatistics.setId(statistics.getId())); featureStatisticsRepository.save(featureStatistics); + featureNameStatistics.add(stat); } - featureNameStatistics.addAll(featureSetStatistics.getFeatureNameStatistics()); } return featureNameStatistics; } @@ -596,4 +621,32 @@ private void validateRequest(GetFeatureStatisticsRequest request) { } } } + + private boolean isEmpty(FeatureNameStatistics featureNameStatistics) { + switch (featureNameStatistics.getType()) { + case STRUCT: + return featureNameStatistics + .getStructStats() + .getCommonStats() + .equals(CommonStatistics.getDefaultInstance()); + case STRING: + return featureNameStatistics + .getStringStats() + .getCommonStats() + .equals(CommonStatistics.getDefaultInstance()); + case BYTES: + return featureNameStatistics + .getBytesStats() + .getCommonStats() + .equals(CommonStatistics.getDefaultInstance()); + case FLOAT: + case INT: + return featureNameStatistics + .getNumStats() + .getCommonStats() + .equals(CommonStatistics.getDefaultInstance()); + default: + return true; + } + } } diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryResult.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryResult.java index 7ed9cc797a0..62108643455 100644 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryResult.java +++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryResult.java @@ -306,8 +306,8 @@ private NumericStatistics getNumericStatistics(Map valuesMap .setMaxNumValues(1) .setAvgNumValues(1) .setTotNumValues(valuesMap.get("feature_count").getLongValue())) - .addHistograms(quantilesBuilder) .addHistograms(histBuilder) + .addHistograms(quantilesBuilder) .build(); } diff --git a/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsQueryResultTest.java b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsQueryResultTest.java index 6c49153c13b..500908291a3 100644 --- a/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsQueryResultTest.java +++ b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsQueryResultTest.java @@ -63,7 +63,7 @@ public class StatsQueryResultTest { com.google.cloud.bigquery.Field.of("count", LegacySQLTypeName.INTEGER))); @Test - public void voidShouldConvertNumericStatsToFeatureNameStatistics() + public void shouldConvertNumericStatsToFeatureNameStatistics() throws InvalidProtocolBufferException { FieldValueList numericFieldValueList = FieldValueList.of( @@ -128,7 +128,7 @@ public void voidShouldConvertNumericStatsToFeatureNameStatistics() .toFeatureNameStatistics(featureSpec.getValueType()); String expectedJson = - "{\"type\":\"FLOAT\",\"numStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"1\",\"maxNumValues\":\"1\",\"avgNumValues\":1,\"totNumValues\":\"20\"},\"mean\":1,\"stdDev\":6,\"min\":-8.5,\"median\":0.5,\"max\":10.5,\"histograms\":[{\"buckets\":[{\"lowValue\":-8.5,\"highValue\":-7.5,\"sampleCount\":2},{\"lowValue\":-7.5,\"highValue\":-5.5,\"sampleCount\":2},{\"lowValue\":-5.5,\"highValue\":-3.5,\"sampleCount\":2},{\"lowValue\":-3.5,\"highValue\":-1.5,\"sampleCount\":2},{\"lowValue\":-1.5,\"highValue\":0.5,\"sampleCount\":2},{\"lowValue\":0.5,\"highValue\":2.5,\"sampleCount\":2},{\"lowValue\":2.5,\"highValue\":4.5,\"sampleCount\":2},{\"lowValue\":4.5,\"highValue\":6.5,\"sampleCount\":2},{\"lowValue\":6.5,\"highValue\":8.5,\"sampleCount\":2},{\"lowValue\":8.5,\"highValue\":10.5,\"sampleCount\":2}],\"type\":\"QUANTILES\"},{\"buckets\":[{\"lowValue\":1,\"highValue\":2,\"sampleCount\":1},{\"lowValue\":2,\"highValue\":3,\"sampleCount\":2}]}]},\"path\":{\"step\":[\"floats\"]}}"; + "{\"type\":\"FLOAT\",\"numStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"1\",\"maxNumValues\":\"1\",\"avgNumValues\":1,\"totNumValues\":\"20\"},\"mean\":1,\"stdDev\":6,\"min\":-8.5,\"median\":0.5,\"max\":10.5,\"histograms\":[{\"buckets\":[{\"lowValue\":1,\"highValue\":2,\"sampleCount\":1},{\"lowValue\":2,\"highValue\":3,\"sampleCount\":2}]},{\"buckets\":[{\"lowValue\":-8.5,\"highValue\":-7.5,\"sampleCount\":2},{\"lowValue\":-7.5,\"highValue\":-5.5,\"sampleCount\":2},{\"lowValue\":-5.5,\"highValue\":-3.5,\"sampleCount\":2},{\"lowValue\":-3.5,\"highValue\":-1.5,\"sampleCount\":2},{\"lowValue\":-1.5,\"highValue\":0.5,\"sampleCount\":2},{\"lowValue\":0.5,\"highValue\":2.5,\"sampleCount\":2},{\"lowValue\":2.5,\"highValue\":4.5,\"sampleCount\":2},{\"lowValue\":4.5,\"highValue\":6.5,\"sampleCount\":2},{\"lowValue\":6.5,\"highValue\":8.5,\"sampleCount\":2},{\"lowValue\":8.5,\"highValue\":10.5,\"sampleCount\":2}],\"type\":\"QUANTILES\"}]},\"path\":{\"step\":[\"floats\"]}}"; FeatureNameStatistics.Builder expected = FeatureNameStatistics.newBuilder(); JsonFormat.parser().merge(expectedJson, expected); assertThat(actual, equalTo(expected.build())); diff --git a/tests/e2e/bq/feature-stats.py b/tests/e2e/bq/feature-stats.py index 0dce7c07256..3970522ea7f 100644 --- a/tests/e2e/bq/feature-stats.py +++ b/tests/e2e/bq/feature-stats.py @@ -2,6 +2,8 @@ import pytest import pytz import uuid +import time +import os from datetime import datetime, timedelta from feast.client import Client @@ -19,6 +21,7 @@ PROJECT_NAME = "batch_" + uuid.uuid4().hex.upper()[0:6] STORE_NAME = "historical" +os.environ['CUDA_VISIBLE_DEVICES'] = "0" @pytest.fixture(scope="module") @@ -92,13 +95,22 @@ def feature_stats_dataset_basic(client, feature_stats_feature_set): ) expected_stats = tfdv.generate_statistics_from_dataframe( - df[["entity_id", "strings", "ints", "floats"]] + df[["strings", "ints", "floats"]] ) clear_unsupported_fields(expected_stats) + # Since TFDV computes population std dev + for feature in expected_stats.datasets[0].features: + if feature.HasField("num_stats"): + name = feature.path.step[0] + std = combined_df[name].std() + feature.num_stats.std_dev = std + + dataset_id = client.ingest(feature_stats_feature_set, df) + time.sleep(10) return { "df": df, - "id": client.ingest(feature_stats_feature_set, df), + "id": dataset_id, "date": datetime(time_offset.year, time_offset.month, time_offset.day).replace( tzinfo=pytz.utc ), @@ -132,17 +144,19 @@ def feature_stats_dataset_agg(client, feature_stats_feature_set): ) dataset_id_2 = client.ingest(feature_stats_feature_set, df2) - combined_df = pd.concat([df1, df2])[["entity_id", "strings", "ints", "floats"]] + combined_df = pd.concat([df1, df2])[["strings", "ints", "floats"]] expected_stats = tfdv.generate_statistics_from_dataframe(combined_df) clear_unsupported_agg_fields(expected_stats) - # Temporary until TFDV fixes their std dev computation + # Since TFDV computes population std dev for feature in expected_stats.datasets[0].features: if feature.HasField("num_stats"): name = feature.path.step[0] std = combined_df[name].std() feature.num_stats.std_dev = std + time.sleep(10) + return { "ids": [dataset_id_1, dataset_id_2], "start_date": datetime( @@ -157,7 +171,7 @@ def feature_stats_dataset_agg(client, feature_stats_feature_set): def test_feature_stats_retrieval_by_single_dataset(client, feature_stats_dataset_basic): stats = client.get_statistics( - f"{PROJECT_NAME}/feature_validation:1", + f"{PROJECT_NAME}/feature_stats:1", features=["strings", "ints", "floats"], store=STORE_NAME, dataset_ids=[feature_stats_dataset_basic["id"]], @@ -168,7 +182,7 @@ def test_feature_stats_retrieval_by_single_dataset(client, feature_stats_dataset def test_feature_stats_by_date(client, feature_stats_dataset_basic): stats = client.get_statistics( - f"{PROJECT_NAME}/feature_validation:1", + f"{PROJECT_NAME}/feature_stats:1", features=["strings", "ints", "floats"], store=STORE_NAME, start_date=feature_stats_dataset_basic["date"], @@ -179,17 +193,17 @@ def test_feature_stats_by_date(client, feature_stats_dataset_basic): def test_feature_stats_agg_over_datasets(client, feature_stats_dataset_agg): stats = client.get_statistics( - f"{PROJECT_NAME}/feature_validation:1", + f"{PROJECT_NAME}/feature_stats:1", features=["strings", "ints", "floats"], store=STORE_NAME, - dataset_ids=[feature_stats_dataset_basic["ids"]], + dataset_ids=feature_stats_dataset_agg["ids"], ) - assert_stats_equal(feature_stats_dataset_basic["stats"], stats) + assert_stats_equal(feature_stats_dataset_agg["stats"], stats) def test_feature_stats_agg_over_dates(client, feature_stats_dataset_agg): stats = client.get_statistics( - f"{PROJECT_NAME}/feature_validation:1", + f"{PROJECT_NAME}/feature_stats:1", features=["strings", "ints", "floats"], store=STORE_NAME, start_date=feature_stats_dataset_agg["start_date"], @@ -213,9 +227,10 @@ def test_feature_stats_force_refresh( } ) client.ingest(feature_stats_feature_set, df2) + time.sleep(10) actual_stats = client.get_statistics( - f"{PROJECT_NAME}/feature_validation:1", + f"{PROJECT_NAME}/feature_stats:1", features=["strings", "ints", "floats"], store="historical", start_date=feature_stats_dataset_basic["date"], @@ -225,8 +240,16 @@ def test_feature_stats_force_refresh( combined_df = pd.concat([df, df2]) expected_stats = tfdv.generate_statistics_from_dataframe(combined_df) + clear_unsupported_fields(expected_stats) + # Since TFDV computes population std dev + for feature in expected_stats.datasets[0].features: + if feature.HasField("num_stats"): + name = feature.path.step[0] + std = combined_df[name].std() + feature.num_stats.std_dev = std + assert_stats_equal(expected_stats, actual_stats) @@ -235,6 +258,8 @@ def clear_unsupported_fields(datasets): for feature in dataset.features: if feature.HasField("num_stats"): feature.num_stats.common_stats.ClearField("num_values_histogram") + for hist in feature.num_stats.histograms: + hist.buckets[:] = sorted(hist.buckets, key=lambda k: k["highValue"]) elif feature.HasField("string_stats"): feature.string_stats.common_stats.ClearField("num_values_histogram") for bucket in feature.string_stats.rank_histogram.buckets: @@ -252,16 +277,17 @@ def clear_unsupported_agg_fields(datasets): if feature.HasField("num_stats"): feature.num_stats.common_stats.ClearField("num_values_histogram") feature.num_stats.ClearField("histograms") + feature.num_stats.ClearField("median") elif feature.HasField("string_stats"): feature.string_stats.common_stats.ClearField("num_values_histogram") - feature.string_stats.ClearField("histograms") feature.string_stats.ClearField("rank_histogram") feature.string_stats.ClearField("top_values") feature.string_stats.ClearField("unique") elif feature.HasField("struct_stats"): - feature.string_stats.struct_stats.ClearField("num_values_histogram") + feature.struct_stats.ClearField("num_values_histogram") elif feature.HasField("bytes_stats"): - feature.string_stats.bytes_stats.ClearField("num_values_histogram") + feature.bytes_stats.ClearField("num_values_histogram") + feature.bytes_stats.ClearField("unique") def assert_stats_equal(left, right): @@ -273,5 +299,5 @@ def assert_stats_equal(left, right): left_features = sorted(left_stats["features"], key=lambda k: k["path"]["step"][0]) right_features = sorted(right_stats["features"], key=lambda k: k["path"]["step"][0]) - diff = DeepDiff(left_features, right_features) - assert len(diff) == 0, f"Statistics do not match: \n{diff}" + diff = DeepDiff(left_features, right_features, significant_digits=4) + assert len(diff) == 0, f"Feature statistics do not match: \nwanted: {left_features}\n got: {right_features}" diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index 92a7ffb3a8f..f1806b67fef 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -25,6 +25,7 @@ FLOAT_TOLERANCE = 0.00001 PROJECT_NAME = "basic_" + uuid.uuid4().hex.upper()[0:6] +ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) @pytest.fixture(scope="module") @@ -77,7 +78,7 @@ def basic_dataframe(): @pytest.mark.run(order=10) def test_basic_register_feature_set_success(client): # Load feature set from file - cust_trans_fs_expected = FeatureSet.from_yaml("basic/cust_trans_fs.yaml") + cust_trans_fs_expected = FeatureSet.from_yaml(os.path.join(ROOT_PATH, "basic/cust_trans_fs.yaml")) client.set_project(PROJECT_NAME) @@ -380,7 +381,7 @@ def large_volume_dataframe(): @pytest.mark.run(order=30) def test_large_volume_register_feature_set_success(client): cust_trans_fs_expected = FeatureSet.from_yaml( - "large_volume/cust_trans_large_fs.yaml" + os.path.join(ROOT_PATH,"large_volume/cust_trans_large_fs.yaml") ) # Register feature set @@ -513,7 +514,7 @@ def all_types_parquet_file(): def test_all_types_parquet_register_feature_set_success(client): # Load feature set from file all_types_parquet_expected = FeatureSet.from_yaml( - "all_types_parquet/all_types_parquet.yaml" + os.path.join(ROOT_PATH, "all_types_parquet/all_types_parquet.yaml") ) # Register feature set From baf4b884b1585aea892ac02d90122fc0823b6848 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Sun, 26 Apr 2020 22:47:48 +0800 Subject: [PATCH 12/12] Clear and assign hist field --- tests/e2e/bq/feature-stats.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/bq/feature-stats.py b/tests/e2e/bq/feature-stats.py index 3970522ea7f..b25ff3949bb 100644 --- a/tests/e2e/bq/feature-stats.py +++ b/tests/e2e/bq/feature-stats.py @@ -259,7 +259,9 @@ def clear_unsupported_fields(datasets): if feature.HasField("num_stats"): feature.num_stats.common_stats.ClearField("num_values_histogram") for hist in feature.num_stats.histograms: - hist.buckets[:] = sorted(hist.buckets, key=lambda k: k["highValue"]) + sorted_buckets = sorted(hist.buckets, key=lambda k: k["highValue"]) + del hist.buckets[:] + hist.buckets.extend(sorted_buckets) elif feature.HasField("string_stats"): feature.string_stats.common_stats.ClearField("num_values_histogram") for bucket in feature.string_stats.rank_histogram.buckets: