newMetrics) {
- metrics.clear();
- metrics.addAll(newMetrics);
- }
-
public String getSinkName() {
return store.getName();
}
diff --git a/core/src/main/java/feast/core/model/Metrics.java b/core/src/main/java/feast/core/model/Metrics.java
deleted file mode 100644
index 0b7514816fa..00000000000
--- a/core/src/main/java/feast/core/model/Metrics.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.core.model;
-
-import javax.persistence.Entity;
-import javax.persistence.FetchType;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
-import javax.persistence.Id;
-import javax.persistence.JoinColumn;
-import javax.persistence.ManyToOne;
-import javax.persistence.Table;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-@NoArgsConstructor
-@Getter
-@Setter
-@Entity
-@Table(name = "metrics")
-public class Metrics extends AbstractTimestampEntity {
-
- @Id
- @GeneratedValue(strategy = GenerationType.AUTO)
- private long id;
-
- @ManyToOne(fetch = FetchType.LAZY)
- @JoinColumn(name = "job_id")
- private Job job;
-
- /** Metrics name */
- private String name;
-
- /** Metrics value */
- private double value;
-
- /**
- * Create a metrics owned by a {@code job}.
- *
- * @param job owner of this metrics.
- * @param metricsName metrics name.
- * @param value metrics value.
- */
- public Metrics(Job job, String metricsName, double value) {
- this.job = job;
- this.name = metricsName;
- this.value = value;
- }
-}
diff --git a/core/src/main/java/feast/core/service/StatsService.java b/core/src/main/java/feast/core/service/StatsService.java
new file mode 100644
index 00000000000..0f1fea1ac37
--- /dev/null
+++ b/core/src/main/java/feast/core/service/StatsService.java
@@ -0,0 +1,652 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.service;
+
+import static java.lang.Math.*;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.Timestamp;
+import feast.core.CoreServiceProto.*;
+import feast.core.CoreServiceProto.ListStoresRequest.Filter;
+import feast.core.FeatureSetProto.FeatureSet;
+import feast.core.FeatureSetProto.FeatureSetSpec;
+import feast.core.FeatureSetProto.FeatureSpec;
+import feast.core.StoreProto.Store;
+import feast.core.StoreProto.Store.StoreType;
+import feast.core.dao.FeatureStatisticsRepository;
+import feast.core.exception.RetrievalException;
+import feast.core.model.Feature;
+import feast.core.model.FeatureReference;
+import feast.core.model.FeatureStatistics;
+import feast.storage.api.statistics.FeatureSetStatistics;
+import feast.storage.api.statistics.StatisticsRetriever;
+import feast.storage.connectors.bigquery.statistics.BigQueryStatisticsRetriever;
+import java.io.IOException;
+import java.time.Instant;
+import java.util.*;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import org.tensorflow.metadata.v0.*;
+import org.tensorflow.metadata.v0.FeatureNameStatistics.Builder;
+
+/** Facilitates the retrieval of feature set statistics from historical stores. */
+@Slf4j
+@Service
+public class StatsService {
+
+ private SpecService specService;
+ private FeatureStatisticsRepository featureStatisticsRepository;
+
+ @Autowired
+ public StatsService(
+ SpecService specService, FeatureStatisticsRepository featureStatisticsRepository) {
+ this.specService = specService;
+ this.featureStatisticsRepository = featureStatisticsRepository;
+ }
+
+ /**
+ * Get {@link DatasetFeatureStatistics} for the requested feature set in the provided datasets or
+ * date range for the store provided. The {@link DatasetFeatureStatistics} will contain a list of
+ * {@link FeatureNameStatistics} for each feature requested. Results retrieved will be cached
+ * indefinitely. To force Feast to recompute the statistics, set forceRefresh to true.
+ *
+ * Only one of datasetIds or startDate/endDate should be provided. If both are provided, the
+ * former will be used over the latter.
+ *
+ *
If multiple datasetIds or if the date ranges over a few days, statistics will be retrieved
+ * for each single unit (dataset id or day) and results aggregated across that set. As a result of
+ * this, in such a scenario, statistics that cannot be aggregated will be dropped. This includes
+ * all histograms and quantiles, unique values, and top value counts.
+ *
+ * @param request {@link GetFeatureStatisticsRequest} containing feature set name, subset of
+ * features, dataset ids or date range, and store to retrieve the data from.
+ * @return {@link GetFeatureStatisticsResponse} containing {@link DatasetFeatureStatistics} with
+ * the feature statistics requested.
+ * @throws IOException
+ */
+ @Transactional
+ public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsRequest request)
+ throws IOException {
+
+ // Validate the request
+ validateRequest(request);
+
+ // Get the stats retriever for the store requested
+ StatisticsRetriever statisticsRetriever = getStatisticsRetriever(request.getStore());
+
+ // 1. Retrieve the feature set spec from the db
+ FeatureSetSpec featureSetSpec;
+ try {
+ featureSetSpec = getFeatureSetSpec(request.getFeatureSetId());
+ } catch (IllegalArgumentException | RetrievalException e) {
+ throw new RetrievalException(
+ String.format("Unable to find feature set %s", request.getFeatureSetId()), e);
+ }
+
+ // 2. Filter out the features requested by the user. If none are provided,
+ // use all features in the feature set.
+ List features = request.getFeaturesList();
+ if (features.size() == 0) {
+ features =
+ featureSetSpec.getFeaturesList().stream()
+ .map(FeatureSpec::getName)
+ .collect(Collectors.toList());
+ }
+
+ // 3. Retrieve the statistics from the StatsRetriever.
+ List> featureNameStatisticsList = new ArrayList<>();
+ if (request.getDatasetIdsCount() == 0) {
+ Timestamp endDate = request.getEndDate();
+ Timestamp startDate = request.getStartDate();
+ // If no dataset provided, retrieve by date
+
+ long timestamp = startDate.getSeconds();
+ while (timestamp < endDate.getSeconds()) {
+ List featureNameStatistics =
+ getFeatureNameStatisticsByDate(
+ statisticsRetriever,
+ featureSetSpec,
+ features,
+ timestamp,
+ request.getForceRefresh());
+ featureNameStatisticsList.add(featureNameStatistics);
+ timestamp += 86400; // advance by a day
+ }
+ if (featureNameStatisticsList.size() == 0) {
+ DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd");
+ DateTime startDateTime = new DateTime(startDate.getSeconds() * 1000, DateTimeZone.UTC);
+ DateTime endDateTime = new DateTime(endDate.getSeconds() * 1000, DateTimeZone.UTC);
+ throw new RetrievalException(
+ String.format(
+ "Unable to find any data over provided dates [%s, %s)",
+ fmt.print(startDateTime), fmt.print(endDateTime)));
+ }
+ } else {
+ // else, retrieve by dataset
+ for (String datasetId : request.getDatasetIdsList()) {
+ List featureNameStatistics =
+ getFeatureNameStatisticsByDataset(
+ statisticsRetriever,
+ featureSetSpec,
+ features,
+ datasetId,
+ request.getForceRefresh());
+ featureNameStatisticsList.add(featureNameStatistics);
+ if (featureNameStatisticsList.size() == 0) {
+ throw new RetrievalException(
+ String.format(
+ "Unable to find any data over provided datasets %s",
+ request.getDatasetIdsList()));
+ }
+ }
+ }
+
+ // Merge statistics values across days/datasets
+ List featureNameStatistics = mergeStatistics(featureNameStatisticsList);
+ long totalCount = getTotalCount(featureNameStatistics.get(0));
+ return GetFeatureStatisticsResponse.newBuilder()
+ .setDatasetFeatureStatisticsList(
+ DatasetFeatureStatisticsList.newBuilder()
+ .addDatasets(
+ DatasetFeatureStatistics.newBuilder()
+ .setNumExamples(totalCount)
+ .addAllFeatures(featureNameStatistics)))
+ .build();
+ }
+
+ /**
+ * Get {@link FeatureNameStatistics} by dataset id.
+ *
+ * @param statisticsRetriever {@link StatisticsRetriever} corresponding to the store to get the
+ * data from.
+ * @param featureSetSpec {@link FeatureSetSpec} of the feature set requested
+ * @param features features to retrieve
+ * @param datasetId dataset id to subset the data by
+ * @param forceRefresh whether to override the values in the cache
+ * @return {@link FeatureNameStatistics} for the data within the dataset id provided
+ * @throws IOException
+ */
+ private List getFeatureNameStatisticsByDataset(
+ StatisticsRetriever statisticsRetriever,
+ FeatureSetSpec featureSetSpec,
+ List features,
+ String datasetId,
+ boolean forceRefresh)
+ throws IOException {
+ List featureNameStatistics = new ArrayList<>();
+ List featuresMissingStats = new ArrayList<>();
+
+ // For each feature requested, check if statistics already exist in the cache
+ // If not refreshing data in the cache, retrieve the cached data and add it to the
+ // list of FeatureNameStatistics for this dataset.
+ // Else, add to the list of features we still need to retrieve statistics for.
+ for (String featureName : features) {
+ FeatureReference featureReference =
+ new FeatureReference(
+ featureSetSpec.getProject(),
+ featureSetSpec.getName(),
+ featureSetSpec.getVersion(),
+ featureName);
+ Feature feature = Feature.withReference(featureReference);
+ Optional cachedFeatureStatistics = Optional.empty();
+ if (!forceRefresh) {
+ cachedFeatureStatistics =
+ featureStatisticsRepository.findFeatureStatisticsByFeatureAndDatasetId(
+ feature, datasetId);
+ }
+ if (cachedFeatureStatistics.isPresent()) {
+ featureNameStatistics.add(cachedFeatureStatistics.get().toProto());
+ } else {
+ featuresMissingStats.add(featureName);
+ }
+ }
+
+ // Retrieve the balance of statistics after checking the cache, and add it to the
+ // list of FeatureNameStatistics.
+ if (featuresMissingStats.size() > 0) {
+ FeatureSetStatistics featureSetStatistics =
+ statisticsRetriever.getFeatureStatistics(featureSetSpec, featuresMissingStats, datasetId);
+
+ // Persist the newly retrieved statistics in the cache.
+ for (FeatureNameStatistics stat : featureSetStatistics.getFeatureNameStatistics()) {
+ if (isEmpty(stat)) {
+ continue;
+ }
+ FeatureStatistics featureStatistics =
+ FeatureStatistics.createForDataset(
+ featureSetSpec.getProject(),
+ featureSetSpec.getName(),
+ featureSetSpec.getVersion(),
+ stat,
+ datasetId);
+ Optional existingRecord =
+ featureStatisticsRepository.findFeatureStatisticsByFeatureAndDatasetId(
+ featureStatistics.getFeature(), datasetId);
+ existingRecord.ifPresent(statistics -> featureStatistics.setId(statistics.getId()));
+ featureStatisticsRepository.save(featureStatistics);
+ featureNameStatistics.add(stat);
+ }
+ }
+ return featureNameStatistics;
+ }
+
+ /**
+ * Get {@link FeatureNameStatistics} by date.
+ *
+ * @param statisticsRetriever {@link StatisticsRetriever} corresponding to the store to get the
+ * data from.
+ * @param featureSetSpec {@link FeatureSetSpec} of the feature set requested
+ * @param features features to retrieve
+ * @param timestamp timestamp of the date to subset the data
+ * @param forceRefresh whether to override the values in the cache
+ * @return {@link FeatureNameStatistics} for the data within the dataset id provided
+ * @throws IOException
+ */
+ private List getFeatureNameStatisticsByDate(
+ StatisticsRetriever statisticsRetriever,
+ FeatureSetSpec featureSetSpec,
+ List features,
+ long timestamp,
+ boolean forceRefresh)
+ throws IOException {
+ Date date = Date.from(Instant.ofEpochSecond(timestamp));
+ List featureNameStatistics = new ArrayList<>();
+ List featuresMissingStats = new ArrayList<>();
+
+ // For each feature requested, check if statistics already exist in the cache
+ // If not refreshing data in the cache, retrieve the cached data and add it to the
+ // list of FeatureNameStatistics for this date.
+ // Else, add to the list of features we still need to retrieve statistics for.
+ for (String featureName : features) {
+ FeatureReference featureReference =
+ new FeatureReference(
+ featureSetSpec.getProject(),
+ featureSetSpec.getName(),
+ featureSetSpec.getVersion(),
+ featureName);
+ Feature feature = Feature.withReference(featureReference);
+ Optional cachedFeatureStatistics = Optional.empty();
+ if (!forceRefresh) {
+ cachedFeatureStatistics =
+ featureStatisticsRepository.findFeatureStatisticsByFeatureAndDate(feature, date);
+ }
+ if (cachedFeatureStatistics.isPresent()) {
+ featureNameStatistics.add(cachedFeatureStatistics.get().toProto());
+ } else {
+ featuresMissingStats.add(featureName);
+ }
+ }
+
+ // Retrieve the balance of statistics after checking the cache, and add it to the
+ // list of FeatureNameStatistics.
+ if (featuresMissingStats.size() > 0) {
+ FeatureSetStatistics featureSetStatistics =
+ statisticsRetriever.getFeatureStatistics(
+ featureSetSpec,
+ featuresMissingStats,
+ Timestamp.newBuilder().setSeconds(timestamp).build());
+
+ // Persist the newly retrieved statistics in the cache.
+ for (FeatureNameStatistics stat : featureSetStatistics.getFeatureNameStatistics()) {
+ if (isEmpty(stat)) {
+ continue;
+ }
+ FeatureStatistics featureStatistics =
+ FeatureStatistics.createForDate(
+ featureSetSpec.getProject(),
+ featureSetSpec.getName(),
+ featureSetSpec.getVersion(),
+ stat,
+ date);
+ Optional existingRecord =
+ featureStatisticsRepository.findFeatureStatisticsByFeatureAndDate(
+ featureStatistics.getFeature(), date);
+ existingRecord.ifPresent(statistics -> featureStatistics.setId(statistics.getId()));
+ featureStatisticsRepository.save(featureStatistics);
+ featureNameStatistics.add(stat);
+ }
+ }
+ return featureNameStatistics;
+ }
+
+ /**
+ * Get the {@link StatisticsRetriever} corresponding to the store name provided.
+ *
+ * @param storeName name of the store to retrieve statistics from
+ * @return {@link StatisticsRetriever}
+ */
+ private StatisticsRetriever getStatisticsRetriever(String storeName) {
+ ListStoresResponse listStoresResponse =
+ specService.listStores(Filter.newBuilder().setName(storeName).build());
+ Store store = listStoresResponse.getStore(0);
+ if (store.getType() != StoreType.BIGQUERY) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Invalid store %s with type %s specified. Batch statistics are only supported for BigQuery stores",
+ store.getName(), store.getType()));
+ }
+ return BigQueryStatisticsRetriever.create(store.getBigqueryConfig());
+ }
+
+ private FeatureSetSpec getFeatureSetSpec(String featureSetId)
+ throws InvalidProtocolBufferException, IllegalArgumentException, RetrievalException {
+ String[] split = featureSetId.split("/");
+ String project = split[0];
+ split = split[1].split(":");
+ FeatureSet featureSet =
+ specService
+ .getFeatureSet(
+ GetFeatureSetRequest.newBuilder()
+ .setProject(project)
+ .setName(split[0])
+ .setVersion(Integer.parseInt(split[1]))
+ .build())
+ .getFeatureSet();
+ return featureSet.getSpec();
+ }
+
+ /**
+ * Merge feature statistics by name. This method is used to merge statistics retrieved over
+ * multiple days or datasets.
+ *
+ * @param featureNameStatistics {@link FeatureNameStatistics} retrieved from the store
+ * @return Merged list of {@link FeatureNameStatistics} by name
+ */
+ @VisibleForTesting
+ public List mergeStatistics(
+ List> featureNameStatistics) {
+ List unnestedList = new ArrayList<>();
+
+ featureNameStatistics.forEach(unnestedList::addAll);
+ Map> groupByPath =
+ unnestedList.stream()
+ .collect(Collectors.groupingBy(FeatureNameStatistics::getPath, Collectors.toList()));
+
+ List merged = new ArrayList<>();
+ for (Path key : groupByPath.keySet()) {
+ List featureNameStatisticsForKey = groupByPath.get(key);
+ if (featureNameStatisticsForKey.size() == 1) {
+ merged.add(featureNameStatisticsForKey.get(0));
+ } else {
+ switch (featureNameStatisticsForKey.get(0).getType()) {
+ case INT:
+ case FLOAT:
+ merged.add(mergeNumStatistics(featureNameStatisticsForKey));
+ break;
+ case STRING:
+ merged.add(mergeCategoricalStatistics(groupByPath.get(key)));
+ break;
+ case BYTES:
+ merged.add(mergeByteStatistics(groupByPath.get(key)));
+ break;
+ case STRUCT:
+ merged.add(mergeStructStats(groupByPath.get(key)));
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Statistics are only supported for string, boolean, bytes and numeric features");
+ }
+ }
+ }
+ return merged;
+ }
+
+ private FeatureNameStatistics mergeStructStats(
+ List featureNameStatisticsList) {
+ Builder mergedFeatureNameStatistics =
+ FeatureNameStatistics.newBuilder()
+ .setPath(featureNameStatisticsList.get(0).getPath())
+ .setType(featureNameStatisticsList.get(0).getType());
+
+ long totalCount = 0;
+ long missingCount = 0;
+ long totalNumValues = 0;
+ long maxNumValues =
+ featureNameStatisticsList.get(0).getStructStats().getCommonStats().getMaxNumValues();
+ long minNumValues =
+ featureNameStatisticsList.get(0).getStructStats().getCommonStats().getMinNumValues();
+
+ for (FeatureNameStatistics featureNameStatistics : featureNameStatisticsList) {
+ StructStatistics structStats = featureNameStatistics.getStructStats();
+ totalCount += structStats.getCommonStats().getNumNonMissing();
+ missingCount += structStats.getCommonStats().getNumMissing();
+ totalNumValues +=
+ structStats.getCommonStats().getAvgNumValues()
+ * structStats.getCommonStats().getNumNonMissing();
+ maxNumValues = max(maxNumValues, structStats.getCommonStats().getMaxNumValues());
+ minNumValues = min(minNumValues, structStats.getCommonStats().getMinNumValues());
+ }
+
+ StructStatistics mergedStructStatistics =
+ StructStatistics.newBuilder()
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(totalNumValues)
+ .setNumNonMissing(totalCount)
+ .setAvgNumValues((float) totalNumValues / totalCount)
+ .setMaxNumValues(maxNumValues)
+ .setMinNumValues(minNumValues)
+ .setNumMissing(missingCount))
+ .build();
+
+ return mergedFeatureNameStatistics.setStructStats(mergedStructStatistics).build();
+ }
+
+ private FeatureNameStatistics mergeNumStatistics(
+ List featureNameStatisticsList) {
+ Builder mergedFeatureNameStatistics =
+ FeatureNameStatistics.newBuilder()
+ .setPath(featureNameStatisticsList.get(0).getPath())
+ .setType(featureNameStatisticsList.get(0).getType());
+
+ FeatureNameStatistics first = featureNameStatisticsList.remove(0);
+ double max = first.getNumStats().getMax();
+ double min = first.getNumStats().getMin();
+ double var = pow(first.getNumStats().getStdDev(), 2);
+ long totalCount = first.getNumStats().getCommonStats().getNumNonMissing();
+ double totalVal = totalCount * first.getNumStats().getMean();
+ long missingCount = first.getNumStats().getCommonStats().getNumMissing();
+ long zeroes = first.getNumStats().getNumZeros();
+
+ for (FeatureNameStatistics featureNameStatistics : featureNameStatisticsList) {
+ NumericStatistics numStats = featureNameStatistics.getNumStats();
+ max = max(numStats.getMax(), max);
+ min = min(numStats.getMin(), min);
+ long count = numStats.getCommonStats().getNumNonMissing();
+ double sampleVar = pow(numStats.getStdDev(), 2);
+ float aggMean = (float) totalVal / totalCount;
+ var = getVar(var, totalCount, aggMean, sampleVar, count, numStats.getMean());
+ totalVal += numStats.getMean() * count;
+ totalCount += count;
+ missingCount += numStats.getCommonStats().getNumMissing();
+ zeroes += numStats.getNumZeros();
+ }
+ NumericStatistics mergedNumericStatistics =
+ NumericStatistics.newBuilder()
+ .setMax(max)
+ .setMin(min)
+ .setMean(totalVal / totalCount)
+ .setNumZeros(zeroes)
+ .setStdDev(sqrt(var))
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(totalCount)
+ .setNumNonMissing(totalCount)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(missingCount))
+ .build();
+ return mergedFeatureNameStatistics.setNumStats(mergedNumericStatistics).build();
+ }
+
+ // Aggregation of sample variance follows the formula described here:
+ // https://www.tandfonline.com/doi/abs/10.1080/00031305.2014.966589
+ private double getVar(
+ double s1Var, long s1Count, double s1Mean, double s2Var, long s2Count, double s2Mean) {
+ long totalCount = s1Count + s2Count;
+ return ((s1Count - 1) * s1Var
+ + (s2Count - 1) * s2Var
+ + ((float) s1Count * s2Count / totalCount) * pow(s1Mean - s2Mean, 2))
+ / (s1Count + s2Count - 1);
+ }
+
+ private FeatureNameStatistics mergeCategoricalStatistics(
+ List featureNameStatisticsList) {
+ Builder mergedFeatureNameStatistics =
+ FeatureNameStatistics.newBuilder()
+ .setPath(featureNameStatisticsList.get(0).getPath())
+ .setType(featureNameStatisticsList.get(0).getType());
+ long totalCount = 0;
+ long missingCount = 0;
+ long totalLen = 0;
+ for (FeatureNameStatistics featureNameStatistics : featureNameStatisticsList) {
+ StringStatistics stringStats = featureNameStatistics.getStringStats();
+ totalCount += stringStats.getCommonStats().getNumNonMissing();
+ missingCount += stringStats.getCommonStats().getNumMissing();
+ totalLen += stringStats.getAvgLength() * stringStats.getCommonStats().getNumNonMissing();
+ }
+ StringStatistics mergedStringStatistics =
+ StringStatistics.newBuilder()
+ .setAvgLength((float) totalLen / totalCount)
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(totalCount)
+ .setNumNonMissing(totalCount)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(missingCount))
+ .build();
+ return mergedFeatureNameStatistics.setStringStats(mergedStringStatistics).build();
+ }
+
+ private FeatureNameStatistics mergeByteStatistics(
+ List featureNameStatisticsList) {
+ Builder mergedFeatureNameStatistics =
+ FeatureNameStatistics.newBuilder()
+ .setPath(featureNameStatisticsList.get(0).getPath())
+ .setType(featureNameStatisticsList.get(0).getType());
+
+ long totalCount = 0;
+ long missingCount = 0;
+ float totalNumBytes = 0;
+ float maxNumBytes = featureNameStatisticsList.get(0).getBytesStats().getMaxNumBytes();
+ float minNumBytes = featureNameStatisticsList.get(0).getBytesStats().getMinNumBytes();
+
+ for (FeatureNameStatistics featureNameStatistics : featureNameStatisticsList) {
+ BytesStatistics bytesStats = featureNameStatistics.getBytesStats();
+ totalCount += bytesStats.getCommonStats().getNumNonMissing();
+ missingCount += bytesStats.getCommonStats().getNumMissing();
+ totalNumBytes += bytesStats.getAvgNumBytes() * bytesStats.getCommonStats().getNumNonMissing();
+ maxNumBytes = max(maxNumBytes, bytesStats.getMaxNumBytes());
+ minNumBytes = min(minNumBytes, bytesStats.getMinNumBytes());
+ }
+
+ BytesStatistics mergedBytesStatistics =
+ BytesStatistics.newBuilder()
+ .setAvgNumBytes(totalNumBytes / totalCount)
+ .setMinNumBytes(minNumBytes)
+ .setMaxNumBytes(maxNumBytes)
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(totalCount)
+ .setNumNonMissing(totalCount)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(missingCount))
+ .build();
+
+ return mergedFeatureNameStatistics.setBytesStats(mergedBytesStatistics).build();
+ }
+
+ private long getTotalCount(FeatureNameStatistics featureNameStatistics) {
+ CommonStatistics commonStats;
+ switch (featureNameStatistics.getType()) {
+ case STRUCT:
+ commonStats = featureNameStatistics.getStructStats().getCommonStats();
+ break;
+ case STRING:
+ commonStats = featureNameStatistics.getStringStats().getCommonStats();
+ break;
+ case BYTES:
+ commonStats = featureNameStatistics.getBytesStats().getCommonStats();
+ break;
+ case FLOAT:
+ case INT:
+ commonStats = featureNameStatistics.getNumStats().getCommonStats();
+ break;
+ default:
+ throw new RuntimeException("Unable to extract dataset size; Invalid type provided");
+ }
+ return commonStats.getNumNonMissing() + commonStats.getNumMissing();
+ }
+
+ private void validateRequest(GetFeatureStatisticsRequest request) {
+ if (request.getDatasetIdsCount() == 0) {
+ Timestamp startDate = request.getStartDate();
+ Timestamp endDate = request.getEndDate();
+ if (!request.hasStartDate() || !request.hasEndDate()) {
+ throw new IllegalArgumentException(
+ "Invalid request. Either provide dataset ids to retrieve statistics over, or a start date and end date.");
+ }
+ if (endDate.getSeconds() < startDate.getSeconds()) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Invalid request. Start timestamp %d is greater than the end timestamp %d",
+ startDate.getSeconds(), endDate.getSeconds()));
+ }
+ }
+ }
+
+ private boolean isEmpty(FeatureNameStatistics featureNameStatistics) {
+ switch (featureNameStatistics.getType()) {
+ case STRUCT:
+ return featureNameStatistics
+ .getStructStats()
+ .getCommonStats()
+ .equals(CommonStatistics.getDefaultInstance());
+ case STRING:
+ return featureNameStatistics
+ .getStringStats()
+ .getCommonStats()
+ .equals(CommonStatistics.getDefaultInstance());
+ case BYTES:
+ return featureNameStatistics
+ .getBytesStats()
+ .getCommonStats()
+ .equals(CommonStatistics.getDefaultInstance());
+ case FLOAT:
+ case INT:
+ return featureNameStatistics
+ .getNumStats()
+ .getCommonStats()
+ .equals(CommonStatistics.getDefaultInstance());
+ default:
+ return true;
+ }
+ }
+}
diff --git a/core/src/test/java/feast/core/service/JobServiceTest.java b/core/src/test/java/feast/core/service/JobServiceTest.java
index c0e90ca43f4..bc3c9f8ba02 100644
--- a/core/src/test/java/feast/core/service/JobServiceTest.java
+++ b/core/src/test/java/feast/core/service/JobServiceTest.java
@@ -34,7 +34,9 @@
import feast.core.CoreServiceProto.RestartIngestionJobResponse;
import feast.core.CoreServiceProto.StopIngestionJobRequest;
import feast.core.CoreServiceProto.StopIngestionJobResponse;
+import feast.core.FeatureSetProto.EntitySpec;
import feast.core.FeatureSetProto.FeatureSetStatus;
+import feast.core.FeatureSetProto.FeatureSpec;
import feast.core.FeatureSetReferenceProto.FeatureSetReference;
import feast.core.IngestionJobProto.IngestionJob;
import feast.core.SourceProto.KafkaSourceConfig;
@@ -44,8 +46,9 @@
import feast.core.dao.JobRepository;
import feast.core.job.JobManager;
import feast.core.job.Runner;
+import feast.core.model.Entity;
+import feast.core.model.Feature;
import feast.core.model.FeatureSet;
-import feast.core.model.Field;
import feast.core.model.Job;
import feast.core.model.JobStatus;
import feast.core.model.Source;
@@ -158,8 +161,12 @@ public void setupJobManager() {
// dummy model constructorss
private FeatureSet newDummyFeatureSet(String name, int version, String project) {
- Field feature = new Field(name + "_feature", Enum.INT64);
- Field entity = new Field(name + "_entity", Enum.STRING);
+ FeatureSpec featureSpec =
+ FeatureSpec.newBuilder().setName(name + "_feature").setValueType(Enum.INT64).build();
+ Feature feature = Feature.fromProto(featureSpec);
+ EntitySpec entitySpec =
+ EntitySpec.newBuilder().setName(name + "_entity").setValueType(Enum.STRING).build();
+ Entity entity = Entity.fromProto(entitySpec);
FeatureSet fs =
new FeatureSet(
diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java
index 43a66135dce..2aa7563cfaa 100644
--- a/core/src/test/java/feast/core/service/SpecServiceTest.java
+++ b/core/src/test/java/feast/core/service/SpecServiceTest.java
@@ -51,11 +51,8 @@
import feast.core.dao.ProjectRepository;
import feast.core.dao.StoreRepository;
import feast.core.exception.RetrievalException;
-import feast.core.model.FeatureSet;
-import feast.core.model.Field;
-import feast.core.model.Project;
-import feast.core.model.Source;
-import feast.core.model.Store;
+import feast.core.model.*;
+import feast.types.ValueProto.ValueType;
import feast.types.ValueProto.ValueType.Enum;
import java.sql.Date;
import java.time.Instant;
@@ -124,9 +121,9 @@ public void setUp() {
FeatureSet featureSet1v3 = newDummyFeatureSet("f1", 3, "project1");
FeatureSet featureSet2v1 = newDummyFeatureSet("f2", 1, "project1");
- Field f3f1 = new Field("f3f1", Enum.INT64);
- Field f3f2 = new Field("f3f2", Enum.INT64);
- Field f3e1 = new Field("f3e1", Enum.STRING);
+ Feature f3f1 = newFeature("f3f1", Enum.INT64);
+ Feature f3f2 = newFeature("f3f2", Enum.INT64);
+ Entity f3e1 = newEntity("f3e1", Enum.STRING);
FeatureSet featureSet3v1 =
new FeatureSet(
"f3",
@@ -490,9 +487,9 @@ public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists()
public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered()
throws InvalidProtocolBufferException {
- Field f3f1 = new Field("f3f1", Enum.INT64);
- Field f3f2 = new Field("f3f2", Enum.INT64);
- Field f3e1 = new Field("f3e1", Enum.STRING);
+ Feature f3f1 = newFeature("f3f1", Enum.INT64);
+ Feature f3f2 = newFeature("f3f2", Enum.INT64);
+ Entity f3e1 = newEntity("f3e1", Enum.STRING);
FeatureSetProto.FeatureSet incomingFeatureSet =
(new FeatureSet(
"f3",
@@ -523,46 +520,11 @@ public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered()
public void applyFeatureSetShouldAcceptPresenceShapeAndDomainConstraints()
throws InvalidProtocolBufferException {
List entitySpecs = new ArrayList<>();
- entitySpecs.add(
- EntitySpec.newBuilder()
- .setName("entity1")
- .setValueType(Enum.INT64)
- .setPresence(FeaturePresence.getDefaultInstance())
- .setShape(FixedShape.getDefaultInstance())
- .setDomain("mydomain")
- .build());
- entitySpecs.add(
- EntitySpec.newBuilder()
- .setName("entity2")
- .setValueType(Enum.INT64)
- .setGroupPresence(FeaturePresenceWithinGroup.getDefaultInstance())
- .setValueCount(ValueCount.getDefaultInstance())
- .setIntDomain(IntDomain.getDefaultInstance())
- .build());
- entitySpecs.add(
- EntitySpec.newBuilder()
- .setName("entity3")
- .setValueType(Enum.FLOAT)
- .setPresence(FeaturePresence.getDefaultInstance())
- .setValueCount(ValueCount.getDefaultInstance())
- .setFloatDomain(FloatDomain.getDefaultInstance())
- .build());
- entitySpecs.add(
- EntitySpec.newBuilder()
- .setName("entity4")
- .setValueType(Enum.STRING)
- .setPresence(FeaturePresence.getDefaultInstance())
- .setValueCount(ValueCount.getDefaultInstance())
- .setStringDomain(StringDomain.getDefaultInstance())
- .build());
- entitySpecs.add(
- EntitySpec.newBuilder()
- .setName("entity5")
- .setValueType(Enum.BOOL)
- .setPresence(FeaturePresence.getDefaultInstance())
- .setValueCount(ValueCount.getDefaultInstance())
- .setBoolDomain(BoolDomain.getDefaultInstance())
- .build());
+ entitySpecs.add(EntitySpec.newBuilder().setName("entity1").setValueType(Enum.INT64).build());
+ entitySpecs.add(EntitySpec.newBuilder().setName("entity2").setValueType(Enum.INT64).build());
+ entitySpecs.add(EntitySpec.newBuilder().setName("entity3").setValueType(Enum.FLOAT).build());
+ entitySpecs.add(EntitySpec.newBuilder().setName("entity4").setValueType(Enum.STRING).build());
+ entitySpecs.add(EntitySpec.newBuilder().setName("entity5").setValueType(Enum.BOOL).build());
List featureSpecs = new ArrayList<>();
featureSpecs.add(
@@ -713,9 +675,9 @@ public void applyFeatureSetShouldUpdateFeatureSetWhenConstraintsAreUpdated()
@Test
public void applyFeatureSetShouldCreateProjectWhenNotAlreadyExists()
throws InvalidProtocolBufferException {
- Field f3f1 = new Field("f3f1", Enum.INT64);
- Field f3f2 = new Field("f3f2", Enum.INT64);
- Field f3e1 = new Field("f3e1", Enum.STRING);
+ Feature f3f1 = newFeature("f3f1", Enum.INT64);
+ Feature f3f2 = newFeature("f3f2", Enum.INT64);
+ Entity f3e1 = newEntity("f3e1", Enum.STRING);
FeatureSetProto.FeatureSet incomingFeatureSet =
(new FeatureSet(
"f3",
@@ -739,9 +701,9 @@ public void applyFeatureSetShouldCreateProjectWhenNotAlreadyExists()
@Test
public void applyFeatureSetShouldFailWhenProjectIsArchived()
throws InvalidProtocolBufferException {
- Field f3f1 = new Field("f3f1", Enum.INT64);
- Field f3f2 = new Field("f3f2", Enum.INT64);
- Field f3e1 = new Field("f3e1", Enum.STRING);
+ Feature f3f1 = newFeature("f3f1", Enum.INT64);
+ Feature f3f2 = newFeature("f3f2", Enum.INT64);
+ Entity f3e1 = newEntity("f3e1", Enum.STRING);
FeatureSetProto.FeatureSet incomingFeatureSet =
(new FeatureSet(
"f3",
@@ -806,8 +768,8 @@ public void shouldFailIfGetFeatureSetWithoutProject() throws InvalidProtocolBuff
}
private FeatureSet newDummyFeatureSet(String name, int version, String project) {
- Field feature = new Field("feature", Enum.INT64);
- Field entity = new Field("entity", Enum.STRING);
+ Feature feature = newFeature("feature", Enum.INT64);
+ Entity entity = newEntity("entity", Enum.STRING);
FeatureSet fs =
new FeatureSet(
@@ -832,4 +794,14 @@ private Store newDummyStore(String name) {
store.setConfig(RedisConfig.newBuilder().setPort(6379).build().toByteArray());
return store;
}
+
+ private Feature newFeature(String name, ValueType.Enum type) {
+ FeatureSpec spec = FeatureSpec.newBuilder().setName(name).setValueType(type).build();
+ return Feature.fromProto(spec);
+ }
+
+ private Entity newEntity(String name, ValueType.Enum type) {
+ EntitySpec spec = EntitySpec.newBuilder().setName(name).setValueType(type).build();
+ return Entity.fromProto(spec);
+ }
}
diff --git a/core/src/test/java/feast/core/service/StatsServiceTest.java b/core/src/test/java/feast/core/service/StatsServiceTest.java
new file mode 100644
index 00000000000..126aa518a7f
--- /dev/null
+++ b/core/src/test/java/feast/core/service/StatsServiceTest.java
@@ -0,0 +1,357 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.service;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+
+import com.google.protobuf.Timestamp;
+import feast.core.CoreServiceProto.GetFeatureStatisticsRequest;
+import feast.core.CoreServiceProto.ListStoresRequest;
+import feast.core.CoreServiceProto.ListStoresResponse;
+import feast.core.StoreProto.Store;
+import feast.core.StoreProto.Store.StoreType;
+import feast.core.dao.FeatureStatisticsRepository;
+import java.io.IOException;
+import java.util.Arrays;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.Mock;
+import org.tensorflow.metadata.v0.*;
+import org.tensorflow.metadata.v0.FeatureNameStatistics.Type;
+
+public class StatsServiceTest {
+
+ private StatsService statsService;
+ @Mock private FeatureStatisticsRepository featureStatisticsRepository;
+ @Mock private SpecService specService;
+
+ @Rule public final ExpectedException expectedException = ExpectedException.none();
+
+ @Before
+ public void setUp() {
+ initMocks(this);
+ statsService = new StatsService(specService, featureStatisticsRepository);
+ }
+
+ @Test
+ public void shouldThrowExceptionIfNeitherDatesNorDatasetsProvided() throws IOException {
+ GetFeatureStatisticsRequest request = GetFeatureStatisticsRequest.newBuilder().build();
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage(
+ "Invalid request. Either provide dataset ids to retrieve statistics over, or a start date and end date.");
+ statsService.getFeatureStatistics(request);
+ }
+
+ @Test
+ public void shouldThrowExceptionIfInvalidDatesProvided() throws IOException {
+ GetFeatureStatisticsRequest request =
+ GetFeatureStatisticsRequest.newBuilder()
+ .setStartDate(Timestamp.newBuilder().setSeconds(1))
+ .setEndDate(Timestamp.newBuilder().setSeconds(0))
+ .build();
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage(
+ "Invalid request. Start timestamp 1 is greater than the end timestamp 0");
+ statsService.getFeatureStatistics(request);
+ }
+
+ @Test
+ public void shouldThrowExceptionIfInvalidStoreProvided() throws IOException {
+ GetFeatureStatisticsRequest request =
+ GetFeatureStatisticsRequest.newBuilder()
+ .setStartDate(Timestamp.newBuilder().setSeconds(0))
+ .setEndDate(Timestamp.newBuilder().setSeconds(1))
+ .setStore("redis")
+ .build();
+
+ when(specService.listStores(ListStoresRequest.Filter.newBuilder().setName("redis").build()))
+ .thenReturn(
+ ListStoresResponse.newBuilder()
+ .addStore(Store.newBuilder().setName("redis").setType(StoreType.REDIS).build())
+ .build());
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage(
+ "Invalid store redis with type REDIS specified. Batch statistics are only supported for BigQuery stores");
+ statsService.getFeatureStatistics(request);
+ }
+
+ @Test
+ public void shouldThrowExceptionIfFeatureSetNotFound() throws IOException {
+ GetFeatureStatisticsRequest request =
+ GetFeatureStatisticsRequest.newBuilder()
+ .setStartDate(Timestamp.newBuilder().setSeconds(1))
+ .setEndDate(Timestamp.newBuilder().setSeconds(0))
+ .build();
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage(
+ "Invalid request. Start timestamp 1 is greater than the end timestamp 0");
+ statsService.getFeatureStatistics(request);
+ }
+
+ @Test
+ public void shouldAggregateNumericStatistics() {
+ FeatureNameStatistics stat1 =
+ FeatureNameStatistics.newBuilder()
+ .setNumStats(
+ NumericStatistics.newBuilder()
+ .setMax(20)
+ .setMin(1)
+ .setMean(6)
+ .setNumZeros(0)
+ .setStdDev(7.90569415)
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(5)
+ .setNumNonMissing(5)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(0)))
+ .setType(Type.INT)
+ .setPath(Path.newBuilder().addStep("feature").build())
+ .build();
+ FeatureNameStatistics stat2 =
+ FeatureNameStatistics.newBuilder()
+ .setNumStats(
+ NumericStatistics.newBuilder()
+ .setMax(10)
+ .setMin(0)
+ .setMean(4)
+ .setNumZeros(1)
+ .setStdDev(3.807886553)
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(5)
+ .setNumNonMissing(5)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(1)))
+ .setPath(Path.newBuilder().addStep("feature").build())
+ .setType(Type.INT)
+ .build();
+
+ FeatureNameStatistics expected =
+ FeatureNameStatistics.newBuilder()
+ .setNumStats(
+ NumericStatistics.newBuilder()
+ .setMax(20)
+ .setMin(0)
+ .setMean(5)
+ .setNumZeros(1)
+ .setStdDev(5.944184833146219)
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(10)
+ .setNumNonMissing(10)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(1)))
+ .setPath(Path.newBuilder().addStep("feature").build())
+ .setType(Type.INT)
+ .build();
+
+ assertThat(
+ statsService.mergeStatistics(Arrays.asList(Arrays.asList(stat1, stat2))),
+ equalTo(Arrays.asList(expected)));
+ }
+
+ @Test
+ public void shouldAggregateCategoricalStatistics() {
+ FeatureNameStatistics stat1 =
+ FeatureNameStatistics.newBuilder()
+ .setStringStats(
+ StringStatistics.newBuilder()
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(5)
+ .setNumNonMissing(5)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(0))
+ .setUnique(4)
+ .setAvgLength(6))
+ .setType(Type.STRING)
+ .setPath(Path.newBuilder().addStep("feature").build())
+ .build();
+ FeatureNameStatistics stat2 =
+ FeatureNameStatistics.newBuilder()
+ .setStringStats(
+ StringStatistics.newBuilder()
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(5)
+ .setNumNonMissing(5)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(1))
+ .setUnique(4)
+ .setAvgLength(4))
+ .setType(Type.STRING)
+ .setPath(Path.newBuilder().addStep("feature").build())
+ .build();
+
+ FeatureNameStatistics expected =
+ FeatureNameStatistics.newBuilder()
+ .setStringStats(
+ StringStatistics.newBuilder()
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(10)
+ .setNumNonMissing(10)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(1))
+ .setAvgLength(5))
+ .setType(Type.STRING)
+ .setPath(Path.newBuilder().addStep("feature").build())
+ .build();
+ assertThat(
+ statsService.mergeStatistics(Arrays.asList(Arrays.asList(stat1, stat2))),
+ equalTo(Arrays.asList(expected)));
+ }
+
+ @Test
+ public void shouldAggregateBytesStatistics() {
+ FeatureNameStatistics stat1 =
+ FeatureNameStatistics.newBuilder()
+ .setBytesStats(
+ BytesStatistics.newBuilder()
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(5)
+ .setNumNonMissing(5)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(0))
+ .setUnique(4)
+ .setAvgNumBytes(6)
+ .setMaxNumBytes(10)
+ .setMinNumBytes(0))
+ .setType(Type.BYTES)
+ .setPath(Path.newBuilder().addStep("feature").build())
+ .build();
+ FeatureNameStatistics stat2 =
+ FeatureNameStatistics.newBuilder()
+ .setBytesStats(
+ BytesStatistics.newBuilder()
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(5)
+ .setNumNonMissing(5)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(1))
+ .setUnique(4)
+ .setAvgNumBytes(4)
+ .setMaxNumBytes(20)
+ .setMinNumBytes(1))
+ .setType(Type.BYTES)
+ .setPath(Path.newBuilder().addStep("feature").build())
+ .build();
+
+ FeatureNameStatistics expected =
+ FeatureNameStatistics.newBuilder()
+ .setBytesStats(
+ BytesStatistics.newBuilder()
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(10)
+ .setNumNonMissing(10)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(1))
+ .setAvgNumBytes(5)
+ .setMaxNumBytes(20)
+ .setMinNumBytes(0))
+ .setType(Type.BYTES)
+ .setPath(Path.newBuilder().addStep("feature").build())
+ .build();
+ assertThat(
+ statsService.mergeStatistics(Arrays.asList(Arrays.asList(stat1, stat2))),
+ equalTo(Arrays.asList(expected)));
+ }
+
+ @Test
+ public void shouldAggregateStructStatistics() {
+ FeatureNameStatistics stat1 =
+ FeatureNameStatistics.newBuilder()
+ .setStructStats(
+ StructStatistics.newBuilder()
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(5)
+ .setNumNonMissing(5)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(0)))
+ .setType(Type.STRUCT)
+ .setPath(Path.newBuilder().addStep("feature").build())
+ .build();
+ FeatureNameStatistics stat2 =
+ FeatureNameStatistics.newBuilder()
+ .setStructStats(
+ StructStatistics.newBuilder()
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(5)
+ .setNumNonMissing(5)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(1)))
+ .setType(Type.STRUCT)
+ .setPath(Path.newBuilder().addStep("feature").build())
+ .build();
+
+ FeatureNameStatistics expected =
+ FeatureNameStatistics.newBuilder()
+ .setStructStats(
+ StructStatistics.newBuilder()
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setTotNumValues(10)
+ .setNumNonMissing(10)
+ .setAvgNumValues(1)
+ .setMaxNumValues(1)
+ .setMinNumValues(1)
+ .setNumMissing(1)))
+ .setType(Type.STRUCT)
+ .setPath(Path.newBuilder().addStep("feature").build())
+ .build();
+ assertThat(
+ statsService.mergeStatistics(Arrays.asList(Arrays.asList(stat1, stat2))),
+ equalTo(Arrays.asList(expected)));
+ }
+}
diff --git a/examples/statistics/Historical Feature Statistics with Feast, TFDV and Facets.ipynb b/examples/statistics/Historical Feature Statistics with Feast, TFDV and Facets.ipynb
new file mode 100644
index 00000000000..2ee48e1b1d5
--- /dev/null
+++ b/examples/statistics/Historical Feature Statistics with Feast, TFDV and Facets.ipynb
@@ -0,0 +1,706 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Historical Feature Statistics with Feast, TFDV and Facets\n",
+ "\n",
+ "This tutorial covers how Feast can be used in conjunction with TFDV and Facets to retrieve statistics about feature datasets. \n",
+ "\n",
+ "The notebook showcases how Feast's integration with TFDV allows users to:\n",
+ "\n",
+ "1. Define TFX feature schemas and persist these properties in the Feature Store\n",
+ "2. Validate new data against the defined schema\n",
+ "3. Validate data already in Feast against the defined schema\n",
+ "\n",
+ "**Prerequisites**:\n",
+ "\n",
+ "- Feast running with at least 1 BigQuery warehouse store"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "setting project to statistics...\n",
+ "project already exists, skipping.\n"
+ ]
+ }
+ ],
+ "source": [
+ "import pandas as pd\n",
+ "import pytest\n",
+ "import pytz\n",
+ "import uuid\n",
+ "import time\n",
+ "from datetime import datetime, timedelta\n",
+ "\n",
+ "from feast.client import Client\n",
+ "from feast.entity import Entity\n",
+ "from feast.feature import Feature\n",
+ "from feast.feature_set import FeatureSet\n",
+ "from feast.type_map import ValueType\n",
+ "from google.protobuf import json_format\n",
+ "from google.protobuf.duration_pb2 import Duration\n",
+ "from tensorflow_metadata.proto.v0 import statistics_pb2\n",
+ "import tensorflow_data_validation as tfdv\n",
+ "\n",
+ "PROJECT_NAME = \"statistics\"\n",
+ "IRIS_DATASET = \"http://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data\"\n",
+ "BIGQUERY_STORE_NAME = \"serving\"\n",
+ "client = Client(core_url=\"localhost:6565\")\n",
+ "print(f\"setting project to {PROJECT_NAME}...\")\n",
+ "try:\n",
+ " client.create_project(PROJECT_NAME)\n",
+ "except:\n",
+ " print(\"project already exists, skipping.\")\n",
+ "client.set_project(PROJECT_NAME)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "In this example, we are using the iris dataset. More information about this dataset can be found [here](http://archive.ics.uci.edu/ml/datasets/iris)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " sepal_length | \n",
+ " sepal_width | \n",
+ " petal_length | \n",
+ " petal_width | \n",
+ " class | \n",
+ " datetime | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 5.1 | \n",
+ " 3.5 | \n",
+ " 1.4 | \n",
+ " 0.2 | \n",
+ " Iris-setosa | \n",
+ " 2020-04-12 07:22:07.065951+00:00 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 4.9 | \n",
+ " 3.0 | \n",
+ " 1.4 | \n",
+ " 0.2 | \n",
+ " Iris-setosa | \n",
+ " 2020-04-12 07:22:07.065951+00:00 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 4.7 | \n",
+ " 3.2 | \n",
+ " 1.3 | \n",
+ " 0.2 | \n",
+ " Iris-setosa | \n",
+ " 2020-04-12 07:22:07.065951+00:00 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 4.6 | \n",
+ " 3.1 | \n",
+ " 1.5 | \n",
+ " 0.2 | \n",
+ " Iris-setosa | \n",
+ " 2020-04-12 07:22:07.065951+00:00 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 5.0 | \n",
+ " 3.6 | \n",
+ " 1.4 | \n",
+ " 0.2 | \n",
+ " Iris-setosa | \n",
+ " 2020-04-12 07:22:07.065951+00:00 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " Anomaly short description | \n",
+ " Anomaly long description | \n",
+ "
\n",
+ " \n",
+ " | Feature name | \n",
+ " | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 'class' | \n",
+ " Unexpected string values | \n",
+ " Examples contain values missing from the schema: Iris-nonsensica (~33%). | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " Anomaly short description | \n",
+ " Anomaly long description | \n",
+ "
\n",
+ " \n",
+ " | Feature name | \n",
+ " | \n",
+ " | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 'class' | \n",
+ " Unexpected string values | \n",
+ " Examples contain values missing from the schema: Iris-nonsensica (~33%). | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ ""
+ ],
+ "text/plain": [
+ " Anomaly short description \\\n",
+ "Feature name \n",
+ "'class' Unexpected string values \n",
+ "\n",
+ " Anomaly long description \n",
+ "Feature name \n",
+ "'class' Examples contain values missing from the schema: Iris-nonsensica (~33%). "
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# Ingest the data into Feast\n",
+ "dataset_id_2 = client.ingest(feature_set, df_2)\n",
+ "time.sleep(10) # Sleep is not necessary if not using DirectRunner\n",
+ "\n",
+ "# Compute statistics over the new dataset\n",
+ "stats_2 = client.get_statistics(\n",
+ " feature_set_id=f'{PROJECT_NAME}/iris:1', \n",
+ " store=BIGQUERY_STORE_NAME, \n",
+ " features=iris_feature_names, \n",
+ " dataset_ids=[dataset_id_2])\n",
+ "\n",
+ "# Detect anomalies in the dataset\n",
+ "anomalies = tfdv.validate_statistics(statistics=stats_2, schema=feature_set.export_tfx_schema())\n",
+ "tfdv.display_anomalies(anomalies)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.2"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/infra/scripts/test-end-to-end-batch.sh b/infra/scripts/test-end-to-end-batch.sh
index 0e7bfe8bf8d..9dcd74877f3 100755
--- a/infra/scripts/test-end-to-end-batch.sh
+++ b/infra/scripts/test-end-to-end-batch.sh
@@ -251,7 +251,7 @@ ORIGINAL_DIR=$(pwd)
cd tests/e2e
set +e
-pytest bq-batch-retrieval.py --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
+pytest bq/* --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
TEST_EXIT_CODE=$?
if [[ ${TEST_EXIT_CODE} != 0 ]]; then
diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh
index c33dadc5413..f5943f2518d 100755
--- a/infra/scripts/test-end-to-end.sh
+++ b/infra/scripts/test-end-to-end.sh
@@ -216,7 +216,7 @@ ORIGINAL_DIR=$(pwd)
cd tests/e2e
set +e
-pytest basic-ingest-redis-serving.py --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
+pytest redis/* --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
TEST_EXIT_CODE=$?
if [[ ${TEST_EXIT_CODE} != 0 ]]; then
diff --git a/ingestion/pom.xml b/ingestion/pom.xml
index 9386d066bfd..9ba371f719c 100644
--- a/ingestion/pom.xml
+++ b/ingestion/pom.xml
@@ -63,30 +63,6 @@
reference.conf
-
-
-
- org.springframework
- org.springframework.vendor
-
-
- io.grpc
- io.grpc.vendor
-
-
- io.opencensus
- io.opencensus.vendor
-
-
- feast.core
- feast.core.vendor
-
-
- com.google.cloud.bigquery
- com.google.cloud.bigquery.vendor
-
-
diff --git a/protos/feast/core/CoreService.proto b/protos/feast/core/CoreService.proto
index b7760d0b9aa..fafefa07d9c 100644
--- a/protos/feast/core/CoreService.proto
+++ b/protos/feast/core/CoreService.proto
@@ -22,6 +22,8 @@ option go_package = "github.com/gojek/feast/sdk/go/protos/feast/core";
option java_outer_classname = "CoreServiceProto";
option java_package = "feast.core";
+import "google/protobuf/timestamp.proto";
+import "tensorflow_metadata/proto/v0/statistics.proto";
import "feast/core/FeatureSet.proto";
import "feast/core/Store.proto";
import "feast/core/FeatureSetReference.proto";
@@ -42,6 +44,11 @@ service CoreService {
// sets currently stored in the registry.
rpc ListFeatureSets (ListFeatureSetsRequest) returns (ListFeatureSetsResponse);
+ // Get feature statistics computed over the data in the batch stores.
+ //
+ // Returns a dataset containing TFDV statistics mapped to each valid historical store.
+ rpc GetFeatureStatistics (GetFeatureStatisticsRequest) returns (GetFeatureStatisticsResponse);
+
// Retrieve store details given a filter.
//
// Returns all stores matching that filter. If none are found, an empty list will be returned.
@@ -274,3 +281,42 @@ message StopIngestionJobRequest {
// Request from stopping an ingestion job
message StopIngestionJobResponse {}
+
+message GetFeatureStatisticsRequest {
+ // Feature set to retrieve the statistics for. A fully qualified feature set
+ // id in the format of project/feature_set:version must be provided.
+ string feature_set_id = 1;
+
+ // Optional filter which filters returned statistics by selected features. These
+ // features must be present in the data that is being processed.
+ repeated string features = 2;
+
+ // Optional filter to select store over which the statistics will retrieved.
+ // Only historical stores are allowed.
+ string store = 3;
+
+ // Optional start and end dates over which to filter statistical data
+ // Start date is inclusive, but end date is not.
+ // Only dates are supported, not times.
+ // Cannot be used with dataset_ids.
+ // If this period spans multiple days, unaggregatable statistics will be dropped.
+ google.protobuf.Timestamp start_date = 4;
+ google.protobuf.Timestamp end_date = 5;
+
+ // Optional list of dataset Ids by which to filter data before
+ // retrieving statistics.
+ // Cannot be used with the date ranges
+ // If multiple dataset ids are provided, unaggregatable statistics will be dropped.
+ repeated string dataset_ids = 6;
+
+ // Setting this flag to true will force a recalculation of statistics and overwrite results currently in the
+ // cache, if any.
+ bool force_refresh = 7;
+}
+
+message GetFeatureStatisticsResponse {
+ // Contains statistics for the requested data.
+ // Due to the limitations of TFDV and Facets, only a single dataset can be returned in,
+ // despite the message being of list type.
+ tensorflow.metadata.v0.DatasetFeatureStatisticsList dataset_feature_statistics_list = 1;
+}
diff --git a/protos/feast/core/FeatureSet.proto b/protos/feast/core/FeatureSet.proto
index 429d99c8547..40eb468a47e 100644
--- a/protos/feast/core/FeatureSet.proto
+++ b/protos/feast/core/FeatureSet.proto
@@ -68,46 +68,6 @@ message EntitySpec {
// Value type of the feature.
feast.types.ValueType.Enum value_type = 2;
-
- // presence_constraints, shape_type and domain_info are referenced from:
- // https://github.com/tensorflow/metadata/blob/36f65d1268cbc92cdbcf812ee03dcf47fb53b91e/tensorflow_metadata/proto/v0/schema.proto#L107
-
- oneof presence_constraints {
- // Constraints on the presence of this feature in the examples.
- tensorflow.metadata.v0.FeaturePresence presence = 3;
- // Only used in the context of a "group" context, e.g., inside a sequence.
- tensorflow.metadata.v0.FeaturePresenceWithinGroup group_presence = 4;
- }
-
- // The shape of the feature which governs the number of values that appear in
- // each example.
- oneof shape_type {
- // The feature has a fixed shape corresponding to a multi-dimensional
- // tensor.
- tensorflow.metadata.v0.FixedShape shape = 5;
- // The feature doesn't have a well defined shape. All we know are limits on
- // the minimum and maximum number of values.
- tensorflow.metadata.v0.ValueCount value_count = 6;
- }
-
- // Domain for the values of the feature.
- oneof domain_info {
- // Reference to a domain defined at the schema level.
- string domain = 7;
- // Inline definitions of domains.
- tensorflow.metadata.v0.IntDomain int_domain = 8;
- tensorflow.metadata.v0.FloatDomain float_domain = 9;
- tensorflow.metadata.v0.StringDomain string_domain = 10;
- tensorflow.metadata.v0.BoolDomain bool_domain = 11;
- tensorflow.metadata.v0.StructDomain struct_domain = 12;
- // Supported semantic domains.
- tensorflow.metadata.v0.NaturalLanguageDomain natural_language_domain = 13;
- tensorflow.metadata.v0.ImageDomain image_domain = 14;
- tensorflow.metadata.v0.MIDDomain mid_domain = 15;
- tensorflow.metadata.v0.URLDomain url_domain = 16;
- tensorflow.metadata.v0.TimeDomain time_domain = 17;
- tensorflow.metadata.v0.TimeOfDayDomain time_of_day_domain = 18;
- }
}
message FeatureSpec {
diff --git a/protos/feast/core/Store.proto b/protos/feast/core/Store.proto
index de9af0a99fe..4ffdbcdf58f 100644
--- a/protos/feast/core/Store.proto
+++ b/protos/feast/core/Store.proto
@@ -69,6 +69,7 @@ message Store {
// ====================|==================|================================
// - event_timestamp | TIMESTAMP | event time of the FeatureRow
// - created_timestamp | TIMESTAMP | processing time of the ingestion of the FeatureRow
+ // - dataset_id | STRING | identifier of the batch dataset a row belongs to
// - job_id | STRING | identifier for the job that writes the FeatureRow to the corresponding BigQuery table
//
// BigQuery table created will be partitioned by the field "event_timestamp"
diff --git a/protos/feast/types/FeatureRow.proto b/protos/feast/types/FeatureRow.proto
index c170cd5d502..c3614e00274 100644
--- a/protos/feast/types/FeatureRow.proto
+++ b/protos/feast/types/FeatureRow.proto
@@ -39,4 +39,8 @@ message FeatureRow {
// /:. This value will be used by the feast ingestion job to filter
// rows, and write the values to the correct tables.
string feature_set = 6;
+
+ // Identifier tying this feature row to a specific ingestion dataset. For
+ // batch loads, this dataset id can be attributed to a single ingestion job.
+ string dataset_id = 7;
}
diff --git a/protos/tensorflow_metadata/proto/v0/statistics.proto b/protos/tensorflow_metadata/proto/v0/statistics.proto
new file mode 100644
index 00000000000..b07f5e5fc29
--- /dev/null
+++ b/protos/tensorflow_metadata/proto/v0/statistics.proto
@@ -0,0 +1,427 @@
+// Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// =============================================================================
+
+// Definitions for aggregated feature statistics for datasets.
+// TODO(b/80075690): make a Javascript build rule for this.
+// TODO(b/80075691): migrate Facets to use this.
+syntax = "proto3";
+option cc_enable_arenas = true;
+
+package tensorflow.metadata.v0;
+
+option java_package = "org.tensorflow.metadata.v0";
+option java_multiple_files = true;
+option go_package = "github.com/gojek/feast/sdk/go/protos/tensorflow_metadata/proto/v0";
+
+import "tensorflow_metadata/proto/v0/path.proto";
+
+// Copied from Facets feature_statistics.proto
+// Must be kept binary-compatible with the original, until all usages
+// are updated to use this version, or we write a proto-to-proto converter.
+
+// A list of features statistics for different datasets. If you wish to compare
+// different datasets using this list, then the DatasetFeatureStatistics
+// entries should all contain the same list of features.
+message DatasetFeatureStatisticsList {
+ repeated DatasetFeatureStatistics datasets = 1;
+}
+
+// The feature statistics for a single dataset.
+message DatasetFeatureStatistics {
+ // The name of the dataset.
+ string name = 1;
+ // The number of examples in the dataset.
+ uint64 num_examples = 2;
+
+ // Only valid if the weight feature was specified.
+ // Treats a missing weighted feature as zero.
+ double weighted_num_examples = 4;
+ // The feature statistics for the dataset.
+ repeated FeatureNameStatistics features = 3;
+
+ // Cross feature statistics for the dataset.
+ repeated CrossFeatureStatistics cross_features = 5;
+}
+
+message CrossFeatureStatistics {
+ // The path of feature x.
+ Path path_x = 1;
+ // The path of feature y.
+ Path path_y = 2;
+
+ // Number of occurrences of this feature cross in the data. If any of
+ // the features in the cross is missing, the example is ignored.
+ uint64 count = 3;
+
+ oneof cross_stats {
+ NumericCrossStatistics num_cross_stats = 4;
+ CategoricalCrossStatistics categorical_cross_stats = 5;
+ }
+}
+
+message NumericCrossStatistics {
+ // Pearson product-moment correlation coefficient.
+ float correlation = 1;
+ // Standard covariance. E[(X-E[X])*(Y-E[Y])]
+ float covariance = 2;
+}
+
+message CategoricalCrossStatistics {
+ LiftStatistics lift = 1;
+}
+
+message LiftStatistics {
+ // Lift information for each value of path_y. Lift is defined for each pair of
+ // values (x,y) as P(path_y=y|path_x=x)/P(path_y=y).
+ repeated LiftSeries lift_series = 1;
+ // Weighted lift information for each value of path_y. Weighted lift is
+ // defined for each pair of values (x,y) as P(path_y=y|path_x=x)/P(path_y=y)
+ // where probabilities are computed over weighted example space.
+ repeated LiftSeries weighted_lift_series = 2;
+}
+
+// Container for lift information for a specific y-value.
+message LiftSeries {
+ // A bucket for referring to binned numeric features.
+ message Bucket {
+ // The low value of the bucket, inclusive.
+ double low_value = 1;
+ // The high value of the bucket, exclusive (unless the high_value is
+ // positive infinity).
+ double high_value = 2;
+ }
+
+ // The particular value of path_y corresponding to this LiftSeries. Each
+ // element in lift_values corresponds to the lift a different x_value and
+ // this specific y_value.
+ oneof y_value {
+ int32 y_int = 1;
+ string y_string = 2;
+ Bucket y_bucket = 3;
+ }
+
+ // The number of examples in which y_value appears.
+ oneof y_count_value {
+ uint64 y_count = 4;
+ double weighted_y_count = 5;
+ }
+
+ // A container for lift information about a specific value of path_x.
+ message LiftValue {
+ oneof x_value {
+ int32 x_int = 1;
+ string x_string = 2;
+ }
+ // P(path_y=y|path_x=x) / P(path_y=y) for x_value and the enclosing y_value.
+ // In terms of concrete fields, this number represents:
+ // (x_and_y_count / x_count) / (y_count / num_examples)
+ double lift = 3;
+ // The number of examples in which x_value appears.
+ oneof x_count_value {
+ uint64 x_count = 4;
+ double weighted_x_count = 5;
+ }
+ // The number of examples in which x_value appears and y_value appears.
+ oneof x_and_y_count_value {
+ uint64 x_and_y_count = 6;
+ double weighted_x_and_y_count = 7;
+ }
+ }
+
+ // The lifts for a each path_x value and this y_value.
+ repeated LiftValue lift_values = 6;
+}
+
+// The complete set of statistics for a given feature name for a dataset.
+message FeatureNameStatistics {
+ // The types supported by the feature statistics. When aggregating
+ // tf.Examples, if the bytelist contains a string, it is recommended to encode
+ // it here as STRING instead of BYTES in order to calculate string-specific
+ // statistical measures.
+ enum Type {
+ INT = 0;
+ FLOAT = 1;
+ STRING = 2;
+ BYTES = 3;
+ STRUCT = 4;
+ }
+
+ // One can identify a field either by the name (for simple fields), or by
+ // a path (for structured fields). Note that:
+ // name: "foo"
+ // is equivalent to:
+ // path: {step:"foo"}
+ // Note: this oneof must be consistently either name or path across all
+ // FeatureNameStatistics in one DatasetFeatureStatistics.
+ oneof field_id {
+ // The feature name
+ string name = 1;
+
+ // The path of the feature.
+ Path path = 8;
+ }
+
+ // The data type of the feature
+ Type type = 2;
+
+ // The statistics of the values of the feature.
+ oneof stats {
+ NumericStatistics num_stats = 3;
+ StringStatistics string_stats = 4;
+ BytesStatistics bytes_stats = 5;
+ StructStatistics struct_stats = 7;
+ }
+
+ // Any custom statistics can be stored in this list.
+ repeated CustomStatistic custom_stats = 6;
+}
+
+// Common weighted statistics for all feature types. Statistics counting number
+// of values (i.e., avg_num_values and tot_num_values) include NaNs.
+// If the weighted column is missing, then this counts as a weight of 1
+// for that example.
+message WeightedCommonStatistics {
+ // Weighted number of examples not missing.
+ double num_non_missing = 1;
+ // Weighted number of examples missing.
+ // Note that if the weighted column is zero, this does not count
+ // as missing.
+ double num_missing = 2;
+ // average number of values, weighted by the number of examples.
+ double avg_num_values = 3;
+ // tot_num_values = avg_num_values * num_non_missing.
+ // This is calculated directly, so should have less numerical error.
+ double tot_num_values = 4;
+}
+
+// Stores the name and value of any custom statistic. The value can be a string,
+// double, or histogram.
+message CustomStatistic {
+ string name = 1;
+ oneof val {
+ double num = 2;
+ string str = 3;
+ Histogram histogram = 4;
+ RankHistogram rank_histogram = 5;
+ }
+}
+
+// Statistics for a numeric feature in a dataset.
+message NumericStatistics {
+ CommonStatistics common_stats = 1;
+ // The mean of the values
+ double mean = 2;
+ // The standard deviation of the values
+ double std_dev = 3;
+ // The number of values that equal 0
+ uint64 num_zeros = 4;
+ // The minimum value
+ double min = 5;
+ // The median value
+ double median = 6;
+ // The maximum value
+ double max = 7;
+ // The histogram(s) of the feature values.
+ repeated Histogram histograms = 8;
+
+ // Weighted statistics for the feature, if the values have weights.
+ WeightedNumericStatistics weighted_numeric_stats = 9;
+}
+
+// Statistics for a string feature in a dataset.
+message StringStatistics {
+ CommonStatistics common_stats = 1;
+ // The number of unique values
+ uint64 unique = 2;
+
+ message FreqAndValue {
+ string value = 2;
+
+ // The number of times the value occurs. Stored as a double to be able to
+ // handle weighted features.
+ double frequency = 3;
+
+ // Deleted fields.
+ reserved 1;
+ }
+ // A sorted list of the most-frequent values and their frequencies, with
+ // the most-frequent being first.
+ repeated FreqAndValue top_values = 3;
+
+ // The average length of the values
+ float avg_length = 4;
+
+ // The rank histogram for the values of the feature.
+ // The rank is used to measure of how commonly the value is found in the
+ // dataset. The most common value would have a rank of 1, with the second-most
+ // common value having a rank of 2, and so on.
+ RankHistogram rank_histogram = 5;
+
+ // Weighted statistics for the feature, if the values have weights.
+ WeightedStringStatistics weighted_string_stats = 6;
+
+ // A vocabulary file, used for vocabularies too large to store in the proto
+ // itself. Note that the file may be relative to some context-dependent
+ // directory. E.g. in TFX the feature statistics will live in a PPP and
+ // vocabulary file names will be relative to this PPP.
+ string vocabulary_file = 7;
+}
+
+// Statistics for a weighted numeric feature in a dataset.
+message WeightedNumericStatistics {
+ // The weighted mean of the values
+ double mean = 1;
+ // The weighted standard deviation of the values
+ double std_dev = 2;
+ // The weighted median of the values
+ double median = 3;
+
+ // The histogram(s) of the weighted feature values.
+ repeated Histogram histograms = 4;
+}
+
+// Statistics for a weighted string feature in a dataset.
+message WeightedStringStatistics {
+ // A sorted list of the most-frequent values and their weighted frequencies,
+ // with the most-frequent being first.
+ repeated StringStatistics.FreqAndValue top_values = 1;
+
+ // The rank histogram for the weighted values of the feature.
+ RankHistogram rank_histogram = 2;
+}
+
+// Statistics for a bytes feature in a dataset.
+message BytesStatistics {
+ CommonStatistics common_stats = 1;
+ // The number of unique values
+ uint64 unique = 2;
+
+ // The average number of bytes in a value
+ float avg_num_bytes = 3;
+ // The minimum number of bytes in a value
+ float min_num_bytes = 4;
+ // The maximum number of bytes in a value
+ float max_num_bytes = 5;
+}
+
+message StructStatistics {
+ CommonStatistics common_stats = 1;
+}
+
+// Common statistics for all feature types. Statistics counting number of values
+// (i.e., min_num_values, max_num_values, avg_num_values, and tot_num_values)
+// include NaNs.
+message CommonStatistics {
+ // The number of examples with at least one value for this feature.
+ uint64 num_non_missing = 1;
+ // The number of examples with no values for this feature.
+ uint64 num_missing = 2;
+ // The minimum number of values in a single example for this feature.
+ uint64 min_num_values = 3;
+ // The maximum number of values in a single example for this feature.
+ uint64 max_num_values = 4;
+ // The average number of values in a single example for this feature.
+ float avg_num_values = 5;
+ // tot_num_values = avg_num_values * num_non_missing.
+ // This is calculated directly, so should have less numerical error.
+ uint64 tot_num_values = 8;
+ // The quantiles histogram for the number of values in this feature.
+ Histogram num_values_histogram = 6;
+ WeightedCommonStatistics weighted_common_stats = 7;
+ // The histogram for the number of features in the feature list (only set if
+ // this feature is a non-context feature from a tf.SequenceExample).
+ // This is different from num_values_histogram, as num_values_histogram tracks
+ // the count of all values for a feature in an example, whereas this tracks
+ // the length of the feature list for this feature in an example (where each
+ // feature list can contain multiple values).
+ Histogram feature_list_length_histogram = 9;
+}
+
+// The data used to create a histogram of a numeric feature for a dataset.
+message Histogram {
+ // Each bucket defines its low and high values along with its count. The
+ // low and high values must be a real number or positive or negative
+ // infinity. They cannot be NaN or undefined. Counts of those special values
+ // can be found in the numNaN and numUndefined fields.
+ message Bucket {
+ // The low value of the bucket, inclusive.
+ double low_value = 1;
+ // The high value of the bucket, exclusive (unless the highValue is
+ // positive infinity).
+ double high_value = 2;
+
+ // The number of items in the bucket. Stored as a double to be able to
+ // handle weighted histograms.
+ double sample_count = 4;
+
+ // Deleted fields.
+ reserved 3;
+ }
+
+ // The number of NaN values in the dataset.
+ uint64 num_nan = 1;
+ // The number of undefined values in the dataset.
+ uint64 num_undefined = 2;
+
+ // A list of buckets in the histogram, sorted from lowest bucket to highest
+ // bucket.
+ repeated Bucket buckets = 3;
+
+ // The type of the histogram. A standard histogram has equal-width buckets.
+ // The quantiles type is used for when the histogram message is used to store
+ // quantile information (by using equal-count buckets with variable widths).
+ enum HistogramType {
+ STANDARD = 0;
+ QUANTILES = 1;
+ }
+
+ // The type of the histogram.
+ HistogramType type = 4;
+
+ // An optional descriptive name of the histogram, to be used for labeling.
+ string name = 5;
+}
+
+// The data used to create a rank histogram of a non-numeric feature of a
+// dataset. The rank of a value in a feature can be used as a measure of how
+// commonly the value is found in the entire dataset. With bucket sizes of one,
+// this becomes a distribution function of all feature values.
+message RankHistogram {
+ // Each bucket defines its start and end ranks along with its count.
+ message Bucket {
+ // The low rank of the bucket, inclusive.
+ uint64 low_rank = 1;
+ // The high rank of the bucket, exclusive.
+ uint64 high_rank = 2;
+
+ // The label for the bucket. Can be used to list or summarize the values in
+ // this rank bucket.
+ string label = 4;
+
+ // The number of items in the bucket. Stored as a double to be able to
+ // handle weighted histograms.
+ double sample_count = 5;
+
+ // Deleted fields.
+ reserved 3;
+ }
+
+ // A list of buckets in the histogram, sorted from lowest-ranked bucket to
+ // highest-ranked bucket.
+ repeated Bucket buckets = 1;
+
+ // An optional descriptive name of the histogram, to be used for labeling.
+ string name = 2;
+}
\ No newline at end of file
diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py
index 0a38236a510..74560e572c7 100644
--- a/sdk/python/feast/client.py
+++ b/sdk/python/feast/client.py
@@ -17,7 +17,9 @@
import os
import shutil
import tempfile
+import datetime
import time
+import uuid
from collections import OrderedDict
from math import ceil
from typing import Dict, List, Optional, Tuple, Union
@@ -26,6 +28,8 @@
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
+from tensorflow_metadata.proto.v0 import statistics_pb2
+from google.protobuf.timestamp_pb2 import Timestamp
from feast.config import Config
from feast.constants import (
@@ -46,6 +50,7 @@
GetFeastCoreVersionRequest,
GetFeatureSetRequest,
GetFeatureSetResponse,
+ GetFeatureStatisticsRequest,
ListFeatureSetsRequest,
ListFeatureSetsResponse,
ListIngestionJobsRequest,
@@ -735,7 +740,7 @@ def ingest(
max_workers: int = max(CPU_COUNT - 1, 1),
disable_progress_bar: bool = False,
timeout: int = KAFKA_CHUNK_PRODUCTION_TIMEOUT,
- ) -> None:
+ ) -> str:
"""
Loads feature data into Feast for a specific feature set.
@@ -771,8 +776,8 @@ def ingest(
Timeout in seconds to wait for completion.
Returns:
- None:
- None
+ str:
+ dataset id of the ingested dataset
"""
if isinstance(feature_set, FeatureSet):
@@ -825,6 +830,7 @@ def ingest(
# Loop optimization declarations
produce = producer.produce
flush = producer.flush
+ dataset_id = _generate_dataset_id(feature_set)
# Transform and push data to Kafka
if feature_set.source.source_type == "Kafka":
@@ -832,6 +838,7 @@ def ingest(
file=dest_path,
row_groups=list(range(pq_file.num_row_groups)),
fs=feature_set,
+ dataset_id=dataset_id,
max_workers=max_workers,
):
@@ -859,7 +866,76 @@ def ingest(
print("Removing temporary file(s)...")
shutil.rmtree(dir_path)
- return None
+ return dataset_id
+
+ def get_statistics(
+ self,
+ feature_set_id: str,
+ store: str,
+ features: List[str] = [],
+ dataset_ids: Optional[List[str]] = None,
+ start_date: Optional[datetime.datetime] = None,
+ end_date: Optional[datetime.datetime] = None,
+ force_refresh: bool = False,
+ default_project: Optional[str] = None,
+ ) -> statistics_pb2.DatasetFeatureStatisticsList:
+ """
+ Retrieves the feature featureStatistics computed over the data in the batch
+ stores.
+
+ Args:
+ feature_set_id: Fully qualified feature set id in the format
+ project/feature_set:version to retrieve batch featureStatistics for.
+ store: Name of the store to retrieve feature featureStatistics over. This
+ store must be a historical store.
+ features: Optional list of feature names to filter from the results.
+ dataset_ids: Optional list of dataset Ids by which to filter data
+ before retrieving featureStatistics. Cannot be used with start_date
+ and end_date.
+ If multiple dataset ids are provided, unaggregatable featureStatistics
+ will be dropped.
+ start_date: Optional start date over which to filter statistical data.
+ Data from this date will be included.
+ Cannot be used with dataset_ids. If the provided period spans
+ multiple days, unaggregatable featureStatistics will be dropped.
+ end_date: Optional end date over which to filter statistical data.
+ Data from this data will not be included.
+ Cannot be used with dataset_ids. If the provided period spans
+ multiple days, unaggregatable featureStatistics will be dropped.
+ force_refresh: Setting this flag to true will force a recalculation
+ of featureStatistics and overwrite results currently in the cache, if any.
+ default_project: This project will be used if the project name is
+ not provided in the feature reference
+
+ Returns:
+ Returns a tensorflow DatasetFeatureStatisticsList containing TFDV featureStatistics.
+ """
+
+ self._connect_core()
+ if dataset_ids is not None and (start_date is not None or end_date is not None):
+ raise ValueError(
+ "Only one of dataset_id or [start_date, end_date] can be provided."
+ )
+
+ request = GetFeatureStatisticsRequest(
+ feature_set_id=feature_set_id,
+ features=features,
+ store=store,
+ force_refresh=force_refresh,
+ )
+ if dataset_ids is not None:
+ request.dataset_ids.extend(dataset_ids)
+ else:
+ if start_date is not None:
+ request.start_date.CopyFrom(
+ Timestamp(seconds=int(start_date.timestamp()))
+ )
+ if end_date is not None:
+ request.end_date.CopyFrom(Timestamp(seconds=int(end_date.timestamp())))
+
+ return self._core_service_stub.GetFeatureStatistics(
+ request
+ ).dataset_feature_statistics_list
def _build_feature_references(
@@ -916,6 +992,20 @@ def _build_feature_references(
return features
+def _generate_dataset_id(feature_set: FeatureSet) -> str:
+ """
+ Generates a UUID from the feature set name, version, and the current time.
+
+ Args:
+ feature_set: Feature set of the dataset to be ingested.
+
+ Returns:
+ UUID unique to current time and the feature set provided.
+ """
+ uuid_str = f"{feature_set.name}_{feature_set.version}_{int(time.time())}"
+ return str(uuid.uuid3(uuid.NAMESPACE_DNS, uuid_str))
+
+
def _read_table_from_source(
source: Union[pd.DataFrame, str], chunk_size: int, max_workers: int
) -> Tuple[str, str]:
diff --git a/sdk/python/feast/entity.py b/sdk/python/feast/entity.py
index 9c5a027b974..012d01631af 100644
--- a/sdk/python/feast/entity.py
+++ b/sdk/python/feast/entity.py
@@ -29,26 +29,7 @@ def to_proto(self) -> EntityProto:
Returns EntitySpec object
"""
value_type = ValueTypeProto.ValueType.Enum.Value(self.dtype.name)
- return EntityProto(
- name=self.name,
- value_type=value_type,
- presence=self.presence,
- group_presence=self.group_presence,
- shape=self.shape,
- value_count=self.value_count,
- domain=self.domain,
- int_domain=self.int_domain,
- float_domain=self.float_domain,
- string_domain=self.string_domain,
- bool_domain=self.bool_domain,
- struct_domain=self.struct_domain,
- natural_language_domain=self.natural_language_domain,
- image_domain=self.image_domain,
- mid_domain=self.mid_domain,
- url_domain=self.url_domain,
- time_domain=self.time_domain,
- time_of_day_domain=self.time_of_day_domain,
- )
+ return EntityProto(name=self.name, value_type=value_type,)
@classmethod
def from_proto(cls, entity_proto: EntityProto):
@@ -62,7 +43,4 @@ def from_proto(cls, entity_proto: EntityProto):
Entity object
"""
entity = cls(name=entity_proto.name, dtype=ValueType(entity_proto.value_type))
- entity.update_presence_constraints(entity_proto)
- entity.update_shape_type(entity_proto)
- entity.update_domain_info(entity_proto)
return entity
diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py
index 760e947318f..ace7f165de1 100644
--- a/sdk/python/feast/feature_set.py
+++ b/sdk/python/feast/feature_set.py
@@ -716,6 +716,8 @@ def export_tfx_schema(self) -> schema_pb2.Schema:
]
for _, field in self._fields.items():
+ if isinstance(field, Entity):
+ continue
feature = schema_pb2.Feature()
for attr in attributes_to_copy_from_field_to_feature:
if getattr(field, attr) is None:
diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py
index b4490f025c5..dc8ad48a746 100644
--- a/sdk/python/feast/loaders/ingest.py
+++ b/sdk/python/feast/loaders/ingest.py
@@ -25,7 +25,9 @@
KAFKA_CHUNK_PRODUCTION_TIMEOUT = 120 # type: int
-def _encode_pa_tables(file: str, fs: FeatureSet, row_group_idx: int) -> List[bytes]:
+def _encode_pa_tables(
+ file: str, fs: FeatureSet, dataset_id: str, row_group_idx: int
+) -> List[bytes]:
"""
Helper function to encode a PyArrow table(s) read from parquet file(s) into
FeatureRows.
@@ -44,6 +46,9 @@ def _encode_pa_tables(file: str, fs: FeatureSet, row_group_idx: int) -> List[byt
fs (feast.feature_set.FeatureSet):
FeatureSet describing parquet files.
+ dataset_id (str):
+ UUID unique to this dataset.
+
row_group_idx(int):
Row group index to read and encode into byte like FeatureRow
protobuf objects.
@@ -78,7 +83,9 @@ def _encode_pa_tables(file: str, fs: FeatureSet, row_group_idx: int) -> List[byt
# Iterate through the rows
for row_idx in range(table.num_rows):
feature_row = FeatureRow(
- event_timestamp=datetime_col[row_idx], feature_set=feature_set
+ event_timestamp=datetime_col[row_idx],
+ feature_set=feature_set,
+ dataset_id=dataset_id,
)
# Loop optimization declaration
ext = feature_row.fields.extend
@@ -94,7 +101,7 @@ def _encode_pa_tables(file: str, fs: FeatureSet, row_group_idx: int) -> List[byt
def get_feature_row_chunks(
- file: str, row_groups: List[int], fs: FeatureSet, max_workers: int
+ file: str, row_groups: List[int], fs: FeatureSet, dataset_id: str, max_workers: int
) -> Iterable[List[bytes]]:
"""
Iterator function to encode a PyArrow table read from a parquet file to
@@ -112,6 +119,9 @@ def get_feature_row_chunks(
fs (feast.feature_set.FeatureSet):
FeatureSet describing parquet files.
+ dataset_id (str):
+ UUID unique to this dataset.
+
max_workers (int):
Maximum number of workers to spawn.
@@ -121,7 +131,7 @@ def get_feature_row_chunks(
"""
pool = Pool(max_workers)
- func = partial(_encode_pa_tables, file, fs)
+ func = partial(_encode_pa_tables, file, fs, dataset_id)
for chunk in pool.imap(func, row_groups):
yield chunk
return
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index 9d8a3786505..10c4c29a8a6 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -42,7 +42,7 @@
"tabulate==0.8.*",
"toml==0.10.*",
"tqdm==4.*",
- "pyarrow>=0.15.1",
+ "pyarrow<0.16.0,>=0.15.1",
"numpy",
"google",
"confluent_kafka",
diff --git a/sdk/python/tests/data/tensorflow_metadata/bikeshare_feature_set.yaml b/sdk/python/tests/data/tensorflow_metadata/bikeshare_feature_set.yaml
index daa0a35f0ab..48c595712cb 100644
--- a/sdk/python/tests/data/tensorflow_metadata/bikeshare_feature_set.yaml
+++ b/sdk/python/tests/data/tensorflow_metadata/bikeshare_feature_set.yaml
@@ -3,15 +3,6 @@ spec:
entities:
- name: station_id
valueType: INT64
- intDomain:
- min: 1
- max: 5000
- presence:
- minFraction: 1.0
- minCount: 1
- shape:
- dim:
- - size: 1
features:
- name: location
valueType: STRING
diff --git a/sdk/python/tests/data/tensorflow_metadata/bikeshare_schema.json b/sdk/python/tests/data/tensorflow_metadata/bikeshare_schema.json
index e7a886053c1..fa9f97cca0d 100644
--- a/sdk/python/tests/data/tensorflow_metadata/bikeshare_schema.json
+++ b/sdk/python/tests/data/tensorflow_metadata/bikeshare_schema.json
@@ -85,25 +85,6 @@
}
]
}
- },
- {
- "name": "station_id",
- "type": "INT",
- "presence": {
- "minFraction": 1.0,
- "minCount": "1"
- },
- "int_domain": {
- "min": 1,
- "max": 5000
- },
- "shape": {
- "dim": [
- {
- "size": "1"
- }
- ]
- }
}
],
"stringDomain": [
diff --git a/sdk/python/tests/test_feature_set.py b/sdk/python/tests/test_feature_set.py
index 0a7d1ebabea..a2cc12fe113 100644
--- a/sdk/python/tests/test_feature_set.py
+++ b/sdk/python/tests/test_feature_set.py
@@ -210,9 +210,6 @@ def test_import_tfx_schema(self):
feature_set.import_tfx_schema(test_input_schema)
# After update
- for entity in feature_set.entities:
- assert entity.presence is not None
- assert entity.shape is not None
for feature in feature_set.features:
assert feature.presence is not None
assert feature.shape is not None
diff --git a/storage/api/src/main/java/feast/storage/api/statistics/FeatureSetStatistics.java b/storage/api/src/main/java/feast/storage/api/statistics/FeatureSetStatistics.java
new file mode 100644
index 00000000000..a328df46b97
--- /dev/null
+++ b/storage/api/src/main/java/feast/storage/api/statistics/FeatureSetStatistics.java
@@ -0,0 +1,48 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.api.statistics;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+import org.tensorflow.metadata.v0.FeatureNameStatistics;
+
+/** Feature statistics for a feature set over a bounded set of data. */
+@AutoValue
+public abstract class FeatureSetStatistics {
+
+ public abstract long getNumExamples();
+
+ public abstract ImmutableList getFeatureNameStatistics();
+
+ public static Builder newBuilder() {
+ return new AutoValue_FeatureSetStatistics.Builder();
+ }
+
+ @AutoValue.Builder
+ public abstract static class Builder {
+ public abstract Builder setNumExamples(long numExamples);
+
+ protected abstract ImmutableList.Builder featureNameStatisticsBuilder();
+
+ public Builder addFeatureNameStatistics(FeatureNameStatistics featureNameStatistics) {
+ featureNameStatisticsBuilder().add(featureNameStatistics);
+ return this;
+ }
+
+ public abstract FeatureSetStatistics build();
+ }
+}
diff --git a/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java b/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java
new file mode 100644
index 00000000000..c527b3c80cd
--- /dev/null
+++ b/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java
@@ -0,0 +1,46 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.api.statistics;
+
+import com.google.protobuf.Timestamp;
+import feast.core.FeatureSetProto.FeatureSetSpec;
+import java.util.List;
+
+public interface StatisticsRetriever {
+
+ /**
+ * Get feature set statistics for a single feature set, for a single dataset id.
+ *
+ * @param featureSetSpec feature set spec of the requested feature set
+ * @param features subset of features to retrieve.
+ * @param dataset dataset id to filter the data by
+ * @return {@link FeatureSetStatistics} containing statistics for the requested features.
+ */
+ FeatureSetStatistics getFeatureStatistics(
+ FeatureSetSpec featureSetSpec, List features, String dataset);
+
+ /**
+ * Get feature set statistics for a single feature set, for a single day.
+ *
+ * @param featureSetSpec feature set spec of the requested feature set
+ * @param features subset of features to retrieve.
+ * @param date date to filter the data by
+ * @return {@link FeatureSetStatistics} containing statistics for the requested features.
+ */
+ FeatureSetStatistics getFeatureStatistics(
+ FeatureSetSpec featureSetSpec, List features, Timestamp date);
+}
diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java
index cba997b6ab0..0c7ef4c5aed 100644
--- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java
+++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java
@@ -38,6 +38,8 @@ public class QueryTemplater {
private static final PebbleEngine engine = new PebbleEngine.Builder().build();
private static final String FEATURESET_TEMPLATE_NAME = "templates/single_featureset_pit_join.sql";
private static final String JOIN_TEMPLATE_NAME = "templates/join_featuresets.sql";
+ private static final String BASIC_STATS_TEMPLATE_NAME = "templates/basic_stats.sql";
+ private static final String HIST_STATS_TEMPLATE_NAME = "templates/hist_stats.sql";
/**
* Get the query for retrieving the earliest and latest timestamps in the entity dataset.
diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/BigQueryStatisticsRetriever.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/BigQueryStatisticsRetriever.java
new file mode 100644
index 00000000000..59a1f546429
--- /dev/null
+++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/BigQueryStatisticsRetriever.java
@@ -0,0 +1,160 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.connectors.bigquery.statistics;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.bigquery.*;
+import com.google.common.collect.Streams;
+import com.google.protobuf.Timestamp;
+import feast.core.FeatureSetProto.FeatureSetSpec;
+import feast.core.FeatureSetProto.FeatureSpec;
+import feast.core.StoreProto.Store.BigQueryConfig;
+import feast.storage.api.statistics.FeatureSetStatistics;
+import feast.storage.api.statistics.StatisticsRetriever;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.tensorflow.metadata.v0.FeatureNameStatistics;
+
+@AutoValue
+public abstract class BigQueryStatisticsRetriever implements StatisticsRetriever {
+
+ public abstract String projectId();
+
+ public abstract String datasetId();
+
+ public abstract BigQuery bigquery();
+
+ public static BigQueryStatisticsRetriever create(BigQueryConfig config) {
+ BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
+ return newBuilder()
+ .setBigquery(bigquery)
+ .setDatasetId(config.getDatasetId())
+ .setProjectId(config.getProjectId())
+ .build();
+ }
+
+ static Builder newBuilder() {
+ return new AutoValue_BigQueryStatisticsRetriever.Builder();
+ }
+
+ @AutoValue.Builder
+ abstract static class Builder {
+
+ public abstract Builder setProjectId(String projectId);
+
+ public abstract Builder setDatasetId(String datasetId);
+
+ public abstract Builder setBigquery(BigQuery bigquery);
+
+ public abstract BigQueryStatisticsRetriever build();
+ }
+
+ @Override
+ public FeatureSetStatistics getFeatureStatistics(
+ FeatureSetSpec featureSetSpec, List features, String dataset) {
+ FeatureSetStatisticsQueryInfo featureSetStatisticsQueryInfo =
+ new FeatureSetStatisticsQueryInfo(
+ featureSetSpec.getProject(),
+ featureSetSpec.getName(),
+ featureSetSpec.getVersion(),
+ dataset);
+ return getFeatureSetStatistics(featureSetSpec, features, featureSetStatisticsQueryInfo);
+ }
+
+ @Override
+ public FeatureSetStatistics getFeatureStatistics(
+ FeatureSetSpec featureSetSpec, List features, Timestamp date) {
+ FeatureSetStatisticsQueryInfo featureSetStatisticsQueryInfo =
+ new FeatureSetStatisticsQueryInfo(
+ featureSetSpec.getProject(),
+ featureSetSpec.getName(),
+ featureSetSpec.getVersion(),
+ date);
+ return getFeatureSetStatistics(featureSetSpec, features, featureSetStatisticsQueryInfo);
+ }
+
+ private FeatureSetStatistics getFeatureSetStatistics(
+ FeatureSetSpec featureSetSpec,
+ List features,
+ FeatureSetStatisticsQueryInfo featureSetStatisticsQueryInfo) {
+ List featuresList = featureSetSpec.getFeaturesList();
+
+ FeatureSetSpec.Builder featureSetSpecBuilder = featureSetSpec.toBuilder().clearFeatures();
+ for (FeatureSpec featureSpec : featuresList) {
+ if (features.contains(featureSpec.getName())) {
+ featureSetStatisticsQueryInfo.addFeature(featureSpec);
+ featureSetSpecBuilder.addFeatures(featureSpec);
+ }
+ }
+ featureSetSpec = featureSetSpecBuilder.build();
+
+ try {
+ // Generate SQL for and retrieve non-histogram statistics
+ String getFeatureSetStatsQuery =
+ StatsQueryTemplater.createGetFeatureSetStatsQuery(
+ featureSetStatisticsQueryInfo, projectId(), datasetId());
+ QueryJobConfiguration queryJobConfiguration =
+ QueryJobConfiguration.newBuilder(getFeatureSetStatsQuery).build();
+ TableResult basicStats = bigquery().query(queryJobConfiguration);
+
+ // Generate SQL for and retrieve histogram statistics
+ String getFeatureSetHistQuery =
+ StatsQueryTemplater.createGetFeatureSetHistQuery(
+ featureSetStatisticsQueryInfo, projectId(), datasetId());
+ queryJobConfiguration = QueryJobConfiguration.newBuilder(getFeatureSetHistQuery).build();
+ TableResult hist = bigquery().query(queryJobConfiguration);
+
+ // Convert to map of feature_name:row containing the statistics
+ Map basicStatsValues = getTableResultByFeatureName(basicStats);
+ Map histValues = getTableResultByFeatureName(hist);
+
+ int totalCountIndex = basicStats.getSchema().getFields().getIndex("total_count");
+ String ref = features.get(0);
+ FeatureSetStatistics.Builder featureSetStatisticsBuilder =
+ FeatureSetStatistics.newBuilder()
+ .setNumExamples(basicStatsValues.get(ref).get(totalCountIndex).getLongValue());
+
+ // Convert BQ rows to FeatureNameStatistics
+ for (FeatureSpec featureSpec : featureSetSpec.getFeaturesList()) {
+ FeatureNameStatistics featureNameStatistics =
+ StatsQueryResult.create()
+ .withBasicStatsResults(
+ basicStats.getSchema(), basicStatsValues.get(featureSpec.getName()))
+ .withHistResults(hist.getSchema(), histValues.get(featureSpec.getName()))
+ .toFeatureNameStatistics(featureSpec.getValueType());
+ featureSetStatisticsBuilder.addFeatureNameStatistics(featureNameStatistics);
+ }
+ return featureSetStatisticsBuilder.build();
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException(
+ String.format(
+ "Unable to retrieve statistics from BigQuery for Feature set %s, features %s",
+ featureSetSpec.getName(), features),
+ e);
+ }
+ }
+
+ private Map getTableResultByFeatureName(TableResult basicStats) {
+ return Streams.stream(basicStats.getValues())
+ .collect(
+ Collectors.toMap(
+ fieldValueList -> fieldValueList.get(0).getStringValue(),
+ fieldValueList -> fieldValueList));
+ }
+}
diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java
new file mode 100644
index 00000000000..5bac2bd7771
--- /dev/null
+++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureSetStatisticsQueryInfo.java
@@ -0,0 +1,111 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.connectors.bigquery.statistics;
+
+import com.google.protobuf.Timestamp;
+import feast.core.FeatureSetProto.EntitySpec;
+import feast.core.FeatureSetProto.FeatureSpec;
+import java.util.ArrayList;
+import java.util.List;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+/**
+ * Value class for Feature Sets containing information necessary to template stats-retrieving
+ * queries.
+ */
+public class FeatureSetStatisticsQueryInfo {
+ // Feast project name
+ private final String project;
+
+ // Feature set name
+ private final String name;
+
+ // Version of the feature set
+ private final int version;
+
+ // Dataset ID to retrieve statistics over
+ private String datasetId = "";
+
+ // Date to retrieve statistics over
+ private String date = "";
+
+ // List of entity names in this feature set
+ private final List entityNames;
+
+ // List of fields to get stats for
+ private final List features;
+
+ public FeatureSetStatisticsQueryInfo(String project, String name, int version, String datasetId) {
+ this.project = project;
+ this.name = name;
+ this.version = version;
+ this.entityNames = new ArrayList<>();
+ this.features = new ArrayList<>();
+ this.datasetId = datasetId;
+ }
+
+ public FeatureSetStatisticsQueryInfo(String project, String name, int version, Timestamp date) {
+ this.project = project;
+ this.name = name;
+ this.version = version;
+ this.entityNames = new ArrayList<>();
+ this.features = new ArrayList<>();
+ DateTime dateTime = new DateTime(date.getSeconds() * 1000, DateTimeZone.UTC);
+ DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd");
+ this.date = fmt.print(dateTime);
+ }
+
+ public void addFeature(FeatureSpec featureSpec) {
+ this.features.add(FeatureStatisticsQueryInfo.fromProto(featureSpec));
+ }
+
+ public void addEntity(EntitySpec entitySpec) {
+ this.entityNames.add(entitySpec.getName());
+ this.features.add(FeatureStatisticsQueryInfo.fromProto(entitySpec));
+ }
+
+ public String getProject() {
+ return project;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public int getVersion() {
+ return version;
+ }
+
+ public String getDatasetId() {
+ return datasetId;
+ }
+
+ public String getDate() {
+ return date;
+ }
+
+ public List getEntityNames() {
+ return entityNames;
+ }
+
+ public List getFeatures() {
+ return features;
+ }
+}
diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureStatisticsQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureStatisticsQueryInfo.java
new file mode 100644
index 00000000000..6461f8350b8
--- /dev/null
+++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureStatisticsQueryInfo.java
@@ -0,0 +1,99 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.connectors.bigquery.statistics;
+
+import feast.core.FeatureSetProto.EntitySpec;
+import feast.core.FeatureSetProto.FeatureSpec;
+import feast.types.ValueProto.ValueType.Enum;
+
+/**
+ * Value class for Features containing information necessary to template stats-retrieving queries.
+ */
+public class FeatureStatisticsQueryInfo {
+ // Name of the field
+ private final String name;
+
+ // Type of the field
+ private final String type;
+
+ private FeatureStatisticsQueryInfo(String name, String type) {
+ this.name = name;
+ this.type = type;
+ }
+
+ public static FeatureStatisticsQueryInfo fromProto(FeatureSpec featureSpec) {
+ Enum valueType = featureSpec.getValueType();
+ switch (valueType) {
+ case FLOAT:
+ case DOUBLE:
+ case INT32:
+ case INT64:
+ case BOOL:
+ return new FeatureStatisticsQueryInfo(featureSpec.getName(), "NUMERIC");
+ case STRING:
+ return new FeatureStatisticsQueryInfo(featureSpec.getName(), "CATEGORICAL");
+ case BYTES:
+ return new FeatureStatisticsQueryInfo(featureSpec.getName(), "BYTES");
+ case BYTES_LIST:
+ case BOOL_LIST:
+ case FLOAT_LIST:
+ case INT32_LIST:
+ case INT64_LIST:
+ case DOUBLE_LIST:
+ case STRING_LIST:
+ return new FeatureStatisticsQueryInfo(featureSpec.getName(), "LIST");
+ default:
+ throw new IllegalArgumentException(
+ String.format("Invalid feature type provided: %s", valueType));
+ }
+ }
+
+ public static FeatureStatisticsQueryInfo fromProto(EntitySpec entitySpec) {
+ Enum valueType = entitySpec.getValueType();
+ switch (valueType) {
+ case FLOAT:
+ case DOUBLE:
+ case INT32:
+ case INT64:
+ case BOOL:
+ return new FeatureStatisticsQueryInfo(entitySpec.getName(), "NUMERIC");
+ case STRING:
+ return new FeatureStatisticsQueryInfo(entitySpec.getName(), "CATEGORICAL");
+ case BYTES:
+ return new FeatureStatisticsQueryInfo(entitySpec.getName(), "BYTES");
+ case BYTES_LIST:
+ case BOOL_LIST:
+ case FLOAT_LIST:
+ case INT32_LIST:
+ case INT64_LIST:
+ case DOUBLE_LIST:
+ case STRING_LIST:
+ return new FeatureStatisticsQueryInfo(entitySpec.getName(), "LIST");
+ default:
+ throw new IllegalArgumentException(
+ String.format("Invalid entity type provided: %s", valueType));
+ }
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getType() {
+ return type;
+ }
+}
diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryResult.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryResult.java
new file mode 100644
index 00000000000..62108643455
--- /dev/null
+++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryResult.java
@@ -0,0 +1,332 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.connectors.bigquery.statistics;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.bigquery.*;
+import com.google.cloud.bigquery.Schema;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.Ordering;
+import feast.types.ValueProto.ValueType;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.tensorflow.metadata.v0.*;
+import org.tensorflow.metadata.v0.Histogram.Bucket;
+import org.tensorflow.metadata.v0.Histogram.HistogramType;
+import org.tensorflow.metadata.v0.StringStatistics.FreqAndValue;
+
+@AutoValue
+public abstract class StatsQueryResult {
+ // Map converting Feast type to TFDV type
+ private static final Map TFDV_TYPE_MAP =
+ new HashMap<>();
+
+ static {
+ TFDV_TYPE_MAP.put(ValueType.Enum.INT64, FeatureNameStatistics.Type.INT);
+ TFDV_TYPE_MAP.put(ValueType.Enum.INT32, FeatureNameStatistics.Type.INT);
+ TFDV_TYPE_MAP.put(ValueType.Enum.BOOL, FeatureNameStatistics.Type.INT);
+ TFDV_TYPE_MAP.put(ValueType.Enum.FLOAT, FeatureNameStatistics.Type.FLOAT);
+ TFDV_TYPE_MAP.put(ValueType.Enum.DOUBLE, FeatureNameStatistics.Type.FLOAT);
+ TFDV_TYPE_MAP.put(ValueType.Enum.STRING, FeatureNameStatistics.Type.STRING);
+ TFDV_TYPE_MAP.put(ValueType.Enum.BYTES, FeatureNameStatistics.Type.BYTES);
+ TFDV_TYPE_MAP.put(ValueType.Enum.BYTES_LIST, FeatureNameStatistics.Type.STRUCT);
+ TFDV_TYPE_MAP.put(ValueType.Enum.STRING_LIST, FeatureNameStatistics.Type.STRUCT);
+ TFDV_TYPE_MAP.put(ValueType.Enum.INT32_LIST, FeatureNameStatistics.Type.STRUCT);
+ TFDV_TYPE_MAP.put(ValueType.Enum.INT64_LIST, FeatureNameStatistics.Type.STRUCT);
+ TFDV_TYPE_MAP.put(ValueType.Enum.BOOL_LIST, FeatureNameStatistics.Type.STRUCT);
+ TFDV_TYPE_MAP.put(ValueType.Enum.FLOAT_LIST, FeatureNameStatistics.Type.STRUCT);
+ TFDV_TYPE_MAP.put(ValueType.Enum.DOUBLE_LIST, FeatureNameStatistics.Type.STRUCT);
+ }
+
+ // Schema of the table returned by the basic stats retrieval query
+ @Nullable
+ abstract Schema basicStatsSchema();
+
+ // Table values returned by the basic stats retrieval query
+ @Nullable
+ abstract FieldValueList basicStatsFieldValues();
+
+ // Schema of the table returned by the histogram retrieval query
+ @Nullable
+ abstract Schema histSchema();
+
+ // Table values returned by the histogram retrieval query
+ @Nullable
+ abstract FieldValueList histFieldValues();
+
+ public static StatsQueryResult create() {
+ return StatsQueryResult.newBuilder().build();
+ }
+
+ private static StatsQueryResult.Builder newBuilder() {
+ return new AutoValue_StatsQueryResult.Builder();
+ }
+
+ abstract Builder toBuilder();
+
+ /**
+ * Add basic stats query results to the StatsQueryResult.
+ *
+ * @param basicStatsSchema BigQuery {@link Schema} of the retrieved statistics row for the
+ * non-histogram statistics. Used to retrieve the column names corresponding to each value in
+ * the row.
+ * @param basicStatsFieldValues BigQuery {@link FieldValueList} containing a single row of
+ * non-histogram statistics retrieved from BigQuery
+ * @return {@link StatsQueryResult}
+ */
+ public StatsQueryResult withBasicStatsResults(
+ Schema basicStatsSchema, FieldValueList basicStatsFieldValues) {
+ return toBuilder()
+ .setBasicStatsSchema(basicStatsSchema)
+ .setBasicStatsFieldValues(basicStatsFieldValues)
+ .build();
+ }
+
+ /**
+ * Add histogram stats query results to the StatsQueryResult.
+ *
+ * @param histSchema BigQuery {@link Schema} of the retrieved statistics row for the histogram
+ * statistics. Used to retrieve the column names corresponding to each value in the row.
+ * @param histFieldValues BigQuery {@link FieldValueList} containing a single row of histogram
+ * statistics retrieved from BigQuery
+ * @return {@link StatsQueryResult}
+ */
+ public StatsQueryResult withHistResults(Schema histSchema, FieldValueList histFieldValues) {
+ return toBuilder().setHistSchema(histSchema).setHistFieldValues(histFieldValues).build();
+ }
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setBasicStatsSchema(Schema basicStatsSchema);
+
+ abstract Builder setBasicStatsFieldValues(FieldValueList basicStatsFieldValues);
+
+ abstract Builder setHistSchema(Schema histSchema);
+
+ abstract Builder setHistFieldValues(FieldValueList histFieldValues);
+
+ public abstract StatsQueryResult build();
+ }
+
+ /**
+ * Convert BQ-retrieved statistics to the corresponding TFDV {@link FeatureNameStatistics}
+ * specific to the feature type.
+ *
+ * @param valueType {@link ValueType.Enum} denoting the value type of the feature
+ * @return {@link FeatureNameStatistics}
+ */
+ public FeatureNameStatistics toFeatureNameStatistics(ValueType.Enum valueType) {
+ Map valuesMap = new HashMap<>();
+
+ // Convert the table values to a map of field name : table value for easy retrieval
+ FieldList basicStatsfields = basicStatsSchema().getFields();
+ for (int i = 0; i < basicStatsSchema().getFields().size(); i++) {
+ valuesMap.put(basicStatsfields.get(i).getName(), basicStatsFieldValues().get(i));
+ }
+
+ FieldList histFields = histSchema().getFields();
+ for (int i = 0; i < histSchema().getFields().size(); i++) {
+ valuesMap.put(histFields.get(i).getName(), histFieldValues().get(i));
+ }
+
+ FeatureNameStatistics.Builder featureNameStatisticsBuilder =
+ FeatureNameStatistics.newBuilder()
+ .setPath(Path.newBuilder().addStep(valuesMap.get("feature_name").getStringValue()))
+ .setType(TFDV_TYPE_MAP.get(valueType));
+
+ switch (valueType) {
+ case FLOAT:
+ case BOOL:
+ case DOUBLE:
+ case INT32:
+ case INT64:
+ NumericStatistics numStats = getNumericStatistics(valuesMap);
+ featureNameStatisticsBuilder.setNumStats(numStats);
+ break;
+ case STRING:
+ StringStatistics stringStats = getStringStatistics(valuesMap);
+ featureNameStatisticsBuilder.setStringStats(stringStats);
+ break;
+ case BYTES:
+ BytesStatistics bytesStats = getBytesStatistics(valuesMap);
+ featureNameStatisticsBuilder.setBytesStats(bytesStats);
+ break;
+ case BYTES_LIST:
+ case BOOL_LIST:
+ case FLOAT_LIST:
+ case INT32_LIST:
+ case INT64_LIST:
+ case DOUBLE_LIST:
+ case STRING_LIST:
+ StructStatistics structStats = getStructStatistics(valuesMap);
+ featureNameStatisticsBuilder.setStructStats(structStats);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Invalid feature type provided. Only statistics for numeric, bytes, string, boolean and list features are supported.");
+ }
+ return featureNameStatisticsBuilder.build();
+ }
+
+ private BytesStatistics getBytesStatistics(Map valuesMap) {
+ if (valuesMap.get("total_count").getLongValue() == 0) {
+ return BytesStatistics.getDefaultInstance();
+ }
+
+ return BytesStatistics.newBuilder()
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setNumMissing(valuesMap.get("missing_count").getLongValue())
+ .setNumNonMissing(valuesMap.get("feature_count").getLongValue())
+ .setMinNumValues(1)
+ .setMaxNumValues(1)
+ .setAvgNumValues(1)
+ .setTotNumValues(valuesMap.get("feature_count").getLongValue()))
+ .setUnique(valuesMap.get("unique").getLongValue())
+ .setMaxNumBytes((float) valuesMap.get("max").getDoubleValue())
+ .setMinNumBytes((float) valuesMap.get("min").getDoubleValue())
+ .setAvgNumBytes((float) valuesMap.get("mean").getDoubleValue())
+ .build();
+ }
+
+ private StringStatistics getStringStatistics(Map valuesMap) {
+ if (valuesMap.get("total_count").getLongValue() == 0) {
+ return StringStatistics.getDefaultInstance();
+ }
+
+ RankHistogram.Builder rankHistogram = RankHistogram.newBuilder();
+ valuesMap
+ .get("cat_hist")
+ .getRepeatedValue()
+ .forEach(
+ v -> {
+ FieldValueList recordValue = v.getRecordValue();
+ rankHistogram.addBuckets(
+ RankHistogram.Bucket.newBuilder()
+ .setLabel(recordValue.get(0).getStringValue())
+ .setSampleCount(recordValue.get(1).getLongValue()));
+ });
+
+ List topCount =
+ rankHistogram.getBucketsList().stream()
+ .sorted(
+ (a, b) ->
+ ComparisonChain.start()
+ .compare(
+ a.getSampleCount(), b.getSampleCount(), Ordering.natural().reverse())
+ .result())
+ .limit(5)
+ .map(
+ bucket ->
+ FreqAndValue.newBuilder()
+ .setValue(bucket.getLabel())
+ .setFrequency(bucket.getSampleCount())
+ .build())
+ .collect(Collectors.toList());
+
+ return StringStatistics.newBuilder()
+ .setUnique(valuesMap.get("unique").getLongValue())
+ .setAvgLength((long) valuesMap.get("mean").getDoubleValue())
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setNumMissing(valuesMap.get("missing_count").getLongValue())
+ .setNumNonMissing(valuesMap.get("feature_count").getLongValue())
+ .setMinNumValues(1)
+ .setMaxNumValues(1)
+ .setAvgNumValues(1)
+ .setTotNumValues(valuesMap.get("feature_count").getLongValue()))
+ .setRankHistogram(rankHistogram)
+ .addAllTopValues(topCount)
+ .build();
+ }
+
+ private NumericStatistics getNumericStatistics(Map valuesMap) {
+ if (valuesMap.get("total_count").getLongValue() == 0) {
+ return NumericStatistics.getDefaultInstance();
+ }
+
+ // Build quantiles
+ long quantileCount = valuesMap.get("feature_count").getLongValue() / 10;
+ Histogram.Builder quantilesBuilder = Histogram.newBuilder().setType(HistogramType.QUANTILES);
+
+ List quantilesRaw = valuesMap.get("quantiles").getRepeatedValue();
+ for (int i = 0; i < quantilesRaw.size() - 1; i++) {
+ quantilesBuilder.addBuckets(
+ Bucket.newBuilder()
+ .setLowValue(quantilesRaw.get(i).getDoubleValue())
+ .setHighValue(quantilesRaw.get(i + 1).getDoubleValue())
+ .setSampleCount(quantileCount));
+ }
+ // Build histogram
+ Histogram.Builder histBuilder = Histogram.newBuilder().setType(HistogramType.STANDARD);
+
+ // Order of histogram records is defined in the query hist_stats.sql:L35
+ valuesMap
+ .get("num_hist")
+ .getRepeatedValue()
+ .forEach(
+ v -> {
+ FieldValueList recordValue = v.getRecordValue();
+ histBuilder.addBuckets(
+ Bucket.newBuilder()
+ .setHighValue(recordValue.get(2).getDoubleValue())
+ .setLowValue(recordValue.get(1).getDoubleValue())
+ .setSampleCount(recordValue.get(0).getLongValue()));
+ });
+
+ return NumericStatistics.newBuilder()
+ .setMax(valuesMap.get("max").getDoubleValue())
+ .setMin(valuesMap.get("min").getDoubleValue())
+ .setMedian(quantilesRaw.get(5).getDoubleValue())
+ .setNumZeros(valuesMap.get("zeroes").getLongValue())
+ .setStdDev(valuesMap.get("stdev").getDoubleValue())
+ .setMean(valuesMap.get("mean").getDoubleValue())
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setNumMissing(valuesMap.get("missing_count").getLongValue())
+ .setNumNonMissing(valuesMap.get("feature_count").getLongValue())
+ .setMinNumValues(1)
+ .setMaxNumValues(1)
+ .setAvgNumValues(1)
+ .setTotNumValues(valuesMap.get("feature_count").getLongValue()))
+ .addHistograms(histBuilder)
+ .addHistograms(quantilesBuilder)
+ .build();
+ }
+
+ private StructStatistics getStructStatistics(Map valuesMap) {
+ if (valuesMap.get("total_count").getLongValue() == 0) {
+ return StructStatistics.getDefaultInstance();
+ }
+
+ return StructStatistics.newBuilder()
+ .setCommonStats(
+ CommonStatistics.newBuilder()
+ .setNumMissing(valuesMap.get("missing_count").getLongValue())
+ .setNumNonMissing(valuesMap.get("feature_count").getLongValue())
+ .setMinNumValues(valuesMap.get("min").getLongValue())
+ .setMaxNumValues(valuesMap.get("max").getLongValue())
+ .setAvgNumValues(valuesMap.get("mean").getLongValue())
+ .setTotNumValues(
+ valuesMap.get("feature_count").getLongValue()
+ * valuesMap.get("mean").getLongValue()))
+ .build();
+ }
+}
diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryTemplater.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryTemplater.java
new file mode 100644
index 00000000000..8103b2cb48a
--- /dev/null
+++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryTemplater.java
@@ -0,0 +1,78 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2019 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.connectors.bigquery.statistics;
+
+import com.mitchellbosecke.pebble.PebbleEngine;
+import com.mitchellbosecke.pebble.template.PebbleTemplate;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.HashMap;
+import java.util.Map;
+
+public class StatsQueryTemplater {
+
+ private static final PebbleEngine engine = new PebbleEngine.Builder().build();
+ private static final String BASIC_STATS_TEMPLATE_NAME = "templates/basic_stats.sql";
+ private static final String HIST_STATS_TEMPLATE_NAME = "templates/hist_stats.sql";
+
+ /**
+ * Generate the query for getting basic statistics about a given feature set
+ *
+ * @param featureSetInfo Information about the feature set necessary for the query templating
+ * @param projectId google project ID
+ * @param datasetId feast bigquery dataset ID
+ * @return point in time correctness join BQ SQL query
+ */
+ public static String createGetFeatureSetStatsQuery(
+ FeatureSetStatisticsQueryInfo featureSetInfo, String projectId, String datasetId)
+ throws IOException {
+
+ PebbleTemplate template = engine.getTemplate(BASIC_STATS_TEMPLATE_NAME);
+ Map context = new HashMap<>();
+ context.put("featureSet", featureSetInfo);
+ context.put("projectId", projectId);
+ context.put("datasetId", datasetId);
+
+ Writer writer = new StringWriter();
+ template.evaluate(writer, context);
+ return writer.toString();
+ }
+
+ /**
+ * Generate the query for getting histograms for features in a given feature set
+ *
+ * @param featureSetInfo Information about the feature set necessary for the query templating
+ * @param projectId google project ID
+ * @param datasetId feast bigquery dataset ID
+ * @return point in time correctness join BQ SQL query
+ */
+ public static String createGetFeatureSetHistQuery(
+ FeatureSetStatisticsQueryInfo featureSetInfo, String projectId, String datasetId)
+ throws IOException {
+
+ PebbleTemplate template = engine.getTemplate(HIST_STATS_TEMPLATE_NAME);
+ Map context = new HashMap<>();
+ context.put("featureSet", featureSetInfo);
+ context.put("projectId", projectId);
+ context.put("datasetId", datasetId);
+
+ Writer writer = new StringWriter();
+ template.evaluate(writer, context);
+ return writer.toString();
+ }
+}
diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java
index d155d3f1f50..33f79f8d97a 100644
--- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java
+++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java
@@ -42,6 +42,8 @@ public abstract class BigQueryFeatureSink implements FeatureSink {
"Event time for the FeatureRow";
public static final String BIGQUERY_CREATED_TIMESTAMP_FIELD_DESCRIPTION =
"Processing time of the FeatureRow ingestion in Feast\"";
+ public static final String BIGQUERY_DATASET_ID_FIELD_DESCRIPTION =
+ "Identifier of the batch dataset a row belongs to";
public static final String BIGQUERY_JOB_ID_FIELD_DESCRIPTION =
"Feast import job ID for the FeatureRow";
@@ -108,10 +110,13 @@ public void prepareWrite(FeatureSetProto.FeatureSet featureSet) {
Table table = bigquery.getTable(tableId);
if (table != null) {
log.info(
- "Writing to existing BigQuery table '{}:{}.{}'",
- getProjectId(),
+ "Updating and writing to existing BigQuery table '{}:{}.{}'",
+ datasetId.getProject(),
datasetId.getDataset(),
tableName);
+ TableDefinition tableDefinition = createBigQueryTableDefinition(featureSet.getSpec());
+ TableInfo tableInfo = TableInfo.of(tableId, tableDefinition);
+ bigquery.update(tableInfo);
return;
}
@@ -166,6 +171,8 @@ private TableDefinition createBigQueryTableDefinition(FeatureSetProto.FeatureSet
"created_timestamp",
Pair.of(
StandardSQLTypeName.TIMESTAMP, BIGQUERY_CREATED_TIMESTAMP_FIELD_DESCRIPTION),
+ "dataset_id",
+ Pair.of(StandardSQLTypeName.STRING, BIGQUERY_DATASET_ID_FIELD_DESCRIPTION),
"job_id",
Pair.of(StandardSQLTypeName.STRING, BIGQUERY_JOB_ID_FIELD_DESCRIPTION));
for (Map.Entry> entry :
diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureRowToTableRow.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureRowToTableRow.java
index 12833b31b85..9eaf504558e 100644
--- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureRowToTableRow.java
+++ b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureRowToTableRow.java
@@ -31,6 +31,7 @@
public class FeatureRowToTableRow implements SerializableFunction {
private static final String EVENT_TIMESTAMP_COLUMN = "event_timestamp";
private static final String CREATED_TIMESTAMP_COLUMN = "created_timestamp";
+ private static final String DATASET_ID_COLUMN = "dataset_id";
private static final String JOB_ID_COLUMN = "job_id";
private final String jobId;
@@ -47,6 +48,7 @@ public TableRow apply(FeatureRow featureRow) {
TableRow tableRow = new TableRow();
tableRow.set(EVENT_TIMESTAMP_COLUMN, Timestamps.toString(featureRow.getEventTimestamp()));
tableRow.set(CREATED_TIMESTAMP_COLUMN, Instant.now().toString());
+ tableRow.set(DATASET_ID_COLUMN, featureRow.getDatasetId());
tableRow.set(JOB_ID_COLUMN, jobId);
for (Field field : featureRow.getFieldsList()) {
diff --git a/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql b/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql
new file mode 100644
index 00000000000..331d9a108ef
--- /dev/null
+++ b/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql
@@ -0,0 +1,85 @@
+WITH subset AS (
+SELECT * FROM `{{ projectId }}.{{ datasetId }}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}`
+{% if featureSet.datasetId == "" %}
+WHERE DATE(event_timestamp) = '{{ featureSet.date }}'
+{% else %}
+WHERE dataset_id='{{ featureSet.datasetId }}'
+{% endif %}
+)
+{% for feature in featureSet.features %}
+SELECT
+ "{{ feature.name }}" as feature_name,
+ -- total count
+ COUNT(*) AS total_count,
+ -- count
+ COUNT({{ feature.name }}) as feature_count,
+ -- missing
+ COUNT(*) - COUNT({{ feature.name }}) as missing_count,
+ {% if feature.type equals "NUMERIC" %}
+ -- mean
+ AVG({{ feature.name }}) as mean,
+ -- stdev
+ STDDEV({{ feature.name }}) as stdev,
+ -- zeroes
+ COUNTIF({{ feature.name }} = 0) as zeroes,
+ -- min
+ MIN({{ feature.name }}) as min,
+ -- max
+ MAX({{ feature.name }}) as max,
+ -- hist will have to be called separately
+ -- quantiles
+ APPROX_QUANTILES(CAST({{ feature.name }} AS FLOAT64), 10) AS quantiles,
+ -- unique
+ null as unique
+ {% elseif feature.type equals "CATEGORICAL" %}
+ -- mean
+ AVG(LENGTH({{ feature.name }})) as mean,
+ -- stdev
+ null as stdev,
+ -- zeroes
+ null as zeroes,
+ -- min
+ null as min,
+ -- max
+ null as max,
+ -- quantiles
+ ARRAY[] AS quantiles,
+ -- unique
+ COUNT(DISTINCT({{ feature.name }})) as unique
+ {% elseif feature.type equals "BYTES" %}
+ -- mean
+ AVG(BIT_COUNT({{ feature.name }})) as mean,
+ -- stdev
+ null as stdev,
+ -- zeroes
+ null as zeroes,
+ -- min
+ MIN(BIT_COUNT({{ feature.name }})) as min,
+ -- max
+ MAX(BIT_COUNT({{ feature.name }})) as max,
+ -- hist will have to be called separately
+ -- quantiles
+ ARRAY[] AS quantiles,
+ -- unique
+ COUNT(DISTINCT({{ feature.name }})) as unique
+ {% elseif feature.type equals "LIST" %}
+ -- mean
+ AVG(ARRAY_LENGTH({{ feature.name }})) as mean,
+ -- stdev
+ null as stdev,
+ -- zeroes
+ null as zeroes,
+ -- min
+ MIN(ARRAY_LENGTH({{ feature.name }})) as min,
+ -- max
+ MAX(ARRAY_LENGTH({{ feature.name }})) as max,
+ -- hist will have to be called separately
+ -- quantiles
+ ARRAY[] AS quantiles,
+ -- unique
+ null as unique
+ {% endif %}
+FROM subset
+{% if loop.last %}{% else %}UNION ALL {% endif %}
+{% endfor %}
+
diff --git a/storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql b/storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql
new file mode 100644
index 00000000000..4908a9124c9
--- /dev/null
+++ b/storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql
@@ -0,0 +1,41 @@
+WITH subset AS (
+SELECT * FROM `{{ projectId }}.{{ datasetId }}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}`
+{% if featureSet.datasetId == "" %}
+WHERE DATE(event_timestamp) = '{{ featureSet.date }}'
+{% else %}
+WHERE dataset_id='{{ featureSet.datasetId }}'
+{% endif %}
+)
+{% for feature in featureSet.features %}
+, {{ feature.name }}_stats AS (
+{% if feature.type == 'NUMERIC' %}
+ WITH stats AS (
+ SELECT min+step*i as min, min+step*(i+1) as max
+ FROM (
+ SELECT MIN({{ feature.name }}) as min, MAX({{ feature.name }}) as max, (MAX({{ feature.name }})-MIN({{ feature.name }}))/10 step, GENERATE_ARRAY(0, 10, 1) i
+ FROM subset
+ ), UNNEST(i) i
+ ), counts as (
+ SELECT COUNT(*) as count, min, max,
+ FROM subset
+ JOIN stats
+ ON subset.{{ feature.name }} >= stats.min AND subset.{{ feature.name }}>[] as cat_hist FROM counts
+{% elseif feature.type == 'CATEGORICAL' %}
+ WITH counts AS (
+ SELECT {{ feature.name }}, COUNT({{ feature.name }}) AS count FROM subset GROUP BY {{ feature.name }}
+ )
+ SELECT '{{ feature.name }}' as feature, ARRAY>[] as num_hist, ARRAY_AGG(STRUCT({{ feature.name }} as value, count as count)) as cat_hist FROM counts
+{% elseif feature.type == 'BYTES' %}
+ SELECT '{{ feature.name }}' as feature, ARRAY>[] as num_hist, ARRAY>[] as cat_hist
+{% elseif feature.type == 'LIST' %}
+ SELECT '{{ feature.name }}' as feature, ARRAY>[] as num_hist, ARRAY>[] as cat_hist
+{% endif %}
+)
+{% endfor %}
+{% for feature in featureSet.features %}
+SELECT * FROM {{ feature.name }}_stats
+{% if loop.last %}{% else %}UNION ALL {% endif %}
+{% endfor %}
\ No newline at end of file
diff --git a/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsQueryResultTest.java b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsQueryResultTest.java
new file mode 100644
index 00000000000..500908291a3
--- /dev/null
+++ b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/statistics/StatsQueryResultTest.java
@@ -0,0 +1,274 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.storage.connectors.bigquery.statistics;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+import com.google.cloud.bigquery.FieldValue;
+import com.google.cloud.bigquery.FieldValue.Attribute;
+import com.google.cloud.bigquery.FieldValueList;
+import com.google.cloud.bigquery.LegacySQLTypeName;
+import com.google.cloud.bigquery.Schema;
+import com.google.common.collect.Lists;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.util.JsonFormat;
+import feast.core.FeatureSetProto.FeatureSpec;
+import feast.types.ValueProto.ValueType;
+import org.junit.Test;
+import org.tensorflow.metadata.v0.FeatureNameStatistics;
+
+public class StatsQueryResultTest {
+ private Schema basicStatsSchema =
+ Schema.of(
+ com.google.cloud.bigquery.Field.of("feature_name", LegacySQLTypeName.STRING),
+ com.google.cloud.bigquery.Field.of("total_count", LegacySQLTypeName.INTEGER),
+ com.google.cloud.bigquery.Field.of("feature_count", LegacySQLTypeName.INTEGER),
+ com.google.cloud.bigquery.Field.of("missing_count", LegacySQLTypeName.INTEGER),
+ com.google.cloud.bigquery.Field.of("mean", LegacySQLTypeName.FLOAT),
+ com.google.cloud.bigquery.Field.of("stdev", LegacySQLTypeName.FLOAT),
+ com.google.cloud.bigquery.Field.of("zeroes", LegacySQLTypeName.INTEGER),
+ com.google.cloud.bigquery.Field.of("min", LegacySQLTypeName.FLOAT),
+ com.google.cloud.bigquery.Field.of("max", LegacySQLTypeName.FLOAT),
+ com.google.cloud.bigquery.Field.of("quantiles", LegacySQLTypeName.NUMERIC),
+ com.google.cloud.bigquery.Field.of("unique", LegacySQLTypeName.INTEGER));
+
+ private Schema histStatsSchema =
+ Schema.of(
+ com.google.cloud.bigquery.Field.of("field", LegacySQLTypeName.STRING),
+ com.google.cloud.bigquery.Field.of(
+ "num_hist",
+ LegacySQLTypeName.RECORD,
+ com.google.cloud.bigquery.Field.of("count", LegacySQLTypeName.INTEGER),
+ com.google.cloud.bigquery.Field.of("low_value", LegacySQLTypeName.FLOAT),
+ com.google.cloud.bigquery.Field.of("high_value", LegacySQLTypeName.FLOAT)),
+ com.google.cloud.bigquery.Field.of(
+ "cat_hist",
+ LegacySQLTypeName.RECORD,
+ com.google.cloud.bigquery.Field.of("value", LegacySQLTypeName.STRING),
+ com.google.cloud.bigquery.Field.of("count", LegacySQLTypeName.INTEGER)));
+
+ @Test
+ public void shouldConvertNumericStatsToFeatureNameStatistics()
+ throws InvalidProtocolBufferException {
+ FieldValueList numericFieldValueList =
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(Attribute.PRIMITIVE, "floats"),
+ FieldValue.of(Attribute.PRIMITIVE, "20"),
+ FieldValue.of(Attribute.PRIMITIVE, "20"),
+ FieldValue.of(Attribute.PRIMITIVE, "0"),
+ FieldValue.of(Attribute.PRIMITIVE, "1"),
+ FieldValue.of(Attribute.PRIMITIVE, "6"),
+ FieldValue.of(Attribute.PRIMITIVE, "0"),
+ FieldValue.of(Attribute.PRIMITIVE, "-8.5"),
+ FieldValue.of(Attribute.PRIMITIVE, "10.5"),
+ FieldValue.of(
+ Attribute.REPEATED,
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(Attribute.PRIMITIVE, "-8.5"),
+ FieldValue.of(Attribute.PRIMITIVE, "-7.5"),
+ FieldValue.of(Attribute.PRIMITIVE, "-5.5"),
+ FieldValue.of(Attribute.PRIMITIVE, "-3.5"),
+ FieldValue.of(Attribute.PRIMITIVE, "-1.5"),
+ FieldValue.of(Attribute.PRIMITIVE, "0.5"),
+ FieldValue.of(Attribute.PRIMITIVE, "2.5"),
+ FieldValue.of(Attribute.PRIMITIVE, "4.5"),
+ FieldValue.of(Attribute.PRIMITIVE, "6.5"),
+ FieldValue.of(Attribute.PRIMITIVE, "8.5"),
+ FieldValue.of(Attribute.PRIMITIVE, "10.5")))),
+ FieldValue.of(Attribute.PRIMITIVE, null)));
+
+ FieldValueList numericHistFieldValueList =
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(Attribute.PRIMITIVE, "floats"),
+ FieldValue.of(
+ Attribute.REPEATED,
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(
+ Attribute.RECORD,
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(Attribute.PRIMITIVE, "1"),
+ FieldValue.of(Attribute.PRIMITIVE, "1"),
+ FieldValue.of(Attribute.PRIMITIVE, "2")))),
+ FieldValue.of(
+ Attribute.RECORD,
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(Attribute.PRIMITIVE, "2"),
+ FieldValue.of(Attribute.PRIMITIVE, "2"),
+ FieldValue.of(Attribute.PRIMITIVE, "3"))))))),
+ FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList()))));
+
+ FeatureSpec featureSpec =
+ FeatureSpec.newBuilder().setName("floats").setValueType(ValueType.Enum.DOUBLE).build();
+
+ FeatureNameStatistics actual =
+ StatsQueryResult.create()
+ .withBasicStatsResults(basicStatsSchema, numericFieldValueList)
+ .withHistResults(histStatsSchema, numericHistFieldValueList)
+ .toFeatureNameStatistics(featureSpec.getValueType());
+
+ String expectedJson =
+ "{\"type\":\"FLOAT\",\"numStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"1\",\"maxNumValues\":\"1\",\"avgNumValues\":1,\"totNumValues\":\"20\"},\"mean\":1,\"stdDev\":6,\"min\":-8.5,\"median\":0.5,\"max\":10.5,\"histograms\":[{\"buckets\":[{\"lowValue\":1,\"highValue\":2,\"sampleCount\":1},{\"lowValue\":2,\"highValue\":3,\"sampleCount\":2}]},{\"buckets\":[{\"lowValue\":-8.5,\"highValue\":-7.5,\"sampleCount\":2},{\"lowValue\":-7.5,\"highValue\":-5.5,\"sampleCount\":2},{\"lowValue\":-5.5,\"highValue\":-3.5,\"sampleCount\":2},{\"lowValue\":-3.5,\"highValue\":-1.5,\"sampleCount\":2},{\"lowValue\":-1.5,\"highValue\":0.5,\"sampleCount\":2},{\"lowValue\":0.5,\"highValue\":2.5,\"sampleCount\":2},{\"lowValue\":2.5,\"highValue\":4.5,\"sampleCount\":2},{\"lowValue\":4.5,\"highValue\":6.5,\"sampleCount\":2},{\"lowValue\":6.5,\"highValue\":8.5,\"sampleCount\":2},{\"lowValue\":8.5,\"highValue\":10.5,\"sampleCount\":2}],\"type\":\"QUANTILES\"}]},\"path\":{\"step\":[\"floats\"]}}";
+ FeatureNameStatistics.Builder expected = FeatureNameStatistics.newBuilder();
+ JsonFormat.parser().merge(expectedJson, expected);
+ assertThat(actual, equalTo(expected.build()));
+ }
+
+ @Test
+ public void voidShouldConvertStringStatsToFeatureNameStatistics()
+ throws InvalidProtocolBufferException {
+ FieldValueList stringFieldValueList =
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(Attribute.PRIMITIVE, "strings"),
+ FieldValue.of(Attribute.PRIMITIVE, "20"),
+ FieldValue.of(Attribute.PRIMITIVE, "20"),
+ FieldValue.of(Attribute.PRIMITIVE, "0"),
+ FieldValue.of(Attribute.PRIMITIVE, "1"),
+ FieldValue.of(Attribute.PRIMITIVE, null),
+ FieldValue.of(Attribute.PRIMITIVE, null),
+ FieldValue.of(Attribute.PRIMITIVE, null),
+ FieldValue.of(Attribute.PRIMITIVE, null),
+ FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())),
+ FieldValue.of(Attribute.PRIMITIVE, "2")));
+
+ FieldValueList stringHistFieldValueList =
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(Attribute.PRIMITIVE, "strings"),
+ FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())),
+ FieldValue.of(
+ Attribute.REPEATED,
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(
+ Attribute.RECORD,
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(Attribute.PRIMITIVE, "a"),
+ FieldValue.of(Attribute.PRIMITIVE, "1")))),
+ FieldValue.of(
+ Attribute.RECORD,
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(Attribute.PRIMITIVE, "b"),
+ FieldValue.of(Attribute.PRIMITIVE, "2")))))))));
+
+ FeatureSpec featureSpec =
+ FeatureSpec.newBuilder().setName("strings").setValueType(ValueType.Enum.STRING).build();
+
+ FeatureNameStatistics actual =
+ StatsQueryResult.create()
+ .withBasicStatsResults(basicStatsSchema, stringFieldValueList)
+ .withHistResults(histStatsSchema, stringHistFieldValueList)
+ .toFeatureNameStatistics(featureSpec.getValueType());
+
+ String expectedJson =
+ "{\"type\":\"STRING\",\"stringStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"1\",\"maxNumValues\":\"1\",\"avgNumValues\":1,\"totNumValues\":\"20\"},\"unique\":\"2\",\"topValues\":[{\"value\":\"b\",\"frequency\":2},{\"value\":\"a\",\"frequency\":1}],\"avgLength\":1,\"rankHistogram\":{\"buckets\":[{\"label\":\"a\",\"sampleCount\":1},{\"label\":\"b\",\"sampleCount\":2}]}},\"path\":{\"step\":[\"strings\"]}}";
+ FeatureNameStatistics.Builder expected = FeatureNameStatistics.newBuilder();
+ JsonFormat.parser().merge(expectedJson, expected);
+ assertThat(actual, equalTo(expected.build()));
+ }
+
+ @Test
+ public void voidShouldConvertBytesStatsToFeatureNameStatistics()
+ throws InvalidProtocolBufferException {
+ FieldValueList stringFieldValueList =
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(Attribute.PRIMITIVE, "bytes"),
+ FieldValue.of(Attribute.PRIMITIVE, "20"),
+ FieldValue.of(Attribute.PRIMITIVE, "20"),
+ FieldValue.of(Attribute.PRIMITIVE, "0"),
+ FieldValue.of(Attribute.PRIMITIVE, "5"),
+ FieldValue.of(Attribute.PRIMITIVE, null),
+ FieldValue.of(Attribute.PRIMITIVE, null),
+ FieldValue.of(Attribute.PRIMITIVE, "3"),
+ FieldValue.of(Attribute.PRIMITIVE, "7"),
+ FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())),
+ FieldValue.of(Attribute.PRIMITIVE, "2")));
+
+ FieldValueList stringHistFieldValueList =
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(Attribute.PRIMITIVE, "bytes"),
+ FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())),
+ FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList()))));
+
+ FeatureSpec featureSpec =
+ FeatureSpec.newBuilder().setName("bytes").setValueType(ValueType.Enum.BYTES).build();
+
+ FeatureNameStatistics actual =
+ StatsQueryResult.create()
+ .withBasicStatsResults(basicStatsSchema, stringFieldValueList)
+ .withHistResults(histStatsSchema, stringHistFieldValueList)
+ .toFeatureNameStatistics(featureSpec.getValueType());
+
+ String expectedJson =
+ "{\"type\":\"BYTES\",\"bytesStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"1\",\"maxNumValues\":\"1\",\"avgNumValues\":1,\"totNumValues\":\"20\"},\"unique\":\"2\",\"avgNumBytes\":5,\"minNumBytes\":3,\"maxNumBytes\":7},\"path\":{\"step\":[\"bytes\"]}}";
+ FeatureNameStatistics.Builder expected = FeatureNameStatistics.newBuilder();
+ JsonFormat.parser().merge(expectedJson, expected);
+ assertThat(actual, equalTo(expected.build()));
+ }
+
+ @Test
+ public void voidShouldConvertStructStatsToFeatureNameStatistics()
+ throws InvalidProtocolBufferException {
+ FieldValueList stringFieldValueList =
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(Attribute.PRIMITIVE, "list"),
+ FieldValue.of(Attribute.PRIMITIVE, "20"),
+ FieldValue.of(Attribute.PRIMITIVE, "20"),
+ FieldValue.of(Attribute.PRIMITIVE, "0"),
+ FieldValue.of(Attribute.PRIMITIVE, "5"),
+ FieldValue.of(Attribute.PRIMITIVE, null),
+ FieldValue.of(Attribute.PRIMITIVE, null),
+ FieldValue.of(Attribute.PRIMITIVE, "3"),
+ FieldValue.of(Attribute.PRIMITIVE, "7"),
+ FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())),
+ FieldValue.of(Attribute.PRIMITIVE, null)));
+
+ FieldValueList stringHistFieldValueList =
+ FieldValueList.of(
+ Lists.newArrayList(
+ FieldValue.of(Attribute.PRIMITIVE, "list"),
+ FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList())),
+ FieldValue.of(Attribute.REPEATED, FieldValueList.of(Lists.newArrayList()))));
+
+ FeatureSpec featureSpec =
+ FeatureSpec.newBuilder().setName("list").setValueType(ValueType.Enum.STRING_LIST).build();
+
+ FeatureNameStatistics actual =
+ StatsQueryResult.create()
+ .withBasicStatsResults(basicStatsSchema, stringFieldValueList)
+ .withHistResults(histStatsSchema, stringHistFieldValueList)
+ .toFeatureNameStatistics(featureSpec.getValueType());
+
+ String expectedJson =
+ "{\"type\":\"STRUCT\",\"structStats\":{\"commonStats\":{\"numNonMissing\":\"20\",\"minNumValues\":\"3\",\"maxNumValues\":\"7\",\"avgNumValues\":5,\"totNumValues\":\"100\"}},\"path\":{\"step\":[\"list\"]}}";
+ FeatureNameStatistics.Builder expected = FeatureNameStatistics.newBuilder();
+ JsonFormat.parser().merge(expectedJson, expected);
+ assertThat(actual, equalTo(expected.build()));
+ }
+}
diff --git a/tests/e2e/bq-batch-retrieval.py b/tests/e2e/bq/bq-batch-retrieval.py
similarity index 70%
rename from tests/e2e/bq-batch-retrieval.py
rename to tests/e2e/bq/bq-batch-retrieval.py
index 0cf05e77e1d..fc37f3faf86 100644
--- a/tests/e2e/bq-batch-retrieval.py
+++ b/tests/e2e/bq/bq-batch-retrieval.py
@@ -18,9 +18,9 @@
from google.protobuf.duration_pb2 import Duration
from pandavro import to_avro
-pd.set_option('display.max_columns', None)
+pd.set_option("display.max_columns", None)
-PROJECT_NAME = 'batch_' + uuid.uuid4().hex.upper()[0:6]
+PROJECT_NAME = "batch_" + uuid.uuid4().hex.upper()[0:6]
@pytest.fixture(scope="module")
@@ -54,60 +54,63 @@ def client(core_url, serving_url, allow_dirty):
if not allow_dirty:
feature_sets = client.list_feature_sets()
if len(feature_sets) > 0:
- raise Exception("Feast cannot have existing feature sets registered. Exiting tests.")
+ raise Exception(
+ "Feast cannot have existing feature sets registered. Exiting tests."
+ )
return client
+
@pytest.mark.first
-def test_apply_all_featuresets(client):
+def test_batch_apply_all_featuresets(client):
client.set_project(PROJECT_NAME)
file_fs1 = FeatureSet(
- "file_feature_set",
- features=[Feature("feature_value1", ValueType.STRING)],
- entities=[Entity("entity_id", ValueType.INT64)],
- max_age=Duration(seconds=100),
- )
+ "file_feature_set",
+ features=[Feature("feature_value1", ValueType.STRING)],
+ entities=[Entity("entity_id", ValueType.INT64)],
+ max_age=Duration(seconds=100),
+ )
client.apply(file_fs1)
gcs_fs1 = FeatureSet(
- "gcs_feature_set",
- features=[Feature("feature_value2", ValueType.STRING)],
- entities=[Entity("entity_id", ValueType.INT64)],
- max_age=Duration(seconds=100),
- )
+ "gcs_feature_set",
+ features=[Feature("feature_value2", ValueType.STRING)],
+ entities=[Entity("entity_id", ValueType.INT64)],
+ max_age=Duration(seconds=100),
+ )
client.apply(gcs_fs1)
proc_time_fs = FeatureSet(
- "processing_time",
- features=[Feature("feature_value3", ValueType.STRING)],
- entities=[Entity("entity_id", ValueType.INT64)],
- max_age=Duration(seconds=100),
- )
+ "processing_time",
+ features=[Feature("feature_value3", ValueType.STRING)],
+ entities=[Entity("entity_id", ValueType.INT64)],
+ max_age=Duration(seconds=100),
+ )
client.apply(proc_time_fs)
add_cols_fs = FeatureSet(
- "additional_columns",
- features=[Feature("feature_value4", ValueType.STRING)],
- entities=[Entity("entity_id", ValueType.INT64)],
- max_age=Duration(seconds=100),
- )
+ "additional_columns",
+ features=[Feature("feature_value4", ValueType.STRING)],
+ entities=[Entity("entity_id", ValueType.INT64)],
+ max_age=Duration(seconds=100),
+ )
client.apply(add_cols_fs)
historical_fs = FeatureSet(
- "historical",
- features=[Feature("feature_value5", ValueType.STRING)],
- entities=[Entity("entity_id", ValueType.INT64)],
- max_age=Duration(seconds=100),
- )
+ "historical",
+ features=[Feature("feature_value5", ValueType.STRING)],
+ entities=[Entity("entity_id", ValueType.INT64)],
+ max_age=Duration(seconds=100),
+ )
client.apply(historical_fs)
fs1 = FeatureSet(
- "feature_set_1",
- features=[Feature("feature_value6", ValueType.STRING)],
- entities=[Entity("entity_id", ValueType.INT64)],
- max_age=Duration(seconds=100),
- )
+ "feature_set_1",
+ features=[Feature("feature_value6", ValueType.STRING)],
+ entities=[Entity("entity_id", ValueType.INT64)],
+ max_age=Duration(seconds=100),
+ )
fs2 = FeatureSet(
"feature_set_2",
@@ -127,7 +130,7 @@ def test_apply_all_featuresets(client):
client.apply(no_max_age_fs)
-def test_get_batch_features_with_file(client):
+def test_batch_get_batch_features_with_file(client):
file_fs1 = client.get_feature_set(name="file_feature_set", version=1)
N_ROWS = 10
@@ -144,20 +147,26 @@ def test_get_batch_features_with_file(client):
# Rename column (datetime -> event_timestamp)
features_1_df = features_1_df.rename(columns={"datetime": "event_timestamp"})
- to_avro(df=features_1_df[["event_timestamp", "entity_id"]], file_path_or_buffer="file_feature_set.avro")
+ to_avro(
+ df=features_1_df[["event_timestamp", "entity_id"]],
+ file_path_or_buffer="file_feature_set.avro",
+ )
time.sleep(15)
feature_retrieval_job = client.get_batch_features(
- entity_rows="file://file_feature_set.avro", feature_refs=[f"{PROJECT_NAME}/feature_value1:1"]
+ entity_rows="file://file_feature_set.avro",
+ feature_refs=[f"{PROJECT_NAME}/feature_value1:1"],
)
output = feature_retrieval_job.to_dataframe()
print(output.head())
- assert output["entity_id"].to_list() == [int(i) for i in output["feature_value1"].to_list()]
+ assert output["entity_id"].to_list() == [
+ int(i) for i in output["feature_value1"].to_list()
+ ]
-def test_get_batch_features_with_gs_path(client, gcs_path):
+def test_batch_get_batch_features_with_gs_path(client, gcs_path):
gcs_fs1 = client.get_feature_set(name="gcs_feature_set", version=1)
N_ROWS = 10
@@ -176,7 +185,10 @@ def test_get_batch_features_with_gs_path(client, gcs_path):
# Output file to local
file_name = "gcs_feature_set.avro"
- to_avro(df=features_1_df[["event_timestamp", "entity_id"]], file_path_or_buffer=file_name)
+ to_avro(
+ df=features_1_df[["event_timestamp", "entity_id"]],
+ file_path_or_buffer=file_name,
+ )
uri = urlparse(gcs_path)
bucket = uri.hostname
@@ -192,16 +204,18 @@ def test_get_batch_features_with_gs_path(client, gcs_path):
time.sleep(15)
feature_retrieval_job = client.get_batch_features(
entity_rows=f"{gcs_path}{ts}/*",
- feature_refs=[f"{PROJECT_NAME}/feature_value2:1"]
+ feature_refs=[f"{PROJECT_NAME}/feature_value2:1"],
)
output = feature_retrieval_job.to_dataframe()
print(output.head())
- assert output["entity_id"].to_list() == [int(i) for i in output["feature_value2"].to_list()]
+ assert output["entity_id"].to_list() == [
+ int(i) for i in output["feature_value2"].to_list()
+ ]
-def test_order_by_creation_time(client):
+def test_batch_order_by_creation_time(client):
proc_time_fs = client.get_feature_set(name="processing_time", version=1)
time_offset = datetime.utcnow().replace(tzinfo=pytz.utc)
@@ -224,7 +238,8 @@ def test_order_by_creation_time(client):
time.sleep(15)
client.ingest(proc_time_fs, correct_df)
feature_retrieval_job = client.get_batch_features(
- entity_rows=incorrect_df[["datetime", "entity_id"]], feature_refs=[f"{PROJECT_NAME}/feature_value3:1"]
+ entity_rows=incorrect_df[["datetime", "entity_id"]],
+ feature_refs=[f"{PROJECT_NAME}/feature_value3:1"],
)
output = feature_retrieval_job.to_dataframe()
print(output.head())
@@ -232,13 +247,17 @@ def test_order_by_creation_time(client):
assert output["feature_value3"].to_list() == ["CORRECT"] * N_ROWS
-def test_additional_columns_in_entity_table(client):
+def test_batch_additional_columns_in_entity_table(client):
add_cols_fs = client.get_feature_set(name="additional_columns", version=1)
N_ROWS = 10
time_offset = datetime.utcnow().replace(tzinfo=pytz.utc)
features_df = pd.DataFrame(
- {"datetime": [time_offset] * N_ROWS, "entity_id": [i for i in range(N_ROWS)], "feature_value4": ["abc"] * N_ROWS}
+ {
+ "datetime": [time_offset] * N_ROWS,
+ "entity_id": [i for i in range(N_ROWS)],
+ "feature_value4": ["abc"] * N_ROWS,
+ }
)
client.ingest(add_cols_fs, features_df)
@@ -258,12 +277,17 @@ def test_additional_columns_in_entity_table(client):
output = feature_retrieval_job.to_dataframe().sort_values(by=["entity_id"])
print(output.head(10))
- assert np.allclose(output["additional_float_col"], entity_df["additional_float_col"])
- assert output["additional_string_col"].to_list() == entity_df["additional_string_col"].to_list()
+ assert np.allclose(
+ output["additional_float_col"], entity_df["additional_float_col"]
+ )
+ assert (
+ output["additional_string_col"].to_list()
+ == entity_df["additional_string_col"].to_list()
+ )
assert output["feature_value4"].to_list() == features_df["feature_value4"].to_list()
-def test_point_in_time_correctness_join(client):
+def test_batch_point_in_time_correctness_join(client):
historical_fs = client.get_feature_set(name="historical", version=1)
time_offset = datetime.utcnow().replace(tzinfo=pytz.utc)
@@ -281,20 +305,25 @@ def test_point_in_time_correctness_join(client):
}
)
entity_df = pd.DataFrame(
- {"datetime": [time_offset - timedelta(seconds=10)] * N_EXAMPLES, "entity_id": [i for i in range(N_EXAMPLES)]}
+ {
+ "datetime": [time_offset - timedelta(seconds=10)] * N_EXAMPLES,
+ "entity_id": [i for i in range(N_EXAMPLES)],
+ }
)
client.ingest(historical_fs, historical_df)
time.sleep(15)
- feature_retrieval_job = client.get_batch_features(entity_rows=entity_df, feature_refs=[f"{PROJECT_NAME}/feature_value5"])
+ feature_retrieval_job = client.get_batch_features(
+ entity_rows=entity_df, feature_refs=[f"{PROJECT_NAME}/feature_value5"]
+ )
output = feature_retrieval_job.to_dataframe()
print(output.head())
assert output["feature_value5"].to_list() == ["CORRECT"] * N_EXAMPLES
-def test_multiple_featureset_joins(client):
+def test_batch_multiple_featureset_joins(client):
fs1 = client.get_feature_set(name="feature_set_1", version=1)
fs2 = client.get_feature_set(name="feature_set_2", version=1)
@@ -328,16 +357,24 @@ def test_multiple_featureset_joins(client):
time.sleep(15)
feature_retrieval_job = client.get_batch_features(
- entity_rows=entity_df, feature_refs=[f"{PROJECT_NAME}/feature_value6:1", f"{PROJECT_NAME}/other_feature_value7:1"]
+ entity_rows=entity_df,
+ feature_refs=[
+ f"{PROJECT_NAME}/feature_value6:1",
+ f"{PROJECT_NAME}/other_feature_value7:1",
+ ],
)
output = feature_retrieval_job.to_dataframe()
print(output.head())
- assert output["entity_id"].to_list() == [int(i) for i in output["feature_value6"].to_list()]
- assert output["other_entity_id"].to_list() == output["other_feature_value7"].to_list()
+ assert output["entity_id"].to_list() == [
+ int(i) for i in output["feature_value6"].to_list()
+ ]
+ assert (
+ output["other_entity_id"].to_list() == output["other_feature_value7"].to_list()
+ )
-def test_no_max_age(client):
+def test_batch_no_max_age(client):
no_max_age_fs = client.get_feature_set(name="no_max_age", version=1)
time_offset = datetime.utcnow().replace(tzinfo=pytz.utc)
@@ -353,10 +390,11 @@ def test_no_max_age(client):
time.sleep(15)
feature_retrieval_job = client.get_batch_features(
- entity_rows=features_8_df[["datetime", "entity_id"]], feature_refs=[f"{PROJECT_NAME}/feature_value8:1"]
+ entity_rows=features_8_df[["datetime", "entity_id"]],
+ feature_refs=[f"{PROJECT_NAME}/feature_value8:1"],
)
output = feature_retrieval_job.to_dataframe()
print(output.head())
- assert output["entity_id"].to_list() == output["feature_value8"].to_list()
\ No newline at end of file
+ assert output["entity_id"].to_list() == output["feature_value8"].to_list()
diff --git a/tests/e2e/bq/feature-stats.py b/tests/e2e/bq/feature-stats.py
new file mode 100644
index 00000000000..b25ff3949bb
--- /dev/null
+++ b/tests/e2e/bq/feature-stats.py
@@ -0,0 +1,305 @@
+import pandas as pd
+import pytest
+import pytz
+import uuid
+import time
+import os
+from datetime import datetime, timedelta
+
+from feast.client import Client
+from feast.entity import Entity
+from feast.feature import Feature
+from feast.feature_set import FeatureSet
+from feast.type_map import ValueType
+from google.protobuf.duration_pb2 import Duration
+import tensorflow_data_validation as tfdv
+from deepdiff import DeepDiff
+from google.protobuf.json_format import MessageToDict
+
+
+pd.set_option("display.max_columns", None)
+
+PROJECT_NAME = "batch_" + uuid.uuid4().hex.upper()[0:6]
+STORE_NAME = "historical"
+os.environ['CUDA_VISIBLE_DEVICES'] = "0"
+
+
+@pytest.fixture(scope="module")
+def core_url(pytestconfig):
+ return pytestconfig.getoption("core_url")
+
+
+@pytest.fixture(scope="module")
+def serving_url(pytestconfig):
+ return pytestconfig.getoption("serving_url")
+
+
+@pytest.fixture(scope="module")
+def allow_dirty(pytestconfig):
+ return True if pytestconfig.getoption("allow_dirty").lower() == "true" else False
+
+
+@pytest.fixture(scope="module")
+def gcs_path(pytestconfig):
+ return pytestconfig.getoption("gcs_path")
+
+
+@pytest.fixture(scope="module")
+def client(core_url, allow_dirty):
+ # Get client for core and serving
+ client = Client(core_url=core_url)
+ client.create_project(PROJECT_NAME)
+ client.set_project(PROJECT_NAME)
+
+ # Ensure Feast core is active, but empty
+ if not allow_dirty:
+ feature_sets = client.list_feature_sets()
+ if len(feature_sets) > 0:
+ raise Exception(
+ "Feast cannot have existing feature sets registered. Exiting tests."
+ )
+
+ return client
+
+
+@pytest.fixture(scope="module")
+def feature_stats_feature_set(client):
+ fv_fs = FeatureSet(
+ "feature_stats",
+ features=[
+ Feature("strings", ValueType.STRING),
+ Feature("ints", ValueType.INT64),
+ Feature("floats", ValueType.FLOAT),
+ ],
+ entities=[Entity("entity_id", ValueType.INT64)],
+ max_age=Duration(seconds=100),
+ )
+ client.apply(fv_fs)
+ return fv_fs
+
+
+@pytest.fixture(scope="module")
+def feature_stats_dataset_basic(client, feature_stats_feature_set):
+
+ N_ROWS = 20
+
+ time_offset = datetime.utcnow().replace(tzinfo=pytz.utc)
+ df = pd.DataFrame(
+ {
+ "datetime": [time_offset] * N_ROWS,
+ "entity_id": [i for i in range(N_ROWS)],
+ "strings": ["a", "b"] * int(N_ROWS / 2),
+ "ints": [int(i) for i in range(N_ROWS)],
+ "floats": [10.5 - i for i in range(N_ROWS)],
+ }
+ )
+
+ expected_stats = tfdv.generate_statistics_from_dataframe(
+ df[["strings", "ints", "floats"]]
+ )
+ clear_unsupported_fields(expected_stats)
+
+ # Since TFDV computes population std dev
+ for feature in expected_stats.datasets[0].features:
+ if feature.HasField("num_stats"):
+ name = feature.path.step[0]
+ std = combined_df[name].std()
+ feature.num_stats.std_dev = std
+
+ dataset_id = client.ingest(feature_stats_feature_set, df)
+ time.sleep(10)
+ return {
+ "df": df,
+ "id": dataset_id,
+ "date": datetime(time_offset.year, time_offset.month, time_offset.day).replace(
+ tzinfo=pytz.utc
+ ),
+ "stats": expected_stats,
+ }
+
+
+@pytest.fixture(scope="module")
+def feature_stats_dataset_agg(client, feature_stats_feature_set):
+ time_offset = datetime.utcnow().replace(tzinfo=pytz.utc)
+ start_date = time_offset - timedelta(days=10)
+ end_date = time_offset - timedelta(days=7)
+ df1 = pd.DataFrame(
+ {
+ "datetime": [start_date] * 5,
+ "entity_id": [i for i in range(5)],
+ "strings": ["a", "b", "b", "b", "a"],
+ "ints": [4, 3, 2, 6, 3],
+ "floats": [2.1, 5.2, 4.3, 0.6, 0.1],
+ }
+ )
+ dataset_id_1 = client.ingest(feature_stats_feature_set, df1)
+ df2 = pd.DataFrame(
+ {
+ "datetime": [start_date + timedelta(days=1)] * 3,
+ "entity_id": [i for i in range(3)],
+ "strings": ["a", "b", "c"],
+ "ints": [2, 6, 7],
+ "floats": [1.6, 2.4, 2],
+ }
+ )
+ dataset_id_2 = client.ingest(feature_stats_feature_set, df2)
+
+ combined_df = pd.concat([df1, df2])[["strings", "ints", "floats"]]
+ expected_stats = tfdv.generate_statistics_from_dataframe(combined_df)
+ clear_unsupported_agg_fields(expected_stats)
+
+ # Since TFDV computes population std dev
+ for feature in expected_stats.datasets[0].features:
+ if feature.HasField("num_stats"):
+ name = feature.path.step[0]
+ std = combined_df[name].std()
+ feature.num_stats.std_dev = std
+
+ time.sleep(10)
+
+ return {
+ "ids": [dataset_id_1, dataset_id_2],
+ "start_date": datetime(
+ start_date.year, start_date.month, start_date.day
+ ).replace(tzinfo=pytz.utc),
+ "end_date": datetime(end_date.year, end_date.month, end_date.day).replace(
+ tzinfo=pytz.utc
+ ),
+ "stats": expected_stats,
+ }
+
+
+def test_feature_stats_retrieval_by_single_dataset(client, feature_stats_dataset_basic):
+ stats = client.get_statistics(
+ f"{PROJECT_NAME}/feature_stats:1",
+ features=["strings", "ints", "floats"],
+ store=STORE_NAME,
+ dataset_ids=[feature_stats_dataset_basic["id"]],
+ )
+
+ assert_stats_equal(feature_stats_dataset_basic["stats"], stats)
+
+
+def test_feature_stats_by_date(client, feature_stats_dataset_basic):
+ stats = client.get_statistics(
+ f"{PROJECT_NAME}/feature_stats:1",
+ features=["strings", "ints", "floats"],
+ store=STORE_NAME,
+ start_date=feature_stats_dataset_basic["date"],
+ end_date=feature_stats_dataset_basic["date"] + timedelta(days=1),
+ )
+ assert_stats_equal(feature_stats_dataset_basic["stats"], stats)
+
+
+def test_feature_stats_agg_over_datasets(client, feature_stats_dataset_agg):
+ stats = client.get_statistics(
+ f"{PROJECT_NAME}/feature_stats:1",
+ features=["strings", "ints", "floats"],
+ store=STORE_NAME,
+ dataset_ids=feature_stats_dataset_agg["ids"],
+ )
+ assert_stats_equal(feature_stats_dataset_agg["stats"], stats)
+
+
+def test_feature_stats_agg_over_dates(client, feature_stats_dataset_agg):
+ stats = client.get_statistics(
+ f"{PROJECT_NAME}/feature_stats:1",
+ features=["strings", "ints", "floats"],
+ store=STORE_NAME,
+ start_date=feature_stats_dataset_agg["start_date"],
+ end_date=feature_stats_dataset_agg["end_date"],
+ )
+ assert_stats_equal(feature_stats_dataset_agg["stats"], stats)
+
+
+def test_feature_stats_force_refresh(
+ client, feature_stats_dataset_basic, feature_stats_feature_set
+):
+ df = feature_stats_dataset_basic["df"]
+
+ df2 = pd.DataFrame(
+ {
+ "datetime": [df.iloc[0].datetime],
+ "entity_id": [10],
+ "strings": ["c"],
+ "ints": [2],
+ "floats": [1.3],
+ }
+ )
+ client.ingest(feature_stats_feature_set, df2)
+ time.sleep(10)
+
+ actual_stats = client.get_statistics(
+ f"{PROJECT_NAME}/feature_stats:1",
+ features=["strings", "ints", "floats"],
+ store="historical",
+ start_date=feature_stats_dataset_basic["date"],
+ end_date=feature_stats_dataset_basic["date"] + timedelta(days=1),
+ force_refresh=True,
+ )
+
+ combined_df = pd.concat([df, df2])
+ expected_stats = tfdv.generate_statistics_from_dataframe(combined_df)
+
+ clear_unsupported_fields(expected_stats)
+
+ # Since TFDV computes population std dev
+ for feature in expected_stats.datasets[0].features:
+ if feature.HasField("num_stats"):
+ name = feature.path.step[0]
+ std = combined_df[name].std()
+ feature.num_stats.std_dev = std
+
+ assert_stats_equal(expected_stats, actual_stats)
+
+
+def clear_unsupported_fields(datasets):
+ dataset = datasets.datasets[0]
+ for feature in dataset.features:
+ if feature.HasField("num_stats"):
+ feature.num_stats.common_stats.ClearField("num_values_histogram")
+ for hist in feature.num_stats.histograms:
+ sorted_buckets = sorted(hist.buckets, key=lambda k: k["highValue"])
+ del hist.buckets[:]
+ hist.buckets.extend(sorted_buckets)
+ elif feature.HasField("string_stats"):
+ feature.string_stats.common_stats.ClearField("num_values_histogram")
+ for bucket in feature.string_stats.rank_histogram.buckets:
+ bucket.ClearField("low_rank")
+ bucket.ClearField("high_rank")
+ elif feature.HasField("struct_stats"):
+ feature.string_stats.struct_stats.ClearField("num_values_histogram")
+ elif feature.HasField("bytes_stats"):
+ feature.string_stats.bytes_stats.ClearField("num_values_histogram")
+
+
+def clear_unsupported_agg_fields(datasets):
+ dataset = datasets.datasets[0]
+ for feature in dataset.features:
+ if feature.HasField("num_stats"):
+ feature.num_stats.common_stats.ClearField("num_values_histogram")
+ feature.num_stats.ClearField("histograms")
+ feature.num_stats.ClearField("median")
+ elif feature.HasField("string_stats"):
+ feature.string_stats.common_stats.ClearField("num_values_histogram")
+ feature.string_stats.ClearField("rank_histogram")
+ feature.string_stats.ClearField("top_values")
+ feature.string_stats.ClearField("unique")
+ elif feature.HasField("struct_stats"):
+ feature.struct_stats.ClearField("num_values_histogram")
+ elif feature.HasField("bytes_stats"):
+ feature.bytes_stats.ClearField("num_values_histogram")
+ feature.bytes_stats.ClearField("unique")
+
+
+def assert_stats_equal(left, right):
+ left_stats = MessageToDict(left)["datasets"][0]
+ right_stats = MessageToDict(right)["datasets"][0]
+ assert (
+ left_stats["numExamples"] == right_stats["numExamples"]
+ ), f"Number of examples do not match. Expected {left_stats['numExamples']}, got {right_stats['numExamples']}"
+
+ left_features = sorted(left_stats["features"], key=lambda k: k["path"]["step"][0])
+ right_features = sorted(right_stats["features"], key=lambda k: k["path"]["step"][0])
+ diff = DeepDiff(left_features, right_features, significant_digits=4)
+ assert len(diff) == 0, f"Feature statistics do not match: \nwanted: {left_features}\n got: {right_features}"
diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py
index 8ea472b6620..c1c3726844f 100644
--- a/tests/e2e/conftest.py
+++ b/tests/e2e/conftest.py
@@ -2,4 +2,6 @@ def pytest_addoption(parser):
parser.addoption("--core_url", action="store", default="localhost:6565")
parser.addoption("--serving_url", action="store", default="localhost:6566")
parser.addoption("--allow_dirty", action="store", default="False")
- parser.addoption("--gcs_path", action="store", default="gs://feast-templocation-kf-feast/")
+ parser.addoption(
+ "--gcs_path", action="store", default="gs://feast-templocation-kf-feast/"
+ )
diff --git a/tests/e2e/all_types_parquet/all_types_parquet.yaml b/tests/e2e/redis/all_types_parquet/all_types_parquet.yaml
similarity index 100%
rename from tests/e2e/all_types_parquet/all_types_parquet.yaml
rename to tests/e2e/redis/all_types_parquet/all_types_parquet.yaml
diff --git a/tests/e2e/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py
similarity index 84%
rename from tests/e2e/basic-ingest-redis-serving.py
rename to tests/e2e/redis/basic-ingest-redis-serving.py
index 8e40794344e..f1806b67fef 100644
--- a/tests/e2e/basic-ingest-redis-serving.py
+++ b/tests/e2e/redis/basic-ingest-redis-serving.py
@@ -24,25 +24,26 @@
import uuid
FLOAT_TOLERANCE = 0.00001
-PROJECT_NAME = 'basic_' + uuid.uuid4().hex.upper()[0:6]
+PROJECT_NAME = "basic_" + uuid.uuid4().hex.upper()[0:6]
+ROOT_PATH = os.path.dirname(os.path.abspath(__file__))
-@pytest.fixture(scope='module')
+
+@pytest.fixture(scope="module")
def core_url(pytestconfig):
return pytestconfig.getoption("core_url")
-@pytest.fixture(scope='module')
+@pytest.fixture(scope="module")
def serving_url(pytestconfig):
return pytestconfig.getoption("serving_url")
-@pytest.fixture(scope='module')
+@pytest.fixture(scope="module")
def allow_dirty(pytestconfig):
- return True if pytestconfig.getoption(
- "allow_dirty").lower() == "true" else False
+ return True if pytestconfig.getoption("allow_dirty").lower() == "true" else False
-@pytest.fixture(scope='module')
+@pytest.fixture(scope="module")
def client(core_url, serving_url, allow_dirty):
# Get client for core and serving
client = Client(core_url=core_url, serving_url=serving_url)
@@ -60,13 +61,12 @@ def client(core_url, serving_url, allow_dirty):
return client
-@pytest.fixture(scope='module')
+@pytest.fixture(scope="module")
def basic_dataframe():
offset = random.randint(1000, 100000) # ensure a unique key space is used
return pd.DataFrame(
{
- "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in
- range(5)],
+ "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in range(5)],
"customer_id": [offset + inc for inc in range(5)],
"daily_transactions": [np.random.rand() for _ in range(5)],
"total_transactions": [512 for _ in range(5)],
@@ -78,7 +78,7 @@ def basic_dataframe():
@pytest.mark.run(order=10)
def test_basic_register_feature_set_success(client):
# Load feature set from file
- cust_trans_fs_expected = FeatureSet.from_yaml("basic/cust_trans_fs.yaml")
+ cust_trans_fs_expected = FeatureSet.from_yaml(os.path.join(ROOT_PATH, "basic/cust_trans_fs.yaml"))
client.set_project(PROJECT_NAME)
@@ -109,6 +109,7 @@ def test_basic_ingest_success(client, basic_dataframe):
client.ingest(cust_trans_fs, basic_dataframe)
time.sleep(5)
+
@pytest.mark.timeout(45)
@pytest.mark.run(order=12)
def test_basic_retrieve_online_success(client, basic_dataframe):
@@ -128,10 +129,7 @@ def test_basic_retrieve_online_success(client, basic_dataframe):
}
)
],
- feature_refs=[
- "daily_transactions",
- "total_transactions",
- ],
+ feature_refs=["daily_transactions", "total_transactions",],
) # type: GetOnlineFeaturesResponse
if response is None:
@@ -139,11 +137,10 @@ def test_basic_retrieve_online_success(client, basic_dataframe):
returned_daily_transactions = float(
response.field_values[0]
- .fields[PROJECT_NAME + "/daily_transactions"]
- .float_val
+ .fields[PROJECT_NAME + "/daily_transactions"]
+ .float_val
)
- sent_daily_transactions = float(
- basic_dataframe.iloc[0]["daily_transactions"])
+ sent_daily_transactions = float(basic_dataframe.iloc[0]["daily_transactions"])
if math.isclose(
sent_daily_transactions,
@@ -152,15 +149,19 @@ def test_basic_retrieve_online_success(client, basic_dataframe):
):
break
+
@pytest.mark.timeout(300)
@pytest.mark.run(order=19)
def test_basic_ingest_jobs(client, basic_dataframe):
# list ingestion jobs given featureset
cust_trans_fs = client.get_feature_set(name="customer_transactions")
ingest_jobs = client.list_ingest_jobs(
- feature_set_ref=FeatureSetRef.from_feature_set(cust_trans_fs))
+ feature_set_ref=FeatureSetRef.from_feature_set(cust_trans_fs)
+ )
# filter ingestion jobs to only those that are running
- ingest_jobs = [job for job in ingest_jobs if job.status == IngestionJobStatus.RUNNING]
+ ingest_jobs = [
+ job for job in ingest_jobs if job.status == IngestionJobStatus.RUNNING
+ ]
assert len(ingest_jobs) >= 1
for ingest_job in ingest_jobs:
@@ -175,18 +176,16 @@ def test_basic_ingest_jobs(client, basic_dataframe):
assert ingest_job.status == IngestionJobStatus.ABORTED
-@pytest.fixture(scope='module')
+@pytest.fixture(scope="module")
def all_types_dataframe():
return pd.DataFrame(
{
- "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in
- range(3)],
+ "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in range(3)],
"user_id": [1001, 1002, 1003],
"int32_feature": [np.int32(1), np.int32(2), np.int32(3)],
"int64_feature": [np.int64(1), np.int64(2), np.int64(3)],
"float_feature": [np.float(0.1), np.float(0.2), np.float(0.3)],
- "double_feature": [np.float64(0.1), np.float64(0.2),
- np.float64(0.3)],
+ "double_feature": [np.float64(0.1), np.float64(0.2), np.float64(0.3)],
"string_feature": ["one", "two", "three"],
"bytes_feature": [b"one", b"two", b"three"],
"bool_feature": [True, False, False],
@@ -248,8 +247,7 @@ def test_all_types_register_feature_set_success(client):
Feature(name="float_list_feature", dtype=ValueType.FLOAT_LIST),
Feature(name="int64_list_feature", dtype=ValueType.INT64_LIST),
Feature(name="int32_list_feature", dtype=ValueType.INT32_LIST),
- Feature(name="string_list_feature",
- dtype=ValueType.STRING_LIST),
+ Feature(name="string_list_feature", dtype=ValueType.STRING_LIST),
Feature(name="bytes_list_feature", dtype=ValueType.BYTES_LIST),
],
max_age=Duration(seconds=3600),
@@ -295,8 +293,11 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe):
response = client.get_online_features(
entity_rows=[
GetOnlineFeaturesRequest.EntityRow(
- fields={"user_id": Value(
- int64_val=all_types_dataframe.iloc[0]["user_id"])}
+ fields={
+ "user_id": Value(
+ int64_val=all_types_dataframe.iloc[0]["user_id"]
+ )
+ }
)
],
feature_refs=[
@@ -319,11 +320,10 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe):
if response is None:
continue
-
returned_float_list = (
response.field_values[0]
- .fields[PROJECT_NAME+"/float_list_feature"]
- .float_list_val.val
+ .fields[PROJECT_NAME + "/float_list_feature"]
+ .float_list_val.val
)
sent_float_list = all_types_dataframe.iloc[0]["float_list_feature"]
@@ -333,15 +333,19 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe):
):
break
+
@pytest.mark.timeout(300)
@pytest.mark.run(order=29)
def test_all_types_ingest_jobs(client, all_types_dataframe):
# list ingestion jobs given featureset
all_types_fs = client.get_feature_set(name="all_types")
ingest_jobs = client.list_ingest_jobs(
- feature_set_ref=FeatureSetRef.from_feature_set(all_types_fs))
+ feature_set_ref=FeatureSetRef.from_feature_set(all_types_fs)
+ )
# filter ingestion jobs to only those that are running
- ingest_jobs = [job for job in ingest_jobs if job.status == IngestionJobStatus.RUNNING]
+ ingest_jobs = [
+ job for job in ingest_jobs if job.status == IngestionJobStatus.RUNNING
+ ]
assert len(ingest_jobs) >= 1
for ingest_job in ingest_jobs:
@@ -355,15 +359,15 @@ def test_all_types_ingest_jobs(client, all_types_dataframe):
ingest_job.wait(IngestionJobStatus.ABORTED)
assert ingest_job.status == IngestionJobStatus.ABORTED
-@pytest.fixture(scope='module')
+
+@pytest.fixture(scope="module")
def large_volume_dataframe():
ROW_COUNT = 100000
offset = random.randint(1000000, 10000000) # ensure a unique key space
customer_data = pd.DataFrame(
{
"datetime": [
- datetime.utcnow().replace(tzinfo=pytz.utc) for _ in
- range(ROW_COUNT)
+ datetime.utcnow().replace(tzinfo=pytz.utc) for _ in range(ROW_COUNT)
],
"customer_id": [offset + inc for inc in range(ROW_COUNT)],
"daily_transactions_large": [np.random.rand() for _ in range(ROW_COUNT)],
@@ -377,7 +381,8 @@ def large_volume_dataframe():
@pytest.mark.run(order=30)
def test_large_volume_register_feature_set_success(client):
cust_trans_fs_expected = FeatureSet.from_yaml(
- "large_volume/cust_trans_large_fs.yaml")
+ os.path.join(ROOT_PATH,"large_volume/cust_trans_large_fs.yaml")
+ )
# Register feature set
client.apply(cust_trans_fs_expected)
@@ -385,8 +390,7 @@ def test_large_volume_register_feature_set_success(client):
# Feast Core needs some time to fully commit the FeatureSet applied
# when there is no existing job yet for the Featureset
time.sleep(10)
- cust_trans_fs_actual = client.get_feature_set(
- name="customer_transactions_large")
+ cust_trans_fs_actual = client.get_feature_set(name="customer_transactions_large")
assert cust_trans_fs_actual == cust_trans_fs_expected
@@ -421,16 +425,12 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe):
GetOnlineFeaturesRequest.EntityRow(
fields={
"customer_id": Value(
- int64_val=large_volume_dataframe.iloc[0][
- "customer_id"]
+ int64_val=large_volume_dataframe.iloc[0]["customer_id"]
)
}
)
],
- feature_refs=[
- "daily_transactions_large",
- "total_transactions_large",
- ],
+ feature_refs=["daily_transactions_large", "total_transactions_large",],
) # type: GetOnlineFeaturesResponse
if response is None:
@@ -438,11 +438,12 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe):
returned_daily_transactions = float(
response.field_values[0]
- .fields[PROJECT_NAME + "/daily_transactions_large"]
- .float_val
+ .fields[PROJECT_NAME + "/daily_transactions_large"]
+ .float_val
)
sent_daily_transactions = float(
- large_volume_dataframe.iloc[0]["daily_transactions_large"])
+ large_volume_dataframe.iloc[0]["daily_transactions_large"]
+ )
if math.isclose(
sent_daily_transactions,
@@ -452,49 +453,47 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe):
break
-@pytest.fixture(scope='module')
+@pytest.fixture(scope="module")
def all_types_parquet_file():
COUNT = 20000
df = pd.DataFrame(
{
"datetime": [datetime.utcnow() for _ in range(COUNT)],
- "customer_id": [np.int32(random.randint(0, 10000)) for _ in
- range(COUNT)],
- "int32_feature_parquet": [np.int32(random.randint(0, 10000)) for _ in
- range(COUNT)],
- "int64_feature_parquet": [np.int64(random.randint(0, 10000)) for _ in
- range(COUNT)],
+ "customer_id": [np.int32(random.randint(0, 10000)) for _ in range(COUNT)],
+ "int32_feature_parquet": [
+ np.int32(random.randint(0, 10000)) for _ in range(COUNT)
+ ],
+ "int64_feature_parquet": [
+ np.int64(random.randint(0, 10000)) for _ in range(COUNT)
+ ],
"float_feature_parquet": [np.float(random.random()) for _ in range(COUNT)],
- "double_feature_parquet": [np.float64(random.random()) for _ in
- range(COUNT)],
- "string_feature_parquet": ["one" + str(random.random()) for _ in
- range(COUNT)],
+ "double_feature_parquet": [
+ np.float64(random.random()) for _ in range(COUNT)
+ ],
+ "string_feature_parquet": [
+ "one" + str(random.random()) for _ in range(COUNT)
+ ],
"bytes_feature_parquet": [b"one" for _ in range(COUNT)],
"int32_list_feature_parquet": [
np.array([1, 2, 3, random.randint(0, 10000)], dtype=np.int32)
- for _
- in range(COUNT)
+ for _ in range(COUNT)
],
"int64_list_feature_parquet": [
np.array([1, random.randint(0, 10000), 3, 4], dtype=np.int64)
- for _
- in range(COUNT)
+ for _ in range(COUNT)
],
"float_list_feature_parquet": [
- np.array([1.1, 1.2, 1.3, random.random()], dtype=np.float32) for
- _
- in range(COUNT)
+ np.array([1.1, 1.2, 1.3, random.random()], dtype=np.float32)
+ for _ in range(COUNT)
],
"double_list_feature_parquet": [
- np.array([1.1, 1.2, 1.3, random.random()], dtype=np.float64) for
- _
- in range(COUNT)
+ np.array([1.1, 1.2, 1.3, random.random()], dtype=np.float64)
+ for _ in range(COUNT)
],
"string_list_feature_parquet": [
- np.array(["one", "two" + str(random.random()), "three"]) for _
- in
- range(COUNT)
+ np.array(["one", "two" + str(random.random()), "three"])
+ for _ in range(COUNT)
],
"bytes_list_feature_parquet": [
np.array([b"one", b"two", b"three"]) for _ in range(COUNT)
@@ -505,16 +504,18 @@ def all_types_parquet_file():
# TODO: Boolean list is not being tested.
# https://github.com/gojek/feast/issues/341
- file_path = os.path.join(tempfile.mkdtemp(), 'all_types.parquet')
+ file_path = os.path.join(tempfile.mkdtemp(), "all_types.parquet")
df.to_parquet(file_path, allow_truncated_timestamps=True)
return file_path
+
@pytest.mark.timeout(300)
@pytest.mark.run(order=40)
def test_all_types_parquet_register_feature_set_success(client):
# Load feature set from file
all_types_parquet_expected = FeatureSet.from_yaml(
- "all_types_parquet/all_types_parquet.yaml")
+ os.path.join(ROOT_PATH, "all_types_parquet/all_types_parquet.yaml")
+ )
# Register feature set
client.apply(all_types_parquet_expected)
@@ -538,11 +539,11 @@ def test_all_types_parquet_register_feature_set_success(client):
@pytest.mark.timeout(600)
@pytest.mark.run(order=41)
-def test_all_types_infer_register_ingest_file_success(client,
- all_types_parquet_file):
+def test_all_types_infer_register_ingest_file_success(client, all_types_parquet_file):
# Get feature set
all_types_fs = client.get_feature_set(name="all_types_parquet")
# Ingest user embedding data
- client.ingest(feature_set=all_types_fs, source=all_types_parquet_file,
- force_update=True)
+ client.ingest(
+ feature_set=all_types_fs, source=all_types_parquet_file, force_update=True
+ )
diff --git a/tests/e2e/basic/cust_trans_fs.yaml b/tests/e2e/redis/basic/cust_trans_fs.yaml
similarity index 100%
rename from tests/e2e/basic/cust_trans_fs.yaml
rename to tests/e2e/redis/basic/cust_trans_fs.yaml
diff --git a/tests/e2e/basic/data.csv b/tests/e2e/redis/basic/data.csv
similarity index 100%
rename from tests/e2e/basic/data.csv
rename to tests/e2e/redis/basic/data.csv
diff --git a/tests/e2e/large_volume/cust_trans_large_fs.yaml b/tests/e2e/redis/large_volume/cust_trans_large_fs.yaml
similarity index 100%
rename from tests/e2e/large_volume/cust_trans_large_fs.yaml
rename to tests/e2e/redis/large_volume/cust_trans_large_fs.yaml
diff --git a/tests/e2e/requirements.txt b/tests/e2e/requirements.txt
index 0ba345a000f..5a21a6a0cea 100644
--- a/tests/e2e/requirements.txt
+++ b/tests/e2e/requirements.txt
@@ -7,3 +7,6 @@ pytest-benchmark==3.2.2
pytest-mock==1.10.4
pytest-timeout==1.3.3
pytest-ordering==0.6.*
+tensorflow-data-validation==0.21.2
+deepdiff==4.3.2
+tensorflow==2.1.0
\ No newline at end of file