From 056c06de3b5de57e966892e65444c205a95b977d Mon Sep 17 00:00:00 2001 From: zhilingc Date: Fri, 20 Mar 2020 12:53:44 +0800 Subject: [PATCH 1/2] Change serving and ingestion to use storage API --- ingestion/pom.xml | 18 + .../main/java/feast/ingestion/ImportJob.java | 61 +-- .../ingestion/transform/ReadFromSource.java | 2 +- .../transform/ValidateFeatureRows.java | 9 +- .../WriteFailedElementToBigQuery.java | 2 +- .../ingestion/transform/WriteToStore.java | 168 --------- .../fn/KafkaRecordToFeatureRowDoFn.java | 3 +- .../transform/fn/ValidateFeatureRowDoFn.java | 2 +- .../WriteDeadletterRowMetricsDoFn.java | 2 +- .../metrics/WriteFailureMetricsTransform.java | 52 +++ ...java => WriteSuccessMetricsTransform.java} | 98 ++--- .../java/feast/ingestion/utils/SpecUtil.java | 7 +- .../java/feast/ingestion/utils/StoreUtil.java | 181 +-------- .../feast/ingestion/values/FeatureSet.java | 6 +- .../java/feast/retry/BackOffExecutor.java | 58 --- .../src/main/java/feast/retry/Retriable.java | 25 -- .../bigquery/FeatureRowToTableRow.java | 109 ------ .../serving/bigquery/GetTableDestination.java | 52 --- .../redis/FeatureRowToRedisMutationDoFn.java | 114 ------ .../store/serving/redis/RedisCustomIO.java | 341 ----------------- .../serving/redis/RedisIngestionClient.java | 49 --- .../redis/RedisStandaloneIngestionClient.java | 122 ------ .../java/feast/ingestion/ImportJobTest.java | 4 +- .../transform/ValidateFeatureRowsTest.java | 157 ++++---- .../feast/ingestion/utils/StoreUtilTest.java | 211 ----------- .../FeatureRowToRedisMutationDoFnTest.java | 345 ----------------- .../serving/redis/RedisCustomIOTest.java | 238 ------------ .../src/test/java/feast/test/TestUtil.java | 45 +-- pom.xml | 4 +- serving/pom.xml | 58 +-- .../configuration/ServingServiceConfig.java | 39 +- .../serving/encoding/FeatureRowDecoder.java | 95 ----- .../serving/service/BatchServingService.java | 89 +++++ .../service/BigQueryServingService.java | 282 -------------- .../serving/service/OnlineServingService.java | 162 ++++++++ .../serving/service/RedisServingService.java | 345 ----------------- .../serving/specs/CachedSpecService.java | 1 + .../serving/specs/FeatureSetRequest.java | 53 --- .../bigquery/BatchRetrievalQueryRunnable.java | 352 ------------------ .../store/bigquery/QueryTemplater.java | 161 -------- .../store/bigquery/SubqueryCallable.java | 80 ---- .../store/bigquery/model/FeatureSetInfo.java | 86 ----- .../service/CachedSpecServiceTest.java | 2 +- ...est.java => OnlineServingServiceTest.java} | 215 +++-------- storage/connectors/bigquery/pom.xml | 13 +- storage/connectors/pom.xml | 15 + .../redis/retrieval/RedisOnlineRetriever.java | 8 +- .../retrieval}/FeatureRowDecoderTest.java | 2 +- .../retrieval/RedisOnlineRetrieverTest.java | 13 - 49 files changed, 650 insertions(+), 3906 deletions(-) delete mode 100644 ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java create mode 100644 ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFailureMetricsTransform.java rename ingestion/src/main/java/feast/ingestion/transform/metrics/{WriteMetricsTransform.java => WriteSuccessMetricsTransform.java} (58%) delete mode 100644 ingestion/src/main/java/feast/retry/BackOffExecutor.java delete mode 100644 ingestion/src/main/java/feast/retry/Retriable.java delete mode 100644 ingestion/src/main/java/feast/store/serving/bigquery/FeatureRowToTableRow.java delete mode 100644 ingestion/src/main/java/feast/store/serving/bigquery/GetTableDestination.java delete mode 100644 ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java delete mode 100644 ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java delete mode 100644 ingestion/src/main/java/feast/store/serving/redis/RedisIngestionClient.java delete mode 100644 ingestion/src/main/java/feast/store/serving/redis/RedisStandaloneIngestionClient.java delete mode 100644 ingestion/src/test/java/feast/ingestion/utils/StoreUtilTest.java delete mode 100644 ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java delete mode 100644 ingestion/src/test/java/feast/store/serving/redis/RedisCustomIOTest.java delete mode 100644 serving/src/main/java/feast/serving/encoding/FeatureRowDecoder.java create mode 100644 serving/src/main/java/feast/serving/service/BatchServingService.java delete mode 100644 serving/src/main/java/feast/serving/service/BigQueryServingService.java create mode 100644 serving/src/main/java/feast/serving/service/OnlineServingService.java delete mode 100644 serving/src/main/java/feast/serving/service/RedisServingService.java delete mode 100644 serving/src/main/java/feast/serving/specs/FeatureSetRequest.java delete mode 100644 serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java delete mode 100644 serving/src/main/java/feast/serving/store/bigquery/QueryTemplater.java delete mode 100644 serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java delete mode 100644 serving/src/main/java/feast/serving/store/bigquery/model/FeatureSetInfo.java rename serving/src/test/java/feast/serving/service/{RedisServingServiceTest.java => OnlineServingServiceTest.java} (72%) rename {serving/src/test/java/feast/serving/encoding => storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retrieval}/FeatureRowDecoderTest.java (98%) diff --git a/ingestion/pom.xml b/ingestion/pom.xml index ccc8ca04510..4f63e1acd2a 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -92,6 +92,24 @@ ${project.version} + + dev.feast + feast-storage-api + ${project.version} + + + + dev.feast + feast-storage-connector-redis + ${project.version} + + + + dev.feast + feast-storage-connector-bigquery + ${project.version} + + com.google.auto.value auto-value-annotations diff --git a/ingestion/src/main/java/feast/ingestion/ImportJob.java b/ingestion/src/main/java/feast/ingestion/ImportJob.java index c4973ce3cae..c8982ca24ad 100644 --- a/ingestion/src/main/java/feast/ingestion/ImportJob.java +++ b/ingestion/src/main/java/feast/ingestion/ImportJob.java @@ -17,9 +17,11 @@ package feast.ingestion; import static feast.ingestion.utils.SpecUtil.getFeatureSetReference; +import static feast.ingestion.utils.StoreUtil.getFeatureSink; import com.google.protobuf.InvalidProtocolBufferException; import feast.core.FeatureSetProto.FeatureSet; +import feast.core.FeatureSetProto.FeatureSetSpec; import feast.core.SourceProto.Source; import feast.core.StoreProto.Store; import feast.ingestion.options.BZip2Decompressor; @@ -28,12 +30,13 @@ import feast.ingestion.transform.ReadFromSource; import feast.ingestion.transform.ValidateFeatureRows; import feast.ingestion.transform.WriteFailedElementToBigQuery; -import feast.ingestion.transform.WriteToStore; -import feast.ingestion.transform.metrics.WriteMetricsTransform; +import feast.ingestion.transform.metrics.WriteFailureMetricsTransform; +import feast.ingestion.transform.metrics.WriteSuccessMetricsTransform; import feast.ingestion.utils.ResourceUtil; import feast.ingestion.utils.SpecUtil; -import feast.ingestion.utils.StoreUtil; -import feast.ingestion.values.FailedElement; +import feast.storage.api.write.FailedElement; +import feast.storage.api.write.FeatureSink; +import feast.storage.api.write.WriteResult; import feast.types.FeatureRowProto.FeatureRow; import java.io.IOException; import java.util.HashMap; @@ -93,17 +96,24 @@ public static PipelineResult runPipeline(ImportOptions options) throws IOExcepti SpecUtil.getSubscribedFeatureSets(store.getSubscriptionsList(), featureSets); // Generate tags by key - Map featureSetsByKey = new HashMap<>(); + Map featureSetSpecsByKey = new HashMap<>(); subscribedFeatureSets.stream() .forEach( fs -> { - String ref = getFeatureSetReference(fs); - featureSetsByKey.put(ref, fs); + String ref = getFeatureSetReference(fs.getSpec()); + featureSetSpecsByKey.put(ref, fs.getSpec()); }); + FeatureSink featureSink = getFeatureSink(store, featureSetSpecsByKey); + // TODO: make the source part of the job initialisation options Source source = subscribedFeatureSets.get(0).getSpec().getSource(); + for (FeatureSet featureSet : subscribedFeatureSets) { + // Ensure Store has valid configuration and Feast can access it. + featureSink.prepareWrite(featureSet); + } + // Step 1. Read messages from Feast Source as FeatureRow. PCollectionTuple convertedFeatureRows = pipeline.apply( @@ -114,28 +124,20 @@ public static PipelineResult runPipeline(ImportOptions options) throws IOExcepti .setFailureTag(DEADLETTER_OUT) .build()); - for (FeatureSet featureSet : subscribedFeatureSets) { - // Ensure Store has valid configuration and Feast can access it. - StoreUtil.setupStore(store, featureSet); - } - // Step 2. Validate incoming FeatureRows PCollectionTuple validatedRows = convertedFeatureRows .get(FEATURE_ROW_OUT) .apply( ValidateFeatureRows.newBuilder() - .setFeatureSets(featureSetsByKey) + .setFeatureSetSpecs(featureSetSpecsByKey) .setSuccessTag(FEATURE_ROW_OUT) .setFailureTag(DEADLETTER_OUT) .build()); // Step 3. Write FeatureRow to the corresponding Store. - validatedRows - .get(FEATURE_ROW_OUT) - .apply( - "WriteFeatureRowToStore", - WriteToStore.newBuilder().setFeatureSets(featureSetsByKey).setStore(store).build()); + WriteResult writeFeatureRows = + validatedRows.get(FEATURE_ROW_OUT).apply("WriteFeatureRowToStore", featureSink.write()); // Step 4. Write FailedElements to a dead letter table in BigQuery. if (options.getDeadLetterTableSpec() != null) { @@ -156,16 +158,25 @@ public static PipelineResult runPipeline(ImportOptions options) throws IOExcepti .setJsonSchema(ResourceUtil.getDeadletterTableSchemaJson()) .setTableSpec(options.getDeadLetterTableSpec()) .build()); + + writeFeatureRows + .getFailedInserts() + .apply( + "WriteFailedElements_WriteFeatureRowToStore", + WriteFailedElementToBigQuery.newBuilder() + .setJsonSchema(ResourceUtil.getDeadletterTableSchemaJson()) + .setTableSpec(options.getDeadLetterTableSpec()) + .build()); } // Step 5. Write metrics to a metrics sink. - validatedRows.apply( - "WriteMetrics", - WriteMetricsTransform.newBuilder() - .setStoreName(store.getName()) - .setSuccessTag(FEATURE_ROW_OUT) - .setFailureTag(DEADLETTER_OUT) - .build()); + writeFeatureRows + .getSuccessfulInserts() + .apply("WriteSuccessMetrics", WriteSuccessMetricsTransform.create(store.getName())); + + writeFeatureRows + .getFailedInserts() + .apply("WriteFailureMetrics", WriteFailureMetricsTransform.create(store.getName())); } return pipeline.run(); diff --git a/ingestion/src/main/java/feast/ingestion/transform/ReadFromSource.java b/ingestion/src/main/java/feast/ingestion/transform/ReadFromSource.java index 65e95b287dc..3536ecdecd4 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ReadFromSource.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ReadFromSource.java @@ -21,7 +21,7 @@ import feast.core.SourceProto.Source; import feast.core.SourceProto.SourceType; import feast.ingestion.transform.fn.KafkaRecordToFeatureRowDoFn; -import feast.ingestion.values.FailedElement; +import feast.storage.api.write.FailedElement; import feast.types.FeatureRowProto.FeatureRow; import org.apache.beam.sdk.io.kafka.KafkaIO; import org.apache.beam.sdk.transforms.PTransform; diff --git a/ingestion/src/main/java/feast/ingestion/transform/ValidateFeatureRows.java b/ingestion/src/main/java/feast/ingestion/transform/ValidateFeatureRows.java index 5ca6a710f62..1e74de92a1d 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ValidateFeatureRows.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ValidateFeatureRows.java @@ -19,8 +19,8 @@ import com.google.auto.value.AutoValue; import feast.core.FeatureSetProto; import feast.ingestion.transform.fn.ValidateFeatureRowDoFn; -import feast.ingestion.values.FailedElement; import feast.ingestion.values.FeatureSet; +import feast.storage.api.write.FailedElement; import feast.types.FeatureRowProto.FeatureRow; import java.util.Map; import java.util.stream.Collectors; @@ -36,7 +36,7 @@ public abstract class ValidateFeatureRows extends PTransform, PCollectionTuple> { - public abstract Map getFeatureSets(); + public abstract Map getFeatureSetSpecs(); public abstract TupleTag getSuccessTag(); @@ -49,7 +49,8 @@ public static Builder newBuilder() { @AutoValue.Builder public abstract static class Builder { - public abstract Builder setFeatureSets(Map featureSets); + public abstract Builder setFeatureSetSpecs( + Map featureSets); public abstract Builder setSuccessTag(TupleTag successTag); @@ -62,7 +63,7 @@ public abstract static class Builder { public PCollectionTuple expand(PCollection input) { Map featureSets = - getFeatureSets().entrySet().stream() + getFeatureSetSpecs().entrySet().stream() .map(e -> Pair.of(e.getKey(), new FeatureSet(e.getValue()))) .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); diff --git a/ingestion/src/main/java/feast/ingestion/transform/WriteFailedElementToBigQuery.java b/ingestion/src/main/java/feast/ingestion/transform/WriteFailedElementToBigQuery.java index cda590b21aa..b4847934787 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/WriteFailedElementToBigQuery.java +++ b/ingestion/src/main/java/feast/ingestion/transform/WriteFailedElementToBigQuery.java @@ -18,7 +18,7 @@ import com.google.api.services.bigquery.model.TableRow; import com.google.auto.value.AutoValue; -import feast.ingestion.values.FailedElement; +import feast.storage.api.write.FailedElement; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition; diff --git a/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java b/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java deleted file mode 100644 index 4e9082f5554..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform; - -import com.google.api.services.bigquery.model.TableDataInsertAllResponse.InsertErrors; -import com.google.api.services.bigquery.model.TableRow; -import com.google.auto.value.AutoValue; -import feast.core.FeatureSetProto.FeatureSet; -import feast.core.StoreProto.Store; -import feast.core.StoreProto.Store.BigQueryConfig; -import feast.core.StoreProto.Store.StoreType; -import feast.ingestion.options.ImportOptions; -import feast.ingestion.utils.ResourceUtil; -import feast.ingestion.values.FailedElement; -import feast.store.serving.bigquery.FeatureRowToTableRow; -import feast.store.serving.bigquery.GetTableDestination; -import feast.store.serving.redis.FeatureRowToRedisMutationDoFn; -import feast.store.serving.redis.RedisCustomIO; -import feast.types.FeatureRowProto.FeatureRow; -import java.io.IOException; -import java.util.Map; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.Method; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryInsertError; -import org.apache.beam.sdk.io.gcp.bigquery.InsertRetryPolicy; -import org.apache.beam.sdk.io.gcp.bigquery.WriteResult; -import org.apache.beam.sdk.metrics.Counter; -import org.apache.beam.sdk.metrics.Metrics; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PDone; -import org.apache.beam.sdk.values.TypeDescriptors; -import org.slf4j.Logger; - -@AutoValue -public abstract class WriteToStore extends PTransform, PDone> { - - private static final Logger log = org.slf4j.LoggerFactory.getLogger(WriteToStore.class); - - public static final String METRIC_NAMESPACE = "WriteToStore"; - public static final String ELEMENTS_WRITTEN_METRIC = "elements_written"; - - private static final Counter elementsWritten = - Metrics.counter(METRIC_NAMESPACE, ELEMENTS_WRITTEN_METRIC); - - public abstract Store getStore(); - - public abstract Map getFeatureSets(); - - public static Builder newBuilder() { - return new AutoValue_WriteToStore.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setStore(Store store); - - public abstract Builder setFeatureSets(Map featureSets); - - public abstract WriteToStore build(); - } - - @Override - public PDone expand(PCollection input) { - ImportOptions options = input.getPipeline().getOptions().as(ImportOptions.class); - StoreType storeType = getStore().getType(); - - switch (storeType) { - case REDIS: - PCollection redisWriteResult = - input - .apply( - "FeatureRowToRedisMutation", - ParDo.of(new FeatureRowToRedisMutationDoFn(getFeatureSets()))) - .apply("WriteRedisMutationToRedis", RedisCustomIO.write(getStore())); - if (options.getDeadLetterTableSpec() != null) { - redisWriteResult.apply( - WriteFailedElementToBigQuery.newBuilder() - .setTableSpec(options.getDeadLetterTableSpec()) - .setJsonSchema(ResourceUtil.getDeadletterTableSchemaJson()) - .build()); - } - break; - case BIGQUERY: - BigQueryConfig bigqueryConfig = getStore().getBigqueryConfig(); - - WriteResult bigqueryWriteResult = - input.apply( - "WriteTableRowToBigQuery", - BigQueryIO.write() - .to( - new GetTableDestination( - bigqueryConfig.getProjectId(), bigqueryConfig.getDatasetId())) - .withFormatFunction(new FeatureRowToTableRow(options.getJobName())) - .withCreateDisposition(CreateDisposition.CREATE_NEVER) - .withWriteDisposition(WriteDisposition.WRITE_APPEND) - .withExtendedErrorInfo() - .withMethod(Method.STREAMING_INSERTS) - .withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors())); - - if (options.getDeadLetterTableSpec() != null) { - bigqueryWriteResult - .getFailedInsertsWithErr() - .apply( - "WrapBigQueryInsertionError", - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement(ProcessContext context) { - InsertErrors error = context.element().getError(); - TableRow row = context.element().getRow(); - try { - context.output( - FailedElement.newBuilder() - .setErrorMessage(error.toPrettyString()) - .setPayload(row.toPrettyString()) - .setJobName(context.getPipelineOptions().getJobName()) - .setTransformName("WriteTableRowToBigQuery") - .build()); - } catch (IOException e) { - log.error(e.getMessage()); - } - } - })) - .apply( - WriteFailedElementToBigQuery.newBuilder() - .setTableSpec(options.getDeadLetterTableSpec()) - .setJsonSchema(ResourceUtil.getDeadletterTableSchemaJson()) - .build()); - } - break; - default: - log.error("Store type '{}' is not supported. No Feature Row will be written.", storeType); - break; - } - - input.apply( - "IncrementWriteToStoreElementsWrittenCounter", - MapElements.into(TypeDescriptors.booleans()) - .via( - (FeatureRow row) -> { - elementsWritten.inc(); - return true; - })); - - return PDone.in(input.getPipeline()); - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/KafkaRecordToFeatureRowDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/KafkaRecordToFeatureRowDoFn.java index 25aafd6ee71..c6ce9855e5f 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/KafkaRecordToFeatureRowDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/KafkaRecordToFeatureRowDoFn.java @@ -18,8 +18,7 @@ import com.google.auto.value.AutoValue; import com.google.protobuf.InvalidProtocolBufferException; -import feast.ingestion.transform.ReadFromSource.Builder; -import feast.ingestion.values.FailedElement; +import feast.storage.api.write.FailedElement; import feast.types.FeatureRowProto.FeatureRow; import java.util.Base64; import org.apache.beam.sdk.io.kafka.KafkaRecord; diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java index c31d3c535e9..933ea7d3894 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java @@ -17,9 +17,9 @@ package feast.ingestion.transform.fn; import com.google.auto.value.AutoValue; -import feast.ingestion.values.FailedElement; import feast.ingestion.values.FeatureSet; import feast.ingestion.values.Field; +import feast.storage.api.write.FailedElement; import feast.types.FeatureRowProto.FeatureRow; import feast.types.FieldProto; import feast.types.ValueProto.Value.ValCase; diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteDeadletterRowMetricsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteDeadletterRowMetricsDoFn.java index 687670c5cf0..6f256442373 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteDeadletterRowMetricsDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteDeadletterRowMetricsDoFn.java @@ -20,7 +20,7 @@ import com.timgroup.statsd.NonBlockingStatsDClient; import com.timgroup.statsd.StatsDClient; import com.timgroup.statsd.StatsDClientException; -import feast.ingestion.values.FailedElement; +import feast.storage.api.write.FailedElement; import org.apache.beam.sdk.transforms.DoFn; import org.slf4j.Logger; diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFailureMetricsTransform.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFailureMetricsTransform.java new file mode 100644 index 00000000000..a4227f6e59d --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFailureMetricsTransform.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.transform.metrics; + +import com.google.auto.value.AutoValue; +import feast.ingestion.options.ImportOptions; +import feast.storage.api.write.FailedElement; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PDone; + +@AutoValue +public abstract class WriteFailureMetricsTransform + extends PTransform, PDone> { + + public abstract String getStoreName(); + + public static WriteFailureMetricsTransform create(String storeName) { + return new AutoValue_WriteFailureMetricsTransform(storeName); + } + + @Override + public PDone expand(PCollection input) { + ImportOptions options = input.getPipeline().getOptions().as(ImportOptions.class); + if ("statsd".equals(options.getMetricsExporterType())) { + input.apply( + "WriteDeadletterMetrics", + ParDo.of( + WriteDeadletterRowMetricsDoFn.newBuilder() + .setStatsdHost(options.getStatsdHost()) + .setStatsdPort(options.getStatsdPort()) + .setStoreName(getStoreName()) + .build())); + } + return PDone.in(input.getPipeline()); + } +} diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteMetricsTransform.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteSuccessMetricsTransform.java similarity index 58% rename from ingestion/src/main/java/feast/ingestion/transform/metrics/WriteMetricsTransform.java rename to ingestion/src/main/java/feast/ingestion/transform/metrics/WriteSuccessMetricsTransform.java index 10322ac812f..098cf526355 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteMetricsTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteSuccessMetricsTransform.java @@ -18,71 +18,56 @@ import com.google.auto.value.AutoValue; import feast.ingestion.options.ImportOptions; -import feast.ingestion.values.FailedElement; import feast.types.FeatureRowProto.FeatureRow; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.GroupByKey; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.metrics.Counter; +import org.apache.beam.sdk.metrics.Metrics; +import org.apache.beam.sdk.transforms.*; import org.apache.beam.sdk.transforms.windowing.FixedWindows; import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TypeDescriptors; import org.joda.time.Duration; @AutoValue -public abstract class WriteMetricsTransform extends PTransform { +public abstract class WriteSuccessMetricsTransform + extends PTransform, PDone> { - public abstract String getStoreName(); - - public abstract TupleTag getSuccessTag(); - - public abstract TupleTag getFailureTag(); - - public static Builder newBuilder() { - return new AutoValue_WriteMetricsTransform.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setStoreName(String storeName); - - public abstract Builder setSuccessTag(TupleTag successTag); + public static final String METRIC_NAMESPACE = "WriteToStoreSuccess"; + public static final String ELEMENTS_WRITTEN_METRIC = "elements_written"; + private static final Counter elementsWritten = + Metrics.counter(METRIC_NAMESPACE, ELEMENTS_WRITTEN_METRIC); - public abstract Builder setFailureTag(TupleTag failureTag); + public abstract String getStoreName(); - public abstract WriteMetricsTransform build(); + public static WriteSuccessMetricsTransform create(String storeName) { + return new AutoValue_WriteSuccessMetricsTransform(storeName); } @Override - public PDone expand(PCollectionTuple input) { + public PDone expand(PCollection input) { ImportOptions options = input.getPipeline().getOptions().as(ImportOptions.class); + + input.apply( + "IncrementSuccessfulWriteToStoreElementsWrittenCounter", + MapElements.into(TypeDescriptors.booleans()) + .via( + (FeatureRow row) -> { + elementsWritten.inc(); + return true; + })); + switch (options.getMetricsExporterType()) { case "statsd": - input - .get(getFailureTag()) - .apply( - "WriteDeadletterMetrics", - ParDo.of( - WriteDeadletterRowMetricsDoFn.newBuilder() - .setStatsdHost(options.getStatsdHost()) - .setStatsdPort(options.getStatsdPort()) - .setStoreName(getStoreName()) - .build())); - - input - .get(getSuccessTag()) - .apply( - "WriteRowMetrics", - ParDo.of( - WriteRowMetricsDoFn.newBuilder() - .setStatsdHost(options.getStatsdHost()) - .setStatsdPort(options.getStatsdPort()) - .setStoreName(getStoreName()) - .build())); + input.apply( + "WriteRowMetrics", + ParDo.of( + WriteRowMetricsDoFn.newBuilder() + .setStatsdHost(options.getStatsdHost()) + .setStatsdPort(options.getStatsdPort()) + .setStoreName(getStoreName()) + .build())); // 1. Apply a fixed window // 2. Group feature row by feature set reference @@ -94,7 +79,6 @@ public PDone expand(PCollectionTuple input) { // metrics data. And for metric data, only statistic of the values are usually required // vs the actual values. input - .get(getSuccessTag()) .apply( "FixedWindow", Window.into( @@ -123,15 +107,13 @@ public void processElement(ProcessContext c, @Element FeatureRow featureRow) { return PDone.in(input.getPipeline()); case "none": default: - input - .get(getSuccessTag()) - .apply( - "Noop", - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement(ProcessContext c) {} - })); + input.apply( + "Noop", + ParDo.of( + new DoFn() { + @ProcessElement + public void processElement(ProcessContext c) {} + })); return PDone.in(input.getPipeline()); } } diff --git a/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java b/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java index 9163c5b2d6f..f28dfc9ee39 100644 --- a/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java +++ b/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java @@ -33,9 +33,10 @@ public class SpecUtil { - public static String getFeatureSetReference(FeatureSet featureSet) { - FeatureSetSpec spec = featureSet.getSpec(); - return String.format("%s/%s:%d", spec.getProject(), spec.getName(), spec.getVersion()); + public static String getFeatureSetReference(FeatureSetSpec featureSetSpec) { + return String.format( + "%s/%s:%d", + featureSetSpec.getProject(), featureSetSpec.getName(), featureSetSpec.getVersion()); } /** Get only feature set specs that matches the subscription */ diff --git a/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java b/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java index a02b8626945..99f3dc46650 100644 --- a/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java +++ b/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java @@ -18,39 +18,16 @@ import static feast.types.ValueProto.ValueType; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQueryOptions; -import com.google.cloud.bigquery.DatasetId; -import com.google.cloud.bigquery.DatasetInfo; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.Field.Builder; -import com.google.cloud.bigquery.Field.Mode; -import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; -import com.google.cloud.bigquery.StandardTableDefinition; -import com.google.cloud.bigquery.Table; -import com.google.cloud.bigquery.TableDefinition; -import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.TableInfo; -import com.google.cloud.bigquery.TimePartitioning; -import com.google.cloud.bigquery.TimePartitioning.Type; -import com.google.common.collect.ImmutableMap; -import feast.core.FeatureSetProto.EntitySpec; -import feast.core.FeatureSetProto.FeatureSet; import feast.core.FeatureSetProto.FeatureSetSpec; -import feast.core.FeatureSetProto.FeatureSpec; import feast.core.StoreProto.Store; -import feast.core.StoreProto.Store.RedisConfig; import feast.core.StoreProto.Store.StoreType; +import feast.storage.api.write.FeatureSink; +import feast.storage.connectors.bigquery.write.BigQueryFeatureSink; +import feast.storage.connectors.redis.write.RedisFeatureSink; import feast.types.ValueProto.ValueType.Enum; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisConnectionException; -import io.lettuce.core.RedisURI; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; -import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; // TODO: Create partitioned table by default @@ -101,155 +78,19 @@ public class StoreUtil { VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(Enum.BOOL_LIST, StandardSQLTypeName.BOOL); } - public static void setupStore(Store store, FeatureSet featureSet) { + public static FeatureSink getFeatureSink( + Store store, Map featureSetSpecs) { StoreType storeType = store.getType(); switch (storeType) { case REDIS: - StoreUtil.checkRedisConnection(store.getRedisConfig()); - break; + return RedisFeatureSink.builder() + .setRedisConfig(store.getRedisConfig()) + .setFeatureSetSpecs(featureSetSpecs) + .build(); case BIGQUERY: - StoreUtil.setupBigQuery( - featureSet, - store.getBigqueryConfig().getProjectId(), - store.getBigqueryConfig().getDatasetId(), - BigQueryOptions.getDefaultInstance().getService()); - break; + return BigQueryFeatureSink.fromConfig(store.getBigqueryConfig()); default: - log.warn("Store type '{}' is unsupported", storeType); - break; + throw new RuntimeException(String.format("Store type '{}' is unsupported", storeType)); } } - - @SuppressWarnings("DuplicatedCode") - public static TableDefinition createBigQueryTableDefinition(FeatureSetSpec featureSetSpec) { - List fields = new ArrayList<>(); - log.info("Table will have the following fields:"); - - for (EntitySpec entitySpec : featureSetSpec.getEntitiesList()) { - Builder builder = - Field.newBuilder( - entitySpec.getName(), VALUE_TYPE_TO_STANDARD_SQL_TYPE.get(entitySpec.getValueType())); - if (entitySpec.getValueType().name().toLowerCase().endsWith("_list")) { - builder.setMode(Mode.REPEATED); - } - Field field = builder.build(); - log.info("- {}", field.toString()); - fields.add(field); - } - for (FeatureSpec featureSpec : featureSetSpec.getFeaturesList()) { - Builder builder = - Field.newBuilder( - featureSpec.getName(), - VALUE_TYPE_TO_STANDARD_SQL_TYPE.get(featureSpec.getValueType())); - if (featureSpec.getValueType().name().toLowerCase().endsWith("_list")) { - builder.setMode(Mode.REPEATED); - } - Field field = builder.build(); - log.info("- {}", field.toString()); - fields.add(field); - } - - // Refer to protos/feast/core/Store.proto for reserved fields in BigQuery. - Map> - reservedFieldNameToPairOfStandardSQLTypeAndDescription = - ImmutableMap.of( - "event_timestamp", - Pair.of(StandardSQLTypeName.TIMESTAMP, BIGQUERY_EVENT_TIMESTAMP_FIELD_DESCRIPTION), - "created_timestamp", - Pair.of( - StandardSQLTypeName.TIMESTAMP, BIGQUERY_CREATED_TIMESTAMP_FIELD_DESCRIPTION), - "job_id", - Pair.of(StandardSQLTypeName.STRING, BIGQUERY_JOB_ID_FIELD_DESCRIPTION)); - for (Map.Entry> entry : - reservedFieldNameToPairOfStandardSQLTypeAndDescription.entrySet()) { - Field field = - Field.newBuilder(entry.getKey(), entry.getValue().getLeft()) - .setDescription(entry.getValue().getRight()) - .build(); - log.info("- {}", field.toString()); - fields.add(field); - } - - TimePartitioning timePartitioning = - TimePartitioning.newBuilder(Type.DAY).setField("event_timestamp").build(); - log.info("Table partitioning: " + timePartitioning.toString()); - - return StandardTableDefinition.newBuilder() - .setTimePartitioning(timePartitioning) - .setSchema(Schema.of(fields)) - .build(); - } - - /** - * This method ensures that, given a FeatureSetSpec object, the relevant BigQuery table is created - * with the correct schema. - * - *

Refer to protos/feast/core/Store.proto for the derivation of the table name and schema from - * a FeatureSetSpec object. - * - * @param featureSet FeatureSet object - * @param bigqueryProjectId BigQuery project id - * @param bigqueryDatasetId BigQuery dataset id - * @param bigquery BigQuery service object - */ - public static void setupBigQuery( - FeatureSet featureSet, - String bigqueryProjectId, - String bigqueryDatasetId, - BigQuery bigquery) { - - FeatureSetSpec featureSetSpec = featureSet.getSpec(); - // Ensure BigQuery dataset exists. - DatasetId datasetId = DatasetId.of(bigqueryProjectId, bigqueryDatasetId); - if (bigquery.getDataset(datasetId) == null) { - log.info("Creating dataset '{}' in project '{}'", datasetId.getDataset(), bigqueryProjectId); - bigquery.create(DatasetInfo.of(datasetId)); - } - - String tableName = - String.format( - "%s_%s_v%d", - featureSetSpec.getProject(), featureSetSpec.getName(), featureSetSpec.getVersion()) - .replaceAll("-", "_"); - TableId tableId = TableId.of(bigqueryProjectId, datasetId.getDataset(), tableName); - - // Return if there is an existing table - Table table = bigquery.getTable(tableId); - if (table != null) { - log.info( - "Writing to existing BigQuery table '{}:{}.{}'", - bigqueryProjectId, - datasetId.getDataset(), - tableName); - return; - } - - log.info( - "Creating table '{}' in dataset '{}' in project '{}'", - tableId.getTable(), - datasetId.getDataset(), - bigqueryProjectId); - TableDefinition tableDefinition = createBigQueryTableDefinition(featureSet.getSpec()); - TableInfo tableInfo = TableInfo.of(tableId, tableDefinition); - bigquery.create(tableInfo); - } - - /** - * Ensure Redis is accessible, else throw a RuntimeException. - * - * @param redisConfig Plase refer to feast.core.Store proto - */ - public static void checkRedisConnection(RedisConfig redisConfig) { - RedisClient redisClient = - RedisClient.create(RedisURI.create(redisConfig.getHost(), redisConfig.getPort())); - try { - redisClient.connect(); - } catch (RedisConnectionException e) { - throw new RuntimeException( - String.format( - "Failed to connect to Redis at host: '%s' port: '%d'. Please check that your Redis is running and accessible from Feast.", - redisConfig.getHost(), redisConfig.getPort())); - } - redisClient.shutdown(); - } } diff --git a/ingestion/src/main/java/feast/ingestion/values/FeatureSet.java b/ingestion/src/main/java/feast/ingestion/values/FeatureSet.java index bf07bcec966..758fbd0ba31 100644 --- a/ingestion/src/main/java/feast/ingestion/values/FeatureSet.java +++ b/ingestion/src/main/java/feast/ingestion/values/FeatureSet.java @@ -34,9 +34,9 @@ public class FeatureSet implements Serializable { private final Map fields; - public FeatureSet(FeatureSetProto.FeatureSet featureSet) { - this.reference = getFeatureSetReference(featureSet); - this.fields = getFieldsByName(featureSet.getSpec()); + public FeatureSet(FeatureSetProto.FeatureSetSpec featureSetSpec) { + this.reference = getFeatureSetReference(featureSetSpec); + this.fields = getFieldsByName(featureSetSpec); } public String getReference() { diff --git a/ingestion/src/main/java/feast/retry/BackOffExecutor.java b/ingestion/src/main/java/feast/retry/BackOffExecutor.java deleted file mode 100644 index 344c65ac424..00000000000 --- a/ingestion/src/main/java/feast/retry/BackOffExecutor.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.retry; - -import java.io.Serializable; -import org.apache.beam.sdk.util.BackOff; -import org.apache.beam.sdk.util.BackOffUtils; -import org.apache.beam.sdk.util.FluentBackoff; -import org.apache.beam.sdk.util.Sleeper; -import org.joda.time.Duration; - -public class BackOffExecutor implements Serializable { - - private final Integer maxRetries; - private final Duration initialBackOff; - - public BackOffExecutor(Integer maxRetries, Duration initialBackOff) { - this.maxRetries = maxRetries; - this.initialBackOff = initialBackOff; - } - - public void execute(Retriable retriable) throws Exception { - FluentBackoff backoff = - FluentBackoff.DEFAULT.withMaxRetries(maxRetries).withInitialBackoff(initialBackOff); - execute(retriable, backoff); - } - - private void execute(Retriable retriable, FluentBackoff backoff) throws Exception { - Sleeper sleeper = Sleeper.DEFAULT; - BackOff backOff = backoff.backoff(); - while (true) { - try { - retriable.execute(); - break; - } catch (Exception e) { - if (retriable.isExceptionRetriable(e) && BackOffUtils.next(sleeper, backOff)) { - retriable.cleanUpAfterFailure(); - } else { - throw e; - } - } - } - } -} diff --git a/ingestion/src/main/java/feast/retry/Retriable.java b/ingestion/src/main/java/feast/retry/Retriable.java deleted file mode 100644 index 30676fe8208..00000000000 --- a/ingestion/src/main/java/feast/retry/Retriable.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.retry; - -public interface Retriable { - void execute() throws Exception; - - Boolean isExceptionRetriable(Exception e); - - void cleanUpAfterFailure(); -} diff --git a/ingestion/src/main/java/feast/store/serving/bigquery/FeatureRowToTableRow.java b/ingestion/src/main/java/feast/store/serving/bigquery/FeatureRowToTableRow.java deleted file mode 100644 index b89cf832910..00000000000 --- a/ingestion/src/main/java/feast/store/serving/bigquery/FeatureRowToTableRow.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.store.serving.bigquery; - -import com.google.api.services.bigquery.model.TableRow; -import com.google.protobuf.util.Timestamps; -import feast.types.FeatureRowProto.FeatureRow; -import feast.types.FieldProto.Field; -import java.util.Base64; -import java.util.stream.Collectors; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.joda.time.Instant; - -// TODO: Validate FeatureRow against FeatureSetSpec -// i.e. that the value types in FeatureRow matches against those in FeatureSetSpec - -public class FeatureRowToTableRow implements SerializableFunction { - private static final String EVENT_TIMESTAMP_COLUMN = "event_timestamp"; - private static final String CREATED_TIMESTAMP_COLUMN = "created_timestamp"; - private static final String JOB_ID_COLUMN = "job_id"; - private final String jobId; - - public FeatureRowToTableRow(String jobId) { - this.jobId = jobId; - } - - public static String getEventTimestampColumn() { - return EVENT_TIMESTAMP_COLUMN; - } - - public TableRow apply(FeatureRow featureRow) { - - TableRow tableRow = new TableRow(); - tableRow.set(EVENT_TIMESTAMP_COLUMN, Timestamps.toString(featureRow.getEventTimestamp())); - tableRow.set(CREATED_TIMESTAMP_COLUMN, Instant.now().toString()); - tableRow.set(JOB_ID_COLUMN, jobId); - - for (Field field : featureRow.getFieldsList()) { - switch (field.getValue().getValCase()) { - case BYTES_VAL: - tableRow.set( - field.getName(), - Base64.getEncoder().encodeToString(field.getValue().getBytesVal().toByteArray())); - break; - case STRING_VAL: - tableRow.set(field.getName(), field.getValue().getStringVal()); - break; - case INT32_VAL: - tableRow.set(field.getName(), field.getValue().getInt32Val()); - break; - case INT64_VAL: - tableRow.set(field.getName(), field.getValue().getInt64Val()); - break; - case DOUBLE_VAL: - tableRow.set(field.getName(), field.getValue().getDoubleVal()); - break; - case FLOAT_VAL: - tableRow.set(field.getName(), field.getValue().getFloatVal()); - break; - case BOOL_VAL: - tableRow.set(field.getName(), field.getValue().getBoolVal()); - break; - case BYTES_LIST_VAL: - tableRow.set( - field.getName(), - field.getValue().getBytesListVal().getValList().stream() - .map(x -> Base64.getEncoder().encodeToString(x.toByteArray())) - .collect(Collectors.toList())); - break; - case STRING_LIST_VAL: - tableRow.set(field.getName(), field.getValue().getStringListVal().getValList()); - break; - case INT32_LIST_VAL: - tableRow.set(field.getName(), field.getValue().getInt32ListVal().getValList()); - break; - case INT64_LIST_VAL: - tableRow.set(field.getName(), field.getValue().getInt64ListVal().getValList()); - break; - case DOUBLE_LIST_VAL: - tableRow.set(field.getName(), field.getValue().getDoubleListVal().getValList()); - break; - case FLOAT_LIST_VAL: - tableRow.set(field.getName(), field.getValue().getFloatListVal().getValList()); - break; - case BOOL_LIST_VAL: - tableRow.set(field.getName(), field.getValue().getBytesListVal().getValList()); - break; - case VAL_NOT_SET: - break; - } - } - - return tableRow; - } -} diff --git a/ingestion/src/main/java/feast/store/serving/bigquery/GetTableDestination.java b/ingestion/src/main/java/feast/store/serving/bigquery/GetTableDestination.java deleted file mode 100644 index eb37db94498..00000000000 --- a/ingestion/src/main/java/feast/store/serving/bigquery/GetTableDestination.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.store.serving.bigquery; - -import com.google.api.services.bigquery.model.TimePartitioning; -import feast.types.FeatureRowProto.FeatureRow; -import org.apache.beam.sdk.io.gcp.bigquery.TableDestination; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.values.ValueInSingleWindow; - -public class GetTableDestination - implements SerializableFunction, TableDestination> { - - private String projectId; - private String datasetId; - - public GetTableDestination(String projectId, String datasetId) { - this.projectId = projectId; - this.datasetId = datasetId; - } - - @Override - public TableDestination apply(ValueInSingleWindow input) { - String[] split = input.getValue().getFeatureSet().split(":"); - String[] splitName = split[0].split("/"); - - TimePartitioning timePartitioning = - new TimePartitioning() - .setType("DAY") - .setField(FeatureRowToTableRow.getEventTimestampColumn()); - - return new TableDestination( - String.format( - "%s:%s.%s_%s_v%s", projectId, datasetId, splitName[0], splitName[1], split[1]), - String.format("Feast table for %s", input.getValue().getFeatureSet()), - timePartitioning); - } -} diff --git a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java deleted file mode 100644 index ca017c1f756..00000000000 --- a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.store.serving.redis; - -import feast.core.FeatureSetProto.EntitySpec; -import feast.core.FeatureSetProto.FeatureSet; -import feast.core.FeatureSetProto.FeatureSetSpec; -import feast.core.FeatureSetProto.FeatureSpec; -import feast.storage.RedisProto.RedisKey; -import feast.storage.RedisProto.RedisKey.Builder; -import feast.store.serving.redis.RedisCustomIO.Method; -import feast.store.serving.redis.RedisCustomIO.RedisMutation; -import feast.types.FeatureRowProto.FeatureRow; -import feast.types.FieldProto.Field; -import feast.types.ValueProto; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.beam.sdk.transforms.DoFn; -import org.slf4j.Logger; - -public class FeatureRowToRedisMutationDoFn extends DoFn { - - private static final Logger log = - org.slf4j.LoggerFactory.getLogger(FeatureRowToRedisMutationDoFn.class); - private Map featureSets; - - public FeatureRowToRedisMutationDoFn(Map featureSets) { - this.featureSets = featureSets; - } - - private RedisKey getKey(FeatureRow featureRow) { - FeatureSet featureSet = featureSets.get(featureRow.getFeatureSet()); - List entityNames = - featureSet.getSpec().getEntitiesList().stream() - .map(EntitySpec::getName) - .sorted() - .collect(Collectors.toList()); - - Map entityFields = new HashMap<>(); - Builder redisKeyBuilder = RedisKey.newBuilder().setFeatureSet(featureRow.getFeatureSet()); - for (Field field : featureRow.getFieldsList()) { - if (entityNames.contains(field.getName())) { - entityFields.putIfAbsent( - field.getName(), - Field.newBuilder().setName(field.getName()).setValue(field.getValue()).build()); - } - } - for (String entityName : entityNames) { - redisKeyBuilder.addEntities(entityFields.get(entityName)); - } - return redisKeyBuilder.build(); - } - - private byte[] getValue(FeatureRow featureRow) { - FeatureSetSpec spec = featureSets.get(featureRow.getFeatureSet()).getSpec(); - - List featureNames = - spec.getFeaturesList().stream().map(FeatureSpec::getName).collect(Collectors.toList()); - Map fieldValueOnlyMap = - featureRow.getFieldsList().stream() - .filter(field -> featureNames.contains(field.getName())) - .distinct() - .collect( - Collectors.toMap( - Field::getName, - field -> Field.newBuilder().setValue(field.getValue()).build())); - - List values = - featureNames.stream() - .sorted() - .map( - featureName -> - fieldValueOnlyMap.getOrDefault( - featureName, - Field.newBuilder().setValue(ValueProto.Value.getDefaultInstance()).build())) - .collect(Collectors.toList()); - - return FeatureRow.newBuilder() - .setEventTimestamp(featureRow.getEventTimestamp()) - .addAllFields(values) - .build() - .toByteArray(); - } - - /** Output a redis mutation object for every feature in the feature row. */ - @ProcessElement - public void processElement(ProcessContext context) { - FeatureRow featureRow = context.element(); - try { - byte[] key = getKey(featureRow).toByteArray(); - byte[] value = getValue(featureRow); - RedisMutation redisMutation = new RedisMutation(Method.SET, key, value, null, null); - context.output(redisMutation); - } catch (Exception e) { - log.error(e.getMessage(), e); - } - } -} diff --git a/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java b/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java deleted file mode 100644 index 633c2eb551d..00000000000 --- a/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java +++ /dev/null @@ -1,341 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.store.serving.redis; - -import feast.core.StoreProto; -import feast.ingestion.values.FailedElement; -import feast.retry.Retriable; -import io.lettuce.core.RedisConnectionException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.ExecutionException; -import org.apache.avro.reflect.Nullable; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.coders.DefaultCoder; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.transforms.windowing.GlobalWindow; -import org.apache.beam.sdk.values.PCollection; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.joda.time.Instant; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class RedisCustomIO { - - private static final int DEFAULT_BATCH_SIZE = 1000; - private static final int DEFAULT_TIMEOUT = 2000; - - private static final Logger log = LoggerFactory.getLogger(RedisCustomIO.class); - - private RedisCustomIO() {} - - public static Write write(StoreProto.Store store) { - return new Write(store); - } - - public enum Method { - - /** - * Use APPEND command. If key already exists and is a string, this command appends the value at - * the end of the string. - */ - APPEND, - - /** Use SET command. If key already holds a value, it is overwritten. */ - SET, - - /** - * Use LPUSH command. Insert value at the head of the list stored at key. If key does not exist, - * it is created as empty list before performing the push operations. When key holds a value - * that is not a list, an error is returned. - */ - LPUSH, - - /** - * Use RPUSH command. Insert value at the tail of the list stored at key. If key does not exist, - * it is created as empty list before performing the push operations. When key holds a value - * that is not a list, an error is returned. - */ - RPUSH, - - /** - * Use SADD command. Insert value into a set with a defined key. If key does not exist, it is - * created as empty set before performing the add operations. When key holds a value that is not - * a set, an error is returned. - */ - SADD, - - /** - * Use ZADD command. Adds all the specified members with the specified scores to the sorted set - * stored at key. It is possible to specify multiple score / member pairs. If a specified member - * is already a member of the sorted set, the score is updated and the element reinserted at the - * right position to ensure the correct ordering. - */ - ZADD - } - - @DefaultCoder(AvroCoder.class) - public static class RedisMutation { - - private Method method; - private byte[] key; - private byte[] value; - @Nullable private Long expiryMillis; - @Nullable private Long score; - - public RedisMutation() {} - - public RedisMutation( - Method method, - byte[] key, - byte[] value, - @Nullable Long expiryMillis, - @Nullable Long score) { - this.method = method; - this.key = key; - this.value = value; - this.expiryMillis = expiryMillis; - this.score = score; - } - - public Method getMethod() { - return method; - } - - public void setMethod(Method method) { - this.method = method; - } - - public byte[] getKey() { - return key; - } - - public void setKey(byte[] key) { - this.key = key; - } - - public byte[] getValue() { - return value; - } - - public void setValue(byte[] value) { - this.value = value; - } - - @Nullable - public Long getExpiryMillis() { - return expiryMillis; - } - - public void setExpiryMillis(@Nullable Long expiryMillis) { - this.expiryMillis = expiryMillis; - } - - @Nullable - public Long getScore() { - return score; - } - - public void setScore(@Nullable Long score) { - this.score = score; - } - } - - /** ServingStoreWrite data to a Redis server. */ - public static class Write - extends PTransform, PCollection> { - - private WriteDoFn dofn; - - private Write(StoreProto.Store store) { - this.dofn = new WriteDoFn(store); - } - - public Write withBatchSize(int batchSize) { - this.dofn.withBatchSize(batchSize); - return this; - } - - public Write withTimeout(int timeout) { - this.dofn.withTimeout(timeout); - return this; - } - - @Override - public PCollection expand(PCollection input) { - return input.apply(ParDo.of(dofn)); - } - - public static class WriteDoFn extends DoFn { - - private final List mutations = new ArrayList<>(); - private int batchSize = DEFAULT_BATCH_SIZE; - private int timeout = DEFAULT_TIMEOUT; - private RedisIngestionClient redisIngestionClient; - - WriteDoFn(StoreProto.Store store) { - if (store.getType() == StoreProto.Store.StoreType.REDIS) - this.redisIngestionClient = new RedisStandaloneIngestionClient(store.getRedisConfig()); - } - - public WriteDoFn withBatchSize(int batchSize) { - if (batchSize > 0) { - this.batchSize = batchSize; - } - return this; - } - - public WriteDoFn withTimeout(int timeout) { - if (timeout > 0) { - this.timeout = timeout; - } - return this; - } - - @Setup - public void setup() { - this.redisIngestionClient.setup(); - } - - @StartBundle - public void startBundle() { - try { - redisIngestionClient.connect(); - } catch (RedisConnectionException e) { - log.error("Connection to redis cannot be established ", e); - } - mutations.clear(); - } - - private void executeBatch() throws Exception { - this.redisIngestionClient - .getBackOffExecutor() - .execute( - new Retriable() { - @Override - public void execute() throws ExecutionException, InterruptedException { - if (!redisIngestionClient.isConnected()) { - redisIngestionClient.connect(); - } - mutations.forEach( - mutation -> { - writeRecord(mutation); - if (mutation.getExpiryMillis() != null - && mutation.getExpiryMillis() > 0) { - redisIngestionClient.pexpire( - mutation.getKey(), mutation.getExpiryMillis()); - } - }); - redisIngestionClient.sync(); - mutations.clear(); - } - - @Override - public Boolean isExceptionRetriable(Exception e) { - return e instanceof RedisConnectionException; - } - - @Override - public void cleanUpAfterFailure() {} - }); - } - - private FailedElement toFailedElement( - RedisMutation mutation, Exception exception, String jobName) { - return FailedElement.newBuilder() - .setJobName(jobName) - .setTransformName("RedisCustomIO") - .setPayload(Arrays.toString(mutation.getValue())) - .setErrorMessage(exception.getMessage()) - .setStackTrace(ExceptionUtils.getStackTrace(exception)) - .build(); - } - - @ProcessElement - public void processElement(ProcessContext context) { - RedisMutation mutation = context.element(); - mutations.add(mutation); - if (mutations.size() >= batchSize) { - try { - executeBatch(); - } catch (Exception e) { - mutations.forEach( - failedMutation -> { - FailedElement failedElement = - toFailedElement(failedMutation, e, context.getPipelineOptions().getJobName()); - context.output(failedElement); - }); - mutations.clear(); - } - } - } - - private void writeRecord(RedisMutation mutation) { - switch (mutation.getMethod()) { - case APPEND: - redisIngestionClient.append(mutation.getKey(), mutation.getValue()); - return; - case SET: - redisIngestionClient.set(mutation.getKey(), mutation.getValue()); - return; - case LPUSH: - redisIngestionClient.lpush(mutation.getKey(), mutation.getValue()); - return; - case RPUSH: - redisIngestionClient.rpush(mutation.getKey(), mutation.getValue()); - return; - case SADD: - redisIngestionClient.sadd(mutation.getKey(), mutation.getValue()); - return; - case ZADD: - redisIngestionClient.zadd(mutation.getKey(), mutation.getScore(), mutation.getValue()); - return; - default: - throw new UnsupportedOperationException( - String.format("Not implemented writing records for %s", mutation.getMethod())); - } - } - - @FinishBundle - public void finishBundle(FinishBundleContext context) - throws IOException, InterruptedException { - if (mutations.size() > 0) { - try { - executeBatch(); - } catch (Exception e) { - mutations.forEach( - failedMutation -> { - FailedElement failedElement = - toFailedElement(failedMutation, e, context.getPipelineOptions().getJobName()); - context.output(failedElement, Instant.now(), GlobalWindow.INSTANCE); - }); - mutations.clear(); - } - } - } - - @Teardown - public void teardown() { - redisIngestionClient.shutdown(); - } - } - } -} diff --git a/ingestion/src/main/java/feast/store/serving/redis/RedisIngestionClient.java b/ingestion/src/main/java/feast/store/serving/redis/RedisIngestionClient.java deleted file mode 100644 index d51eead53fb..00000000000 --- a/ingestion/src/main/java/feast/store/serving/redis/RedisIngestionClient.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.store.serving.redis; - -import feast.retry.BackOffExecutor; -import java.io.Serializable; - -public interface RedisIngestionClient extends Serializable { - - void setup(); - - BackOffExecutor getBackOffExecutor(); - - void shutdown(); - - void connect(); - - boolean isConnected(); - - void sync(); - - void pexpire(byte[] key, Long expiryMillis); - - void append(byte[] key, byte[] value); - - void set(byte[] key, byte[] value); - - void lpush(byte[] key, byte[] value); - - void rpush(byte[] key, byte[] value); - - void sadd(byte[] key, byte[] value); - - void zadd(byte[] key, Long score, byte[] value); -} diff --git a/ingestion/src/main/java/feast/store/serving/redis/RedisStandaloneIngestionClient.java b/ingestion/src/main/java/feast/store/serving/redis/RedisStandaloneIngestionClient.java deleted file mode 100644 index d95ebbbf64a..00000000000 --- a/ingestion/src/main/java/feast/store/serving/redis/RedisStandaloneIngestionClient.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.store.serving.redis; - -import com.google.common.collect.Lists; -import feast.core.StoreProto; -import feast.retry.BackOffExecutor; -import io.lettuce.core.*; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.async.RedisAsyncCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.joda.time.Duration; - -public class RedisStandaloneIngestionClient implements RedisIngestionClient { - private final String host; - private final int port; - private final BackOffExecutor backOffExecutor; - private RedisClient redisclient; - private static final int DEFAULT_TIMEOUT = 2000; - private StatefulRedisConnection connection; - private RedisAsyncCommands commands; - private List futures = Lists.newArrayList(); - - public RedisStandaloneIngestionClient(StoreProto.Store.RedisConfig redisConfig) { - this.host = redisConfig.getHost(); - this.port = redisConfig.getPort(); - long backoffMs = redisConfig.getInitialBackoffMs() > 0 ? redisConfig.getInitialBackoffMs() : 1; - this.backOffExecutor = - new BackOffExecutor(redisConfig.getMaxRetries(), Duration.millis(backoffMs)); - } - - @Override - public void setup() { - this.redisclient = - RedisClient.create(new RedisURI(host, port, java.time.Duration.ofMillis(DEFAULT_TIMEOUT))); - } - - @Override - public BackOffExecutor getBackOffExecutor() { - return this.backOffExecutor; - } - - @Override - public void shutdown() { - this.redisclient.shutdown(); - } - - @Override - public void connect() { - if (!isConnected()) { - this.connection = this.redisclient.connect(new ByteArrayCodec()); - this.commands = connection.async(); - } - } - - @Override - public boolean isConnected() { - return connection != null; - } - - @Override - public void sync() { - // Wait for some time for futures to complete - // TODO: should this be configurable? - try { - LettuceFutures.awaitAll(60, TimeUnit.SECONDS, futures.toArray(new RedisFuture[0])); - } finally { - futures.clear(); - } - } - - @Override - public void pexpire(byte[] key, Long expiryMillis) { - commands.pexpire(key, expiryMillis); - } - - @Override - public void append(byte[] key, byte[] value) { - futures.add(commands.append(key, value)); - } - - @Override - public void set(byte[] key, byte[] value) { - futures.add(commands.set(key, value)); - } - - @Override - public void lpush(byte[] key, byte[] value) { - futures.add(commands.lpush(key, value)); - } - - @Override - public void rpush(byte[] key, byte[] value) { - futures.add(commands.rpush(key, value)); - } - - @Override - public void sadd(byte[] key, byte[] value) { - futures.add(commands.sadd(key, value)); - } - - @Override - public void zadd(byte[] key, Long score, byte[] value) { - futures.add(commands.zadd(key, score, value)); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java index 0b000df0f59..13df73e96a4 100644 --- a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java +++ b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java @@ -188,8 +188,8 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow() IntStream.range(0, IMPORT_JOB_SAMPLE_FEATURE_ROW_SIZE) .forEach( i -> { - FeatureRow randomRow = TestUtil.createRandomFeatureRow(featureSet); - RedisKey redisKey = TestUtil.createRedisKey(featureSet, randomRow); + FeatureRow randomRow = TestUtil.createRandomFeatureRow(featureSet.getSpec()); + RedisKey redisKey = TestUtil.createRedisKey(featureSet.getSpec(), randomRow); input.add(randomRow); List fields = randomRow.getFieldsList().stream() diff --git a/ingestion/src/test/java/feast/ingestion/transform/ValidateFeatureRowsTest.java b/ingestion/src/test/java/feast/ingestion/transform/ValidateFeatureRowsTest.java index a78852ae6c6..ab5dbd93492 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/ValidateFeatureRowsTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/ValidateFeatureRowsTest.java @@ -17,10 +17,9 @@ package feast.ingestion.transform; import feast.core.FeatureSetProto.EntitySpec; -import feast.core.FeatureSetProto.FeatureSet; import feast.core.FeatureSetProto.FeatureSetSpec; import feast.core.FeatureSetProto.FeatureSpec; -import feast.ingestion.values.FailedElement; +import feast.storage.api.write.FailedElement; import feast.test.TestUtil; import feast.types.FeatureRowProto.FeatureRow; import feast.types.FieldProto.Field; @@ -50,73 +49,57 @@ public class ValidateFeatureRowsTest { @Test public void shouldWriteSuccessAndFailureTagsCorrectly() { - FeatureSet fs1 = - FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("feature_set") - .setVersion(1) - .setProject("myproject") - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_primary") - .setValueType(Enum.INT32) - .build()) - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_secondary") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature_1") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature_2") - .setValueType(Enum.INT64) - .build())) + FeatureSetSpec fs1 = + FeatureSetSpec.newBuilder() + .setName("feature_set") + .setVersion(1) + .setProject("myproject") + .addEntities( + EntitySpec.newBuilder() + .setName("entity_id_primary") + .setValueType(Enum.INT32) + .build()) + .addEntities( + EntitySpec.newBuilder() + .setName("entity_id_secondary") + .setValueType(Enum.STRING) + .build()) + .addFeatures( + FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build()) + .addFeatures( + FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build()) .build(); - FeatureSet fs2 = - FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("feature_set") - .setVersion(2) - .setProject("myproject") - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_primary") - .setValueType(Enum.INT32) - .build()) - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_secondary") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature_1") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature_2") - .setValueType(Enum.INT64) - .build())) + FeatureSetSpec fs2 = + FeatureSetSpec.newBuilder() + .setName("feature_set") + .setVersion(2) + .setProject("myproject") + .addEntities( + EntitySpec.newBuilder() + .setName("entity_id_primary") + .setValueType(Enum.INT32) + .build()) + .addEntities( + EntitySpec.newBuilder() + .setName("entity_id_secondary") + .setValueType(Enum.STRING) + .build()) + .addFeatures( + FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build()) + .addFeatures( + FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build()) .build(); - Map featureSets = new HashMap<>(); - featureSets.put("myproject/feature_set:1", fs1); - featureSets.put("myproject/feature_set:2", fs2); + Map featureSetSpecs = new HashMap<>(); + featureSetSpecs.put("myproject/feature_set:1", fs1); + featureSetSpecs.put("myproject/feature_set:2", fs2); List input = new ArrayList<>(); List expected = new ArrayList<>(); - for (FeatureSet featureSet : featureSets.values()) { - FeatureRow randomRow = TestUtil.createRandomFeatureRow(featureSet); + for (FeatureSetSpec featureSetSpec : featureSetSpecs.values()) { + FeatureRow randomRow = TestUtil.createRandomFeatureRow(featureSetSpec); input.add(randomRow); expected.add(randomRow); } @@ -130,7 +113,7 @@ public void shouldWriteSuccessAndFailureTagsCorrectly() { ValidateFeatureRows.newBuilder() .setFailureTag(FAILURE_TAG) .setSuccessTag(SUCCESS_TAG) - .setFeatureSets(featureSets) + .setFeatureSetSpecs(featureSetSpecs) .build()); PAssert.that(output.get(SUCCESS_TAG)).containsInAnyOrder(expected); @@ -141,36 +124,28 @@ public void shouldWriteSuccessAndFailureTagsCorrectly() { @Test public void shouldExcludeUnregisteredFields() { - FeatureSet fs1 = - FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("feature_set") - .setVersion(1) - .setProject("myproject") - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_primary") - .setValueType(Enum.INT32) - .build()) - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_secondary") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature_1") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature_2") - .setValueType(Enum.INT64) - .build())) + FeatureSetSpec fs1 = + FeatureSetSpec.newBuilder() + .setName("feature_set") + .setVersion(1) + .setProject("myproject") + .addEntities( + EntitySpec.newBuilder() + .setName("entity_id_primary") + .setValueType(Enum.INT32) + .build()) + .addEntities( + EntitySpec.newBuilder() + .setName("entity_id_secondary") + .setValueType(Enum.STRING) + .build()) + .addFeatures( + FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build()) + .addFeatures( + FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build()) .build(); - Map featureSets = new HashMap<>(); + Map featureSets = new HashMap<>(); featureSets.put("myproject/feature_set:1", fs1); List input = new ArrayList<>(); @@ -194,7 +169,7 @@ public void shouldExcludeUnregisteredFields() { ValidateFeatureRows.newBuilder() .setFailureTag(FAILURE_TAG) .setSuccessTag(SUCCESS_TAG) - .setFeatureSets(featureSets) + .setFeatureSetSpecs(featureSets) .build()); PAssert.that(output.get(SUCCESS_TAG)).containsInAnyOrder(expected); diff --git a/ingestion/src/test/java/feast/ingestion/utils/StoreUtilTest.java b/ingestion/src/test/java/feast/ingestion/utils/StoreUtilTest.java deleted file mode 100644 index 82988121bc8..00000000000 --- a/ingestion/src/test/java/feast/ingestion/utils/StoreUtilTest.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.utils; - -import static feast.types.ValueProto.ValueType.Enum.*; - -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.Field.Mode; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardSQLTypeName; -import feast.core.FeatureSetProto.EntitySpec; -import feast.core.FeatureSetProto.FeatureSet; -import feast.core.FeatureSetProto.FeatureSetSpec; -import feast.core.FeatureSetProto.FeatureSpec; -import java.util.Arrays; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; - -public class StoreUtilTest { - - @Test - public void setupBigQuery_shouldCreateTable_givenValidFeatureSetSpec() { - FeatureSet featureSet = - FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("feature_set_1") - .setVersion(1) - .setProject("feast-project") - .addEntities(EntitySpec.newBuilder().setName("entity_1").setValueType(INT32)) - .addFeatures(FeatureSpec.newBuilder().setName("feature_1").setValueType(INT32)) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_2").setValueType(STRING_LIST))) - .build(); - BigQuery mockedBigquery = Mockito.mock(BigQuery.class); - StoreUtil.setupBigQuery(featureSet, "project-1", "dataset_1", mockedBigquery); - } - - @Test - public void createBigQueryTableDefinition_shouldCreateCorrectSchema_givenValidFeatureSetSpec() { - FeatureSetSpec input = - FeatureSetSpec.newBuilder() - .addAllEntities( - Arrays.asList( - EntitySpec.newBuilder().setName("bytes_entity").setValueType(BYTES).build(), - EntitySpec.newBuilder().setName("string_entity").setValueType(STRING).build(), - EntitySpec.newBuilder().setName("int32_entity").setValueType(INT32).build(), - EntitySpec.newBuilder().setName("int64_entity").setValueType(INT64).build(), - EntitySpec.newBuilder().setName("double_entity").setValueType(DOUBLE).build(), - EntitySpec.newBuilder().setName("float_entity").setValueType(FLOAT).build(), - EntitySpec.newBuilder().setName("bool_entity").setValueType(BOOL).build(), - EntitySpec.newBuilder() - .setName("bytes_list_entity") - .setValueType(BYTES_LIST) - .build(), - EntitySpec.newBuilder() - .setName("string_list_entity") - .setValueType(STRING_LIST) - .build(), - EntitySpec.newBuilder() - .setName("int32_list_entity") - .setValueType(INT32_LIST) - .build(), - EntitySpec.newBuilder() - .setName("int64_list_entity") - .setValueType(INT64_LIST) - .build(), - EntitySpec.newBuilder() - .setName("double_list_entity") - .setValueType(DOUBLE_LIST) - .build(), - EntitySpec.newBuilder() - .setName("float_list_entity") - .setValueType(FLOAT_LIST) - .build(), - EntitySpec.newBuilder() - .setName("bool_list_entity") - .setValueType(BOOL_LIST) - .build())) - .addAllFeatures( - Arrays.asList( - FeatureSpec.newBuilder().setName("bytes_feature").setValueType(BYTES).build(), - FeatureSpec.newBuilder().setName("string_feature").setValueType(STRING).build(), - FeatureSpec.newBuilder().setName("int32_feature").setValueType(INT32).build(), - FeatureSpec.newBuilder().setName("int64_feature").setValueType(INT64).build(), - FeatureSpec.newBuilder().setName("double_feature").setValueType(DOUBLE).build(), - FeatureSpec.newBuilder().setName("float_feature").setValueType(FLOAT).build(), - FeatureSpec.newBuilder().setName("bool_feature").setValueType(BOOL).build(), - FeatureSpec.newBuilder() - .setName("bytes_list_feature") - .setValueType(BYTES_LIST) - .build(), - FeatureSpec.newBuilder() - .setName("string_list_feature") - .setValueType(STRING_LIST) - .build(), - FeatureSpec.newBuilder() - .setName("int32_list_feature") - .setValueType(INT32_LIST) - .build(), - FeatureSpec.newBuilder() - .setName("int64_list_feature") - .setValueType(INT64_LIST) - .build(), - FeatureSpec.newBuilder() - .setName("double_list_feature") - .setValueType(DOUBLE_LIST) - .build(), - FeatureSpec.newBuilder() - .setName("float_list_feature") - .setValueType(FLOAT_LIST) - .build(), - FeatureSpec.newBuilder() - .setName("bool_list_feature") - .setValueType(BOOL_LIST) - .build())) - .build(); - - Schema actual = StoreUtil.createBigQueryTableDefinition(input).getSchema(); - - Schema expected = - Schema.of( - Arrays.asList( - // Fields from entity - Field.newBuilder("bytes_entity", StandardSQLTypeName.BYTES).build(), - Field.newBuilder("string_entity", StandardSQLTypeName.STRING).build(), - Field.newBuilder("int32_entity", StandardSQLTypeName.INT64).build(), - Field.newBuilder("int64_entity", StandardSQLTypeName.INT64).build(), - Field.newBuilder("double_entity", StandardSQLTypeName.FLOAT64).build(), - Field.newBuilder("float_entity", StandardSQLTypeName.FLOAT64).build(), - Field.newBuilder("bool_entity", StandardSQLTypeName.BOOL).build(), - Field.newBuilder("bytes_list_entity", StandardSQLTypeName.BYTES) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("string_list_entity", StandardSQLTypeName.STRING) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("int32_list_entity", StandardSQLTypeName.INT64) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("int64_list_entity", StandardSQLTypeName.INT64) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("double_list_entity", StandardSQLTypeName.FLOAT64) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("float_list_entity", StandardSQLTypeName.FLOAT64) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("bool_list_entity", StandardSQLTypeName.BOOL) - .setMode(Mode.REPEATED) - .build(), - // Fields from feature - Field.newBuilder("bytes_feature", StandardSQLTypeName.BYTES).build(), - Field.newBuilder("string_feature", StandardSQLTypeName.STRING).build(), - Field.newBuilder("int32_feature", StandardSQLTypeName.INT64).build(), - Field.newBuilder("int64_feature", StandardSQLTypeName.INT64).build(), - Field.newBuilder("double_feature", StandardSQLTypeName.FLOAT64).build(), - Field.newBuilder("float_feature", StandardSQLTypeName.FLOAT64).build(), - Field.newBuilder("bool_feature", StandardSQLTypeName.BOOL).build(), - Field.newBuilder("bytes_list_feature", StandardSQLTypeName.BYTES) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("string_list_feature", StandardSQLTypeName.STRING) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("int32_list_feature", StandardSQLTypeName.INT64) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("int64_list_feature", StandardSQLTypeName.INT64) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("double_list_feature", StandardSQLTypeName.FLOAT64) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("float_list_feature", StandardSQLTypeName.FLOAT64) - .setMode(Mode.REPEATED) - .build(), - Field.newBuilder("bool_list_feature", StandardSQLTypeName.BOOL) - .setMode(Mode.REPEATED) - .build(), - // Reserved fields - Field.newBuilder("event_timestamp", StandardSQLTypeName.TIMESTAMP) - .setDescription(StoreUtil.BIGQUERY_EVENT_TIMESTAMP_FIELD_DESCRIPTION) - .build(), - Field.newBuilder("created_timestamp", StandardSQLTypeName.TIMESTAMP) - .setDescription(StoreUtil.BIGQUERY_CREATED_TIMESTAMP_FIELD_DESCRIPTION) - .build(), - Field.newBuilder("job_id", StandardSQLTypeName.STRING) - .setDescription(StoreUtil.BIGQUERY_JOB_ID_FIELD_DESCRIPTION) - .build())); - - Assert.assertEquals(expected, actual); - } -} diff --git a/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java deleted file mode 100644 index 86b4feae05f..00000000000 --- a/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java +++ /dev/null @@ -1,345 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.store.serving.redis; - -import static org.junit.Assert.*; - -import com.google.protobuf.Timestamp; -import feast.core.FeatureSetProto; -import feast.core.FeatureSetProto.EntitySpec; -import feast.core.FeatureSetProto.FeatureSetSpec; -import feast.core.FeatureSetProto.FeatureSpec; -import feast.storage.RedisProto.RedisKey; -import feast.store.serving.redis.RedisCustomIO.RedisMutation; -import feast.types.FeatureRowProto.FeatureRow; -import feast.types.FieldProto.Field; -import feast.types.ValueProto.Value; -import feast.types.ValueProto.ValueType.Enum; -import java.util.*; -import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.values.PCollection; -import org.junit.Rule; -import org.junit.Test; - -public class FeatureRowToRedisMutationDoFnTest { - - @Rule public transient TestPipeline p = TestPipeline.create(); - - private FeatureSetProto.FeatureSet fs = - FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("feature_set") - .setVersion(1) - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_primary") - .setValueType(Enum.INT32) - .build()) - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_secondary") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature_1") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature_2") - .setValueType(Enum.INT64) - .build())) - .build(); - - @Test - public void shouldConvertRowWithDuplicateEntitiesToValidKey() { - Map featureSets = new HashMap<>(); - featureSets.put("feature_set", fs); - - FeatureRow offendingRow = - FeatureRow.newBuilder() - .setFeatureSet("feature_set") - .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) - .addFields( - Field.newBuilder() - .setName("entity_id_primary") - .setValue(Value.newBuilder().setInt32Val(1))) - .addFields( - Field.newBuilder() - .setName("entity_id_primary") - .setValue(Value.newBuilder().setInt32Val(2))) - .addFields( - Field.newBuilder() - .setName("entity_id_secondary") - .setValue(Value.newBuilder().setStringVal("a"))) - .addFields( - Field.newBuilder() - .setName("feature_1") - .setValue(Value.newBuilder().setStringVal("strValue1"))) - .addFields( - Field.newBuilder() - .setName("feature_2") - .setValue(Value.newBuilder().setInt64Val(1001))) - .build(); - - PCollection output = - p.apply(Create.of(Collections.singletonList(offendingRow))) - .setCoder(ProtoCoder.of(FeatureRow.class)) - .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSets))); - - RedisKey expectedKey = - RedisKey.newBuilder() - .setFeatureSet("feature_set") - .addEntities( - Field.newBuilder() - .setName("entity_id_primary") - .setValue(Value.newBuilder().setInt32Val(1))) - .addEntities( - Field.newBuilder() - .setName("entity_id_secondary") - .setValue(Value.newBuilder().setStringVal("a"))) - .build(); - - FeatureRow expectedValue = - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) - .addFields(Field.newBuilder().setValue(Value.newBuilder().setStringVal("strValue1"))) - .addFields(Field.newBuilder().setValue(Value.newBuilder().setInt64Val(1001))) - .build(); - - PAssert.that(output) - .satisfies( - (SerializableFunction, Void>) - input -> { - input.forEach( - rm -> { - assert (Arrays.equals(rm.getKey(), expectedKey.toByteArray())); - assert (Arrays.equals(rm.getValue(), expectedValue.toByteArray())); - }); - return null; - }); - p.run(); - } - - @Test - public void shouldConvertRowWithOutOfOrderFieldsToValidKey() { - Map featureSets = new HashMap<>(); - featureSets.put("feature_set", fs); - - FeatureRow offendingRow = - FeatureRow.newBuilder() - .setFeatureSet("feature_set") - .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) - .addFields( - Field.newBuilder() - .setName("entity_id_secondary") - .setValue(Value.newBuilder().setStringVal("a"))) - .addFields( - Field.newBuilder() - .setName("entity_id_primary") - .setValue(Value.newBuilder().setInt32Val(1))) - .addFields( - Field.newBuilder() - .setName("feature_2") - .setValue(Value.newBuilder().setInt64Val(1001))) - .addFields( - Field.newBuilder() - .setName("feature_1") - .setValue(Value.newBuilder().setStringVal("strValue1"))) - .build(); - - PCollection output = - p.apply(Create.of(Collections.singletonList(offendingRow))) - .setCoder(ProtoCoder.of(FeatureRow.class)) - .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSets))); - - RedisKey expectedKey = - RedisKey.newBuilder() - .setFeatureSet("feature_set") - .addEntities( - Field.newBuilder() - .setName("entity_id_primary") - .setValue(Value.newBuilder().setInt32Val(1))) - .addEntities( - Field.newBuilder() - .setName("entity_id_secondary") - .setValue(Value.newBuilder().setStringVal("a"))) - .build(); - - List expectedFields = - Arrays.asList( - Field.newBuilder().setValue(Value.newBuilder().setStringVal("strValue1")).build(), - Field.newBuilder().setValue(Value.newBuilder().setInt64Val(1001)).build()); - FeatureRow expectedValue = - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) - .addAllFields(expectedFields) - .build(); - - PAssert.that(output) - .satisfies( - (SerializableFunction, Void>) - input -> { - input.forEach( - rm -> { - assert (Arrays.equals(rm.getKey(), expectedKey.toByteArray())); - assert (Arrays.equals(rm.getValue(), expectedValue.toByteArray())); - }); - return null; - }); - p.run(); - } - - @Test - public void shouldMergeDuplicateFeatureFields() { - Map featureSets = new HashMap<>(); - featureSets.put("feature_set", fs); - - FeatureRow featureRowWithDuplicatedFeatureFields = - FeatureRow.newBuilder() - .setFeatureSet("feature_set") - .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) - .addFields( - Field.newBuilder() - .setName("entity_id_primary") - .setValue(Value.newBuilder().setInt32Val(1))) - .addFields( - Field.newBuilder() - .setName("entity_id_secondary") - .setValue(Value.newBuilder().setStringVal("a"))) - .addFields( - Field.newBuilder() - .setName("feature_1") - .setValue(Value.newBuilder().setStringVal("strValue1"))) - .addFields( - Field.newBuilder() - .setName("feature_1") - .setValue(Value.newBuilder().setStringVal("strValue1"))) - .addFields( - Field.newBuilder() - .setName("feature_2") - .setValue(Value.newBuilder().setInt64Val(1001))) - .build(); - - PCollection output = - p.apply(Create.of(Collections.singletonList(featureRowWithDuplicatedFeatureFields))) - .setCoder(ProtoCoder.of(FeatureRow.class)) - .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSets))); - - RedisKey expectedKey = - RedisKey.newBuilder() - .setFeatureSet("feature_set") - .addEntities( - Field.newBuilder() - .setName("entity_id_primary") - .setValue(Value.newBuilder().setInt32Val(1))) - .addEntities( - Field.newBuilder() - .setName("entity_id_secondary") - .setValue(Value.newBuilder().setStringVal("a"))) - .build(); - - FeatureRow expectedValue = - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) - .addFields(Field.newBuilder().setValue(Value.newBuilder().setStringVal("strValue1"))) - .addFields(Field.newBuilder().setValue(Value.newBuilder().setInt64Val(1001))) - .build(); - - PAssert.that(output) - .satisfies( - (SerializableFunction, Void>) - input -> { - input.forEach( - rm -> { - assert (Arrays.equals(rm.getKey(), expectedKey.toByteArray())); - assert (Arrays.equals(rm.getValue(), expectedValue.toByteArray())); - }); - return null; - }); - p.run(); - } - - @Test - public void shouldPopulateMissingFeatureValuesWithDefaultInstance() { - Map featureSets = new HashMap<>(); - featureSets.put("feature_set", fs); - - FeatureRow featureRowWithDuplicatedFeatureFields = - FeatureRow.newBuilder() - .setFeatureSet("feature_set") - .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) - .addFields( - Field.newBuilder() - .setName("entity_id_primary") - .setValue(Value.newBuilder().setInt32Val(1))) - .addFields( - Field.newBuilder() - .setName("entity_id_secondary") - .setValue(Value.newBuilder().setStringVal("a"))) - .addFields( - Field.newBuilder() - .setName("feature_1") - .setValue(Value.newBuilder().setStringVal("strValue1"))) - .build(); - - PCollection output = - p.apply(Create.of(Collections.singletonList(featureRowWithDuplicatedFeatureFields))) - .setCoder(ProtoCoder.of(FeatureRow.class)) - .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSets))); - - RedisKey expectedKey = - RedisKey.newBuilder() - .setFeatureSet("feature_set") - .addEntities( - Field.newBuilder() - .setName("entity_id_primary") - .setValue(Value.newBuilder().setInt32Val(1))) - .addEntities( - Field.newBuilder() - .setName("entity_id_secondary") - .setValue(Value.newBuilder().setStringVal("a"))) - .build(); - - FeatureRow expectedValue = - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) - .addFields(Field.newBuilder().setValue(Value.newBuilder().setStringVal("strValue1"))) - .addFields(Field.newBuilder().setValue(Value.getDefaultInstance())) - .build(); - - PAssert.that(output) - .satisfies( - (SerializableFunction, Void>) - input -> { - input.forEach( - rm -> { - assert (Arrays.equals(rm.getKey(), expectedKey.toByteArray())); - assert (Arrays.equals(rm.getValue(), expectedValue.toByteArray())); - }); - return null; - }); - p.run(); - } -} diff --git a/ingestion/src/test/java/feast/store/serving/redis/RedisCustomIOTest.java b/ingestion/src/test/java/feast/store/serving/redis/RedisCustomIOTest.java deleted file mode 100644 index 75663d24a6a..00000000000 --- a/ingestion/src/test/java/feast/store/serving/redis/RedisCustomIOTest.java +++ /dev/null @@ -1,238 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.store.serving.redis; - -import static feast.test.TestUtil.field; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.MatcherAssert.assertThat; - -import feast.core.StoreProto; -import feast.storage.RedisProto.RedisKey; -import feast.store.serving.redis.RedisCustomIO.Method; -import feast.store.serving.redis.RedisCustomIO.RedisMutation; -import feast.types.FeatureRowProto.FeatureRow; -import feast.types.ValueProto.ValueType.Enum; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisStringCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Count; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.values.PCollection; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import redis.embedded.Redis; -import redis.embedded.RedisServer; - -public class RedisCustomIOTest { - @Rule public transient TestPipeline p = TestPipeline.create(); - - private static String REDIS_HOST = "localhost"; - private static int REDIS_PORT = 51234; - private Redis redis; - private RedisClient redisClient; - private RedisStringCommands sync; - - @Before - public void setUp() throws IOException { - redis = new RedisServer(REDIS_PORT); - redis.start(); - redisClient = - RedisClient.create(new RedisURI(REDIS_HOST, REDIS_PORT, java.time.Duration.ofMillis(2000))); - StatefulRedisConnection connection = redisClient.connect(new ByteArrayCodec()); - sync = connection.sync(); - } - - @After - public void teardown() { - redisClient.shutdown(); - redis.stop(); - } - - @Test - public void shouldWriteToRedis() { - StoreProto.Store.RedisConfig redisConfig = - StoreProto.Store.RedisConfig.newBuilder().setHost(REDIS_HOST).setPort(REDIS_PORT).build(); - HashMap kvs = new LinkedHashMap<>(); - kvs.put( - RedisKey.newBuilder() - .setFeatureSet("fs:1") - .addEntities(field("entity", 1, Enum.INT64)) - .build(), - FeatureRow.newBuilder() - .setFeatureSet("fs:1") - .addFields(field("entity", 1, Enum.INT64)) - .addFields(field("feature", "one", Enum.STRING)) - .build()); - kvs.put( - RedisKey.newBuilder() - .setFeatureSet("fs:1") - .addEntities(field("entity", 2, Enum.INT64)) - .build(), - FeatureRow.newBuilder() - .setFeatureSet("fs:1") - .addFields(field("entity", 2, Enum.INT64)) - .addFields(field("feature", "two", Enum.STRING)) - .build()); - - List featureRowWrites = - kvs.entrySet().stream() - .map( - kv -> - new RedisMutation( - Method.SET, - kv.getKey().toByteArray(), - kv.getValue().toByteArray(), - null, - null)) - .collect(Collectors.toList()); - - StoreProto.Store store = - StoreProto.Store.newBuilder() - .setRedisConfig(redisConfig) - .setType(StoreProto.Store.StoreType.REDIS) - .build(); - p.apply(Create.of(featureRowWrites)).apply(RedisCustomIO.write(store)); - p.run(); - - kvs.forEach( - (key, value) -> { - byte[] actual = sync.get(key.toByteArray()); - assertThat(actual, equalTo(value.toByteArray())); - }); - } - - @Test(timeout = 10000) - public void shouldRetryFailConnection() throws InterruptedException { - StoreProto.Store.RedisConfig redisConfig = - StoreProto.Store.RedisConfig.newBuilder() - .setHost(REDIS_HOST) - .setPort(REDIS_PORT) - .setMaxRetries(4) - .setInitialBackoffMs(2000) - .build(); - HashMap kvs = new LinkedHashMap<>(); - kvs.put( - RedisKey.newBuilder() - .setFeatureSet("fs:1") - .addEntities(field("entity", 1, Enum.INT64)) - .build(), - FeatureRow.newBuilder() - .setFeatureSet("fs:1") - .addFields(field("entity", 1, Enum.INT64)) - .addFields(field("feature", "one", Enum.STRING)) - .build()); - - List featureRowWrites = - kvs.entrySet().stream() - .map( - kv -> - new RedisMutation( - Method.SET, - kv.getKey().toByteArray(), - kv.getValue().toByteArray(), - null, - null)) - .collect(Collectors.toList()); - - StoreProto.Store store = - StoreProto.Store.newBuilder() - .setRedisConfig(redisConfig) - .setType(StoreProto.Store.StoreType.REDIS) - .build(); - PCollection failedElementCount = - p.apply(Create.of(featureRowWrites)) - .apply(RedisCustomIO.write(store)) - .apply(Count.globally()); - - redis.stop(); - final ScheduledThreadPoolExecutor redisRestartExecutor = new ScheduledThreadPoolExecutor(1); - ScheduledFuture scheduledRedisRestart = - redisRestartExecutor.schedule( - () -> { - redis.start(); - }, - 3, - TimeUnit.SECONDS); - - PAssert.that(failedElementCount).containsInAnyOrder(0L); - p.run(); - scheduledRedisRestart.cancel(true); - - kvs.forEach( - (key, value) -> { - byte[] actual = sync.get(key.toByteArray()); - assertThat(actual, equalTo(value.toByteArray())); - }); - } - - @Test - public void shouldProduceFailedElementIfRetryExceeded() { - StoreProto.Store.RedisConfig redisConfig = - StoreProto.Store.RedisConfig.newBuilder().setHost(REDIS_HOST).setPort(REDIS_PORT).build(); - HashMap kvs = new LinkedHashMap<>(); - kvs.put( - RedisKey.newBuilder() - .setFeatureSet("fs:1") - .addEntities(field("entity", 1, Enum.INT64)) - .build(), - FeatureRow.newBuilder() - .setFeatureSet("fs:1") - .addFields(field("entity", 1, Enum.INT64)) - .addFields(field("feature", "one", Enum.STRING)) - .build()); - - List featureRowWrites = - kvs.entrySet().stream() - .map( - kv -> - new RedisMutation( - Method.SET, - kv.getKey().toByteArray(), - kv.getValue().toByteArray(), - null, - null)) - .collect(Collectors.toList()); - - StoreProto.Store store = - StoreProto.Store.newBuilder() - .setRedisConfig(redisConfig) - .setType(StoreProto.Store.StoreType.REDIS) - .build(); - PCollection failedElementCount = - p.apply(Create.of(featureRowWrites)) - .apply(RedisCustomIO.write(store)) - .apply(Count.globally()); - - redis.stop(); - PAssert.that(failedElementCount).containsInAnyOrder(1L); - p.run(); - } -} diff --git a/ingestion/src/test/java/feast/test/TestUtil.java b/ingestion/src/test/java/feast/test/TestUtil.java index 5c16d7e9e31..1c9e5bdc555 100644 --- a/ingestion/src/test/java/feast/test/TestUtil.java +++ b/ingestion/src/test/java/feast/test/TestUtil.java @@ -21,20 +21,13 @@ import com.google.protobuf.ByteString; import com.google.protobuf.util.Timestamps; import feast.core.FeatureSetProto.FeatureSet; -import feast.ingestion.transform.WriteToStore; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.ingestion.transform.metrics.WriteSuccessMetricsTransform; import feast.storage.RedisProto.RedisKey; import feast.types.FeatureRowProto.FeatureRow; import feast.types.FeatureRowProto.FeatureRow.Builder; import feast.types.FieldProto.Field; -import feast.types.ValueProto.BoolList; -import feast.types.ValueProto.BytesList; -import feast.types.ValueProto.DoubleList; -import feast.types.ValueProto.FloatList; -import feast.types.ValueProto.Int32List; -import feast.types.ValueProto.Int64List; -import feast.types.ValueProto.StringList; -import feast.types.ValueProto.Value; -import feast.types.ValueProto.ValueType; +import feast.types.ValueProto.*; import java.io.IOException; import java.util.List; import java.util.Properties; @@ -168,12 +161,12 @@ public static void publishFeatureRowsToKafka( /** * Create a Feature Row with random value according to the FeatureSetSpec * - *

See {@link #createRandomFeatureRow(FeatureSet, int)} + *

See {@link #createRandomFeatureRow(FeatureSetSpec, int)} */ - public static FeatureRow createRandomFeatureRow(FeatureSet featureSet) { + public static FeatureRow createRandomFeatureRow(FeatureSetSpec featureSetSpec) { ThreadLocalRandom random = ThreadLocalRandom.current(); int randomStringSizeMaxSize = 12; - return createRandomFeatureRow(featureSet, random.nextInt(0, randomStringSizeMaxSize) + 4); + return createRandomFeatureRow(featureSetSpec, random.nextInt(0, randomStringSizeMaxSize) + 4); } /** @@ -182,18 +175,18 @@ public static FeatureRow createRandomFeatureRow(FeatureSet featureSet) { *

The Feature Row created contains fields according to the entities and features defined in * FeatureSet, matching the value type of the field, with randomized value for testing. * - * @param featureSet {@link FeatureSet} + * @param featureSetSpec {@link FeatureSetSpec} * @param randomStringSize number of characters for the generated random string * @return {@link FeatureRow} */ - public static FeatureRow createRandomFeatureRow(FeatureSet featureSet, int randomStringSize) { + public static FeatureRow createRandomFeatureRow( + FeatureSetSpec featureSetSpec, int randomStringSize) { Builder builder = FeatureRow.newBuilder() - .setFeatureSet(getFeatureSetReference(featureSet)) + .setFeatureSet(getFeatureSetReference(featureSetSpec)) .setEventTimestamp(Timestamps.fromMillis(System.currentTimeMillis())); - featureSet - .getSpec() + featureSetSpec .getEntitiesList() .forEach( field -> { @@ -204,8 +197,7 @@ public static FeatureRow createRandomFeatureRow(FeatureSet featureSet, int rando .build()); }); - featureSet - .getSpec() + featureSetSpec .getFeaturesList() .forEach( field -> { @@ -295,15 +287,14 @@ public static Value createRandomValue(ValueType.Enum type, int randomStringSize) *

The entities in the created {@link RedisKey} will contain the value with matching field name * in the {@link FeatureRow} * - * @param featureSet {@link FeatureSet} + * @param featureSetSpec {@link FeatureSetSpec} * @param row {@link FeatureSet} * @return {@link RedisKey} */ - public static RedisKey createRedisKey(FeatureSet featureSet, FeatureRow row) { + public static RedisKey createRedisKey(FeatureSetSpec featureSetSpec, FeatureRow row) { RedisKey.Builder builder = - RedisKey.newBuilder().setFeatureSet(getFeatureSetReference(featureSet)); - featureSet - .getSpec() + RedisKey.newBuilder().setFeatureSet(getFeatureSetReference(featureSetSpec)); + featureSetSpec .getEntitiesList() .forEach( entityField -> @@ -389,7 +380,9 @@ public static void waitUntilAllElementsAreWrittenToStore( } String writeToStoreMetric = - WriteToStore.METRIC_NAMESPACE + ":" + WriteToStore.ELEMENTS_WRITTEN_METRIC; + WriteSuccessMetricsTransform.METRIC_NAMESPACE + + ":" + + WriteSuccessMetricsTransform.ELEMENTS_WRITTEN_METRIC; long committed = 0; long maxSystemTimeMillis = System.currentTimeMillis() + maxWaitDuration.getMillis(); diff --git a/pom.xml b/pom.xml index 307e6918982..29354ef4c1b 100644 --- a/pom.xml +++ b/pom.xml @@ -29,12 +29,12 @@ datatypes/java + storage/api + storage/connectors ingestion core serving sdk/java - storage/api - storage/connectors diff --git a/serving/pom.xml b/serving/pom.xml index 4cc02dc4510..3c33d6ab3d5 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -76,6 +76,39 @@ ${project.version} + + dev.feast + feast-storage-api + ${project.version} + + + + dev.feast + feast-storage-connector-redis + ${project.version} + + + + dev.feast + feast-storage-connector-bigquery + ${project.version} + + + org.apache.beam + * + + + org.apache.beam + * + + + + + + com.google.cloud + google-cloud-storage + + org.slf4j @@ -114,6 +147,7 @@ io.github.lognet grpc-spring-boot-starter + org.springframework.boot @@ -136,17 +170,6 @@ protobuf-java-util - - io.pebbletemplates - pebble - 3.1.0 - - - - io.lettuce - lettuce-core - - com.google.guava @@ -180,12 +203,14 @@ simpleclient 0.8.0 + io.prometheus simpleclient_hotspot 0.8.0 + io.prometheus @@ -198,17 +223,6 @@ 0.8.0 - - - com.google.cloud - google-cloud-bigquery - - - - com.google.cloud - google-cloud-storage - - com.google.auto.value auto-value-annotations diff --git a/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java b/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java index d0ea058baf4..e9819c275fe 100644 --- a/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java +++ b/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java @@ -25,12 +25,16 @@ import feast.core.StoreProto.Store.RedisConfig; import feast.core.StoreProto.Store.Subscription; import feast.serving.FeastProperties; -import feast.serving.service.BigQueryServingService; +import feast.serving.service.BatchServingService; import feast.serving.service.JobService; import feast.serving.service.NoopJobService; -import feast.serving.service.RedisServingService; +import feast.serving.service.OnlineServingService; import feast.serving.service.ServingService; import feast.serving.specs.CachedSpecService; +import feast.storage.api.retrieval.BatchRetriever; +import feast.storage.api.retrieval.OnlineRetriever; +import feast.storage.connectors.bigquery.retrieval.BigQueryBatchRetriever; +import feast.storage.connectors.redis.retrieval.RedisOnlineRetriever; import io.opentracing.Tracer; import java.util.Map; import org.slf4j.Logger; @@ -79,9 +83,9 @@ public ServingService servingService( switch (store.getType()) { case REDIS: - servingService = - new RedisServingService( - storeConfiguration.getServingRedisConnection(), specService, tracer); + OnlineRetriever redisRetriever = + new RedisOnlineRetriever(storeConfiguration.getServingRedisConnection()); + servingService = new OnlineServingService(redisRetriever, specService, tracer); break; case BIGQUERY: BigQueryConfig bqConfig = store.getBigqueryConfig(); @@ -104,17 +108,20 @@ public ServingService servingService( throw new IllegalArgumentException( "Unable to instantiate jobService for BigQuery store."); } - servingService = - new BigQueryServingService( - bigquery, - bqConfig.getProjectId(), - bqConfig.getDatasetId(), - specService, - jobService, - jobStagingLocation, - feastProperties.getJobs().getBigqueryInitialRetryDelaySecs(), - feastProperties.getJobs().getBigqueryTotalTimeoutSecs(), - storage); + + BatchRetriever bqRetriever = + BigQueryBatchRetriever.builder() + .setBigquery(bigquery) + .setDatasetId(bqConfig.getDatasetId()) + .setProjectId(bqConfig.getProjectId()) + .setJobStagingLocation(jobStagingLocation) + .setInitialRetryDelaySecs( + feastProperties.getJobs().getBigqueryInitialRetryDelaySecs()) + .setTotalTimeoutSecs(feastProperties.getJobs().getBigqueryTotalTimeoutSecs()) + .setStorage(storage) + .build(); + + servingService = new BatchServingService(bqRetriever, specService, jobService); break; case CASSANDRA: case UNRECOGNIZED: diff --git a/serving/src/main/java/feast/serving/encoding/FeatureRowDecoder.java b/serving/src/main/java/feast/serving/encoding/FeatureRowDecoder.java deleted file mode 100644 index e70695d8c64..00000000000 --- a/serving/src/main/java/feast/serving/encoding/FeatureRowDecoder.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.encoding; - -import feast.core.FeatureSetProto.FeatureSetSpec; -import feast.core.FeatureSetProto.FeatureSpec; -import feast.types.FeatureRowProto.FeatureRow; -import feast.types.FieldProto.Field; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -public class FeatureRowDecoder { - - private final String featureSetRef; - private final FeatureSetSpec spec; - - public FeatureRowDecoder(String featureSetRef, FeatureSetSpec spec) { - this.featureSetRef = featureSetRef; - this.spec = spec; - } - - /** - * A feature row is considered encoded if the feature set and field names are not set. This method - * is required for backward compatibility purposes, to allow Feast serving to continue serving non - * encoded Feature Row ingested by an older version of Feast. - * - * @param featureRow Feature row - * @return boolean - */ - public Boolean isEncoded(FeatureRow featureRow) { - return featureRow.getFeatureSet().isEmpty() - && featureRow.getFieldsList().stream().allMatch(field -> field.getName().isEmpty()); - } - - /** - * Validates if an encoded feature row can be decoded without exception. - * - * @param featureRow Feature row - * @return boolean - */ - public Boolean isEncodingValid(FeatureRow featureRow) { - return featureRow.getFieldsList().size() == spec.getFeaturesList().size(); - } - - /** - * Decoding feature row by repopulating the field names based on the corresponding feature set - * spec. - * - * @param encodedFeatureRow Feature row - * @return boolean - */ - public FeatureRow decode(FeatureRow encodedFeatureRow) { - final List fieldsWithoutName = encodedFeatureRow.getFieldsList(); - - List featureNames = - spec.getFeaturesList().stream() - .sorted(Comparator.comparing(FeatureSpec::getName)) - .map(FeatureSpec::getName) - .collect(Collectors.toList()); - List fields = - IntStream.range(0, featureNames.size()) - .mapToObj( - featureNameIndex -> { - String featureName = featureNames.get(featureNameIndex); - return fieldsWithoutName - .get(featureNameIndex) - .toBuilder() - .setName(featureName) - .build(); - }) - .collect(Collectors.toList()); - return encodedFeatureRow - .toBuilder() - .clearFields() - .setFeatureSet(featureSetRef) - .addAllFields(fields) - .build(); - } -} diff --git a/serving/src/main/java/feast/serving/service/BatchServingService.java b/serving/src/main/java/feast/serving/service/BatchServingService.java new file mode 100644 index 00000000000..b33a9b4ad7a --- /dev/null +++ b/serving/src/main/java/feast/serving/service/BatchServingService.java @@ -0,0 +1,89 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.service; + +import feast.serving.ServingAPIProto; +import feast.serving.ServingAPIProto.*; +import feast.serving.specs.CachedSpecService; +import feast.storage.api.retrieval.BatchRetriever; +import feast.storage.api.retrieval.FeatureSetRequest; +import feast.storage.connectors.bigquery.retrieval.BigQueryBatchRetriever; +import io.grpc.Status; +import java.util.List; +import java.util.Optional; +import org.slf4j.Logger; + +public class BatchServingService implements ServingService { + + private static final Logger log = org.slf4j.LoggerFactory.getLogger(BatchServingService.class); + + private final BatchRetriever retriever; + private final CachedSpecService specService; + private final JobService jobService; + + public BatchServingService( + BatchRetriever retriever, CachedSpecService specService, JobService jobService) { + this.retriever = retriever; + this.specService = specService; + this.jobService = jobService; + } + + /** {@inheritDoc} */ + @Override + public GetFeastServingInfoResponse getFeastServingInfo( + GetFeastServingInfoRequest getFeastServingInfoRequest) { + try { + BigQueryBatchRetriever bigQueryBatchRetriever = (BigQueryBatchRetriever) retriever; + return GetFeastServingInfoResponse.newBuilder() + .setType(FeastServingType.FEAST_SERVING_TYPE_BATCH) + .setJobStagingLocation(bigQueryBatchRetriever.jobStagingLocation()) + .build(); + } catch (Exception e) { + return GetFeastServingInfoResponse.newBuilder() + .setType(FeastServingType.FEAST_SERVING_TYPE_BATCH) + .build(); + } + } + + /** {@inheritDoc} */ + @Override + public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest getFeaturesRequest) { + throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); + } + + /** {@inheritDoc} */ + @Override + public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { + List featureSetRequests = + specService.getFeatureSets(getFeaturesRequest.getFeaturesList()); + Job feastJob = retriever.getBatchFeatures(getFeaturesRequest, featureSetRequests); + jobService.upsert(feastJob); + return GetBatchFeaturesResponse.newBuilder().setJob(feastJob).build(); + } + + /** {@inheritDoc} */ + @Override + public GetJobResponse getJob(GetJobRequest getJobRequest) { + Optional job = jobService.get(getJobRequest.getJob().getId()); + if (!job.isPresent()) { + throw Status.NOT_FOUND + .withDescription(String.format("Job not found: %s", getJobRequest.getJob().getId())) + .asRuntimeException(); + } + return GetJobResponse.newBuilder().setJob(job.get()).build(); + } +} diff --git a/serving/src/main/java/feast/serving/service/BigQueryServingService.java b/serving/src/main/java/feast/serving/service/BigQueryServingService.java deleted file mode 100644 index 8e3b7ae53e4..00000000000 --- a/serving/src/main/java/feast/serving/service/BigQueryServingService.java +++ /dev/null @@ -1,282 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import static feast.serving.store.bigquery.QueryTemplater.createEntityTableUUIDQuery; -import static feast.serving.store.bigquery.QueryTemplater.generateFullTableName; - -import com.google.cloud.RetryOption; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.FormatOptions; -import com.google.cloud.bigquery.Job; -import com.google.cloud.bigquery.JobInfo; -import com.google.cloud.bigquery.LoadJobConfiguration; -import com.google.cloud.bigquery.QueryJobConfiguration; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.Table; -import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.TableInfo; -import com.google.cloud.storage.Storage; -import feast.serving.ServingAPIProto; -import feast.serving.ServingAPIProto.DataFormat; -import feast.serving.ServingAPIProto.DatasetSource; -import feast.serving.ServingAPIProto.FeastServingType; -import feast.serving.ServingAPIProto.GetBatchFeaturesRequest; -import feast.serving.ServingAPIProto.GetBatchFeaturesResponse; -import feast.serving.ServingAPIProto.GetFeastServingInfoRequest; -import feast.serving.ServingAPIProto.GetFeastServingInfoResponse; -import feast.serving.ServingAPIProto.GetJobRequest; -import feast.serving.ServingAPIProto.GetJobResponse; -import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest; -import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.serving.ServingAPIProto.JobStatus; -import feast.serving.ServingAPIProto.JobType; -import feast.serving.specs.CachedSpecService; -import feast.serving.specs.FeatureSetRequest; -import feast.serving.store.bigquery.BatchRetrievalQueryRunnable; -import feast.serving.store.bigquery.QueryTemplater; -import feast.serving.store.bigquery.model.FeatureSetInfo; -import io.grpc.Status; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.threeten.bp.Duration; - -public class BigQueryServingService implements ServingService { - - public static final long TEMP_TABLE_EXPIRY_DURATION_MS = Duration.ofDays(1).toMillis(); - private static final Logger log = org.slf4j.LoggerFactory.getLogger(BigQueryServingService.class); - - private final BigQuery bigquery; - private final String projectId; - private final String datasetId; - private final CachedSpecService specService; - private final JobService jobService; - private final String jobStagingLocation; - private final int initialRetryDelaySecs; - private final int totalTimeoutSecs; - private final Storage storage; - - public BigQueryServingService( - BigQuery bigquery, - String projectId, - String datasetId, - CachedSpecService specService, - JobService jobService, - String jobStagingLocation, - int initialRetryDelaySecs, - int totalTimeoutSecs, - Storage storage) { - this.bigquery = bigquery; - this.projectId = projectId; - this.datasetId = datasetId; - this.specService = specService; - this.jobService = jobService; - this.jobStagingLocation = jobStagingLocation; - this.initialRetryDelaySecs = initialRetryDelaySecs; - this.totalTimeoutSecs = totalTimeoutSecs; - this.storage = storage; - } - - /** {@inheritDoc} */ - @Override - public GetFeastServingInfoResponse getFeastServingInfo( - GetFeastServingInfoRequest getFeastServingInfoRequest) { - return GetFeastServingInfoResponse.newBuilder() - .setType(FeastServingType.FEAST_SERVING_TYPE_BATCH) - .setJobStagingLocation(jobStagingLocation) - .build(); - } - - /** {@inheritDoc} */ - @Override - public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest getFeaturesRequest) { - throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); - } - - /** {@inheritDoc} */ - @Override - public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { - List featureSetRequests = - specService.getFeatureSets(getFeaturesRequest.getFeaturesList()); - - Table entityTable; - String entityTableName; - try { - entityTable = loadEntities(getFeaturesRequest.getDatasetSource()); - - TableId entityTableWithUUIDs = generateUUIDs(entityTable); - entityTableName = generateFullTableName(entityTableWithUUIDs); - } catch (Exception e) { - throw Status.INTERNAL - .withDescription("Unable to load entity dataset to Bigquery") - .asRuntimeException(); - } - - Schema entityTableSchema = entityTable.getDefinition().getSchema(); - List entityNames = - entityTableSchema.getFields().stream() - .map(Field::getName) - .filter(name -> !name.equals("event_timestamp")) - .collect(Collectors.toList()); - - List featureSetInfos = QueryTemplater.getFeatureSetInfos(featureSetRequests); - - String feastJobId = UUID.randomUUID().toString(); - ServingAPIProto.Job feastJob = - ServingAPIProto.Job.newBuilder() - .setId(feastJobId) - .setType(JobType.JOB_TYPE_DOWNLOAD) - .setStatus(JobStatus.JOB_STATUS_PENDING) - .build(); - jobService.upsert(feastJob); - - new Thread( - BatchRetrievalQueryRunnable.builder() - .setEntityTableName(entityTableName) - .setBigquery(bigquery) - .setStorage(storage) - .setJobService(jobService) - .setProjectId(projectId) - .setDatasetId(datasetId) - .setFeastJobId(feastJobId) - .setEntityTableColumnNames(entityNames) - .setFeatureSetInfos(featureSetInfos) - .setJobStagingLocation(jobStagingLocation) - .setInitialRetryDelaySecs(initialRetryDelaySecs) - .setTotalTimeoutSecs(totalTimeoutSecs) - .build()) - .start(); - - return GetBatchFeaturesResponse.newBuilder().setJob(feastJob).build(); - } - - /** {@inheritDoc} */ - @Override - public GetJobResponse getJob(GetJobRequest getJobRequest) { - Optional job = jobService.get(getJobRequest.getJob().getId()); - if (!job.isPresent()) { - throw Status.NOT_FOUND - .withDescription(String.format("Job not found: %s", getJobRequest.getJob().getId())) - .asRuntimeException(); - } - return GetJobResponse.newBuilder().setJob(job.get()).build(); - } - - private Table loadEntities(DatasetSource datasetSource) { - Table loadedEntityTable; - switch (datasetSource.getDatasetSourceCase()) { - case FILE_SOURCE: - try { - // Currently only AVRO format is supported - - if (datasetSource.getFileSource().getDataFormat() != DataFormat.DATA_FORMAT_AVRO) { - throw Status.INVALID_ARGUMENT - .withDescription("Invalid file format, only AVRO is supported.") - .asRuntimeException(); - } - - TableId tableId = TableId.of(projectId, datasetId, createTempTableName()); - log.info("Loading entity rows to: {}.{}.{}", projectId, datasetId, tableId.getTable()); - - LoadJobConfiguration loadJobConfiguration = - LoadJobConfiguration.of( - tableId, datasetSource.getFileSource().getFileUrisList(), FormatOptions.avro()); - loadJobConfiguration = - loadJobConfiguration.toBuilder().setUseAvroLogicalTypes(true).build(); - Job job = bigquery.create(JobInfo.of(loadJobConfiguration)); - waitForJob(job); - - TableInfo expiry = - bigquery - .getTable(tableId) - .toBuilder() - .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) - .build(); - bigquery.update(expiry); - - loadedEntityTable = bigquery.getTable(tableId); - if (!loadedEntityTable.exists()) { - throw new RuntimeException( - "Unable to create entity dataset table, table already exists"); - } - return loadedEntityTable; - } catch (Exception e) { - log.error("Exception has occurred in loadEntities method: ", e); - throw Status.INTERNAL - .withDescription("Failed to load entity dataset into store: " + e.toString()) - .withCause(e) - .asRuntimeException(); - } - case DATASETSOURCE_NOT_SET: - default: - throw Status.INVALID_ARGUMENT - .withDescription("Data source must be set.") - .asRuntimeException(); - } - } - - private TableId generateUUIDs(Table loadedEntityTable) { - try { - String uuidQuery = - createEntityTableUUIDQuery(generateFullTableName(loadedEntityTable.getTableId())); - QueryJobConfiguration queryJobConfig = - QueryJobConfiguration.newBuilder(uuidQuery) - .setDestinationTable(TableId.of(projectId, datasetId, createTempTableName())) - .build(); - Job queryJob = bigquery.create(JobInfo.of(queryJobConfig)); - Job completedJob = waitForJob(queryJob); - TableInfo expiry = - bigquery - .getTable(queryJobConfig.getDestinationTable()) - .toBuilder() - .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) - .build(); - bigquery.update(expiry); - queryJobConfig = completedJob.getConfiguration(); - return queryJobConfig.getDestinationTable(); - } catch (InterruptedException | BigQueryException e) { - throw Status.INTERNAL - .withDescription("Failed to load entity dataset into store") - .withCause(e) - .asRuntimeException(); - } - } - - private Job waitForJob(Job queryJob) throws InterruptedException { - Job completedJob = - queryJob.waitFor( - RetryOption.initialRetryDelay(Duration.ofSeconds(initialRetryDelaySecs)), - RetryOption.totalTimeout(Duration.ofSeconds(totalTimeoutSecs))); - if (completedJob == null) { - throw Status.INTERNAL.withDescription("Job no longer exists").asRuntimeException(); - } else if (completedJob.getStatus().getError() != null) { - throw Status.INTERNAL - .withDescription("Job failed: " + completedJob.getStatus().getError()) - .asRuntimeException(); - } - return completedJob; - } - - public static String createTempTableName() { - return "_" + UUID.randomUUID().toString().replace("-", ""); - } -} diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java new file mode 100644 index 00000000000..4e0baed17b4 --- /dev/null +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -0,0 +1,162 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.service; + +import static feast.serving.util.Metrics.requestCount; +import static feast.serving.util.Metrics.staleKeyCount; +import static feast.serving.util.RefUtil.generateFeatureStringRef; + +import com.google.common.collect.Maps; +import com.google.protobuf.Duration; +import feast.serving.ServingAPIProto.*; +import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; +import feast.serving.specs.CachedSpecService; +import feast.serving.util.RefUtil; +import feast.storage.api.retrieval.FeatureSetRequest; +import feast.storage.api.retrieval.OnlineRetriever; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.ValueProto.Value; +import io.grpc.Status; +import io.opentracing.Scope; +import io.opentracing.Tracer; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.slf4j.Logger; + +public class OnlineServingService implements ServingService { + + private static final Logger log = org.slf4j.LoggerFactory.getLogger(OnlineServingService.class); + private final CachedSpecService specService; + private final Tracer tracer; + private final OnlineRetriever retriever; + + public OnlineServingService( + OnlineRetriever retriever, CachedSpecService specService, Tracer tracer) { + this.retriever = retriever; + this.specService = specService; + this.tracer = tracer; + } + + /** {@inheritDoc} */ + @Override + public GetFeastServingInfoResponse getFeastServingInfo( + GetFeastServingInfoRequest getFeastServingInfoRequest) { + return GetFeastServingInfoResponse.newBuilder() + .setType(FeastServingType.FEAST_SERVING_TYPE_ONLINE) + .build(); + } + + /** {@inheritDoc} */ + @Override + public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) { + try (Scope scope = tracer.buildSpan("Redis-getOnlineFeatures").startActive(true)) { + GetOnlineFeaturesResponse.Builder getOnlineFeaturesResponseBuilder = + GetOnlineFeaturesResponse.newBuilder(); + List featureSetRequests = + specService.getFeatureSets(request.getFeaturesList()); + List entityRows = request.getEntityRowsList(); + Map> featureValuesMap = + entityRows.stream() + .collect(Collectors.toMap(row -> row, row -> Maps.newHashMap(row.getFieldsMap()))); + + List> featureRows = + retriever.getOnlineFeatures(entityRows, featureSetRequests); + + for (var fsIdx = 0; fsIdx < featureRows.size(); fsIdx++) { + List featureRowsForFs = featureRows.get(fsIdx); + FeatureSetRequest featureSetRequest = featureSetRequests.get(fsIdx); + Map featureNames = + featureSetRequest.getFeatureReferences().stream() + .collect( + Collectors.toMap( + FeatureReference::getName, featureReference -> featureReference)); + for (var entityRowIdx = 0; entityRowIdx < entityRows.size(); entityRowIdx++) { + FeatureRow featureRow = featureRowsForFs.get(entityRowIdx); + EntityRow entityRow = entityRows.get(entityRowIdx); + if (isStale(featureSetRequest, entityRow, featureRow)) { + featureSetRequest + .getFeatureReferences() + .parallelStream() + .forEach( + ref -> { + staleKeyCount + .labels( + featureSetRequest.getSpec().getProject(), + String.format("%s:%d", ref.getName(), ref.getVersion())) + .inc(); + featureValuesMap + .get(entityRow) + .put(RefUtil.generateFeatureStringRef(ref), Value.newBuilder().build()); + }); + + } else { + featureSetRequest + .getFeatureReferences() + .parallelStream() + .forEach( + ref -> + requestCount + .labels( + featureSetRequest.getSpec().getProject(), + String.format("%s:%d", ref.getName(), ref.getVersion())) + .inc()); + + featureRow.getFieldsList().stream() + .filter(field -> featureNames.containsKey(field.getName())) + .forEach( + field -> { + FeatureReference ref = featureNames.get(field.getName()); + String id = generateFeatureStringRef(ref); + featureValuesMap.get(entityRow).put(id, field.getValue()); + }); + } + } + } + + List fieldValues = + featureValuesMap.values().stream() + .map(valueMap -> FieldValues.newBuilder().putAllFields(valueMap).build()) + .collect(Collectors.toList()); + return getOnlineFeaturesResponseBuilder.addAllFieldValues(fieldValues).build(); + } + } + + @Override + public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { + throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); + } + + @Override + public GetJobResponse getJob(GetJobRequest getJobRequest) { + throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); + } + + private boolean isStale( + FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) { + if (featureSetRequest.getSpec().getMaxAge().equals(Duration.getDefaultInstance())) { + return false; + } + long givenTimestamp = entityRow.getEntityTimestamp().getSeconds(); + if (givenTimestamp == 0) { + givenTimestamp = System.currentTimeMillis() / 1000; + } + long timeDifference = givenTimestamp - featureRow.getEventTimestamp().getSeconds(); + return timeDifference > featureSetRequest.getSpec().getMaxAge().getSeconds(); + } +} diff --git a/serving/src/main/java/feast/serving/service/RedisServingService.java b/serving/src/main/java/feast/serving/service/RedisServingService.java deleted file mode 100644 index 78d9d9cebe4..00000000000 --- a/serving/src/main/java/feast/serving/service/RedisServingService.java +++ /dev/null @@ -1,345 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import static feast.serving.util.Metrics.invalidEncodingCount; -import static feast.serving.util.Metrics.missingKeyCount; -import static feast.serving.util.Metrics.requestCount; -import static feast.serving.util.Metrics.requestLatency; -import static feast.serving.util.Metrics.staleKeyCount; -import static feast.serving.util.RefUtil.generateFeatureSetStringRef; -import static feast.serving.util.RefUtil.generateFeatureStringRef; - -import com.google.common.collect.Maps; -import com.google.protobuf.AbstractMessageLite; -import com.google.protobuf.Duration; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.core.FeatureSetProto.EntitySpec; -import feast.core.FeatureSetProto.FeatureSetSpec; -import feast.serving.ServingAPIProto.FeastServingType; -import feast.serving.ServingAPIProto.FeatureReference; -import feast.serving.ServingAPIProto.GetBatchFeaturesRequest; -import feast.serving.ServingAPIProto.GetBatchFeaturesResponse; -import feast.serving.ServingAPIProto.GetFeastServingInfoRequest; -import feast.serving.ServingAPIProto.GetFeastServingInfoResponse; -import feast.serving.ServingAPIProto.GetJobRequest; -import feast.serving.ServingAPIProto.GetJobResponse; -import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest; -import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; -import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; -import feast.serving.encoding.FeatureRowDecoder; -import feast.serving.specs.CachedSpecService; -import feast.serving.specs.FeatureSetRequest; -import feast.serving.util.RefUtil; -import feast.storage.RedisProto.RedisKey; -import feast.types.FeatureRowProto.FeatureRow; -import feast.types.FieldProto.Field; -import feast.types.ValueProto.Value; -import io.grpc.Status; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisCommands; -import io.opentracing.Scope; -import io.opentracing.Tracer; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import org.slf4j.Logger; - -public class RedisServingService implements ServingService { - - private static final Logger log = org.slf4j.LoggerFactory.getLogger(RedisServingService.class); - private final CachedSpecService specService; - private final Tracer tracer; - private final RedisCommands syncCommands; - - public RedisServingService( - StatefulRedisConnection connection, - CachedSpecService specService, - Tracer tracer) { - this.syncCommands = connection.sync(); - this.specService = specService; - this.tracer = tracer; - } - - /** {@inheritDoc} */ - @Override - public GetFeastServingInfoResponse getFeastServingInfo( - GetFeastServingInfoRequest getFeastServingInfoRequest) { - return GetFeastServingInfoResponse.newBuilder() - .setType(FeastServingType.FEAST_SERVING_TYPE_ONLINE) - .build(); - } - - /** {@inheritDoc} */ - @Override - public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) { - try (Scope scope = tracer.buildSpan("Redis-getOnlineFeatures").startActive(true)) { - GetOnlineFeaturesResponse.Builder getOnlineFeaturesResponseBuilder = - GetOnlineFeaturesResponse.newBuilder(); - - List entityRows = request.getEntityRowsList(); - Map> featureValuesMap = - entityRows.stream() - .collect(Collectors.toMap(row -> row, row -> Maps.newHashMap(row.getFieldsMap()))); - List featureSetRequests = - specService.getFeatureSets(request.getFeaturesList()); - for (FeatureSetRequest featureSetRequest : featureSetRequests) { - - List featureSetEntityNames = - featureSetRequest.getSpec().getEntitiesList().stream() - .map(EntitySpec::getName) - .collect(Collectors.toList()); - - List redisKeys = - getRedisKeys(featureSetEntityNames, entityRows, featureSetRequest.getSpec()); - - try { - sendAndProcessMultiGet(redisKeys, entityRows, featureValuesMap, featureSetRequest); - } catch (InvalidProtocolBufferException | ExecutionException e) { - throw Status.INTERNAL - .withDescription("Unable to parse protobuf while retrieving feature") - .withCause(e) - .asRuntimeException(); - } - } - List fieldValues = - featureValuesMap.values().stream() - .map(valueMap -> FieldValues.newBuilder().putAllFields(valueMap).build()) - .collect(Collectors.toList()); - return getOnlineFeaturesResponseBuilder.addAllFieldValues(fieldValues).build(); - } - } - - @Override - public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { - throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); - } - - @Override - public GetJobResponse getJob(GetJobRequest getJobRequest) { - throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); - } - - /** - * Build the redis keys for retrieval from the store. - * - * @param featureSetEntityNames entity names that actually belong to the featureSet - * @param entityRows entity values to retrieve for - * @param featureSetSpec featureSetSpec of the features to retrieve - * @return list of RedisKeys - */ - private List getRedisKeys( - List featureSetEntityNames, - List entityRows, - FeatureSetSpec featureSetSpec) { - try (Scope scope = tracer.buildSpan("Redis-makeRedisKeys").startActive(true)) { - String featureSetRef = generateFeatureSetStringRef(featureSetSpec); - List redisKeys = - entityRows.stream() - .map(row -> makeRedisKey(featureSetRef, featureSetEntityNames, row)) - .collect(Collectors.toList()); - return redisKeys; - } - } - - /** - * Create {@link RedisKey} - * - * @param featureSet featureSet reference of the feature. E.g. feature_set_1:1 - * @param featureSetEntityNames entity names that belong to the featureSet - * @param entityRow entityRow to build the key from - * @return {@link RedisKey} - */ - private RedisKey makeRedisKey( - String featureSet, List featureSetEntityNames, EntityRow entityRow) { - RedisKey.Builder builder = RedisKey.newBuilder().setFeatureSet(featureSet); - Map fieldsMap = entityRow.getFieldsMap(); - featureSetEntityNames.sort(String::compareTo); - for (int i = 0; i < featureSetEntityNames.size(); i++) { - String entityName = featureSetEntityNames.get(i); - - if (!fieldsMap.containsKey(entityName)) { - throw Status.INVALID_ARGUMENT - .withDescription( - String.format( - "Entity row fields \"%s\" does not contain required entity field \"%s\"", - fieldsMap.keySet().toString(), entityName)) - .asRuntimeException(); - } - - builder.addEntities( - Field.newBuilder().setName(entityName).setValue(fieldsMap.get(entityName))); - } - return builder.build(); - } - - private void sendAndProcessMultiGet( - List redisKeys, - List entityRows, - Map> featureValuesMap, - FeatureSetRequest featureSetRequest) - throws InvalidProtocolBufferException, ExecutionException { - - List values = sendMultiGet(redisKeys); - long startTime = System.currentTimeMillis(); - try (Scope scope = tracer.buildSpan("Redis-processResponse").startActive(true)) { - FeatureSetSpec spec = featureSetRequest.getSpec(); - - Map nullValues = - featureSetRequest.getFeatureReferences().stream() - .collect( - Collectors.toMap( - RefUtil::generateFeatureStringRef, - featureReference -> Value.newBuilder().build())); - - for (int i = 0; i < values.size(); i++) { - EntityRow entityRow = entityRows.get(i); - Map featureValues = featureValuesMap.get(entityRow); - - byte[] value = values.get(i); - if (value == null) { - featureSetRequest - .getFeatureReferences() - .parallelStream() - .forEach( - request -> - missingKeyCount - .labels( - spec.getProject(), - String.format("%s:%d", request.getName(), request.getVersion())) - .inc()); - featureValues.putAll(nullValues); - continue; - } - - FeatureRow featureRow = FeatureRow.parseFrom(value); - String featureSetRef = redisKeys.get(i).getFeatureSet(); - FeatureRowDecoder decoder = - new FeatureRowDecoder(featureSetRef, specService.getFeatureSetSpec(featureSetRef)); - if (decoder.isEncoded(featureRow)) { - if (decoder.isEncodingValid(featureRow)) { - featureRow = decoder.decode(featureRow); - } else { - featureSetRequest - .getFeatureReferences() - .parallelStream() - .forEach( - request -> - invalidEncodingCount - .labels( - spec.getProject(), - String.format("%s:%d", request.getName(), request.getVersion())) - .inc()); - featureValues.putAll(nullValues); - continue; - } - } - - boolean stale = isStale(featureSetRequest, entityRow, featureRow); - if (stale) { - featureSetRequest - .getFeatureReferences() - .parallelStream() - .forEach( - request -> - staleKeyCount - .labels( - spec.getProject(), - String.format("%s:%d", request.getName(), request.getVersion())) - .inc()); - featureValues.putAll(nullValues); - continue; - } - - featureSetRequest - .getFeatureReferences() - .parallelStream() - .forEach( - request -> - requestCount - .labels( - spec.getProject(), - String.format("%s:%d", request.getName(), request.getVersion())) - .inc()); - - Map featureNames = - featureSetRequest.getFeatureReferences().stream() - .collect( - Collectors.toMap( - FeatureReference::getName, featureReference -> featureReference)); - featureRow.getFieldsList().stream() - .filter(field -> featureNames.keySet().contains(field.getName())) - .forEach( - field -> { - FeatureReference ref = featureNames.get(field.getName()); - String id = generateFeatureStringRef(ref); - featureValues.put(id, field.getValue()); - }); - } - } finally { - requestLatency - .labels("processResponse") - .observe((System.currentTimeMillis() - startTime) / 1000); - } - } - - private boolean isStale( - FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) { - if (featureSetRequest.getSpec().getMaxAge().equals(Duration.getDefaultInstance())) { - return false; - } - long givenTimestamp = entityRow.getEntityTimestamp().getSeconds(); - if (givenTimestamp == 0) { - givenTimestamp = System.currentTimeMillis() / 1000; - } - long timeDifference = givenTimestamp - featureRow.getEventTimestamp().getSeconds(); - return timeDifference > featureSetRequest.getSpec().getMaxAge().getSeconds(); - } - - /** - * Send a list of get request as an mget - * - * @param keys list of {@link RedisKey} - * @return list of {@link FeatureRow} in primitive byte representation for each {@link RedisKey} - */ - private List sendMultiGet(List keys) { - try (Scope scope = tracer.buildSpan("Redis-sendMultiGet").startActive(true)) { - long startTime = System.currentTimeMillis(); - try { - byte[][] binaryKeys = - keys.stream() - .map(AbstractMessageLite::toByteArray) - .collect(Collectors.toList()) - .toArray(new byte[0][0]); - return syncCommands.mget(binaryKeys).stream() - .map(keyValue -> keyValue.getValueOrElse(null)) - .collect(Collectors.toList()); - } catch (Exception e) { - throw Status.NOT_FOUND - .withDescription("Unable to retrieve feature from Redis") - .withCause(e) - .asRuntimeException(); - } finally { - requestLatency - .labels("sendMultiGet") - .observe((System.currentTimeMillis() - startTime) / 1000d); - } - } - } -} diff --git a/serving/src/main/java/feast/serving/specs/CachedSpecService.java b/serving/src/main/java/feast/serving/specs/CachedSpecService.java index 12a8242da13..11e9f42263e 100644 --- a/serving/src/main/java/feast/serving/specs/CachedSpecService.java +++ b/serving/src/main/java/feast/serving/specs/CachedSpecService.java @@ -36,6 +36,7 @@ import feast.core.StoreProto.Store.Subscription; import feast.serving.ServingAPIProto.FeatureReference; import feast.serving.exception.SpecRetrievalException; +import feast.storage.api.retrieval.FeatureSetRequest; import io.grpc.StatusRuntimeException; import io.prometheus.client.Gauge; import java.io.IOException; diff --git a/serving/src/main/java/feast/serving/specs/FeatureSetRequest.java b/serving/src/main/java/feast/serving/specs/FeatureSetRequest.java deleted file mode 100644 index 904630659d7..00000000000 --- a/serving/src/main/java/feast/serving/specs/FeatureSetRequest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.specs; - -import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableSet; -import feast.core.FeatureSetProto.FeatureSetSpec; -import feast.serving.ServingAPIProto.FeatureReference; -import java.util.List; - -@AutoValue -public abstract class FeatureSetRequest { - public abstract FeatureSetSpec getSpec(); - - public abstract ImmutableSet getFeatureReferences(); - - public static Builder newBuilder() { - return new AutoValue_FeatureSetRequest.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setSpec(FeatureSetSpec spec); - - abstract ImmutableSet.Builder featureReferencesBuilder(); - - public Builder addAllFeatureReferences(List featureReferenceList) { - featureReferencesBuilder().addAll(featureReferenceList); - return this; - } - - public Builder addFeatureReference(FeatureReference featureReference) { - featureReferencesBuilder().add(featureReference); - return this; - } - - public abstract FeatureSetRequest build(); - } -} diff --git a/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java b/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java deleted file mode 100644 index 61103af1092..00000000000 --- a/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java +++ /dev/null @@ -1,352 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.store.bigquery; - -import static feast.serving.service.BigQueryServingService.TEMP_TABLE_EXPIRY_DURATION_MS; -import static feast.serving.service.BigQueryServingService.createTempTableName; -import static feast.serving.store.bigquery.QueryTemplater.createTimestampLimitQuery; - -import com.google.auto.value.AutoValue; -import com.google.cloud.RetryOption; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.DatasetId; -import com.google.cloud.bigquery.ExtractJobConfiguration; -import com.google.cloud.bigquery.FieldValueList; -import com.google.cloud.bigquery.Job; -import com.google.cloud.bigquery.JobInfo; -import com.google.cloud.bigquery.QueryJobConfiguration; -import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.TableInfo; -import com.google.cloud.bigquery.TableResult; -import com.google.cloud.storage.Blob; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.Storage.BlobListOption; -import feast.serving.ServingAPIProto; -import feast.serving.ServingAPIProto.DataFormat; -import feast.serving.ServingAPIProto.JobStatus; -import feast.serving.ServingAPIProto.JobType; -import feast.serving.service.JobService; -import feast.serving.store.bigquery.model.FeatureSetInfo; -import io.grpc.Status; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorCompletionService; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import org.threeten.bp.Duration; - -/** - * BatchRetrievalQueryRunnable is a Runnable for running a BigQuery Feast batch retrieval job async. - * - *

It does the following, in sequence: - * - *

1. Retrieve the temporal bounds of the entity dataset provided. This will be used to filter - * the feature set tables when performing the feature retrieval. - * - *

2. For each of the feature sets requested, generate the subquery for doing a point-in-time - * correctness join of the features in the feature set to the entity table. - * - *

3. Run each of the subqueries in parallel and wait for them to complete. If any of the jobs - * are unsuccessful, the thread running the BatchRetrievalQueryRunnable catches the error and - * updates the job database. - * - *

4. When all the subquery jobs are complete, join the outputs of all the subqueries into a - * single table. - * - *

5. Extract the output of the join to a remote file, and write the location of the remote file - * to the job database, and mark the retrieval job as successful. - */ -@AutoValue -public abstract class BatchRetrievalQueryRunnable implements Runnable { - - private static final long SUBQUERY_TIMEOUT_SECS = 900; // 15 minutes - - public abstract JobService jobService(); - - public abstract String projectId(); - - public abstract String datasetId(); - - public abstract String feastJobId(); - - public abstract BigQuery bigquery(); - - public abstract List entityTableColumnNames(); - - public abstract List featureSetInfos(); - - public abstract String entityTableName(); - - public abstract String jobStagingLocation(); - - public abstract int initialRetryDelaySecs(); - - public abstract int totalTimeoutSecs(); - - public abstract Storage storage(); - - public static Builder builder() { - return new AutoValue_BatchRetrievalQueryRunnable.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setJobService(JobService jobService); - - public abstract Builder setProjectId(String projectId); - - public abstract Builder setDatasetId(String datasetId); - - public abstract Builder setFeastJobId(String feastJobId); - - public abstract Builder setBigquery(BigQuery bigquery); - - public abstract Builder setEntityTableColumnNames(List entityTableColumnNames); - - public abstract Builder setFeatureSetInfos(List featureSetInfos); - - public abstract Builder setEntityTableName(String entityTableName); - - public abstract Builder setJobStagingLocation(String jobStagingLocation); - - public abstract Builder setInitialRetryDelaySecs(int initialRetryDelaySecs); - - public abstract Builder setTotalTimeoutSecs(int totalTimeoutSecs); - - public abstract Builder setStorage(Storage storage); - - public abstract BatchRetrievalQueryRunnable build(); - } - - @Override - public void run() { - - // 1. Retrieve the temporal bounds of the entity dataset provided - FieldValueList timestampLimits = getTimestampLimits(entityTableName()); - - // 2. Generate the subqueries - List featureSetQueries = generateQueries(timestampLimits); - - QueryJobConfiguration queryConfig; - - try { - // 3 & 4. Run the subqueries in parallel then collect the outputs - Job queryJob = runBatchQuery(featureSetQueries); - queryConfig = queryJob.getConfiguration(); - String exportTableDestinationUri = - String.format("%s/%s/*.avro", jobStagingLocation(), feastJobId()); - - // 5. Export the table - // Hardcode the format to Avro for now - ExtractJobConfiguration extractConfig = - ExtractJobConfiguration.of( - queryConfig.getDestinationTable(), exportTableDestinationUri, "Avro"); - Job extractJob = bigquery().create(JobInfo.of(extractConfig)); - waitForJob(extractJob); - } catch (BigQueryException | InterruptedException | IOException e) { - jobService() - .upsert( - ServingAPIProto.Job.newBuilder() - .setId(feastJobId()) - .setType(JobType.JOB_TYPE_DOWNLOAD) - .setStatus(JobStatus.JOB_STATUS_DONE) - .setError(e.getMessage()) - .build()); - return; - } - - List fileUris = parseOutputFileURIs(); - - // 5. Update the job database - jobService() - .upsert( - ServingAPIProto.Job.newBuilder() - .setId(feastJobId()) - .setType(JobType.JOB_TYPE_DOWNLOAD) - .setStatus(JobStatus.JOB_STATUS_DONE) - .addAllFileUris(fileUris) - .setDataFormat(DataFormat.DATA_FORMAT_AVRO) - .build()); - } - - private List parseOutputFileURIs() { - String scheme = jobStagingLocation().substring(0, jobStagingLocation().indexOf("://")); - String stagingLocationNoScheme = - jobStagingLocation().substring(jobStagingLocation().indexOf("://") + 3); - String bucket = stagingLocationNoScheme.split("/")[0]; - List prefixParts = new ArrayList<>(); - prefixParts.add( - stagingLocationNoScheme.contains("/") && !stagingLocationNoScheme.endsWith("/") - ? stagingLocationNoScheme.substring(stagingLocationNoScheme.indexOf("/") + 1) - : ""); - prefixParts.add(feastJobId()); - String prefix = String.join("/", prefixParts) + "/"; - - List fileUris = new ArrayList<>(); - for (Blob blob : storage().list(bucket, BlobListOption.prefix(prefix)).iterateAll()) { - fileUris.add(String.format("%s://%s/%s", scheme, blob.getBucket(), blob.getName())); - } - return fileUris; - } - - Job runBatchQuery(List featureSetQueries) - throws BigQueryException, InterruptedException, IOException { - ExecutorService executorService = Executors.newFixedThreadPool(featureSetQueries.size()); - ExecutorCompletionService executorCompletionService = - new ExecutorCompletionService<>(executorService); - - List featureSetInfos = new ArrayList<>(); - - // For each of the feature sets requested, start an async job joining the features in that - // feature set to the provided entity table - for (int i = 0; i < featureSetQueries.size(); i++) { - QueryJobConfiguration queryJobConfig = - QueryJobConfiguration.newBuilder(featureSetQueries.get(i)) - .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName())) - .build(); - Job subqueryJob = bigquery().create(JobInfo.of(queryJobConfig)); - executorCompletionService.submit( - SubqueryCallable.builder() - .setBigquery(bigquery()) - .setFeatureSetInfo(featureSetInfos().get(i)) - .setSubqueryJob(subqueryJob) - .build()); - } - - for (int i = 0; i < featureSetQueries.size(); i++) { - try { - // Try to retrieve the outputs of all the jobs. The timeout here is a formality; - // a stricter timeout is implemented in the actual SubqueryCallable. - FeatureSetInfo featureSetInfo = - executorCompletionService.take().get(SUBQUERY_TIMEOUT_SECS, TimeUnit.SECONDS); - featureSetInfos.add(featureSetInfo); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - jobService() - .upsert( - ServingAPIProto.Job.newBuilder() - .setId(feastJobId()) - .setType(JobType.JOB_TYPE_DOWNLOAD) - .setStatus(JobStatus.JOB_STATUS_DONE) - .setError(e.getMessage()) - .build()); - - executorService.shutdownNow(); - throw Status.INTERNAL - .withDescription("Error running batch query") - .withCause(e) - .asRuntimeException(); - } - } - - // Generate and run a join query to collect the outputs of all the - // subqueries into a single table. - String joinQuery = - QueryTemplater.createJoinQuery( - featureSetInfos, entityTableColumnNames(), entityTableName()); - QueryJobConfiguration queryJobConfig = - QueryJobConfiguration.newBuilder(joinQuery) - .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName())) - .build(); - Job queryJob = bigquery().create(JobInfo.of(queryJobConfig)); - Job completedQueryJob = waitForJob(queryJob); - - TableInfo expiry = - bigquery() - .getTable(queryJobConfig.getDestinationTable()) - .toBuilder() - .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) - .build(); - bigquery().update(expiry); - - return completedQueryJob; - } - - private List generateQueries(FieldValueList timestampLimits) { - List featureSetQueries = new ArrayList<>(); - try { - for (FeatureSetInfo featureSetInfo : featureSetInfos()) { - String query = - QueryTemplater.createFeatureSetPointInTimeQuery( - featureSetInfo, - projectId(), - datasetId(), - entityTableName(), - timestampLimits.get("min").getStringValue(), - timestampLimits.get("max").getStringValue()); - featureSetQueries.add(query); - } - } catch (IOException e) { - throw Status.INTERNAL - .withDescription("Unable to generate query for batch retrieval") - .withCause(e) - .asRuntimeException(); - } - return featureSetQueries; - } - - private FieldValueList getTimestampLimits(String entityTableName) { - QueryJobConfiguration getTimestampLimitsQuery = - QueryJobConfiguration.newBuilder(createTimestampLimitQuery(entityTableName)) - .setDefaultDataset(DatasetId.of(projectId(), datasetId())) - .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName())) - .build(); - try { - Job job = bigquery().create(JobInfo.of(getTimestampLimitsQuery)); - TableResult getTimestampLimitsQueryResult = waitForJob(job).getQueryResults(); - TableInfo expiry = - bigquery() - .getTable(getTimestampLimitsQuery.getDestinationTable()) - .toBuilder() - .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) - .build(); - bigquery().update(expiry); - FieldValueList result = null; - for (FieldValueList fields : getTimestampLimitsQueryResult.getValues()) { - result = fields; - } - if (result == null || result.get("min").isNull() || result.get("max").isNull()) { - throw new RuntimeException("query returned insufficient values"); - } - return result; - } catch (InterruptedException e) { - throw Status.INTERNAL - .withDescription("Unable to extract min and max timestamps from query") - .withCause(e) - .asRuntimeException(); - } - } - - private Job waitForJob(Job queryJob) throws InterruptedException { - Job completedJob = - queryJob.waitFor( - RetryOption.initialRetryDelay(Duration.ofSeconds(initialRetryDelaySecs())), - RetryOption.totalTimeout(Duration.ofSeconds(totalTimeoutSecs()))); - if (completedJob == null) { - throw Status.INTERNAL.withDescription("Job no longer exists").asRuntimeException(); - } else if (completedJob.getStatus().getError() != null) { - throw Status.INTERNAL - .withDescription("Job failed: " + completedJob.getStatus().getError()) - .asRuntimeException(); - } - return completedJob; - } -} diff --git a/serving/src/main/java/feast/serving/store/bigquery/QueryTemplater.java b/serving/src/main/java/feast/serving/store/bigquery/QueryTemplater.java deleted file mode 100644 index e3f1138db89..00000000000 --- a/serving/src/main/java/feast/serving/store/bigquery/QueryTemplater.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.store.bigquery; - -import com.google.cloud.bigquery.TableId; -import com.google.protobuf.Duration; -import com.mitchellbosecke.pebble.PebbleEngine; -import com.mitchellbosecke.pebble.template.PebbleTemplate; -import feast.core.FeatureSetProto.EntitySpec; -import feast.core.FeatureSetProto.FeatureSetSpec; -import feast.serving.ServingAPIProto.FeatureReference; -import feast.serving.specs.FeatureSetRequest; -import feast.serving.store.bigquery.model.FeatureSetInfo; -import java.io.IOException; -import java.io.StringWriter; -import java.io.Writer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class QueryTemplater { - - private static final PebbleEngine engine = new PebbleEngine.Builder().build(); - private static final String FEATURESET_TEMPLATE_NAME = "templates/single_featureset_pit_join.sql"; - private static final String JOIN_TEMPLATE_NAME = "templates/join_featuresets.sql"; - - /** - * Get the query for retrieving the earliest and latest timestamps in the entity dataset. - * - * @param leftTableName full entity dataset name - * @return timestamp limit BQ SQL query - */ - public static String createTimestampLimitQuery(String leftTableName) { - return String.format( - "SELECT DATETIME(MAX(event_timestamp)) as max, DATETIME(MIN(event_timestamp)) as min FROM `%s`", - leftTableName); - } - - /** - * Creates a query that generates a UUID for the entity table, for left joins later on. - * - * @param leftTableName full entity dataset name - * @return uuid generation query - */ - public static String createEntityTableUUIDQuery(String leftTableName) { - return String.format( - "SELECT GENERATE_UUID() as uuid, `%s`.* from `%s`", leftTableName, leftTableName); - } - - /** - * Generate the information necessary for the sql templating for point in time correctness join to - * the entity dataset for each feature set requested. - * - * @param featureSetRequests List of feature sets requested - * @return List of FeatureSetInfos - */ - public static List getFeatureSetInfos(List featureSetRequests) - throws IllegalArgumentException { - - List featureSetInfos = new ArrayList<>(); - for (FeatureSetRequest featureSetRequest : featureSetRequests) { - FeatureSetSpec spec = featureSetRequest.getSpec(); - Duration maxAge = spec.getMaxAge(); - List fsEntities = - spec.getEntitiesList().stream().map(EntitySpec::getName).collect(Collectors.toList()); - List features = - featureSetRequest.getFeatureReferences().stream() - .map(FeatureReference::getName) - .collect(Collectors.toList()); - featureSetInfos.add( - new FeatureSetInfo( - spec.getProject(), - spec.getName(), - spec.getVersion(), - maxAge.getSeconds(), - fsEntities, - features, - "")); - } - return featureSetInfos; - } - - /** - * Generate the query for point in time correctness join of data for a single feature set to the - * entity dataset. - * - * @param featureSetInfo Information about the feature set necessary for the query templating - * @param projectId google project ID - * @param datasetId feast bigquery dataset ID - * @param leftTableName entity dataset name - * @param minTimestamp earliest allowed timestamp for the historical data in feast - * @param maxTimestamp latest allowed timestamp for the historical data in feast - * @return point in time correctness join BQ SQL query - */ - public static String createFeatureSetPointInTimeQuery( - FeatureSetInfo featureSetInfo, - String projectId, - String datasetId, - String leftTableName, - String minTimestamp, - String maxTimestamp) - throws IOException { - - PebbleTemplate template = engine.getTemplate(FEATURESET_TEMPLATE_NAME); - Map context = new HashMap<>(); - context.put("featureSet", featureSetInfo); - context.put("projectId", projectId); - context.put("datasetId", datasetId); - context.put("minTimestamp", minTimestamp); - context.put("maxTimestamp", maxTimestamp); - context.put("leftTableName", leftTableName); - - Writer writer = new StringWriter(); - template.evaluate(writer, context); - return writer.toString(); - } - - /** - * @param featureSetInfos List of FeatureSetInfos containing information about the feature set - * necessary for the query templating - * @param entityTableColumnNames list of column names in entity table - * @param leftTableName entity dataset name - * @return query to join temporary feature set tables to the entity table - */ - public static String createJoinQuery( - List featureSetInfos, - List entityTableColumnNames, - String leftTableName) - throws IOException { - PebbleTemplate template = engine.getTemplate(JOIN_TEMPLATE_NAME); - Map context = new HashMap<>(); - context.put("entities", entityTableColumnNames); - context.put("featureSets", featureSetInfos); - context.put("leftTableName", leftTableName); - - Writer writer = new StringWriter(); - template.evaluate(writer, context); - return writer.toString(); - } - - public static String generateFullTableName(TableId tableId) { - return String.format( - "%s.%s.%s", tableId.getProject(), tableId.getDataset(), tableId.getTable()); - } -} diff --git a/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java b/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java deleted file mode 100644 index 14026030b42..00000000000 --- a/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.store.bigquery; - -import static feast.serving.service.BigQueryServingService.TEMP_TABLE_EXPIRY_DURATION_MS; -import static feast.serving.store.bigquery.QueryTemplater.generateFullTableName; - -import com.google.auto.value.AutoValue; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.Job; -import com.google.cloud.bigquery.QueryJobConfiguration; -import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.TableInfo; -import feast.serving.store.bigquery.model.FeatureSetInfo; -import java.util.concurrent.Callable; - -/** - * Waits for a point-in-time correctness join to complete. On completion, returns a featureSetInfo - * updated with the reference to the table containing the results of the query. - */ -@AutoValue -public abstract class SubqueryCallable implements Callable { - - public abstract BigQuery bigquery(); - - public abstract FeatureSetInfo featureSetInfo(); - - public abstract Job subqueryJob(); - - public static Builder builder() { - return new AutoValue_SubqueryCallable.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setBigquery(BigQuery bigquery); - - public abstract Builder setFeatureSetInfo(FeatureSetInfo featureSetInfo); - - public abstract Builder setSubqueryJob(Job subqueryJob); - - public abstract SubqueryCallable build(); - } - - @Override - public FeatureSetInfo call() throws BigQueryException, InterruptedException { - QueryJobConfiguration subqueryConfig; - subqueryJob().waitFor(); - subqueryConfig = subqueryJob().getConfiguration(); - TableId destinationTable = subqueryConfig.getDestinationTable(); - - TableInfo expiry = - bigquery() - .getTable(destinationTable) - .toBuilder() - .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) - .build(); - bigquery().update(expiry); - - String fullTablePath = generateFullTableName(destinationTable); - - return new FeatureSetInfo(featureSetInfo(), fullTablePath); - } -} diff --git a/serving/src/main/java/feast/serving/store/bigquery/model/FeatureSetInfo.java b/serving/src/main/java/feast/serving/store/bigquery/model/FeatureSetInfo.java deleted file mode 100644 index 77c80ead0ea..00000000000 --- a/serving/src/main/java/feast/serving/store/bigquery/model/FeatureSetInfo.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.store.bigquery.model; - -import java.util.List; - -public class FeatureSetInfo { - - private final String project; - private final String name; - private final int version; - private final long maxAge; - private final List entities; - private final List features; - private final String table; - - public FeatureSetInfo( - String project, - String name, - int version, - long maxAge, - List entities, - List features, - String table) { - this.project = project; - this.name = name; - this.version = version; - this.maxAge = maxAge; - this.entities = entities; - this.features = features; - this.table = table; - } - - public FeatureSetInfo(FeatureSetInfo featureSetInfo, String table) { - - this.project = featureSetInfo.getProject(); - this.name = featureSetInfo.getName(); - this.version = featureSetInfo.getVersion(); - this.maxAge = featureSetInfo.getMaxAge(); - this.entities = featureSetInfo.getEntities(); - this.features = featureSetInfo.getFeatures(); - this.table = table; - } - - public String getProject() { - return project; - } - - public String getName() { - return name; - } - - public int getVersion() { - return version; - } - - public long getMaxAge() { - return maxAge; - } - - public List getEntities() { - return entities; - } - - public List getFeatures() { - return features; - } - - public String getTable() { - return table; - } -} diff --git a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java index abeb44bd731..759e923fc07 100644 --- a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java +++ b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java @@ -37,7 +37,7 @@ import feast.serving.ServingAPIProto.FeatureReference; import feast.serving.specs.CachedSpecService; import feast.serving.specs.CoreSpecService; -import feast.serving.specs.FeatureSetRequest; +import feast.storage.api.retrieval.FeatureSetRequest; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; diff --git a/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java similarity index 72% rename from serving/src/test/java/feast/serving/service/RedisServingServiceTest.java rename to serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 05a24d3fe6a..10c26ff0567 100644 --- a/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -22,7 +22,6 @@ import static org.mockito.MockitoAnnotations.initMocks; import com.google.common.collect.Lists; -import com.google.protobuf.AbstractMessageLite; import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; import feast.core.FeatureSetProto.EntitySpec; @@ -33,17 +32,16 @@ import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.serving.specs.CachedSpecService; -import feast.serving.specs.FeatureSetRequest; -import feast.storage.RedisProto.RedisKey; +import feast.storage.api.retrieval.FeatureSetRequest; +import feast.storage.connectors.redis.retrieval.RedisOnlineRetriever; import feast.types.FeatureRowProto.FeatureRow; import feast.types.FieldProto.Field; import feast.types.ValueProto.Value; -import io.lettuce.core.KeyValue; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisCommands; import io.opentracing.Tracer; import io.opentracing.Tracer.SpanBuilder; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.junit.Before; import org.junit.Test; @@ -51,44 +49,20 @@ import org.mockito.Mock; import org.mockito.Mockito; -public class RedisServingServiceTest { +public class OnlineServingServiceTest { @Mock CachedSpecService specService; @Mock Tracer tracer; - @Mock StatefulRedisConnection connection; + @Mock RedisOnlineRetriever retriever; - @Mock RedisCommands syncCommands; - - private RedisServingService redisServingService; - private byte[][] redisKeyList; + private OnlineServingService onlineServingService; @Before public void setUp() { initMocks(this); - when(connection.sync()).thenReturn(syncCommands); - redisServingService = new RedisServingService(connection, specService, tracer); - redisKeyList = - Lists.newArrayList( - RedisKey.newBuilder() - .setFeatureSet("project/featureSet:1") - .addAllEntities( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build())) - .build(), - RedisKey.newBuilder() - .setFeatureSet("project/featureSet:1") - .addAllEntities( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("b")).build())) - .build()) - .stream() - .map(AbstractMessageLite::toByteArray) - .collect(Collectors.toList()) - .toArray(new byte[0][0]); + onlineServingService = new OnlineServingService(retriever, specService, tracer); } @Test @@ -148,14 +122,11 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { .setSpec(getFeatureSetSpec()) .build(); - List> featureRowBytes = - featureRows.stream() - .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) - .collect(Collectors.toList()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(connection.sync()).thenReturn(syncCommands); - when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); + when(retriever.getOnlineFeatures( + request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) + .thenReturn(Collections.singletonList(featureRows)); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -173,100 +144,13 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { .putFields("project/feature1:1", intValue(2)) .putFields("project/feature2:1", intValue(2))) .build(); - GetOnlineFeaturesResponse actual = redisServingService.getOnlineFeatures(request); + GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); } @Test - public void shouldReturnResponseWithValuesWhenFeatureSetSpecHasUnspecifiedMaxAge() { - GetOnlineFeaturesRequest request = - GetOnlineFeaturesRequest.newBuilder() - .addFeatures( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) - .addFeatures( - FeatureReference.newBuilder() - .setName("feature2") - .setVersion(1) - .setProject("project") - .build()) - .addEntityRows( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a"))) - .addEntityRows( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b"))) - .build(); - - List featureRows = - Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(2)) // much older timestamp - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .setFeatureSet("featureSet:1") - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(15)) // much older timestamp - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .setFeatureSet("featureSet:1") - .build()); - - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .addAllFeatureReferences(request.getFeaturesList()) - .setSpec(getFeatureSetSpecWithNoMaxAge()) - .build(); - - List> featureRowBytes = - featureRows.stream() - .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) - .collect(Collectors.toList()); - when(specService.getFeatureSets(request.getFeaturesList())) - .thenReturn(Collections.singletonList(featureSetRequest)); - when(connection.sync()).thenReturn(syncCommands); - when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); - - GetOnlineFeaturesResponse expected = - GetOnlineFeaturesResponse.newBuilder() - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a")) - .putFields("project/feature1:1", intValue(1)) - .putFields("project/feature2:1", intValue(1))) - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b")) - .putFields("project/feature1:1", intValue(2)) - .putFields("project/feature2:1", intValue(2))) - .build(); - GetOnlineFeaturesResponse actual = redisServingService.getOnlineFeatures(request); - assertThat( - responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); - } - - @Test - public void shouldReturnKeysWithoutVersionifNotProvided() { + public void shouldReturnKeysWithoutVersionIfNotProvided() { GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() .addFeatures( @@ -318,14 +202,11 @@ public void shouldReturnKeysWithoutVersionifNotProvided() { .setSpec(getFeatureSetSpec()) .build(); - List> featureRowBytes = - featureRows.stream() - .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) - .collect(Collectors.toList()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(connection.sync()).thenReturn(syncCommands); - when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); + when(retriever.getOnlineFeatures( + request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) + .thenReturn(Collections.singletonList(featureRows)); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -343,7 +224,7 @@ public void shouldReturnKeysWithoutVersionifNotProvided() { .putFields("project/feature1:1", intValue(2)) .putFields("project/feature2", intValue(2))) .build(); - GetOnlineFeaturesResponse actual = redisServingService.getOnlineFeatures(request); + GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); } @@ -383,27 +264,29 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { .setSpec(getFeatureSetSpec()) .build(); - FeatureRow featureRowPresent = - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .setFeatureSet("featureSet:1") - .build(); - - List> featureRowBytes = + List featureRows = Lists.newArrayList( - KeyValue.from(new byte[1], Optional.of(featureRowPresent.toByteArray())), - KeyValue.from(new byte[1], Optional.empty())); + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet:1") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + .build(), + FeatureRow.newBuilder() + .setFeatureSet("project/featureSet:1") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").build(), + Field.newBuilder().setName("feature2").build())) + .build()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(connection.sync()).thenReturn(syncCommands); - when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); + when(retriever.getOnlineFeatures( + request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) + .thenReturn(Collections.singletonList(featureRows)); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -421,7 +304,7 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { .putFields("project/feature1:1", Value.newBuilder().build()) .putFields("project/feature2:1", Value.newBuilder().build())) .build(); - GetOnlineFeaturesResponse actual = redisServingService.getOnlineFeatures(request); + GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); } @@ -487,14 +370,11 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { .setSpec(spec) .build(); - List> featureRowBytes = - featureRows.stream() - .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) - .collect(Collectors.toList()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(connection.sync()).thenReturn(syncCommands); - when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); + when(retriever.getOnlineFeatures( + request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) + .thenReturn(Collections.singletonList(featureRows)); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -512,7 +392,7 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { .putFields("project/feature1:1", Value.newBuilder().build()) .putFields("project/feature2:1", Value.newBuilder().build())) .build(); - GetOnlineFeaturesResponse actual = redisServingService.getOnlineFeatures(request); + GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); } @@ -569,14 +449,11 @@ public void shouldFilterOutUndesiredRows() { .setSpec(getFeatureSetSpec()) .build(); - List> featureRowBytes = - featureRows.stream() - .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) - .collect(Collectors.toList()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(connection.sync()).thenReturn(syncCommands); - when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); + when(retriever.getOnlineFeatures( + request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) + .thenReturn(Collections.singletonList(featureRows)); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -592,7 +469,7 @@ public void shouldFilterOutUndesiredRows() { .putFields("entity2", strValue("b")) .putFields("project/feature1:1", intValue(2))) .build(); - GetOnlineFeaturesResponse actual = redisServingService.getOnlineFeatures(request); + GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); } diff --git a/storage/connectors/bigquery/pom.xml b/storage/connectors/bigquery/pom.xml index bca3da671c4..469bf857341 100644 --- a/storage/connectors/bigquery/pom.xml +++ b/storage/connectors/bigquery/pom.xml @@ -30,6 +30,12 @@ google-cloud-storage + + org.apache.beam + beam-sdks-java-io-google-cloud-platform + ${org.apache.beam.version} + + com.google.auto.value auto-value-annotations @@ -67,13 +73,6 @@ hamcrest-library test - - - org.apache.beam - beam-sdks-java-io-google-cloud-platform - 2.16.0 - compile - diff --git a/storage/connectors/pom.xml b/storage/connectors/pom.xml index bb6883a0b01..2cb0b106081 100644 --- a/storage/connectors/pom.xml +++ b/storage/connectors/pom.xml @@ -19,6 +19,21 @@ bigquery + + + + org.apache.maven.plugins + maven-dependency-plugin + + + + javax.annotation + + + + + + dev.feast diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retrieval/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retrieval/RedisOnlineRetriever.java index 7ff925a7223..17d7af4bdbc 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retrieval/RedisOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retrieval/RedisOnlineRetriever.java @@ -177,7 +177,13 @@ private List sendMultiGet(List keys) { .collect(Collectors.toList()) .toArray(new byte[0][0]); return syncCommands.mget(binaryKeys).stream() - .map(keyValue -> keyValue.getValueOrElse(null)) + .map( + keyValue -> { + if (keyValue == null) { + return null; + } + return keyValue.getValueOrElse(null); + }) .collect(Collectors.toList()); } catch (Exception e) { throw Status.NOT_FOUND diff --git a/serving/src/test/java/feast/serving/encoding/FeatureRowDecoderTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retrieval/FeatureRowDecoderTest.java similarity index 98% rename from serving/src/test/java/feast/serving/encoding/FeatureRowDecoderTest.java rename to storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retrieval/FeatureRowDecoderTest.java index 8f6c79ad66c..8f0a8b48afc 100644 --- a/serving/src/test/java/feast/serving/encoding/FeatureRowDecoderTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retrieval/FeatureRowDecoderTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.serving.encoding; +package feast.storage.connectors.redis.retrieval; import static org.junit.Assert.*; diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retrieval/RedisOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retrieval/RedisOnlineRetrieverTest.java index 9fabc5fae73..fb69fe75f7f 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retrieval/RedisOnlineRetrieverTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retrieval/RedisOnlineRetrieverTest.java @@ -259,17 +259,4 @@ private FeatureSetSpec getFeatureSetSpec() { .setMaxAge(Duration.newBuilder().setSeconds(30)) // default .build(); } - - private FeatureSetSpec getFeatureSetSpecWithNoMaxAge() { - return FeatureSetSpec.newBuilder() - .setProject("project") - .setName("featureSet") - .setVersion(1) - .addEntities(EntitySpec.newBuilder().setName("entity1")) - .addEntities(EntitySpec.newBuilder().setName("entity2")) - .addFeatures(FeatureSpec.newBuilder().setName("feature1")) - .addFeatures(FeatureSpec.newBuilder().setName("feature2")) - .setMaxAge(Duration.newBuilder().setSeconds(0).setNanos(0).build()) - .build(); - } } From 0b323a06490092a2dc5ad520cfc6d62d5fc5583c Mon Sep 17 00:00:00 2001 From: zhilingc Date: Tue, 24 Mar 2020 11:54:47 +0800 Subject: [PATCH 2/2] Remove extra exclusion clause --- serving/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/serving/pom.xml b/serving/pom.xml index 3c33d6ab3d5..9495b29d264 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -97,10 +97,6 @@ org.apache.beam * - - org.apache.beam - * -