From db5a3707e93d5c16d119619ccdecf076b54024c0 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Mon, 5 Apr 2021 13:37:11 +0800 Subject: [PATCH 01/21] Add cassandra storage module and scaffold IT Signed-off-by: Terence Lim --- serving/pom.xml | 12 + serving/src/main/resources/application.yml | 6 + .../java/feast/serving/it/BaseAuthIT.java | 19 ++ .../serving/it/ServingServiceCassandraIT.java | 268 ++++++++++++++++++ .../docker-compose-cassandra-it.yml | 31 ++ storage/connectors/cassandra/pom.xml | 39 +++ storage/connectors/pom.xml | 1 + 7 files changed, 376 insertions(+) create mode 100644 serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java create mode 100644 serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml create mode 100644 storage/connectors/cassandra/pom.xml diff --git a/serving/pom.xml b/serving/pom.xml index 6eca569..b8f8ce7 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -90,6 +90,12 @@ ${project.version} + + dev.feast + feast-storage-connector-cassandra + ${project.version} + + dev.feast feast-common @@ -342,6 +348,12 @@ 1.15.2 test + + org.testcontainers + cassandra + 1.15.2 + test + org.awaitility awaitility diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index b23a345..a960385 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -59,6 +59,12 @@ feast: config: project_id: instance_id: + - name: cassandra + type: CASSANDRA + config: + host: localhost + port: 9094 + data_center: datacenter1 tracing: # If true, Feast will provide tracing data (using OpenTracing API) for various RPC method calls # which can be useful to debug performance issues and perform benchmarking diff --git a/serving/src/test/java/feast/serving/it/BaseAuthIT.java b/serving/src/test/java/feast/serving/it/BaseAuthIT.java index ab6e169..93c2e58 100644 --- a/serving/src/test/java/feast/serving/it/BaseAuthIT.java +++ b/serving/src/test/java/feast/serving/it/BaseAuthIT.java @@ -54,6 +54,10 @@ public class BaseAuthIT { static final String BIGTABLE = "bigtable_1"; static final int BIGTABLE_PORT = 8086; + static final String CASSANDRA = "cassandra_1"; + static final int CASSANDRA_PORT = 9042; + static final String CASSANDRA_DATACENTER = "datacenter1"; + static final int FEAST_CORE_PORT = 6565; @DynamicPropertySource @@ -80,6 +84,21 @@ static void properties(DynamicPropertyRegistry registry) { registry.add("feast.stores[1].config.project_id", () -> "test-project"); registry.add("feast.stores[1].config.instance_id", () -> "test-instance"); + registry.add("feast.stores[2].name", () -> "cassandra"); + registry.add("feast.stores[2].type", () -> "CASSANDRA"); + registry.add( + "feast.stores[2].config.host", + () -> { + try { + return InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + e.printStackTrace(); + return ""; + } + }); + registry.add("feast.stores[2].config.port", () -> CASSANDRA_PORT); + registry.add("feast.stores[2].config.data_center", () -> CASSANDRA_DATACENTER); + registry.add("feast.core-authentication.options.oauth_url", () -> TOKEN_URL); registry.add("feast.core-authentication.options.grant_type", () -> GRANT_TYPE); registry.add("feast.core-authentication.options.client_id", () -> CLIENT_ID); diff --git a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java new file mode 100644 index 0000000..2831430 --- /dev/null +++ b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java @@ -0,0 +1,268 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.it; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.BoundStatement; +import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.core.cql.Row; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hashing; +import feast.common.it.DataGenerator; +import feast.proto.core.EntityProto; +import feast.proto.serving.ServingAPIProto; +import feast.proto.serving.ServingServiceGrpc; +import feast.proto.types.ValueProto; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; +import org.junit.ClassRule; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@ActiveProfiles("it") +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = { + "feast.core-cache-refresh-interval=1", + "feast.active_store=cassandra", + "spring.main.allow-bean-definition-overriding=true" + }) +@Testcontainers +public class ServingServiceCassandraIT extends BaseAuthIT { + + static final Map options = new HashMap<>(); + static CoreSimpleAPIClient coreClient; + static ServingServiceGrpc.ServingServiceBlockingStub servingStub; + + static CqlSession cqlSession; + // static Session session; + static final int FEAST_SERVING_PORT = 6570; + + static final ServingAPIProto.FeatureReferenceV2 feature1Reference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + static final ServingAPIProto.FeatureReferenceV2 feature2Reference = + DataGenerator.createFeatureReference("rides", "trip_distance"); + static final ServingAPIProto.FeatureReferenceV2 feature3Reference = + DataGenerator.createFeatureReference("rides", "trip_empty"); + static final ServingAPIProto.FeatureReferenceV2 feature4Reference = + DataGenerator.createFeatureReference("rides", "trip_wrong_type"); + static final String KEYSPACE = "feast"; + + @ClassRule @Container + public static DockerComposeContainer environment = + new DockerComposeContainer( + new File("src/test/resources/docker-compose/docker-compose-cassandra-it.yml")) + .withExposedService( + CORE, + FEAST_CORE_PORT, + Wait.forLogMessage(".*gRPC Server started.*\\n", 1) + .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) + .withExposedService(CASSANDRA, CASSANDRA_PORT); + + @DynamicPropertySource + static void initialize(DynamicPropertyRegistry registry) { + registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); + } + + @BeforeAll + static void globalSetup() throws IOException { + coreClient = TestUtils.getApiClientForCore(FEAST_CORE_PORT); + servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); + + cqlSession = + CqlSession.builder() + .addContactPoint( + new InetSocketAddress( + environment.getServiceHost("cassandra_1", CASSANDRA_PORT), + environment.getServicePort("cassandra_1", CASSANDRA_PORT))) + .withLocalDatacenter(CASSANDRA_DATACENTER) + .build(); + + /** Feast resource creation Workflow */ + String projectName = "default"; + // Apply Entity (driver_id) + String driverEntityName = "driver_id"; + String driverEntityDescription = "My driver id"; + ValueProto.ValueType.Enum driverEntityType = ValueProto.ValueType.Enum.INT64; + EntityProto.EntitySpecV2 driverEntitySpec = + EntityProto.EntitySpecV2.newBuilder() + .setName(driverEntityName) + .setDescription(driverEntityDescription) + .setValueType(driverEntityType) + .build(); + TestUtils.applyEntity(coreClient, projectName, driverEntitySpec); + + // Apply Entity (merchant_id) + String merchantEntityName = "merchant_id"; + String merchantEntityDescription = "My driver id"; + ValueProto.ValueType.Enum merchantEntityType = ValueProto.ValueType.Enum.INT64; + EntityProto.EntitySpecV2 merchantEntitySpec = + EntityProto.EntitySpecV2.newBuilder() + .setName(merchantEntityName) + .setDescription(merchantEntityDescription) + .setValueType(merchantEntityType) + .build(); + TestUtils.applyEntity(coreClient, projectName, merchantEntitySpec); + + // Apply FeatureTable (rides) + String ridesFeatureTableName = "rides"; + ImmutableList ridesEntities = ImmutableList.of(driverEntityName); + ImmutableMap ridesFeatures = + ImmutableMap.of( + "trip_cost", + ValueProto.ValueType.Enum.INT64, + "trip_distance", + ValueProto.ValueType.Enum.DOUBLE, + "trip_empty", + ValueProto.ValueType.Enum.DOUBLE, + "trip_wrong_type", + ValueProto.ValueType.Enum.STRING); + TestUtils.applyFeatureTable( + coreClient, projectName, ridesFeatureTableName, ridesEntities, ridesFeatures, 7200); + + cqlSession.execute(String.format("DROP KEYSPACE IF EXISTS %s", KEYSPACE)); + + cqlSession.execute( + String.format( + "CREATE KEYSPACE %s WITH replication = \n" + + "{'class':'SimpleStrategy','replication_factor':'1'};", + KEYSPACE)); + + ImmutableList.of(driverEntityName, merchantEntityName); + String cassandraTableName = String.format("%s__%s", projectName, driverEntityName); + + cqlSession.execute( + String.format( + "CREATE TABLE IF NOT EXISTS %s.%s (key BLOB, schema_ref BLOB, PRIMARY KEY (key));", + KEYSPACE, cassandraTableName)); + + // Add column families + cqlSession.execute( + String.format("ALTER TABLE %s.%s ADD (rides BLOB)", KEYSPACE, cassandraTableName)); + + /** Single Entity Ingestion Workflow */ + Schema ftSchema = + SchemaBuilder.record("DriverData") + .namespace(ridesFeatureTableName) + .fields() + .requiredInt(feature1Reference.getName()) + .requiredDouble(feature2Reference.getName()) + .nullableString(feature3Reference.getName(), "null") + .requiredString(feature4Reference.getName()) + .endRecord(); + byte[] schemaReference = + Hashing.murmur3_32().hashBytes(ftSchema.toString().getBytes()).asBytes(); + + GenericRecord record = + new GenericRecordBuilder(ftSchema) + .set("trip_cost", 5) + .set("trip_distance", 3.5) + .set("trip_empty", null) + .set("trip_wrong_type", "test") + .build(); + byte[] entityFeatureKey = + String.valueOf(DataGenerator.createInt64Value(1).getInt64Val()).getBytes(); + byte[] entityFeatureValue = createEntityValue(ftSchema, schemaReference, record); + byte[] schemaKey = createSchemaKey(schemaReference); + + PreparedStatement statement = + cqlSession.prepare( + String.format( + "INSERT INTO %s.%s (key, schema_ref, rides) VALUES (?, ?, ?)", + KEYSPACE, cassandraTableName)); + cqlSession.execute( + statement.bind( + ByteBuffer.wrap(entityFeatureKey), + ByteBuffer.wrap(schemaKey), + ByteBuffer.wrap(entityFeatureValue))); + } + + private static byte[] createSchemaKey(byte[] schemaReference) throws IOException { + ByteArrayOutputStream concatOutputStream = new ByteArrayOutputStream(); + concatOutputStream.write(schemaReference); + byte[] schemaKey = concatOutputStream.toByteArray(); + + return schemaKey; + } + + private static byte[] createEntityValue( + Schema schema, byte[] schemaReference, GenericRecord record) throws IOException { + // Entity-Feature Row + byte[] avroSerializedFeatures = recordToAvro(record, schema); + + ByteArrayOutputStream concatOutputStream = new ByteArrayOutputStream(); + concatOutputStream.write(schemaReference); + concatOutputStream.write("".getBytes()); + concatOutputStream.write(avroSerializedFeatures); + byte[] entityFeatureValue = concatOutputStream.toByteArray(); + + return entityFeatureValue; + } + + private static byte[] recordToAvro(GenericRecord datum, Schema schema) throws IOException { + GenericDatumWriter writer = new GenericDatumWriter<>(schema); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Encoder encoder = EncoderFactory.get().binaryEncoder(output, null); + writer.write(datum, encoder); + encoder.flush(); + + return output.toByteArray(); + } + + @Test + public void shouldRegisterSingleEntityAndGetOnlineFeatures() { + String projectName = "default"; + String entityName = "driver_id"; + String cassandraTableName = String.format("%s__%s", projectName, entityName); + byte[] entityFeatureKey = + String.valueOf(DataGenerator.createInt64Value(1).getInt64Val()).getBytes(); + String featureTableName = "rides"; + + BoundStatement statement = + cqlSession + .prepare( + String.format("SELECT * FROM %s.%s WHERE key = ?", KEYSPACE, cassandraTableName)) + .bind(ByteBuffer.wrap(entityFeatureKey)); + Row row = cqlSession.execute(statement).one(); + + assertEquals(ByteBuffer.wrap(entityFeatureKey), row.getByteBuffer("key")); + } +} diff --git a/serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml b/serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml new file mode 100644 index 0000000..15afad0 --- /dev/null +++ b/serving/src/test/resources/docker-compose/docker-compose-cassandra-it.yml @@ -0,0 +1,31 @@ +version: '3' + +services: + core: + image: gcr.io/kf-feast/feast-core:develop + volumes: + - ./core/application-it.yml:/etc/feast/application.yml + environment: + DB_HOST: db + restart: on-failure + depends_on: + - db + ports: + - 6565:6565 + command: + - java + - -jar + - /opt/feast/feast-core.jar + - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml + + db: + image: postgres:12-alpine + environment: + POSTGRES_PASSWORD: password + ports: + - "5432:5432" + + cassandra: + image: datastax/cassandra:4.0 + ports: + - "9042:9042" \ No newline at end of file diff --git a/storage/connectors/cassandra/pom.xml b/storage/connectors/cassandra/pom.xml new file mode 100644 index 0000000..6fdc7a8 --- /dev/null +++ b/storage/connectors/cassandra/pom.xml @@ -0,0 +1,39 @@ + + + + feast-storage-connectors + dev.feast + ${revision} + + + 4.0.0 + feast-storage-connector-cassandra + + + 11 + 11 + + + + + org.apache.avro + avro + 1.10.2 + + + + com.datastax.oss + java-driver-core + 4.11.0 + + + + com.datastax.oss + java-driver-query-builder + 4.11.0 + + + + \ No newline at end of file diff --git a/storage/connectors/pom.xml b/storage/connectors/pom.xml index efa82c5..5be4caf 100644 --- a/storage/connectors/pom.xml +++ b/storage/connectors/pom.xml @@ -17,6 +17,7 @@ redis bigtable + cassandra From 51550efdc19830ecf43d9f8bcf07b3d79692a3d4 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Mon, 5 Apr 2021 13:38:16 +0800 Subject: [PATCH 02/21] Add partial implementation of Cassandra retriever Signed-off-by: Terence Lim --- .../feast/serving/config/FeastProperties.java | 11 +- .../config/ServingServiceConfigV2.java | 19 ++ .../retriever/CassandraOnlineRetriever.java | 164 ++++++++++++++++++ .../retriever/CassandraSchemaRegistry.java | 86 +++++++++ .../retriever/CassandraStoreConfig.java | 42 +++++ 5 files changed, 321 insertions(+), 1 deletion(-) create mode 100644 storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java create mode 100644 storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java create mode 100644 storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java diff --git a/serving/src/main/java/feast/serving/config/FeastProperties.java b/serving/src/main/java/feast/serving/config/FeastProperties.java index 6794b2d..081ed8f 100644 --- a/serving/src/main/java/feast/serving/config/FeastProperties.java +++ b/serving/src/main/java/feast/serving/config/FeastProperties.java @@ -27,6 +27,7 @@ import feast.common.auth.credentials.CoreAuthenticationProperties; import feast.common.logging.config.LoggingProperties; import feast.storage.connectors.bigtable.retriever.BigTableStoreConfig; +import feast.storage.connectors.cassandra.retriever.CassandraStoreConfig; import feast.storage.connectors.redis.retriever.RedisClusterStoreConfig; import feast.storage.connectors.redis.retriever.RedisStoreConfig; import io.lettuce.core.ReadFrom; @@ -270,7 +271,7 @@ public void setName(String name) { } /** - * Gets the store type. Example are REDIS, REDIS_CLUSTER or BIGTABLE + * Gets the store type. Example are REDIS, REDIS_CLUSTER, BIGTABLE or CASSANDRA * * @return the store type as a String. */ @@ -316,6 +317,13 @@ public BigTableStoreConfig getBigtableConfig() { return new BigTableStoreConfig(this.config.get("project_id"), this.config.get("instance_id")); } + public CassandraStoreConfig getCassandraConfig() { + return new CassandraStoreConfig( + this.config.get("host"), + Integer.valueOf(this.config.get("port")), + this.config.get("data_center")); + } + /** * Sets the store config. Please protos/feast/core/Store.proto for the specific options for each * store. @@ -329,6 +337,7 @@ public void setConfig(Map config) { public enum StoreType { BIGTABLE, + CASSANDRA, REDIS, REDIS_CLUSTER; } diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index d6ed6db..b4a0a6e 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -16,6 +16,8 @@ */ package feast.serving.config; +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.CqlSessionBuilder; import com.google.cloud.bigtable.data.v2.BigtableDataClient; import com.google.cloud.bigtable.data.v2.BigtableDataSettings; import feast.serving.service.OnlineServingServiceV2; @@ -24,9 +26,12 @@ import feast.storage.api.retriever.OnlineRetrieverV2; import feast.storage.connectors.bigtable.retriever.BigTableOnlineRetriever; import feast.storage.connectors.bigtable.retriever.BigTableStoreConfig; +import feast.storage.connectors.cassandra.retriever.CassandraOnlineRetriever; +import feast.storage.connectors.cassandra.retriever.CassandraStoreConfig; import feast.storage.connectors.redis.retriever.*; import io.opentracing.Tracer; import java.io.IOException; +import java.net.InetSocketAddress; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; @@ -77,6 +82,20 @@ public ServingServiceV2 servingServiceV2( OnlineRetrieverV2 bigtableRetriever = new BigTableOnlineRetriever(bigtableClient); servingService = new OnlineServingServiceV2(bigtableRetriever, specService, tracer); break; + case CASSANDRA: + CassandraStoreConfig config = feastProperties.getActiveStore().getCassandraConfig(); + String host = config.getHost(); + Integer port = config.getPort(); + String dataCenter = config.getDataCenter(); + + CqlSession session = + new CqlSessionBuilder() + .addContactPoint(new InetSocketAddress(host, port)) + .withLocalDatacenter(dataCenter) + .build(); + OnlineRetrieverV2 cassandraRetriever = new CassandraOnlineRetriever(session); + servingService = new OnlineServingServiceV2(cassandraRetriever, specService, tracer); + break; } return servingService; diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java new file mode 100644 index 0000000..5cb4318 --- /dev/null +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -0,0 +1,164 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.cassandra.retriever; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import com.datastax.oss.driver.api.querybuilder.QueryBuilder; +import com.datastax.oss.driver.api.querybuilder.select.Select; +import com.datastax.oss.driver.api.querybuilder.select.Selector; +import com.google.protobuf.ByteString; +import feast.proto.serving.ServingAPIProto; +import feast.proto.types.ValueProto; +import feast.storage.api.retriever.Feature; +import feast.storage.api.retriever.OnlineRetrieverV2; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class CassandraOnlineRetriever implements OnlineRetrieverV2 { + + private CqlSession session; + private CassandraSchemaRegistry schemaRegistry; + + private static String ENTITY_KEY = "key"; + + public CassandraOnlineRetriever(CqlSession session) { + this.session = session; + this.schemaRegistry = new CassandraSchemaRegistry(session); + } + + /** + * Generate name of Cassandra table in the form of __ + * + * @param project Name of Feast project + * @param entityNames List of entities used in retrieval call + * @return Name of Cassandra table + */ + private String getTableName(String project, List entityNames) { + String tableName = + String.format("%s__%s", project, entityNames.stream().collect(Collectors.joining("__"))); + + return tableName; + } + + /** + * Convert Entity value from Feast valueType to String type. Currently only supports STRING_VAL, + * INT64_VAL, INT32_VAL and BYTES_VAL. + * + * @param v Entity value of Feast valueType + * @return String representation of Entity value + */ + private String valueToString(ValueProto.Value v) { + String stringRepr; + switch (v.getValCase()) { + case STRING_VAL: + stringRepr = v.getStringVal(); + break; + case INT64_VAL: + stringRepr = String.valueOf(v.getInt64Val()); + break; + case INT32_VAL: + stringRepr = String.valueOf(v.getInt32Val()); + break; + case BYTES_VAL: + stringRepr = v.getBytesVal().toString(); + break; + default: + throw new RuntimeException("Type is not supported to be entity"); + } + + return stringRepr; + } + + /** + * Generate Cassandra key in the form of entity values joined by #. + * + * @param entityRow Single EntityRow representation in feature retrieval call + * @param entityNames List of entities related to feature references in retrieval call + * @return Cassandra key for retrieval + */ + private ByteString convertEntityValueToCassandraKey( + ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow entityRow, List entityNames) { + return ByteString.copyFrom( + entityNames.stream() + .map(entity -> entityRow.getFieldsMap().get(entity)) + .map(this::valueToString) + .collect(Collectors.joining("#")) + .getBytes()); + } + + /** + * Retrieve BigTable table column families based on FeatureTable names. + * + * @param featureReferences List of feature references of features in retrieval call + * @return List of String of FeatureTable names + */ + private List getColumnFamilies( + List featureReferences) { + return featureReferences.stream() + .map(ServingAPIProto.FeatureReferenceV2::getFeatureTable) + .collect(Collectors.toList()); + } + + @Override + public List> getOnlineFeatures( + String project, + List entityRows, + List featureReferences, + List entityNames) { + + List columnFamilies = getColumnFamilies(featureReferences); + String tableName = getTableName(project, entityNames); + List rowKeys = + entityRows.stream() + .map(row -> convertEntityValueToCassandraKey(row, entityNames)) + .collect(Collectors.toList()); + + Map rowsFromCassandra = + getFeaturesFromCassandra(tableName, rowKeys, columnFamilies); + + return Collections.emptyList(); + } + + /** + * Retrieve rows for each row entity key by generating Cassandra Query with filters based on + * columns. + * + * @param tableName Name of Cassandra table + * @param rowKeys List of keys of rows to retrieve + * @param columnFamilies List of FeatureTable names + * @return Map of retrieved features for each rowKey + */ + private Map getFeaturesFromCassandra( + String tableName, List rowKeys, List columnFamilies) { + List selectors = + columnFamilies.stream().map(cf -> Selector.column(cf)).collect(Collectors.toList()); + + Select query = + QueryBuilder.selectFrom(tableName) + .listOf(selectors) + .whereColumn(ENTITY_KEY) + .in(QueryBuilder.bindMarker()); + + SimpleStatement statement = query.build(); + + return Collections.emptyMap(); + } +} diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java new file mode 100644 index 0000000..faf76e0 --- /dev/null +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.cassandra.retriever; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.ResultSet; +import com.datastax.oss.driver.api.core.cql.Row; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.protobuf.ByteString; +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutionException; +import org.apache.avro.Schema; + +public class CassandraSchemaRegistry { + private final CqlSession session; + private final LoadingCache cache; + + private static String KEYSPACE = "feast"; + private static String SCHEMA_COLUMN = "avro_schema"; + + public static class SchemaReference { + private final String tableName; + private final ByteString schemaHash; + + public SchemaReference(String tableName, ByteString schemaHash) { + this.tableName = tableName; + this.schemaHash = schemaHash; + } + + public String getTableName() { + return tableName; + } + + public ByteString getSchemaHash() { + return schemaHash; + } + } + + public CassandraSchemaRegistry(CqlSession session) { + this.session = session; + + CacheLoader schemaCacheLoader = CacheLoader.from(this::loadSchema); + + cache = CacheBuilder.newBuilder().build(schemaCacheLoader); + } + + public Schema getSchema(SchemaReference reference) { + Schema schema; + try { + schema = this.cache.get(reference); + } catch (ExecutionException | CacheLoader.InvalidCacheLoadException e) { + throw new RuntimeException(String.format("Unable to find Schema"), e); + } + return schema; + } + + private Schema loadSchema(SchemaReference reference) { + ResultSet rs = + session.execute( + String.format( + "SELECT %s FROM %s.%s WHERE schema_ref = '%s'", + SCHEMA_COLUMN, + KEYSPACE, + reference.getTableName(), + ByteBuffer.wrap(reference.getSchemaHash().toByteArray()))); + Row row = rs.one(); + + return new Schema.Parser().parse(row.getTupleValue(SCHEMA_COLUMN).toString()); + } +} diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java new file mode 100644 index 0000000..1f9af44 --- /dev/null +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.cassandra.retriever; + +public class CassandraStoreConfig { + + private final String host; + private final Integer port; + private final String dataCenter; + + public CassandraStoreConfig(String host, Integer port, String dataCenter) { + this.host = host; + this.port = port; + this.dataCenter = dataCenter; + } + + public String getHost() { + return this.host; + } + + public Integer getPort() { + return this.port; + } + + public String getDataCenter() { + return this.dataCenter; + } +} From b0c88719bfdbe37bb31aa8149d7a3c7353fca188 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Mon, 5 Apr 2021 16:57:45 +0800 Subject: [PATCH 03/21] Add partial implementation Signed-off-by: Terence Lim --- serving/pom.xml | 6 - .../feast/serving/config/FeastProperties.java | 3 +- .../config/ServingServiceConfigV2.java | 2 + serving/src/main/resources/application.yml | 3 +- .../java/feast/serving/it/BaseAuthIT.java | 2 + .../serving/it/ServingServiceCassandraIT.java | 103 +++++++++++-- .../retriever/CassandraOnlineRetriever.java | 137 ++++++++++++++++-- .../retriever/CassandraSchemaRegistry.java | 38 +++-- .../retriever/CassandraStoreConfig.java | 8 +- 9 files changed, 248 insertions(+), 54 deletions(-) diff --git a/serving/pom.xml b/serving/pom.xml index b8f8ce7..dfcc6e5 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -348,12 +348,6 @@ 1.15.2 test - - org.testcontainers - cassandra - 1.15.2 - test - org.awaitility awaitility diff --git a/serving/src/main/java/feast/serving/config/FeastProperties.java b/serving/src/main/java/feast/serving/config/FeastProperties.java index 081ed8f..31276c5 100644 --- a/serving/src/main/java/feast/serving/config/FeastProperties.java +++ b/serving/src/main/java/feast/serving/config/FeastProperties.java @@ -321,7 +321,8 @@ public CassandraStoreConfig getCassandraConfig() { return new CassandraStoreConfig( this.config.get("host"), Integer.valueOf(this.config.get("port")), - this.config.get("data_center")); + this.config.get("data_center"), + this.config.get("keyspace")); } /** diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index b4a0a6e..53d0812 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -87,11 +87,13 @@ public ServingServiceV2 servingServiceV2( String host = config.getHost(); Integer port = config.getPort(); String dataCenter = config.getDataCenter(); + String keySpace = config.getKeySpace(); CqlSession session = new CqlSessionBuilder() .addContactPoint(new InetSocketAddress(host, port)) .withLocalDatacenter(dataCenter) + .withKeyspace(keySpace) .build(); OnlineRetrieverV2 cassandraRetriever = new CassandraOnlineRetriever(session); servingService = new OnlineServingServiceV2(cassandraRetriever, specService, tracer); diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index a960385..d58dd46 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -63,8 +63,9 @@ feast: type: CASSANDRA config: host: localhost - port: 9094 + port: 9042 data_center: datacenter1 + keyspace: feast tracing: # If true, Feast will provide tracing data (using OpenTracing API) for various RPC method calls # which can be useful to debug performance issues and perform benchmarking diff --git a/serving/src/test/java/feast/serving/it/BaseAuthIT.java b/serving/src/test/java/feast/serving/it/BaseAuthIT.java index 93c2e58..9352b80 100644 --- a/serving/src/test/java/feast/serving/it/BaseAuthIT.java +++ b/serving/src/test/java/feast/serving/it/BaseAuthIT.java @@ -57,6 +57,7 @@ public class BaseAuthIT { static final String CASSANDRA = "cassandra_1"; static final int CASSANDRA_PORT = 9042; static final String CASSANDRA_DATACENTER = "datacenter1"; + static final String CASSANDRA_KEYSPACE = "feast"; static final int FEAST_CORE_PORT = 6565; @@ -98,6 +99,7 @@ static void properties(DynamicPropertyRegistry registry) { }); registry.add("feast.stores[2].config.port", () -> CASSANDRA_PORT); registry.add("feast.stores[2].config.data_center", () -> CASSANDRA_DATACENTER); + registry.add("feast.stores[2].config.keyspace", () -> CASSANDRA_KEYSPACE); registry.add("feast.core-authentication.options.oauth_url", () -> TOKEN_URL); registry.add("feast.core-authentication.options.grant_type", () -> GRANT_TYPE); diff --git a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java index 2831430..3cdd60d 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java @@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.hash.Hashing; import feast.common.it.DataGenerator; +import feast.common.models.FeatureV2; import feast.proto.core.EntityProto; import feast.proto.serving.ServingAPIProto; import feast.proto.serving.ServingServiceGrpc; @@ -38,6 +39,7 @@ import java.time.Duration; import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; import org.apache.avro.generic.GenericDatumWriter; @@ -73,7 +75,6 @@ public class ServingServiceCassandraIT extends BaseAuthIT { static ServingServiceGrpc.ServingServiceBlockingStub servingStub; static CqlSession cqlSession; - // static Session session; static final int FEAST_SERVING_PORT = 6570; static final ServingAPIProto.FeatureReferenceV2 feature1Reference = @@ -84,7 +85,6 @@ public class ServingServiceCassandraIT extends BaseAuthIT { DataGenerator.createFeatureReference("rides", "trip_empty"); static final ServingAPIProto.FeatureReferenceV2 feature4Reference = DataGenerator.createFeatureReference("rides", "trip_wrong_type"); - static final String KEYSPACE = "feast"; @ClassRule @Container public static DockerComposeContainer environment = @@ -158,25 +158,41 @@ static void globalSetup() throws IOException { TestUtils.applyFeatureTable( coreClient, projectName, ridesFeatureTableName, ridesEntities, ridesFeatures, 7200); - cqlSession.execute(String.format("DROP KEYSPACE IF EXISTS %s", KEYSPACE)); + // Apply FeatureTable (rides_merchant) + String rideMerchantFeatureTableName = "rides_merchant"; + ImmutableList ridesMerchantEntities = + ImmutableList.of(driverEntityName, merchantEntityName); + TestUtils.applyFeatureTable( + coreClient, + projectName, + rideMerchantFeatureTableName, + ridesMerchantEntities, + ridesFeatures, + 7200); + + // Cassandra Table names + String cassandraTableName = String.format("%s__%s", projectName, driverEntityName); + String compoundCassandraTableName = + String.format( + "%s__%s", + projectName, ridesMerchantEntities.stream().collect(Collectors.joining("__"))); + cqlSession.execute(String.format("DROP KEYSPACE IF EXISTS %s", CASSANDRA_KEYSPACE)); cqlSession.execute( String.format( "CREATE KEYSPACE %s WITH replication = \n" + "{'class':'SimpleStrategy','replication_factor':'1'};", - KEYSPACE)); - - ImmutableList.of(driverEntityName, merchantEntityName); - String cassandraTableName = String.format("%s__%s", projectName, driverEntityName); + CASSANDRA_KEYSPACE)); cqlSession.execute( String.format( "CREATE TABLE IF NOT EXISTS %s.%s (key BLOB, schema_ref BLOB, PRIMARY KEY (key));", - KEYSPACE, cassandraTableName)); + CASSANDRA_KEYSPACE, cassandraTableName)); // Add column families cqlSession.execute( - String.format("ALTER TABLE %s.%s ADD (rides BLOB)", KEYSPACE, cassandraTableName)); + String.format( + "ALTER TABLE %s.%s ADD (rides BLOB)", CASSANDRA_KEYSPACE, cassandraTableName)); /** Single Entity Ingestion Workflow */ Schema ftSchema = @@ -207,12 +223,20 @@ static void globalSetup() throws IOException { cqlSession.prepare( String.format( "INSERT INTO %s.%s (key, schema_ref, rides) VALUES (?, ?, ?)", - KEYSPACE, cassandraTableName)); + CASSANDRA_KEYSPACE, cassandraTableName)); cqlSession.execute( statement.bind( ByteBuffer.wrap(entityFeatureKey), ByteBuffer.wrap(schemaKey), ByteBuffer.wrap(entityFeatureValue))); + + // set up options for call credentials + options.put("oauth_url", TOKEN_URL); + options.put(CLIENT_ID, CLIENT_ID); + options.put(CLIENT_SECRET, CLIENT_SECRET); + options.put("jwkEndpointURI", JWK_URI); + options.put("audience", AUDIENCE); + options.put("grant_type", GRANT_TYPE); } private static byte[] createSchemaKey(byte[] schemaReference) throws IOException { @@ -249,6 +273,62 @@ private static byte[] recordToAvro(GenericRecord datum, Schema schema) throws IO @Test public void shouldRegisterSingleEntityAndGetOnlineFeatures() { + String projectName = "default"; + String entityName = "driver_id"; + ValueProto.Value entityValue = DataGenerator.createInt64Value(1); + + // Instantiate EntityRows + ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow entityRow = + DataGenerator.createEntityRow(entityName, entityValue, 100); + ImmutableList entityRows = + ImmutableList.of(entityRow); + + // Instantiate FeatureReferences + ServingAPIProto.FeatureReferenceV2 featureReference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + ServingAPIProto.FeatureReferenceV2 notFoundFeatureReference = + DataGenerator.createFeatureReference("rides", "trip_transaction"); + + ImmutableList featureReferences = + ImmutableList.of(featureReference, notFoundFeatureReference); + + // Build GetOnlineFeaturesRequestV2 + ServingAPIProto.GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + ServingAPIProto.GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + ImmutableMap expectedValueMap = + ImmutableMap.of( + entityName, + entityValue, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createInt64Value(5), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap = + ImmutableMap.of( + entityName, + ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); + + ServingAPIProto.GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + ServingAPIProto.GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap) + .putAllStatuses(expectedStatusMap) + .build(); + ImmutableList expectedFieldValuesList = + ImmutableList.of(expectedFieldValues); + + assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); + } + + @Test + public void shouldRegisterSingleEntityAndGetOnlineFeatures2() { String projectName = "default"; String entityName = "driver_id"; String cassandraTableName = String.format("%s__%s", projectName, entityName); @@ -259,7 +339,8 @@ public void shouldRegisterSingleEntityAndGetOnlineFeatures() { BoundStatement statement = cqlSession .prepare( - String.format("SELECT * FROM %s.%s WHERE key = ?", KEYSPACE, cassandraTableName)) + String.format( + "SELECT * FROM %s.%s WHERE key = ?", CASSANDRA_KEYSPACE, cassandraTableName)) .bind(ByteBuffer.wrap(entityFeatureKey)); Row row = cqlSession.execute(statement).one(); diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java index 5cb4318..a9ed7bd 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -17,20 +17,32 @@ package feast.storage.connectors.cassandra.retriever; import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.Row; -import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.querybuilder.QueryBuilder; import com.datastax.oss.driver.api.querybuilder.select.Select; import com.datastax.oss.driver.api.querybuilder.select.Selector; -import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; import feast.proto.serving.ServingAPIProto; import feast.proto.types.ValueProto; import feast.storage.api.retriever.Feature; +import feast.storage.api.retriever.NativeFeature; import feast.storage.api.retriever.OnlineRetrieverV2; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import org.apache.avro.AvroRuntimeException; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.Decoder; +import org.apache.avro.io.DecoderFactory; public class CassandraOnlineRetriever implements OnlineRetrieverV2 { @@ -38,6 +50,8 @@ public class CassandraOnlineRetriever implements OnlineRetrieverV2 { private CassandraSchemaRegistry schemaRegistry; private static String ENTITY_KEY = "key"; + private static String SCHEMA_REF_KEY = "schema_ref"; + private static String TIMESTAMP_COLUMN = String.format("writetime(%s)", SCHEMA_REF_KEY); public CassandraOnlineRetriever(CqlSession session) { this.session = session; @@ -94,9 +108,9 @@ private String valueToString(ValueProto.Value v) { * @param entityNames List of entities related to feature references in retrieval call * @return Cassandra key for retrieval */ - private ByteString convertEntityValueToCassandraKey( + private ByteBuffer convertEntityValueToCassandraKey( ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow entityRow, List entityNames) { - return ByteString.copyFrom( + return ByteBuffer.wrap( entityNames.stream() .map(entity -> entityRow.getFieldsMap().get(entity)) .map(this::valueToString) @@ -105,7 +119,7 @@ private ByteString convertEntityValueToCassandraKey( } /** - * Retrieve BigTable table column families based on FeatureTable names. + * Retrieve Cassandra table column families based on FeatureTable names. * * @param featureReferences List of feature references of features in retrieval call * @return List of String of FeatureTable names @@ -117,6 +131,46 @@ private List getColumnFamilies( .collect(Collectors.toList()); } + private List decodeFeatures( + ByteBuffer schemaRefKey, + ByteBuffer value, + List featureReferences, + long timestamp) + throws IOException { + + CassandraSchemaRegistry.SchemaReference schemaReference = + new CassandraSchemaRegistry.SchemaReference(schemaRefKey); + + Schema schema = schemaRegistry.getSchema(schemaReference); + GenericDatumReader reader = new GenericDatumReader<>(schema); + Decoder decoder = DecoderFactory.get().binaryDecoder(value.array(), null); + GenericRecord record = reader.read(null, decoder); + + return featureReferences.stream() + .map( + featureReference -> { + Object featureValue; + try { + featureValue = record.get(featureReference.getName()); + } catch (AvroRuntimeException e) { + // Feature is not found in schema + return null; + } + if (featureValue != null) { + return new NativeFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + featureValue); + } + return new NativeFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + new Object()); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + @Override public List> getOnlineFeatures( String project, @@ -126,15 +180,18 @@ public List> getOnlineFeatures( List columnFamilies = getColumnFamilies(featureReferences); String tableName = getTableName(project, entityNames); - List rowKeys = + + List rowKeys = entityRows.stream() .map(row -> convertEntityValueToCassandraKey(row, entityNames)) .collect(Collectors.toList()); - Map rowsFromCassandra = + Map rowsFromCassandra = getFeaturesFromCassandra(tableName, rowKeys, columnFamilies); + List> features = + convertRowToFeature(rowKeys, rowsFromCassandra, featureReferences, columnFamilies); - return Collections.emptyList(); + return features; } /** @@ -146,19 +203,71 @@ public List> getOnlineFeatures( * @param columnFamilies List of FeatureTable names * @return Map of retrieved features for each rowKey */ - private Map getFeaturesFromCassandra( - String tableName, List rowKeys, List columnFamilies) { + private Map getFeaturesFromCassandra( + String tableName, List rowKeys, List columnFamilies) { + // Specify columns to retrieve List selectors = - columnFamilies.stream().map(cf -> Selector.column(cf)).collect(Collectors.toList()); + columnFamilies.stream() + .map(cf -> Selector.column(cf)) + .distinct() + .collect(Collectors.toList()); + selectors.add(Selector.column(ENTITY_KEY)); + selectors.add(Selector.column(SCHEMA_REF_KEY)); + selectors.add(Selector.writeTime(SCHEMA_REF_KEY)); Select query = - QueryBuilder.selectFrom(tableName) + QueryBuilder.selectFrom(String.format("\"%s\"", tableName)) .listOf(selectors) .whereColumn(ENTITY_KEY) .in(QueryBuilder.bindMarker()); - SimpleStatement statement = query.build(); + BoundStatement statement = session.prepare(query.build()).bind(rowKeys); + + return StreamSupport.stream(session.execute(statement).spliterator(), false) + .collect(Collectors.toMap((Row row) -> row.getByteBuffer(ENTITY_KEY), Function.identity())); + } - return Collections.emptyMap(); + /** + * Converts rowCell feature value into @NativeFeature type. + * + * @param rowKeys List of keys of rows to retrieve + * @param rows Map of rowKey to Row related to it + * @param featureReferences List of feature references + * @return List of List of Features associated with respective rowKey + */ + private List> convertRowToFeature( + List rowKeys, + Map rows, + List featureReferences, + List columnFamilies) { + + return rowKeys.stream() + .map( + rowKey -> { + if (!rows.containsKey(rowKey)) { + return Collections.emptyList(); + } else { + Row row = rows.get(rowKey); + + String featureTableColumn = columnFamilies.get(0); + ByteBuffer schemaRefKey = row.getByteBuffer(SCHEMA_REF_KEY); + ByteBuffer featureValues = row.getByteBuffer(featureTableColumn); + + List features; + try { + features = + decodeFeatures( + schemaRefKey, + featureValues, + featureReferences, + row.getLong(TIMESTAMP_COLUMN)); + } catch (IOException e) { + throw new RuntimeException("Failed to decode features from Cassandra"); + } + + return features.stream().collect(Collectors.toList()); + } + }) + .collect(Collectors.toList()); } } diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java index faf76e0..4d054eb 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java @@ -17,12 +17,13 @@ package feast.storage.connectors.cassandra.retriever; import com.datastax.oss.driver.api.core.CqlSession; -import com.datastax.oss.driver.api.core.cql.ResultSet; +import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.querybuilder.QueryBuilder; +import com.datastax.oss.driver.api.querybuilder.select.Select; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.google.protobuf.ByteString; import java.nio.ByteBuffer; import java.util.concurrent.ExecutionException; import org.apache.avro.Schema; @@ -32,22 +33,18 @@ public class CassandraSchemaRegistry { private final LoadingCache cache; private static String KEYSPACE = "feast"; + private static String SCHEMA_REF_TABLE = "feast_schema_reference"; + private static String SCHEMA_REF_COLUMN = "schema_ref"; private static String SCHEMA_COLUMN = "avro_schema"; public static class SchemaReference { - private final String tableName; - private final ByteString schemaHash; + private final ByteBuffer schemaHash; - public SchemaReference(String tableName, ByteString schemaHash) { - this.tableName = tableName; + public SchemaReference(ByteBuffer schemaHash) { this.schemaHash = schemaHash; } - public String getTableName() { - return tableName; - } - - public ByteString getSchemaHash() { + public ByteBuffer getSchemaHash() { return schemaHash; } } @@ -71,15 +68,16 @@ public Schema getSchema(SchemaReference reference) { } private Schema loadSchema(SchemaReference reference) { - ResultSet rs = - session.execute( - String.format( - "SELECT %s FROM %s.%s WHERE schema_ref = '%s'", - SCHEMA_COLUMN, - KEYSPACE, - reference.getTableName(), - ByteBuffer.wrap(reference.getSchemaHash().toByteArray()))); - Row row = rs.one(); + String tableName = String.format("%s.%s", KEYSPACE, SCHEMA_REF_TABLE); + Select query = + QueryBuilder.selectFrom(tableName) + .column(SCHEMA_COLUMN) + .whereColumn(SCHEMA_REF_COLUMN) + .in(QueryBuilder.bindMarker()); + + BoundStatement statement = session.prepare(query.build()).bind(reference.getSchemaHash()); + + Row row = session.execute(statement).one(); return new Schema.Parser().parse(row.getTupleValue(SCHEMA_COLUMN).toString()); } diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java index 1f9af44..fe82a20 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java @@ -21,11 +21,13 @@ public class CassandraStoreConfig { private final String host; private final Integer port; private final String dataCenter; + private final String keySpace; - public CassandraStoreConfig(String host, Integer port, String dataCenter) { + public CassandraStoreConfig(String host, Integer port, String dataCenter, String keySpace) { this.host = host; this.port = port; this.dataCenter = dataCenter; + this.keySpace = keySpace; } public String getHost() { @@ -39,4 +41,8 @@ public Integer getPort() { public String getDataCenter() { return this.dataCenter; } + + public String getKeySpace() { + return this.keySpace; + } } From 5bf02552fd4e79d12be66c251845e526669ee19a Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Mon, 5 Apr 2021 21:25:46 +0800 Subject: [PATCH 04/21] Update retrieval query and create schema table in IT Signed-off-by: Terence Lim --- .../java/feast/serving/it/BaseAuthIT.java | 1 + .../serving/it/ServingServiceCassandraIT.java | 23 ++++++++++--- .../retriever/CassandraOnlineRetriever.java | 32 ++++++++----------- .../retriever/CassandraSchemaRegistry.java | 9 +++--- 4 files changed, 39 insertions(+), 26 deletions(-) diff --git a/serving/src/test/java/feast/serving/it/BaseAuthIT.java b/serving/src/test/java/feast/serving/it/BaseAuthIT.java index 9352b80..46271c2 100644 --- a/serving/src/test/java/feast/serving/it/BaseAuthIT.java +++ b/serving/src/test/java/feast/serving/it/BaseAuthIT.java @@ -58,6 +58,7 @@ public class BaseAuthIT { static final int CASSANDRA_PORT = 9042; static final String CASSANDRA_DATACENTER = "datacenter1"; static final String CASSANDRA_KEYSPACE = "feast"; + static final String CASSANDRA_SCHEMA_TABLE = "feast_schema_reference"; static final int FEAST_CORE_PORT = 6565; diff --git a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java index 3cdd60d..48bb91f 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java @@ -170,7 +170,7 @@ static void globalSetup() throws IOException { ridesFeatures, 7200); - // Cassandra Table names + /** Create Cassandra Tables Workflow */ String cassandraTableName = String.format("%s__%s", projectName, driverEntityName); String compoundCassandraTableName = String.format( @@ -184,15 +184,15 @@ static void globalSetup() throws IOException { + "{'class':'SimpleStrategy','replication_factor':'1'};", CASSANDRA_KEYSPACE)); + // Single Entity Cassandra Table cqlSession.execute( String.format( - "CREATE TABLE IF NOT EXISTS %s.%s (key BLOB, schema_ref BLOB, PRIMARY KEY (key));", + "CREATE TABLE %s.%s (key BLOB PRIMARY KEY, schema_ref BLOB);", CASSANDRA_KEYSPACE, cassandraTableName)); // Add column families cqlSession.execute( - String.format( - "ALTER TABLE %s.%s ADD (rides BLOB)", CASSANDRA_KEYSPACE, cassandraTableName)); + String.format("ALTER TABLE %s.%s ADD rides BLOB;", CASSANDRA_KEYSPACE, cassandraTableName)); /** Single Entity Ingestion Workflow */ Schema ftSchema = @@ -230,6 +230,21 @@ static void globalSetup() throws IOException { ByteBuffer.wrap(schemaKey), ByteBuffer.wrap(entityFeatureValue))); + /** Schema Ingestion Workflow */ + cqlSession.execute( + String.format( + "CREATE TABLE %s.%s (schema_ref BLOB PRIMARY KEY, avro_schema BLOB);", + CASSANDRA_KEYSPACE, CASSANDRA_SCHEMA_TABLE)); + + PreparedStatement schemaStatement = + cqlSession.prepare( + String.format( + "INSERT INTO %s.%s (schema_ref, avro_schema) VALUES (?, ?);", + CASSANDRA_KEYSPACE, CASSANDRA_SCHEMA_TABLE)); + cqlSession.execute( + schemaStatement.bind( + ByteBuffer.wrap(schemaKey), ByteBuffer.wrap(ftSchema.toString().getBytes()))); + // set up options for call credentials options.put("oauth_url", TOKEN_URL); options.put(CLIENT_ID, CLIENT_ID); diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java index a9ed7bd..25c2788 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -20,8 +20,7 @@ import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.Row; import com.datastax.oss.driver.api.querybuilder.QueryBuilder; -import com.datastax.oss.driver.api.querybuilder.select.Select; -import com.datastax.oss.driver.api.querybuilder.select.Selector; +import com.datastax.oss.driver.api.querybuilder.select.SelectFrom; import com.google.protobuf.Timestamp; import feast.proto.serving.ServingAPIProto; import feast.proto.types.ValueProto; @@ -205,23 +204,20 @@ public List> getOnlineFeatures( */ private Map getFeaturesFromCassandra( String tableName, List rowKeys, List columnFamilies) { - // Specify columns to retrieve - List selectors = - columnFamilies.stream() - .map(cf -> Selector.column(cf)) - .distinct() - .collect(Collectors.toList()); - selectors.add(Selector.column(ENTITY_KEY)); - selectors.add(Selector.column(SCHEMA_REF_KEY)); - selectors.add(Selector.writeTime(SCHEMA_REF_KEY)); - - Select query = - QueryBuilder.selectFrom(String.format("\"%s\"", tableName)) - .listOf(selectors) - .whereColumn(ENTITY_KEY) - .in(QueryBuilder.bindMarker()); + SelectFrom query = QueryBuilder.selectFrom(String.format("\"%s\"", tableName)); - BoundStatement statement = session.prepare(query.build()).bind(rowKeys); + BoundStatement statement = + session + .prepare( + query + .columns(columnFamilies) + .column(SCHEMA_REF_KEY) + .column(ENTITY_KEY) + .writeTime(SCHEMA_REF_KEY) + .whereColumn(ENTITY_KEY) + .in(QueryBuilder.bindMarker()) + .build()) + .bind(rowKeys); return StreamSupport.stream(session.execute(statement).spliterator(), false) .collect(Collectors.toMap((Row row) -> row.getByteBuffer(ENTITY_KEY), Function.identity())); diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java index 4d054eb..42109c0 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java @@ -25,6 +25,7 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.concurrent.ExecutionException; import org.apache.avro.Schema; @@ -32,7 +33,6 @@ public class CassandraSchemaRegistry { private final CqlSession session; private final LoadingCache cache; - private static String KEYSPACE = "feast"; private static String SCHEMA_REF_TABLE = "feast_schema_reference"; private static String SCHEMA_REF_COLUMN = "schema_ref"; private static String SCHEMA_COLUMN = "avro_schema"; @@ -68,17 +68,18 @@ public Schema getSchema(SchemaReference reference) { } private Schema loadSchema(SchemaReference reference) { - String tableName = String.format("%s.%s", KEYSPACE, SCHEMA_REF_TABLE); + String tableName = String.format("\"%s\"", SCHEMA_REF_TABLE); Select query = QueryBuilder.selectFrom(tableName) .column(SCHEMA_COLUMN) .whereColumn(SCHEMA_REF_COLUMN) - .in(QueryBuilder.bindMarker()); + .isEqualTo(QueryBuilder.bindMarker()); BoundStatement statement = session.prepare(query.build()).bind(reference.getSchemaHash()); Row row = session.execute(statement).one(); - return new Schema.Parser().parse(row.getTupleValue(SCHEMA_COLUMN).toString()); + return new Schema.Parser() + .parse(StandardCharsets.UTF_8.decode(row.getByteBuffer(SCHEMA_COLUMN)).toString()); } } From f799aacacbd921077a44cab10d6bca345006da50 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Mon, 5 Apr 2021 22:04:19 +0800 Subject: [PATCH 05/21] Fix IT Signed-off-by: Terence Lim --- .../java/feast/serving/it/BaseAuthIT.java | 1 + .../serving/it/ServingServiceCassandraIT.java | 29 ++++++++++++------- .../retriever/CassandraOnlineRetriever.java | 27 ++++++++++------- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/serving/src/test/java/feast/serving/it/BaseAuthIT.java b/serving/src/test/java/feast/serving/it/BaseAuthIT.java index 46271c2..2f86e96 100644 --- a/serving/src/test/java/feast/serving/it/BaseAuthIT.java +++ b/serving/src/test/java/feast/serving/it/BaseAuthIT.java @@ -59,6 +59,7 @@ public class BaseAuthIT { static final String CASSANDRA_DATACENTER = "datacenter1"; static final String CASSANDRA_KEYSPACE = "feast"; static final String CASSANDRA_SCHEMA_TABLE = "feast_schema_reference"; + static final String CASSANDRA_ENTITY_KEY = "key"; static final int FEAST_CORE_PORT = 6565; diff --git a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java index 48bb91f..38ea69d 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java @@ -31,6 +31,7 @@ import feast.proto.serving.ServingAPIProto; import feast.proto.serving.ServingServiceGrpc; import feast.proto.types.ValueProto; +import io.grpc.ManagedChannel; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -48,6 +49,7 @@ import org.apache.avro.io.Encoder; import org.apache.avro.io.EncoderFactory; import org.junit.ClassRule; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; @@ -187,12 +189,13 @@ static void globalSetup() throws IOException { // Single Entity Cassandra Table cqlSession.execute( String.format( - "CREATE TABLE %s.%s (key BLOB PRIMARY KEY, schema_ref BLOB);", - CASSANDRA_KEYSPACE, cassandraTableName)); + "CREATE TABLE %s.%s (key BLOB PRIMARY KEY);", CASSANDRA_KEYSPACE, cassandraTableName)); // Add column families cqlSession.execute( - String.format("ALTER TABLE %s.%s ADD rides BLOB;", CASSANDRA_KEYSPACE, cassandraTableName)); + String.format( + "ALTER TABLE %s.%s ADD (%s BLOB, %s__schema_ref BLOB);", + CASSANDRA_KEYSPACE, cassandraTableName, ridesFeatureTableName, ridesFeatureTableName)); /** Single Entity Ingestion Workflow */ Schema ftSchema = @@ -216,14 +219,18 @@ static void globalSetup() throws IOException { .build(); byte[] entityFeatureKey = String.valueOf(DataGenerator.createInt64Value(1).getInt64Val()).getBytes(); - byte[] entityFeatureValue = createEntityValue(ftSchema, schemaReference, record); + byte[] entityFeatureValue = createEntityValue(ftSchema, record); byte[] schemaKey = createSchemaKey(schemaReference); PreparedStatement statement = cqlSession.prepare( String.format( - "INSERT INTO %s.%s (key, schema_ref, rides) VALUES (?, ?, ?)", - CASSANDRA_KEYSPACE, cassandraTableName)); + "INSERT INTO %s.%s (%s, %s__schema_ref, %s) VALUES (?, ?, ?)", + CASSANDRA_KEYSPACE, + cassandraTableName, + CASSANDRA_ENTITY_KEY, + ridesFeatureTableName, + ridesFeatureTableName)); cqlSession.execute( statement.bind( ByteBuffer.wrap(entityFeatureKey), @@ -262,14 +269,11 @@ private static byte[] createSchemaKey(byte[] schemaReference) throws IOException return schemaKey; } - private static byte[] createEntityValue( - Schema schema, byte[] schemaReference, GenericRecord record) throws IOException { + private static byte[] createEntityValue(Schema schema, GenericRecord record) throws IOException { // Entity-Feature Row byte[] avroSerializedFeatures = recordToAvro(record, schema); ByteArrayOutputStream concatOutputStream = new ByteArrayOutputStream(); - concatOutputStream.write(schemaReference); - concatOutputStream.write("".getBytes()); concatOutputStream.write(avroSerializedFeatures); byte[] entityFeatureValue = concatOutputStream.toByteArray(); @@ -286,6 +290,11 @@ private static byte[] recordToAvro(GenericRecord datum, Schema schema) throws IO return output.toByteArray(); } + @AfterAll + static void tearDown() { + ((ManagedChannel) servingStub.getChannel()).shutdown(); + } + @Test public void shouldRegisterSingleEntityAndGetOnlineFeatures() { String projectName = "default"; diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java index 25c2788..6d122a4 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -29,10 +29,7 @@ import feast.storage.api.retriever.OnlineRetrieverV2; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -50,7 +47,6 @@ public class CassandraOnlineRetriever implements OnlineRetrieverV2 { private static String ENTITY_KEY = "key"; private static String SCHEMA_REF_KEY = "schema_ref"; - private static String TIMESTAMP_COLUMN = String.format("writetime(%s)", SCHEMA_REF_KEY); public CassandraOnlineRetriever(CqlSession session) { this.session = session; @@ -140,9 +136,12 @@ private List decodeFeatures( CassandraSchemaRegistry.SchemaReference schemaReference = new CassandraSchemaRegistry.SchemaReference(schemaRefKey); + // Convert ByteBuffer to ByteArray + byte[] bytesArray = new byte[value.remaining()]; + value.get(bytesArray, 0, bytesArray.length); Schema schema = schemaRegistry.getSchema(schemaReference); GenericDatumReader reader = new GenericDatumReader<>(schema); - Decoder decoder = DecoderFactory.get().binaryDecoder(value.array(), null); + Decoder decoder = DecoderFactory.get().binaryDecoder(bytesArray, null); GenericRecord record = reader.read(null, decoder); return featureReferences.stream() @@ -205,15 +204,19 @@ public List> getOnlineFeatures( private Map getFeaturesFromCassandra( String tableName, List rowKeys, List columnFamilies) { SelectFrom query = QueryBuilder.selectFrom(String.format("\"%s\"", tableName)); + List schemaRefKeyColumns = + columnFamilies.stream() + .map(cf -> String.format("%s__%s", cf, SCHEMA_REF_KEY)) + .collect(Collectors.toList()); BoundStatement statement = session .prepare( query .columns(columnFamilies) - .column(SCHEMA_REF_KEY) + .columns(schemaRefKeyColumns) .column(ENTITY_KEY) - .writeTime(SCHEMA_REF_KEY) + .writeTime(schemaRefKeyColumns.get(0)) .whereColumn(ENTITY_KEY) .in(QueryBuilder.bindMarker()) .build()) @@ -246,7 +249,11 @@ private List> convertRowToFeature( Row row = rows.get(rowKey); String featureTableColumn = columnFamilies.get(0); - ByteBuffer schemaRefKey = row.getByteBuffer(SCHEMA_REF_KEY); + String schemaRefKeyColumn = + String.format("%s__%s", featureTableColumn, SCHEMA_REF_KEY); + String timestampColumn = String.format("writetime(%s)", schemaRefKeyColumn); + + ByteBuffer schemaRefKey = row.getByteBuffer(schemaRefKeyColumn); ByteBuffer featureValues = row.getByteBuffer(featureTableColumn); List features; @@ -256,7 +263,7 @@ private List> convertRowToFeature( schemaRefKey, featureValues, featureReferences, - row.getLong(TIMESTAMP_COLUMN)); + row.getLong(timestampColumn)); } catch (IOException e) { throw new RuntimeException("Failed to decode features from Cassandra"); } From 62519e36bb50110464d98e9b8bb769fd9c6aed6d Mon Sep 17 00:00:00 2001 From: Khor Shu Heng Date: Tue, 6 Apr 2021 01:09:38 +0800 Subject: [PATCH 06/21] Retrieve timestamp and schema ref based on feature table Signed-off-by: Khor Shu Heng --- .../java/feast/common/it/DataGenerator.java | 4 ++ .../serving/it/ServingServiceCassandraIT.java | 9 +-- .../retriever/CassandraOnlineRetriever.java | 63 ++++++++----------- 3 files changed, 33 insertions(+), 43 deletions(-) diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java index ef31f54..8a0dbb0 100644 --- a/common-test/src/main/java/feast/common/it/DataGenerator.java +++ b/common-test/src/main/java/feast/common/it/DataGenerator.java @@ -249,6 +249,10 @@ public static ValueProto.Value createDoubleValue(double value) { return ValueProto.Value.newBuilder().setDoubleVal(value).build(); } + public static ValueProto.Value createInt32Value(int value) { + return ValueProto.Value.newBuilder().setInt32Val(value).build(); + } + public static ValueProto.Value createInt64Value(long value) { return ValueProto.Value.newBuilder().setInt64Val(value).build(); } diff --git a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java index 38ea69d..5301255 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java @@ -40,7 +40,6 @@ import java.time.Duration; import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; import org.apache.avro.generic.GenericDatumWriter; @@ -150,7 +149,7 @@ static void globalSetup() throws IOException { ImmutableMap ridesFeatures = ImmutableMap.of( "trip_cost", - ValueProto.ValueType.Enum.INT64, + ValueProto.ValueType.Enum.INT32, "trip_distance", ValueProto.ValueType.Enum.DOUBLE, "trip_empty", @@ -175,9 +174,7 @@ static void globalSetup() throws IOException { /** Create Cassandra Tables Workflow */ String cassandraTableName = String.format("%s__%s", projectName, driverEntityName); String compoundCassandraTableName = - String.format( - "%s__%s", - projectName, ridesMerchantEntities.stream().collect(Collectors.joining("__"))); + String.format("%s__%s", projectName, String.join("__", ridesMerchantEntities)); cqlSession.execute(String.format("DROP KEYSPACE IF EXISTS %s", CASSANDRA_KEYSPACE)); cqlSession.execute( @@ -327,7 +324,7 @@ public void shouldRegisterSingleEntityAndGetOnlineFeatures() { entityName, entityValue, FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt64Value(5), + DataGenerator.createInt32Value(5), FeatureV2.getFeatureStringRef(notFoundFeatureReference), DataGenerator.createEmptyValue()); diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java index 6d122a4..a9a544c 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -20,7 +20,7 @@ import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.Row; import com.datastax.oss.driver.api.querybuilder.QueryBuilder; -import com.datastax.oss.driver.api.querybuilder.select.SelectFrom; +import com.datastax.oss.driver.api.querybuilder.select.Select; import com.google.protobuf.Timestamp; import feast.proto.serving.ServingAPIProto; import feast.proto.types.ValueProto; @@ -42,11 +42,12 @@ public class CassandraOnlineRetriever implements OnlineRetrieverV2 { - private CqlSession session; - private CassandraSchemaRegistry schemaRegistry; + private final CqlSession session; + private final CassandraSchemaRegistry schemaRegistry; - private static String ENTITY_KEY = "key"; - private static String SCHEMA_REF_KEY = "schema_ref"; + private static final String ENTITY_KEY = "key"; + private static final String SCHEMA_REF_SUFFIX = "__schema_ref"; + private static final String EVENT_TIMESTAMP_SUFFIX = "__event_timestamp"; public CassandraOnlineRetriever(CqlSession session) { this.session = session; @@ -61,10 +62,8 @@ public CassandraOnlineRetriever(CqlSession session) { * @return Name of Cassandra table */ private String getTableName(String project, List entityNames) { - String tableName = - String.format("%s__%s", project, entityNames.stream().collect(Collectors.joining("__"))); - return tableName; + return String.format("%s__%s", project, String.join("__", entityNames)); } /** @@ -123,6 +122,7 @@ private List getColumnFamilies( List featureReferences) { return featureReferences.stream() .map(ServingAPIProto.FeatureReferenceV2::getFeatureTable) + .distinct() .collect(Collectors.toList()); } @@ -186,10 +186,8 @@ public List> getOnlineFeatures( Map rowsFromCassandra = getFeaturesFromCassandra(tableName, rowKeys, columnFamilies); - List> features = - convertRowToFeature(rowKeys, rowsFromCassandra, featureReferences, columnFamilies); - return features; + return convertRowToFeature(rowKeys, rowsFromCassandra, featureReferences, columnFamilies); } /** @@ -198,29 +196,24 @@ public List> getOnlineFeatures( * * @param tableName Name of Cassandra table * @param rowKeys List of keys of rows to retrieve - * @param columnFamilies List of FeatureTable names + * @param featureTables List of FeatureTable names * @return Map of retrieved features for each rowKey */ private Map getFeaturesFromCassandra( - String tableName, List rowKeys, List columnFamilies) { - SelectFrom query = QueryBuilder.selectFrom(String.format("\"%s\"", tableName)); - List schemaRefKeyColumns = - columnFamilies.stream() - .map(cf -> String.format("%s__%s", cf, SCHEMA_REF_KEY)) - .collect(Collectors.toList()); + String tableName, List rowKeys, List featureTables) { + List schemaRefColumns = + featureTables.stream().map(c -> c + SCHEMA_REF_SUFFIX).collect(Collectors.toList()); + Select query = + QueryBuilder.selectFrom(tableName) + .columns(featureTables) + .columns(schemaRefColumns) + .column(ENTITY_KEY); + for (String featureTable : featureTables) { + query = query.writeTime(featureTable).as(featureTable + EVENT_TIMESTAMP_SUFFIX); + } + query = query.whereColumn(ENTITY_KEY).in(QueryBuilder.bindMarker()); - BoundStatement statement = - session - .prepare( - query - .columns(columnFamilies) - .columns(schemaRefKeyColumns) - .column(ENTITY_KEY) - .writeTime(schemaRefKeyColumns.get(0)) - .whereColumn(ENTITY_KEY) - .in(QueryBuilder.bindMarker()) - .build()) - .bind(rowKeys); + BoundStatement statement = session.prepare(query.build()).bind(rowKeys); return StreamSupport.stream(session.execute(statement).spliterator(), false) .collect(Collectors.toMap((Row row) -> row.getByteBuffer(ENTITY_KEY), Function.identity())); @@ -249,11 +242,7 @@ private List> convertRowToFeature( Row row = rows.get(rowKey); String featureTableColumn = columnFamilies.get(0); - String schemaRefKeyColumn = - String.format("%s__%s", featureTableColumn, SCHEMA_REF_KEY); - String timestampColumn = String.format("writetime(%s)", schemaRefKeyColumn); - - ByteBuffer schemaRefKey = row.getByteBuffer(schemaRefKeyColumn); + ByteBuffer schemaRefKey = row.getByteBuffer(featureTableColumn + SCHEMA_REF_SUFFIX); ByteBuffer featureValues = row.getByteBuffer(featureTableColumn); List features; @@ -263,12 +252,12 @@ private List> convertRowToFeature( schemaRefKey, featureValues, featureReferences, - row.getLong(timestampColumn)); + row.getLong(featureTableColumn + EVENT_TIMESTAMP_SUFFIX)); } catch (IOException e) { throw new RuntimeException("Failed to decode features from Cassandra"); } - return features.stream().collect(Collectors.toList()); + return new ArrayList<>(features); } }) .collect(Collectors.toList()); From 343f5515eee21acf48e29fe19cb51d3ced6d3c74 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Tue, 6 Apr 2021 10:22:15 +0800 Subject: [PATCH 07/21] Fix retrieval logic Signed-off-by: Terence Lim --- .../retriever/CassandraOnlineRetriever.java | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java index a9a544c..23f3b56 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -187,7 +187,7 @@ public List> getOnlineFeatures( Map rowsFromCassandra = getFeaturesFromCassandra(tableName, rowKeys, columnFamilies); - return convertRowToFeature(rowKeys, rowsFromCassandra, featureReferences, columnFamilies); + return convertRowToFeature(rowKeys, rowsFromCassandra, featureReferences); } /** @@ -230,8 +230,7 @@ private Map getFeaturesFromCassandra( private List> convertRowToFeature( List rowKeys, Map rows, - List featureReferences, - List columnFamilies) { + List featureReferences) { return rowKeys.stream() .map( @@ -240,24 +239,30 @@ private List> convertRowToFeature( return Collections.emptyList(); } else { Row row = rows.get(rowKey); + return featureReferences.stream() + .map(ServingAPIProto.FeatureReferenceV2::getFeatureTable) + .distinct() + .flatMap( + featureTableColumn -> { + ByteBuffer featureValues = row.getByteBuffer(featureTableColumn); + ByteBuffer schemaRefKey = + row.getByteBuffer(featureTableColumn + SCHEMA_REF_SUFFIX); - String featureTableColumn = columnFamilies.get(0); - ByteBuffer schemaRefKey = row.getByteBuffer(featureTableColumn + SCHEMA_REF_SUFFIX); - ByteBuffer featureValues = row.getByteBuffer(featureTableColumn); - - List features; - try { - features = - decodeFeatures( - schemaRefKey, - featureValues, - featureReferences, - row.getLong(featureTableColumn + EVENT_TIMESTAMP_SUFFIX)); - } catch (IOException e) { - throw new RuntimeException("Failed to decode features from Cassandra"); - } + List features; + try { + features = + decodeFeatures( + schemaRefKey, + featureValues, + featureReferences, + row.getLong(featureTableColumn + EVENT_TIMESTAMP_SUFFIX)); + } catch (IOException e) { + throw new RuntimeException("Failed to decode features from Cassandra"); + } - return new ArrayList<>(features); + return features.stream(); + }) + .collect(Collectors.toList()); } }) .collect(Collectors.toList()); From 49aa4551a72e8401ebad7f2414d8869c585e4eb0 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Tue, 6 Apr 2021 10:26:23 +0800 Subject: [PATCH 08/21] Refactor IT Signed-off-by: Terence Lim --- .../serving/it/ServingServiceCassandraIT.java | 172 ++++++++++++------ 1 file changed, 120 insertions(+), 52 deletions(-) diff --git a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java index 5301255..53d374a 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java @@ -19,16 +19,16 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.datastax.oss.driver.api.core.CqlSession; -import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.PreparedStatement; -import com.datastax.oss.driver.api.core.cql.Row; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.hash.Hashing; import feast.common.it.DataGenerator; import feast.common.models.FeatureV2; import feast.proto.core.EntityProto; -import feast.proto.serving.ServingAPIProto; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc; import feast.proto.types.ValueProto; import io.grpc.ManagedChannel; @@ -40,6 +40,7 @@ import java.time.Duration; import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; import org.apache.avro.generic.GenericDatumWriter; @@ -78,13 +79,13 @@ public class ServingServiceCassandraIT extends BaseAuthIT { static CqlSession cqlSession; static final int FEAST_SERVING_PORT = 6570; - static final ServingAPIProto.FeatureReferenceV2 feature1Reference = + static final FeatureReferenceV2 feature1Reference = DataGenerator.createFeatureReference("rides", "trip_cost"); - static final ServingAPIProto.FeatureReferenceV2 feature2Reference = + static final FeatureReferenceV2 feature2Reference = DataGenerator.createFeatureReference("rides", "trip_distance"); - static final ServingAPIProto.FeatureReferenceV2 feature3Reference = + static final FeatureReferenceV2 feature3Reference = DataGenerator.createFeatureReference("rides", "trip_empty"); - static final ServingAPIProto.FeatureReferenceV2 feature4Reference = + static final FeatureReferenceV2 feature4Reference = DataGenerator.createFeatureReference("rides", "trip_wrong_type"); @ClassRule @Container @@ -183,16 +184,13 @@ static void globalSetup() throws IOException { + "{'class':'SimpleStrategy','replication_factor':'1'};", CASSANDRA_KEYSPACE)); - // Single Entity Cassandra Table - cqlSession.execute( - String.format( - "CREATE TABLE %s.%s (key BLOB PRIMARY KEY);", CASSANDRA_KEYSPACE, cassandraTableName)); + // Create Cassandra Tables + createCassandraTable(cassandraTableName); + createCassandraTable(compoundCassandraTableName); // Add column families - cqlSession.execute( - String.format( - "ALTER TABLE %s.%s ADD (%s BLOB, %s__schema_ref BLOB);", - CASSANDRA_KEYSPACE, cassandraTableName, ridesFeatureTableName, ridesFeatureTableName)); + addCassandraTableColumn(cassandraTableName, ridesFeatureTableName); + addCassandraTableColumn(compoundCassandraTableName, rideMerchantFeatureTableName); /** Single Entity Ingestion Workflow */ Schema ftSchema = @@ -219,20 +217,50 @@ static void globalSetup() throws IOException { byte[] entityFeatureValue = createEntityValue(ftSchema, record); byte[] schemaKey = createSchemaKey(schemaReference); - PreparedStatement statement = - cqlSession.prepare( - String.format( - "INSERT INTO %s.%s (%s, %s__schema_ref, %s) VALUES (?, ?, ?)", - CASSANDRA_KEYSPACE, - cassandraTableName, - CASSANDRA_ENTITY_KEY, - ridesFeatureTableName, - ridesFeatureTableName)); - cqlSession.execute( - statement.bind( - ByteBuffer.wrap(entityFeatureKey), - ByteBuffer.wrap(schemaKey), - ByteBuffer.wrap(entityFeatureValue))); + ingestData( + ridesFeatureTableName, cassandraTableName, entityFeatureKey, entityFeatureValue, schemaKey); + + /** Compound Entity Ingestion Workflow */ + Schema compoundFtSchema = + SchemaBuilder.record("DriverMerchantData") + .namespace(rideMerchantFeatureTableName) + .fields() + .requiredLong(feature1Reference.getName()) + .requiredDouble(feature2Reference.getName()) + .nullableString(feature3Reference.getName(), "null") + .requiredString(feature4Reference.getName()) + .endRecord(); + byte[] compoundSchemaReference = + Hashing.murmur3_32().hashBytes(compoundFtSchema.toString().getBytes()).asBytes(); + + GenericRecord compoundEntityRecord = + new GenericRecordBuilder(compoundFtSchema) + .set("trip_cost", 10L) + .set("trip_distance", 5.5) + .set("trip_empty", null) + .set("trip_wrong_type", "wrong_type") + .build(); + ValueProto.Value driverEntityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); + ValueProto.Value merchantEntityValue = ValueProto.Value.newBuilder().setInt64Val(1234).build(); + ImmutableMap compoundEntityMap = + ImmutableMap.of( + driverEntityName, driverEntityValue, merchantEntityName, merchantEntityValue); + GetOnlineFeaturesRequestV2.EntityRow entityRow = + DataGenerator.createCompoundEntityRow(compoundEntityMap, 100); + byte[] compoundEntityFeatureKey = + ridesMerchantEntities.stream() + .map(entity -> DataGenerator.valueToString(entityRow.getFieldsMap().get(entity))) + .collect(Collectors.joining("#")) + .getBytes(); + byte[] compoundEntityFeatureValue = createEntityValue(compoundFtSchema, compoundEntityRecord); + byte[] compoundSchemaKey = createSchemaKey(compoundSchemaReference); + + ingestData( + rideMerchantFeatureTableName, + compoundCassandraTableName, + compoundEntityFeatureKey, + compoundEntityFeatureValue, + compoundSchemaKey); /** Schema Ingestion Workflow */ cqlSession.execute( @@ -240,14 +268,8 @@ static void globalSetup() throws IOException { "CREATE TABLE %s.%s (schema_ref BLOB PRIMARY KEY, avro_schema BLOB);", CASSANDRA_KEYSPACE, CASSANDRA_SCHEMA_TABLE)); - PreparedStatement schemaStatement = - cqlSession.prepare( - String.format( - "INSERT INTO %s.%s (schema_ref, avro_schema) VALUES (?, ?);", - CASSANDRA_KEYSPACE, CASSANDRA_SCHEMA_TABLE)); - cqlSession.execute( - schemaStatement.bind( - ByteBuffer.wrap(schemaKey), ByteBuffer.wrap(ftSchema.toString().getBytes()))); + ingestSchema(schemaKey, ftSchema); + ingestSchema(compoundSchemaKey, compoundFtSchema); // set up options for call credentials options.put("oauth_url", TOKEN_URL); @@ -277,6 +299,53 @@ private static byte[] createEntityValue(Schema schema, GenericRecord record) thr return entityFeatureValue; } + private static void createCassandraTable(String cassandraTableName) { + cqlSession.execute( + String.format( + "CREATE TABLE %s.%s (key BLOB PRIMARY KEY);", CASSANDRA_KEYSPACE, cassandraTableName)); + } + + private static void addCassandraTableColumn(String cassandraTableName, String featureTableName) { + cqlSession.execute( + String.format( + "ALTER TABLE %s.%s ADD (%s BLOB, %s__schema_ref BLOB);", + CASSANDRA_KEYSPACE, cassandraTableName, featureTableName, featureTableName)); + } + + private static void ingestData( + String featureTableName, + String cassandraTableName, + byte[] entityFeatureKey, + byte[] entityFeatureValue, + byte[] schemaKey) { + PreparedStatement statement = + cqlSession.prepare( + String.format( + "INSERT INTO %s.%s (%s, %s__schema_ref, %s) VALUES (?, ?, ?)", + CASSANDRA_KEYSPACE, + cassandraTableName, + CASSANDRA_ENTITY_KEY, + featureTableName, + featureTableName)); + + cqlSession.execute( + statement.bind( + ByteBuffer.wrap(entityFeatureKey), + ByteBuffer.wrap(schemaKey), + ByteBuffer.wrap(entityFeatureValue))); + } + + private static void ingestSchema(byte[] schemaKey, Schema schema) { + PreparedStatement schemaStatement = + cqlSession.prepare( + String.format( + "INSERT INTO %s.%s (schema_ref, avro_schema) VALUES (?, ?);", + CASSANDRA_KEYSPACE, CASSANDRA_SCHEMA_TABLE)); + cqlSession.execute( + schemaStatement.bind( + ByteBuffer.wrap(schemaKey), ByteBuffer.wrap(schema.toString().getBytes()))); + } + private static byte[] recordToAvro(GenericRecord datum, Schema schema) throws IOException { GenericDatumWriter writer = new GenericDatumWriter<>(schema); ByteArrayOutputStream output = new ByteArrayOutputStream(); @@ -299,24 +368,23 @@ public void shouldRegisterSingleEntityAndGetOnlineFeatures() { ValueProto.Value entityValue = DataGenerator.createInt64Value(1); // Instantiate EntityRows - ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow entityRow = + GetOnlineFeaturesRequestV2.EntityRow entityRow = DataGenerator.createEntityRow(entityName, entityValue, 100); - ImmutableList entityRows = - ImmutableList.of(entityRow); + ImmutableList entityRows = ImmutableList.of(entityRow); // Instantiate FeatureReferences - ServingAPIProto.FeatureReferenceV2 featureReference = + FeatureReferenceV2 featureReference = DataGenerator.createFeatureReference("rides", "trip_cost"); - ServingAPIProto.FeatureReferenceV2 notFoundFeatureReference = + FeatureReferenceV2 notFoundFeatureReference = DataGenerator.createFeatureReference("rides", "trip_transaction"); - ImmutableList featureReferences = + ImmutableList featureReferences = ImmutableList.of(featureReference, notFoundFeatureReference); // Build GetOnlineFeaturesRequestV2 - ServingAPIProto.GetOnlineFeaturesRequestV2 onlineFeatureRequest = + GetOnlineFeaturesRequestV2 onlineFeatureRequest = TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); - ServingAPIProto.GetOnlineFeaturesResponse featureResponse = + GetOnlineFeaturesResponse featureResponse = servingStub.getOnlineFeaturesV2(onlineFeatureRequest); ImmutableMap expectedValueMap = @@ -328,21 +396,21 @@ public void shouldRegisterSingleEntityAndGetOnlineFeatures() { FeatureV2.getFeatureStringRef(notFoundFeatureReference), DataGenerator.createEmptyValue()); - ImmutableMap expectedStatusMap = + ImmutableMap expectedStatusMap = ImmutableMap.of( entityName, - ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, FeatureV2.getFeatureStringRef(featureReference), - ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.PRESENT, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, FeatureV2.getFeatureStringRef(notFoundFeatureReference), - ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); - ServingAPIProto.GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - ServingAPIProto.GetOnlineFeaturesResponse.FieldValues.newBuilder() + GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + GetOnlineFeaturesResponse.FieldValues.newBuilder() .putAllFields(expectedValueMap) .putAllStatuses(expectedStatusMap) .build(); - ImmutableList expectedFieldValuesList = + ImmutableList expectedFieldValuesList = ImmutableList.of(expectedFieldValues); assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); From 3f848199389e967e7396cc5b3285817321d83a8a Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Tue, 6 Apr 2021 10:26:42 +0800 Subject: [PATCH 09/21] Add more tests Signed-off-by: Terence Lim --- .../serving/it/ServingServiceCassandraIT.java | 165 ++++++++++++++++-- 1 file changed, 152 insertions(+), 13 deletions(-) diff --git a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java index 53d374a..156e934 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java @@ -417,22 +417,161 @@ public void shouldRegisterSingleEntityAndGetOnlineFeatures() { } @Test - public void shouldRegisterSingleEntityAndGetOnlineFeatures2() { + public void shouldRegisterCompoundEntityAndGetOnlineFeatures() { + String projectName = "default"; + String driverEntityName = "driver_id"; + String merchantEntityName = "merchant_id"; + ValueProto.Value driverEntityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); + ValueProto.Value merchantEntityValue = ValueProto.Value.newBuilder().setInt64Val(1234).build(); + + ImmutableMap compoundEntityMap = + ImmutableMap.of( + driverEntityName, driverEntityValue, merchantEntityName, merchantEntityValue); + + // Instantiate EntityRows + GetOnlineFeaturesRequestV2.EntityRow entityRow = + DataGenerator.createCompoundEntityRow(compoundEntityMap, 100); + ImmutableList entityRows = ImmutableList.of(entityRow); + + // Instantiate FeatureReferences + FeatureReferenceV2 featureReference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + FeatureReferenceV2 notFoundFeatureReference = + DataGenerator.createFeatureReference("rides", "trip_transaction"); + + ImmutableList featureReferences = + ImmutableList.of(featureReference, notFoundFeatureReference); + + // Build GetOnlineFeaturesRequestV2 + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + ImmutableMap expectedValueMap = + ImmutableMap.of( + driverEntityName, + driverEntityValue, + merchantEntityName, + merchantEntityValue, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createInt32Value(5), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap = + ImmutableMap.of( + driverEntityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + merchantEntityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap) + .putAllStatuses(expectedStatusMap) + .build(); + ImmutableList expectedFieldValuesList = + ImmutableList.of(expectedFieldValues); + + assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); + } + + @Test + public void shouldReturnCorrectRowCount() { + // getOnlineFeatures Information String projectName = "default"; String entityName = "driver_id"; - String cassandraTableName = String.format("%s__%s", projectName, entityName); - byte[] entityFeatureKey = - String.valueOf(DataGenerator.createInt64Value(1).getInt64Val()).getBytes(); - String featureTableName = "rides"; + ValueProto.Value entityValue1 = ValueProto.Value.newBuilder().setInt64Val(1).build(); + ValueProto.Value entityValue2 = ValueProto.Value.newBuilder().setInt64Val(2).build(); - BoundStatement statement = - cqlSession - .prepare( - String.format( - "SELECT * FROM %s.%s WHERE key = ?", CASSANDRA_KEYSPACE, cassandraTableName)) - .bind(ByteBuffer.wrap(entityFeatureKey)); - Row row = cqlSession.execute(statement).one(); + // Instantiate EntityRows + GetOnlineFeaturesRequestV2.EntityRow entityRow1 = + DataGenerator.createEntityRow(entityName, entityValue1, 100); + GetOnlineFeaturesRequestV2.EntityRow entityRow2 = + DataGenerator.createEntityRow(entityName, entityValue2, 100); + ImmutableList entityRows = + ImmutableList.of(entityRow1, entityRow2); - assertEquals(ByteBuffer.wrap(entityFeatureKey), row.getByteBuffer("key")); + // Instantiate FeatureReferences + FeatureReferenceV2 featureReference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + FeatureReferenceV2 notFoundFeatureReference = + DataGenerator.createFeatureReference("rides", "trip_transaction"); + FeatureReferenceV2 emptyFeatureReference = + DataGenerator.createFeatureReference("rides", "trip_empty"); + + ImmutableList featureReferences = + ImmutableList.of(featureReference, notFoundFeatureReference, emptyFeatureReference); + + // Build GetOnlineFeaturesRequestV2 + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + ImmutableMap expectedValueMap = + ImmutableMap.of( + entityName, + entityValue1, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createInt32Value(5), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue(), + FeatureV2.getFeatureStringRef(emptyFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap = + ImmutableMap.of( + entityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, + FeatureV2.getFeatureStringRef(emptyFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap) + .putAllStatuses(expectedStatusMap) + .build(); + + ImmutableMap expectedValueMap2 = + ImmutableMap.of( + entityName, + entityValue2, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createEmptyValue(), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + DataGenerator.createEmptyValue(), + FeatureV2.getFeatureStringRef(emptyFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedStatusMap2 = + ImmutableMap.of( + entityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(featureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, + FeatureV2.getFeatureStringRef(notFoundFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, + FeatureV2.getFeatureStringRef(emptyFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues2 = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap2) + .putAllStatuses(expectedStatusMap2) + .build(); + ImmutableList expectedFieldValuesList = + ImmutableList.of(expectedFieldValues, expectedFieldValues2); + + assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); } } From 9bdba71646e5b13161dc229eef9acb6847fa3fe0 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Tue, 6 Apr 2021 11:54:36 +0800 Subject: [PATCH 10/21] Fix retrieval logic for same entity with multiple featuretable Signed-off-by: Terence Lim --- .../retriever/CassandraOnlineRetriever.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java index 23f3b56..f74ccff 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -248,13 +248,23 @@ private List> convertRowToFeature( ByteBuffer schemaRefKey = row.getByteBuffer(featureTableColumn + SCHEMA_REF_SUFFIX); + // Prevent retrieval of features from incorrect FeatureTable + List localFeatureReferences = + featureReferences.stream() + .filter( + featureReference -> + featureReference + .getFeatureTable() + .equals(featureTableColumn)) + .collect(Collectors.toList()); + List features; try { features = decodeFeatures( schemaRefKey, featureValues, - featureReferences, + localFeatureReferences, row.getLong(featureTableColumn + EVENT_TIMESTAMP_SUFFIX)); } catch (IOException e) { throw new RuntimeException("Failed to decode features from Cassandra"); From 2d99ac94988e3666d0b7fa48d3674a433a2d56ac Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Tue, 6 Apr 2021 11:55:27 +0800 Subject: [PATCH 11/21] Add IT for same entity with multiple featuretable retrieval Signed-off-by: Terence Lim --- .../serving/it/ServingServiceCassandraIT.java | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java index 156e934..020f592 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java @@ -160,6 +160,18 @@ static void globalSetup() throws IOException { TestUtils.applyFeatureTable( coreClient, projectName, ridesFeatureTableName, ridesEntities, ridesFeatures, 7200); + // Apply FeatureTable (food) + String foodFeatureTableName = "food"; + ImmutableList foodEntities = ImmutableList.of(driverEntityName); + ImmutableMap foodFeatures = + ImmutableMap.of( + "trip_cost", + ValueProto.ValueType.Enum.INT32, + "trip_distance", + ValueProto.ValueType.Enum.DOUBLE); + TestUtils.applyFeatureTable( + coreClient, projectName, foodFeatureTableName, foodEntities, foodFeatures, 7200); + // Apply FeatureTable (rides_merchant) String rideMerchantFeatureTableName = "rides_merchant"; ImmutableList ridesMerchantEntities = @@ -190,6 +202,7 @@ static void globalSetup() throws IOException { // Add column families addCassandraTableColumn(cassandraTableName, ridesFeatureTableName); + addCassandraTableColumn(cassandraTableName, foodFeatureTableName); addCassandraTableColumn(compoundCassandraTableName, rideMerchantFeatureTableName); /** Single Entity Ingestion Workflow */ @@ -220,6 +233,33 @@ static void globalSetup() throws IOException { ingestData( ridesFeatureTableName, cassandraTableName, entityFeatureKey, entityFeatureValue, schemaKey); + Schema foodFtSchema = + SchemaBuilder.record("FoodDriverData") + .namespace(foodFeatureTableName) + .fields() + .requiredInt(feature1Reference.getName()) + .requiredDouble(feature2Reference.getName()) + .endRecord(); + byte[] foodSchemaReference = + Hashing.murmur3_32().hashBytes(foodFtSchema.toString().getBytes()).asBytes(); + + GenericRecord foodRecord = + new GenericRecordBuilder(foodFtSchema) + .set("trip_cost", 12) + .set("trip_distance", 7.5) + .build(); + byte[] foodEntityFeatureKey = + String.valueOf(DataGenerator.createInt64Value(1).getInt64Val()).getBytes(); + byte[] foodEntityFeatureValue = createEntityValue(foodFtSchema, foodRecord); + byte[] foodSchemaKey = createSchemaKey(foodSchemaReference); + + ingestData( + foodFeatureTableName, + cassandraTableName, + foodEntityFeatureKey, + foodEntityFeatureValue, + foodSchemaKey); + /** Compound Entity Ingestion Workflow */ Schema compoundFtSchema = SchemaBuilder.record("DriverMerchantData") @@ -269,6 +309,7 @@ static void globalSetup() throws IOException { CASSANDRA_KEYSPACE, CASSANDRA_SCHEMA_TABLE)); ingestSchema(schemaKey, ftSchema); + ingestSchema(foodSchemaKey, foodFtSchema); ingestSchema(compoundSchemaKey, compoundFtSchema); // set up options for call credentials @@ -574,4 +615,75 @@ public void shouldReturnCorrectRowCount() { assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); } + + @Test + public void shouldReturnFeaturesFromDiffFeatureTable() { + String projectName = "default"; + String entityName = "driver_id"; + ValueProto.Value entityValue = DataGenerator.createInt64Value(1); + + // Instantiate EntityRows + GetOnlineFeaturesRequestV2.EntityRow entityRow = + DataGenerator.createEntityRow(entityName, entityValue, 100); + ImmutableList entityRows = ImmutableList.of(entityRow); + + // Instantiate FeatureReferences + FeatureReferenceV2 rideFeatureReference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + FeatureReferenceV2 rideFeatureReference2 = + DataGenerator.createFeatureReference("rides", "trip_distance"); + FeatureReferenceV2 foodFeatureReference = + DataGenerator.createFeatureReference("food", "trip_cost"); + FeatureReferenceV2 foodFeatureReference2 = + DataGenerator.createFeatureReference("food", "trip_distance"); + + ImmutableList featureReferences = + ImmutableList.of( + rideFeatureReference, + rideFeatureReference2, + foodFeatureReference, + foodFeatureReference2); + + // Build GetOnlineFeaturesRequestV2 + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + ImmutableMap expectedValueMap = + ImmutableMap.of( + entityName, + entityValue, + FeatureV2.getFeatureStringRef(rideFeatureReference), + DataGenerator.createInt32Value(5), + FeatureV2.getFeatureStringRef(rideFeatureReference2), + DataGenerator.createDoubleValue(3.5), + FeatureV2.getFeatureStringRef(foodFeatureReference), + DataGenerator.createInt32Value(12), + FeatureV2.getFeatureStringRef(foodFeatureReference2), + DataGenerator.createDoubleValue(7.5)); + + ImmutableMap expectedStatusMap = + ImmutableMap.of( + entityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(rideFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(rideFeatureReference2), + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(foodFeatureReference), + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(foodFeatureReference2), + GetOnlineFeaturesResponse.FieldStatus.PRESENT); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap) + .putAllStatuses(expectedStatusMap) + .build(); + ImmutableList expectedFieldValuesList = + ImmutableList.of(expectedFieldValues); + + assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); + } } From bef0cb86676f67056202334544b0a466ca8fc164 Mon Sep 17 00:00:00 2001 From: Khor Shu Heng Date: Tue, 6 Apr 2021 13:57:24 +0800 Subject: [PATCH 12/21] Use connection string instead of single host port pair Signed-off-by: Khor Shu Heng --- .../feast/serving/config/FeastProperties.java | 3 +-- .../config/ServingServiceConfigV2.java | 19 ++++++++++++++++--- .../java/feast/serving/it/BaseAuthIT.java | 16 +++++++++++++--- .../retriever/CassandraStoreConfig.java | 16 +++++----------- 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/serving/src/main/java/feast/serving/config/FeastProperties.java b/serving/src/main/java/feast/serving/config/FeastProperties.java index 31276c5..b9029a0 100644 --- a/serving/src/main/java/feast/serving/config/FeastProperties.java +++ b/serving/src/main/java/feast/serving/config/FeastProperties.java @@ -319,8 +319,7 @@ public BigTableStoreConfig getBigtableConfig() { public CassandraStoreConfig getCassandraConfig() { return new CassandraStoreConfig( - this.config.get("host"), - Integer.valueOf(this.config.get("port")), + this.config.get("connection_string"), this.config.get("data_center"), this.config.get("keyspace")); } diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index 53d0812..4c26f4a 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -32,6 +32,9 @@ import io.opentracing.Tracer; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; @@ -84,14 +87,24 @@ public ServingServiceV2 servingServiceV2( break; case CASSANDRA: CassandraStoreConfig config = feastProperties.getActiveStore().getCassandraConfig(); - String host = config.getHost(); - Integer port = config.getPort(); + String connectionString = config.getConnectionString(); String dataCenter = config.getDataCenter(); String keySpace = config.getKeySpace(); + List contactPoints = + Arrays.stream(connectionString.split(",")) + .map(String::trim) + .map(cs -> cs.split(":")) + .map( + hostPort -> { + int port = hostPort.length > 1 ? Integer.parseInt(hostPort[1]) : 9042; + return new InetSocketAddress(hostPort[0], port); + }) + .collect(Collectors.toList()); + CqlSession session = new CqlSessionBuilder() - .addContactPoint(new InetSocketAddress(host, port)) + .addContactPoints(contactPoints) .withLocalDatacenter(dataCenter) .withKeyspace(keySpace) .build(); diff --git a/serving/src/test/java/feast/serving/it/BaseAuthIT.java b/serving/src/test/java/feast/serving/it/BaseAuthIT.java index 2f86e96..d49ac41 100644 --- a/serving/src/test/java/feast/serving/it/BaseAuthIT.java +++ b/serving/src/test/java/feast/serving/it/BaseAuthIT.java @@ -79,8 +79,6 @@ static void properties(DynamicPropertyRegistry registry) { } }); registry.add("feast.stores[0].config.port", () -> REDIS_PORT); - registry.add("feast.stores[0].subscriptions[0].name", () -> "*"); - registry.add("feast.stores[0].subscriptions[0].project", () -> "*"); registry.add("feast.stores[1].name", () -> "bigtable"); registry.add("feast.stores[1].type", () -> "BIGTABLE"); @@ -99,7 +97,19 @@ static void properties(DynamicPropertyRegistry registry) { return ""; } }); - registry.add("feast.stores[2].config.port", () -> CASSANDRA_PORT); + + registry.add( + "feast.stores[2].config.connection_string", + () -> { + String hostAddress = ""; + try { + hostAddress = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + + return String.format("%s:%s", hostAddress, CASSANDRA_PORT); + }); registry.add("feast.stores[2].config.data_center", () -> CASSANDRA_DATACENTER); registry.add("feast.stores[2].config.keyspace", () -> CASSANDRA_KEYSPACE); diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java index fe82a20..3ee25df 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraStoreConfig.java @@ -18,24 +18,18 @@ public class CassandraStoreConfig { - private final String host; - private final Integer port; + private final String connectionString; private final String dataCenter; private final String keySpace; - public CassandraStoreConfig(String host, Integer port, String dataCenter, String keySpace) { - this.host = host; - this.port = port; + public CassandraStoreConfig(String connectionString, String dataCenter, String keySpace) { + this.connectionString = connectionString; this.dataCenter = dataCenter; this.keySpace = keySpace; } - public String getHost() { - return this.host; - } - - public Integer getPort() { - return this.port; + public String getConnectionString() { + return this.connectionString; } public String getDataCenter() { From 4025ff5f06df82f18d234dbb0207ed926c065e9d Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Wed, 7 Apr 2021 14:05:47 +0800 Subject: [PATCH 13/21] Update application.yml configuration Signed-off-by: Terence Lim --- serving/src/main/resources/application.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index d58dd46..91e27de 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -62,8 +62,7 @@ feast: - name: cassandra type: CASSANDRA config: - host: localhost - port: 9042 + connection_string: localhost:9042 data_center: datacenter1 keyspace: feast tracing: From f113edd212a3eda4718ccf73ccea4d1374cb2df9 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Wed, 7 Apr 2021 16:16:59 +0800 Subject: [PATCH 14/21] Update tests Signed-off-by: Terence Lim --- .../serving/it/ServingServiceCassandraIT.java | 127 ++++++++++++------ 1 file changed, 83 insertions(+), 44 deletions(-) diff --git a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java index 020f592..93ee5f5 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceCassandraIT.java @@ -41,6 +41,7 @@ import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; import org.apache.avro.generic.GenericDatumWriter; @@ -217,21 +218,9 @@ static void globalSetup() throws IOException { .endRecord(); byte[] schemaReference = Hashing.murmur3_32().hashBytes(ftSchema.toString().getBytes()).asBytes(); - - GenericRecord record = - new GenericRecordBuilder(ftSchema) - .set("trip_cost", 5) - .set("trip_distance", 3.5) - .set("trip_empty", null) - .set("trip_wrong_type", "test") - .build(); - byte[] entityFeatureKey = - String.valueOf(DataGenerator.createInt64Value(1).getInt64Val()).getBytes(); - byte[] entityFeatureValue = createEntityValue(ftSchema, record); byte[] schemaKey = createSchemaKey(schemaReference); - ingestData( - ridesFeatureTableName, cassandraTableName, entityFeatureKey, entityFeatureValue, schemaKey); + ingestBulk(ridesFeatureTableName, cassandraTableName, ftSchema, 20); Schema foodFtSchema = SchemaBuilder.record("FoodDriverData") @@ -239,26 +228,14 @@ static void globalSetup() throws IOException { .fields() .requiredInt(feature1Reference.getName()) .requiredDouble(feature2Reference.getName()) + .nullableString(feature3Reference.getName(), "null") + .requiredString(feature4Reference.getName()) .endRecord(); byte[] foodSchemaReference = Hashing.murmur3_32().hashBytes(foodFtSchema.toString().getBytes()).asBytes(); - - GenericRecord foodRecord = - new GenericRecordBuilder(foodFtSchema) - .set("trip_cost", 12) - .set("trip_distance", 7.5) - .build(); - byte[] foodEntityFeatureKey = - String.valueOf(DataGenerator.createInt64Value(1).getInt64Val()).getBytes(); - byte[] foodEntityFeatureValue = createEntityValue(foodFtSchema, foodRecord); byte[] foodSchemaKey = createSchemaKey(foodSchemaReference); - ingestData( - foodFeatureTableName, - cassandraTableName, - foodEntityFeatureKey, - foodEntityFeatureValue, - foodSchemaKey); + ingestBulk(foodFeatureTableName, cassandraTableName, foodFtSchema, 20); /** Compound Entity Ingestion Workflow */ Schema compoundFtSchema = @@ -376,6 +353,40 @@ private static void ingestData( ByteBuffer.wrap(entityFeatureValue))); } + private static void ingestBulk( + String featureTableName, String cassandraTableName, Schema schema, Integer counts) { + + IntStream.range(0, counts) + .forEach( + i -> { + try { + GenericRecord record = + new GenericRecordBuilder(schema) + .set("trip_cost", i) + .set("trip_distance", (double) i) + .set("trip_empty", null) + .set("trip_wrong_type", "test") + .build(); + byte[] schemaReference = + Hashing.murmur3_32().hashBytes(schema.toString().getBytes()).asBytes(); + + byte[] entityFeatureKey = + String.valueOf(DataGenerator.createInt64Value(i).getInt64Val()).getBytes(); + byte[] entityFeatureValue = createEntityValue(schema, record); + + byte[] schemaKey = createSchemaKey(schemaReference); + ingestData( + featureTableName, + cassandraTableName, + entityFeatureKey, + entityFeatureValue, + schemaKey); + } catch (IOException e) { + e.printStackTrace(); + } + }); + } + private static void ingestSchema(byte[] schemaKey, Schema schema) { PreparedStatement schemaStatement = cqlSession.prepare( @@ -433,7 +444,7 @@ public void shouldRegisterSingleEntityAndGetOnlineFeatures() { entityName, entityValue, FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt32Value(5), + DataGenerator.createInt32Value(1), FeatureV2.getFeatureStringRef(notFoundFeatureReference), DataGenerator.createEmptyValue()); @@ -496,7 +507,7 @@ public void shouldRegisterCompoundEntityAndGetOnlineFeatures() { merchantEntityName, merchantEntityValue, FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt32Value(5), + DataGenerator.createInt32Value(1), FeatureV2.getFeatureStringRef(notFoundFeatureReference), DataGenerator.createEmptyValue()); @@ -523,20 +534,26 @@ public void shouldRegisterCompoundEntityAndGetOnlineFeatures() { } @Test - public void shouldReturnCorrectRowCount() { + public void shouldReturnCorrectRowCountAndOrder() { // getOnlineFeatures Information String projectName = "default"; String entityName = "driver_id"; ValueProto.Value entityValue1 = ValueProto.Value.newBuilder().setInt64Val(1).build(); ValueProto.Value entityValue2 = ValueProto.Value.newBuilder().setInt64Val(2).build(); + ValueProto.Value entityValue3 = ValueProto.Value.newBuilder().setInt64Val(3).build(); + ValueProto.Value entityValue4 = ValueProto.Value.newBuilder().setInt64Val(4).build(); // Instantiate EntityRows GetOnlineFeaturesRequestV2.EntityRow entityRow1 = DataGenerator.createEntityRow(entityName, entityValue1, 100); GetOnlineFeaturesRequestV2.EntityRow entityRow2 = DataGenerator.createEntityRow(entityName, entityValue2, 100); + GetOnlineFeaturesRequestV2.EntityRow entityRow3 = + DataGenerator.createEntityRow(entityName, entityValue3, 100); + GetOnlineFeaturesRequestV2.EntityRow entityRow4 = + DataGenerator.createEntityRow(entityName, entityValue4, 100); ImmutableList entityRows = - ImmutableList.of(entityRow1, entityRow2); + ImmutableList.of(entityRow1, entityRow2, entityRow4, entityRow3); // Instantiate FeatureReferences FeatureReferenceV2 featureReference = @@ -560,7 +577,7 @@ public void shouldReturnCorrectRowCount() { entityName, entityValue1, FeatureV2.getFeatureStringRef(featureReference), - DataGenerator.createInt32Value(5), + DataGenerator.createInt32Value(1), FeatureV2.getFeatureStringRef(notFoundFeatureReference), DataGenerator.createEmptyValue(), FeatureV2.getFeatureStringRef(emptyFeatureReference), @@ -588,30 +605,52 @@ public void shouldReturnCorrectRowCount() { entityName, entityValue2, FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createInt32Value(2), + FeatureV2.getFeatureStringRef(notFoundFeatureReference), DataGenerator.createEmptyValue(), + FeatureV2.getFeatureStringRef(emptyFeatureReference), + DataGenerator.createEmptyValue()); + + ImmutableMap expectedValueMap3 = + ImmutableMap.of( + entityName, + entityValue3, + FeatureV2.getFeatureStringRef(featureReference), + DataGenerator.createInt32Value(3), FeatureV2.getFeatureStringRef(notFoundFeatureReference), DataGenerator.createEmptyValue(), FeatureV2.getFeatureStringRef(emptyFeatureReference), DataGenerator.createEmptyValue()); - ImmutableMap expectedStatusMap2 = + ImmutableMap expectedValueMap4 = ImmutableMap.of( entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, + entityValue4, FeatureV2.getFeatureStringRef(featureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, + DataGenerator.createInt32Value(4), FeatureV2.getFeatureStringRef(notFoundFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND, + DataGenerator.createEmptyValue(), FeatureV2.getFeatureStringRef(emptyFeatureReference), - GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND); + DataGenerator.createEmptyValue()); GetOnlineFeaturesResponse.FieldValues expectedFieldValues2 = GetOnlineFeaturesResponse.FieldValues.newBuilder() .putAllFields(expectedValueMap2) - .putAllStatuses(expectedStatusMap2) + .putAllStatuses(expectedStatusMap) + .build(); + GetOnlineFeaturesResponse.FieldValues expectedFieldValues3 = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap3) + .putAllStatuses(expectedStatusMap) + .build(); + GetOnlineFeaturesResponse.FieldValues expectedFieldValues4 = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap4) + .putAllStatuses(expectedStatusMap) .build(); ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues, expectedFieldValues2); + ImmutableList.of( + expectedFieldValues, expectedFieldValues2, expectedFieldValues4, expectedFieldValues3); assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); } @@ -655,13 +694,13 @@ public void shouldReturnFeaturesFromDiffFeatureTable() { entityName, entityValue, FeatureV2.getFeatureStringRef(rideFeatureReference), - DataGenerator.createInt32Value(5), + DataGenerator.createInt32Value(1), FeatureV2.getFeatureStringRef(rideFeatureReference2), - DataGenerator.createDoubleValue(3.5), + DataGenerator.createDoubleValue(1.0), FeatureV2.getFeatureStringRef(foodFeatureReference), - DataGenerator.createInt32Value(12), + DataGenerator.createInt32Value(1), FeatureV2.getFeatureStringRef(foodFeatureReference2), - DataGenerator.createDoubleValue(7.5)); + DataGenerator.createDoubleValue(1.0)); ImmutableMap expectedStatusMap = ImmutableMap.of( From c9ce09cf569610d411c5863f17216bc1d4dfca35 Mon Sep 17 00:00:00 2001 From: Khor Shu Heng Date: Thu, 8 Apr 2021 11:26:18 +0800 Subject: [PATCH 15/21] Fix schema caching for Cassandra Signed-off-by: Khor Shu Heng --- .../retriever/CassandraOnlineRetriever.java | 12 ++++--- .../retriever/CassandraSchemaRegistry.java | 35 ++++++++++++++----- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java index f74ccff..a39d4aa 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -37,6 +37,7 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.Decoder; import org.apache.avro.io.DecoderFactory; @@ -130,6 +131,7 @@ private List decodeFeatures( ByteBuffer schemaRefKey, ByteBuffer value, List featureReferences, + BinaryDecoder reusedDecoder, long timestamp) throws IOException { @@ -139,10 +141,9 @@ private List decodeFeatures( // Convert ByteBuffer to ByteArray byte[] bytesArray = new byte[value.remaining()]; value.get(bytesArray, 0, bytesArray.length); - Schema schema = schemaRegistry.getSchema(schemaReference); - GenericDatumReader reader = new GenericDatumReader<>(schema); - Decoder decoder = DecoderFactory.get().binaryDecoder(bytesArray, null); - GenericRecord record = reader.read(null, decoder); + GenericDatumReader reader = schemaRegistry.getReader(schemaReference); + reusedDecoder = DecoderFactory.get().binaryDecoder(bytesArray, reusedDecoder); + GenericRecord record = reader.read(null, reusedDecoder); return featureReferences.stream() .map( @@ -232,6 +233,8 @@ private List> convertRowToFeature( Map rows, List featureReferences) { + BinaryDecoder reusedDecoder = DecoderFactory.get().binaryDecoder(new byte[0], null); + return rowKeys.stream() .map( rowKey -> { @@ -265,6 +268,7 @@ private List> convertRowToFeature( schemaRefKey, featureValues, localFeatureReferences, + reusedDecoder, row.getLong(featureTableColumn + EVENT_TIMESTAMP_SUFFIX)); } catch (IOException e) { throw new RuntimeException("Failed to decode features from Cassandra"); diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java index 42109c0..eaa1978 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java @@ -26,12 +26,15 @@ import com.google.common.cache.LoadingCache; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.util.Objects; import java.util.concurrent.ExecutionException; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; public class CassandraSchemaRegistry { private final CqlSession session; - private final LoadingCache cache; + private final LoadingCache> cache; private static String SCHEMA_REF_TABLE = "feast_schema_reference"; private static String SCHEMA_REF_COLUMN = "schema_ref"; @@ -47,27 +50,40 @@ public SchemaReference(ByteBuffer schemaHash) { public ByteBuffer getSchemaHash() { return schemaHash; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SchemaReference that = (SchemaReference) o; + return Objects.equals(schemaHash, that.schemaHash); + } + + @Override + public int hashCode() { + return Objects.hash(schemaHash); + } } public CassandraSchemaRegistry(CqlSession session) { this.session = session; - CacheLoader schemaCacheLoader = CacheLoader.from(this::loadSchema); + CacheLoader> schemaCacheLoader = CacheLoader.from(this::loadReader); cache = CacheBuilder.newBuilder().build(schemaCacheLoader); } - public Schema getSchema(SchemaReference reference) { - Schema schema; + public GenericDatumReader getReader(SchemaReference reference) { + GenericDatumReader reader; try { - schema = this.cache.get(reference); + reader = this.cache.get(reference); } catch (ExecutionException | CacheLoader.InvalidCacheLoadException e) { - throw new RuntimeException(String.format("Unable to find Schema"), e); + throw new RuntimeException("Unable to find Schema"); } - return schema; + return reader; } - private Schema loadSchema(SchemaReference reference) { + private GenericDatumReader loadReader(SchemaReference reference) { String tableName = String.format("\"%s\"", SCHEMA_REF_TABLE); Select query = QueryBuilder.selectFrom(tableName) @@ -79,7 +95,8 @@ private Schema loadSchema(SchemaReference reference) { Row row = session.execute(statement).one(); - return new Schema.Parser() + Schema schema = new Schema.Parser() .parse(StandardCharsets.UTF_8.decode(row.getByteBuffer(SCHEMA_COLUMN)).toString()); + return new GenericDatumReader<>(schema); } } From 8098f1439f7de16db5e33fee1b6e1744ebddc31e Mon Sep 17 00:00:00 2001 From: Khor Shu Heng Date: Thu, 8 Apr 2021 11:38:16 +0800 Subject: [PATCH 16/21] Fix formatting Signed-off-by: Khor Shu Heng --- .../cassandra/retriever/CassandraOnlineRetriever.java | 2 -- .../cassandra/retriever/CassandraSchemaRegistry.java | 8 +++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java index a39d4aa..338d69c 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -34,11 +34,9 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.avro.AvroRuntimeException; -import org.apache.avro.Schema; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.BinaryDecoder; -import org.apache.avro.io.Decoder; import org.apache.avro.io.DecoderFactory; public class CassandraOnlineRetriever implements OnlineRetrieverV2 { diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java index eaa1978..8001a6f 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraSchemaRegistry.java @@ -68,7 +68,8 @@ public int hashCode() { public CassandraSchemaRegistry(CqlSession session) { this.session = session; - CacheLoader> schemaCacheLoader = CacheLoader.from(this::loadReader); + CacheLoader> schemaCacheLoader = + CacheLoader.from(this::loadReader); cache = CacheBuilder.newBuilder().build(schemaCacheLoader); } @@ -95,8 +96,9 @@ private GenericDatumReader loadReader(SchemaReference reference) Row row = session.execute(statement).one(); - Schema schema = new Schema.Parser() - .parse(StandardCharsets.UTF_8.decode(row.getByteBuffer(SCHEMA_COLUMN)).toString()); + Schema schema = + new Schema.Parser() + .parse(StandardCharsets.UTF_8.decode(row.getByteBuffer(SCHEMA_COLUMN)).toString()); return new GenericDatumReader<>(schema); } } From 2932766399a0b22e13b00bc00974739b03373cd7 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Thu, 8 Apr 2021 12:44:53 +0800 Subject: [PATCH 17/21] Refactor common functionality Signed-off-by: Terence Lim --- .../api/retriever/StorageRetriever.java | 79 +++++++++++++++++++ .../retriever/BigTableOnlineRetriever.java | 64 +-------------- .../retriever/CassandraOnlineRetriever.java | 63 +-------------- 3 files changed, 87 insertions(+), 119 deletions(-) create mode 100644 storage/api/src/main/java/feast/storage/api/retriever/StorageRetriever.java diff --git a/storage/api/src/main/java/feast/storage/api/retriever/StorageRetriever.java b/storage/api/src/main/java/feast/storage/api/retriever/StorageRetriever.java new file mode 100644 index 0000000..af5ce8c --- /dev/null +++ b/storage/api/src/main/java/feast/storage/api/retriever/StorageRetriever.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.api.retriever; + +import feast.proto.serving.ServingAPIProto; +import feast.proto.types.ValueProto; +import java.util.List; +import java.util.stream.Collectors; + +public class StorageRetriever { + + /** + * Generate name of Cassandra table in the form of __ + * + * @param project Name of Feast project + * @param entityNames List of entities used in retrieval call + * @return Name of Cassandra table + */ + protected String getTableName(String project, List entityNames) { + return String.format("%s__%s", project, entityNames.stream().collect(Collectors.joining("__"))); + } + + /** + * Convert Entity value from Feast valueType to String type. Currently only supports STRING_VAL, + * INT64_VAL, INT32_VAL and BYTES_VAL. + * + * @param v Entity value of Feast valueType + * @return String representation of Entity value + */ + protected String valueToString(ValueProto.Value v) { + String stringRepr; + switch (v.getValCase()) { + case STRING_VAL: + stringRepr = v.getStringVal(); + break; + case INT64_VAL: + stringRepr = String.valueOf(v.getInt64Val()); + break; + case INT32_VAL: + stringRepr = String.valueOf(v.getInt32Val()); + break; + case BYTES_VAL: + stringRepr = v.getBytesVal().toString(); + break; + default: + throw new RuntimeException("Type is not supported to be entity"); + } + + return stringRepr; + } + + /** + * Retrieve Cassandra table column families based on FeatureTable names. + * + * @param featureReferences List of feature references of features in retrieval call + * @return List of String of FeatureTable names + */ + protected List getColumnFamilies( + List featureReferences) { + return featureReferences.stream() + .map(ServingAPIProto.FeatureReferenceV2::getFeatureTable) + .distinct() + .collect(Collectors.toList()); + } +} diff --git a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java index b9b707e..1a620f9 100644 --- a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java +++ b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java @@ -25,10 +25,10 @@ import com.google.protobuf.Timestamp; import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; -import feast.proto.types.ValueProto; import feast.storage.api.retriever.Feature; import feast.storage.api.retriever.NativeFeature; import feast.storage.api.retriever.OnlineRetrieverV2; +import feast.storage.api.retriever.StorageRetriever; import java.io.IOException; import java.util.*; import java.util.function.Function; @@ -39,7 +39,7 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.*; -public class BigTableOnlineRetriever implements OnlineRetrieverV2 { +public class BigTableOnlineRetriever extends StorageRetriever implements OnlineRetrieverV2 { private BigtableDataClient client; private BigTableSchemaRegistry schemaRegistry; @@ -49,49 +49,6 @@ public BigTableOnlineRetriever(BigtableDataClient client) { this.schemaRegistry = new BigTableSchemaRegistry(client); } - /** - * Generate name of BigTable table in the form of __ - * - * @param project Name of Feast project - * @param entityNames List of entities used in retrieval call - * @return Name of BigTable table - */ - private String getTableName(String project, List entityNames) { - String tableName = - String.format("%s__%s", project, entityNames.stream().collect(Collectors.joining("__"))); - - return tableName; - } - - /** - * Convert Entity value from Feast valueType to String type. Currently only supports STRING_VAL, - * INT64_VAL, INT32_VAL and BYTES_VAL. - * - * @param v Entity value of Feast valueType - * @return String representation of Entity value - */ - private String valueToString(ValueProto.Value v) { - String stringRepr; - switch (v.getValCase()) { - case STRING_VAL: - stringRepr = v.getStringVal(); - break; - case INT64_VAL: - stringRepr = String.valueOf(v.getInt64Val()); - break; - case INT32_VAL: - stringRepr = String.valueOf(v.getInt32Val()); - break; - case BYTES_VAL: - stringRepr = v.getBytesVal().toString(); - break; - default: - throw new RuntimeException("Type is not supported to be entity"); - } - - return stringRepr; - } - /** * Generate BigTable key in the form of entity values joined by #. * @@ -99,8 +56,7 @@ private String valueToString(ValueProto.Value v) { * @param entityNames List of entities related to feature references in retrieval call * @return BigTable key for retrieval */ - private ByteString convertEntityValueToBigTableKey( - EntityRow entityRow, List entityNames) { + private ByteString convertEntityValueToKey(EntityRow entityRow, List entityNames) { return ByteString.copyFrom( entityNames.stream() .map(entity -> entityRow.getFieldsMap().get(entity)) @@ -109,18 +65,6 @@ private ByteString convertEntityValueToBigTableKey( .getBytes()); } - /** - * Retrieve BigTable table column families based on FeatureTable names. - * - * @param featureReferences List of feature references of features in retrieval call - * @return List of String of FeatureTable names - */ - private List getColumnFamilies(List featureReferences) { - return featureReferences.stream() - .map(FeatureReferenceV2::getFeatureTable) - .collect(Collectors.toList()); - } - /** * AvroRuntimeException is thrown if feature name does not exist in avro schema. Empty Object is * returned when null is retrieved from BigTable RowCell. @@ -187,7 +131,7 @@ public List> getOnlineFeatures( List rowKeys = entityRows.stream() - .map(row -> convertEntityValueToBigTableKey(row, entityNames)) + .map(row -> convertEntityValueToKey(row, entityNames)) .collect(Collectors.toList()); Map rowsFromBigTable = getFeaturesFromBigTable(tableName, rowKeys, columnFamilies); diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java index 338d69c..c6d0cb1 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -23,10 +23,10 @@ import com.datastax.oss.driver.api.querybuilder.select.Select; import com.google.protobuf.Timestamp; import feast.proto.serving.ServingAPIProto; -import feast.proto.types.ValueProto; import feast.storage.api.retriever.Feature; import feast.storage.api.retriever.NativeFeature; import feast.storage.api.retriever.OnlineRetrieverV2; +import feast.storage.api.retriever.StorageRetriever; import java.io.IOException; import java.nio.ByteBuffer; import java.util.*; @@ -39,7 +39,7 @@ import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.DecoderFactory; -public class CassandraOnlineRetriever implements OnlineRetrieverV2 { +public class CassandraOnlineRetriever extends StorageRetriever implements OnlineRetrieverV2 { private final CqlSession session; private final CassandraSchemaRegistry schemaRegistry; @@ -53,47 +53,6 @@ public CassandraOnlineRetriever(CqlSession session) { this.schemaRegistry = new CassandraSchemaRegistry(session); } - /** - * Generate name of Cassandra table in the form of __ - * - * @param project Name of Feast project - * @param entityNames List of entities used in retrieval call - * @return Name of Cassandra table - */ - private String getTableName(String project, List entityNames) { - - return String.format("%s__%s", project, String.join("__", entityNames)); - } - - /** - * Convert Entity value from Feast valueType to String type. Currently only supports STRING_VAL, - * INT64_VAL, INT32_VAL and BYTES_VAL. - * - * @param v Entity value of Feast valueType - * @return String representation of Entity value - */ - private String valueToString(ValueProto.Value v) { - String stringRepr; - switch (v.getValCase()) { - case STRING_VAL: - stringRepr = v.getStringVal(); - break; - case INT64_VAL: - stringRepr = String.valueOf(v.getInt64Val()); - break; - case INT32_VAL: - stringRepr = String.valueOf(v.getInt32Val()); - break; - case BYTES_VAL: - stringRepr = v.getBytesVal().toString(); - break; - default: - throw new RuntimeException("Type is not supported to be entity"); - } - - return stringRepr; - } - /** * Generate Cassandra key in the form of entity values joined by #. * @@ -101,7 +60,7 @@ private String valueToString(ValueProto.Value v) { * @param entityNames List of entities related to feature references in retrieval call * @return Cassandra key for retrieval */ - private ByteBuffer convertEntityValueToCassandraKey( + private ByteBuffer convertEntityValueToKey( ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow entityRow, List entityNames) { return ByteBuffer.wrap( entityNames.stream() @@ -111,20 +70,6 @@ private ByteBuffer convertEntityValueToCassandraKey( .getBytes()); } - /** - * Retrieve Cassandra table column families based on FeatureTable names. - * - * @param featureReferences List of feature references of features in retrieval call - * @return List of String of FeatureTable names - */ - private List getColumnFamilies( - List featureReferences) { - return featureReferences.stream() - .map(ServingAPIProto.FeatureReferenceV2::getFeatureTable) - .distinct() - .collect(Collectors.toList()); - } - private List decodeFeatures( ByteBuffer schemaRefKey, ByteBuffer value, @@ -180,7 +125,7 @@ public List> getOnlineFeatures( List rowKeys = entityRows.stream() - .map(row -> convertEntityValueToCassandraKey(row, entityNames)) + .map(row -> convertEntityValueToKey(row, entityNames)) .collect(Collectors.toList()); Map rowsFromCassandra = From a36309943b198149c304c723474edb1d4cb67974 Mon Sep 17 00:00:00 2001 From: Khor Shu Heng Date: Thu, 8 Apr 2021 13:48:26 +0800 Subject: [PATCH 18/21] Use default interface instead of concrete base class for sstable based retrievers Signed-off-by: Khor Shu Heng --- storage/connectors/bigtable/pom.xml | 6 ++++++ .../retriever/BigTableOnlineRetriever.java | 6 +++--- storage/connectors/cassandra/pom.xml | 6 ++++++ .../retriever/CassandraOnlineRetriever.java | 6 +++--- storage/connectors/pom.xml | 1 + storage/connectors/sstable/pom.xml | 19 +++++++++++++++++ .../retriever/SSTableOnlineRetriever.java} | 21 +++++++++---------- 7 files changed, 48 insertions(+), 17 deletions(-) create mode 100644 storage/connectors/sstable/pom.xml rename storage/{api/src/main/java/feast/storage/api/retriever/StorageRetriever.java => connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java} (73%) diff --git a/storage/connectors/bigtable/pom.xml b/storage/connectors/bigtable/pom.xml index a53d907..81cd450 100644 --- a/storage/connectors/bigtable/pom.xml +++ b/storage/connectors/bigtable/pom.xml @@ -30,6 +30,12 @@ 1.10.2 + + dev.feast + feast-storage-connector-sstable + ${project.version} + + com.google.guava guava diff --git a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java index 1a620f9..3f1e03c 100644 --- a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java +++ b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java @@ -28,7 +28,7 @@ import feast.storage.api.retriever.Feature; import feast.storage.api.retriever.NativeFeature; import feast.storage.api.retriever.OnlineRetrieverV2; -import feast.storage.api.retriever.StorageRetriever; +import feast.storage.connectors.sstable.retriever.SSTableOnlineRetriever; import java.io.IOException; import java.util.*; import java.util.function.Function; @@ -39,7 +39,7 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.*; -public class BigTableOnlineRetriever extends StorageRetriever implements OnlineRetrieverV2 { +public class BigTableOnlineRetriever implements SSTableOnlineRetriever, OnlineRetrieverV2 { private BigtableDataClient client; private BigTableSchemaRegistry schemaRegistry; @@ -126,7 +126,7 @@ public List> getOnlineFeatures( List entityRows, List featureReferences, List entityNames) { - List columnFamilies = getColumnFamilies(featureReferences); + List columnFamilies = getColumns(featureReferences); String tableName = getTableName(project, entityNames); List rowKeys = diff --git a/storage/connectors/cassandra/pom.xml b/storage/connectors/cassandra/pom.xml index 6fdc7a8..32e4d73 100644 --- a/storage/connectors/cassandra/pom.xml +++ b/storage/connectors/cassandra/pom.xml @@ -23,6 +23,12 @@ 1.10.2 + + dev.feast + feast-storage-connector-sstable + ${project.version} + + com.datastax.oss java-driver-core diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java index c6d0cb1..bd646c7 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -26,7 +26,7 @@ import feast.storage.api.retriever.Feature; import feast.storage.api.retriever.NativeFeature; import feast.storage.api.retriever.OnlineRetrieverV2; -import feast.storage.api.retriever.StorageRetriever; +import feast.storage.connectors.sstable.retriever.SSTableOnlineRetriever; import java.io.IOException; import java.nio.ByteBuffer; import java.util.*; @@ -39,7 +39,7 @@ import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.DecoderFactory; -public class CassandraOnlineRetriever extends StorageRetriever implements OnlineRetrieverV2 { +public class CassandraOnlineRetriever implements SSTableOnlineRetriever, OnlineRetrieverV2 { private final CqlSession session; private final CassandraSchemaRegistry schemaRegistry; @@ -120,7 +120,7 @@ public List> getOnlineFeatures( List featureReferences, List entityNames) { - List columnFamilies = getColumnFamilies(featureReferences); + List columnFamilies = getColumns(featureReferences); String tableName = getTableName(project, entityNames); List rowKeys = diff --git a/storage/connectors/pom.xml b/storage/connectors/pom.xml index 5be4caf..1c4b75a 100644 --- a/storage/connectors/pom.xml +++ b/storage/connectors/pom.xml @@ -18,6 +18,7 @@ redis bigtable cassandra + sstable diff --git a/storage/connectors/sstable/pom.xml b/storage/connectors/sstable/pom.xml new file mode 100644 index 0000000..8c7a271 --- /dev/null +++ b/storage/connectors/sstable/pom.xml @@ -0,0 +1,19 @@ + + + + feast-storage-connectors + dev.feast + ${revision} + + + 4.0.0 + feast-storage-connector-sstable + + + 11 + 11 + + + \ No newline at end of file diff --git a/storage/api/src/main/java/feast/storage/api/retriever/StorageRetriever.java b/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java similarity index 73% rename from storage/api/src/main/java/feast/storage/api/retriever/StorageRetriever.java rename to storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java index af5ce8c..0d3fe53 100644 --- a/storage/api/src/main/java/feast/storage/api/retriever/StorageRetriever.java +++ b/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java @@ -14,24 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.storage.api.retriever; +package feast.storage.connectors.sstable.retriever; import feast.proto.serving.ServingAPIProto; import feast.proto.types.ValueProto; import java.util.List; import java.util.stream.Collectors; -public class StorageRetriever { +public interface SSTableOnlineRetriever { /** - * Generate name of Cassandra table in the form of __ + * Retrieve name of SSTable corresponding to entities in retrieval call * * @param project Name of Feast project * @param entityNames List of entities used in retrieval call * @return Name of Cassandra table */ - protected String getTableName(String project, List entityNames) { - return String.format("%s__%s", project, entityNames.stream().collect(Collectors.joining("__"))); + default String getTableName(String project, List entityNames) { + return String.format("%s__%s", project, String.join("__", entityNames)); } /** @@ -41,7 +41,7 @@ protected String getTableName(String project, List entityNames) { * @param v Entity value of Feast valueType * @return String representation of Entity value */ - protected String valueToString(ValueProto.Value v) { + default String valueToString(ValueProto.Value v) { String stringRepr; switch (v.getValCase()) { case STRING_VAL: @@ -64,13 +64,12 @@ protected String valueToString(ValueProto.Value v) { } /** - * Retrieve Cassandra table column families based on FeatureTable names. + * Retrieve SSTable columns based on Feature references. * - * @param featureReferences List of feature references of features in retrieval call - * @return List of String of FeatureTable names + * @param featureReferences List of feature references in retrieval call + * @return List of String of column names */ - protected List getColumnFamilies( - List featureReferences) { + default List getColumns(List featureReferences) { return featureReferences.stream() .map(ServingAPIProto.FeatureReferenceV2::getFeatureTable) .distinct() From b03504b2046da9b63e6cbf96ac5d6eaf933322c7 Mon Sep 17 00:00:00 2001 From: Khor Shu Heng Date: Thu, 8 Apr 2021 16:09:09 +0800 Subject: [PATCH 19/21] Extract common functionalities for bigttable and cassandra retriever Signed-off-by: Khor Shu Heng --- .../retriever/BigTableOnlineRetriever.java | 188 ++++++++---------- .../retriever/CassandraOnlineRetriever.java | 188 ++++++++---------- .../retriever/SSTableOnlineRetriever.java | 74 ++++++- 3 files changed, 238 insertions(+), 212 deletions(-) diff --git a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java index 3f1e03c..d7d5945 100644 --- a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java +++ b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java @@ -27,7 +27,6 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; import feast.storage.api.retriever.Feature; import feast.storage.api.retriever.NativeFeature; -import feast.storage.api.retriever.OnlineRetrieverV2; import feast.storage.connectors.sstable.retriever.SSTableOnlineRetriever; import java.io.IOException; import java.util.*; @@ -39,7 +38,7 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.*; -public class BigTableOnlineRetriever implements SSTableOnlineRetriever, OnlineRetrieverV2 { +public class BigTableOnlineRetriever implements SSTableOnlineRetriever { private BigtableDataClient client; private BigTableSchemaRegistry schemaRegistry; @@ -56,7 +55,8 @@ public BigTableOnlineRetriever(BigtableDataClient client) { * @param entityNames List of entities related to feature references in retrieval call * @return BigTable key for retrieval */ - private ByteString convertEntityValueToKey(EntityRow entityRow, List entityNames) { + @Override + public ByteString convertEntityValueToKey(EntityRow entityRow, List entityNames) { return ByteString.copyFrom( entityNames.stream() .map(entity -> entityRow.getFieldsMap().get(entity)) @@ -65,106 +65,6 @@ private ByteString convertEntityValueToKey(EntityRow entityRow, List ent .getBytes()); } - /** - * AvroRuntimeException is thrown if feature name does not exist in avro schema. Empty Object is - * returned when null is retrieved from BigTable RowCell. - * - * @param tableName Name of BigTable table - * @param value Value of BigTable cell where first 4 bytes represent the schema reference and - * remaining bytes represent avro-serialized features - * @param featureReferences List of feature references - * @param timestamp Timestamp of rowCell - * @return @NativeFeature with retrieved value stored in BigTable RowCell - * @throws IOException - */ - private List decodeFeatures( - String tableName, - ByteString value, - List featureReferences, - BinaryDecoder reusedDecoder, - long timestamp) - throws IOException { - ByteString schemaReferenceBytes = value.substring(0, 4); - byte[] featureValueBytes = value.substring(4).toByteArray(); - - BigTableSchemaRegistry.SchemaReference schemaReference = - new BigTableSchemaRegistry.SchemaReference(tableName, schemaReferenceBytes); - - GenericDatumReader reader = schemaRegistry.getReader(schemaReference); - - reusedDecoder = DecoderFactory.get().binaryDecoder(featureValueBytes, reusedDecoder); - GenericRecord record = reader.read(null, reusedDecoder); - - return featureReferences.stream() - .map( - featureReference -> { - Object featureValue; - try { - featureValue = record.get(featureReference.getName()); - } catch (AvroRuntimeException e) { - // Feature is not found in schema - return null; - } - if (featureValue != null) { - return new NativeFeature( - featureReference, - Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), - featureValue); - } - return new NativeFeature( - featureReference, - Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), - new Object()); - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - - @Override - public List> getOnlineFeatures( - String project, - List entityRows, - List featureReferences, - List entityNames) { - List columnFamilies = getColumns(featureReferences); - String tableName = getTableName(project, entityNames); - - List rowKeys = - entityRows.stream() - .map(row -> convertEntityValueToKey(row, entityNames)) - .collect(Collectors.toList()); - Map rowsFromBigTable = - getFeaturesFromBigTable(tableName, rowKeys, columnFamilies); - List> features = - convertRowToFeature(tableName, rowKeys, rowsFromBigTable, featureReferences); - - return features; - } - - /** - * Retrieve rows for each row entity key by generating BigTable rowQuery with filters based on - * column families. - * - * @param tableName Name of BigTable table - * @param rowKeys List of keys of rows to retrieve - * @param columnFamilies List of FeatureTable names - * @return Map of retrieved features for each rowKey - */ - private Map getFeaturesFromBigTable( - String tableName, List rowKeys, List columnFamilies) { - - Query rowQuery = Query.create(tableName); - Filters.InterleaveFilter familyFilter = Filters.FILTERS.interleave(); - columnFamilies.forEach(cf -> familyFilter.filter(Filters.FILTERS.family().exactMatch(cf))); - - for (ByteString rowKey : rowKeys) { - rowQuery.rowKey(rowKey); - } - - return StreamSupport.stream(client.readRows(rowQuery).spliterator(), false) - .collect(Collectors.toMap(Row::getKey, Function.identity())); - } - /** * Converts rowCell feature value into @NativeFeature type. * @@ -174,7 +74,8 @@ private Map getFeaturesFromBigTable( * @param featureReferences List of feature references * @return List of List of Features associated with respective rowKey */ - private List> convertRowToFeature( + @Override + public List> convertRowToFeature( String tableName, List rowKeys, Map rows, @@ -227,4 +128,83 @@ private List> convertRowToFeature( }) .collect(Collectors.toList()); } + + /** + * Retrieve rows for each row entity key by generating BigTable rowQuery with filters based on + * column families. + * + * @param tableName Name of BigTable table + * @param rowKeys List of keys of rows to retrieve + * @param columnFamilies List of FeatureTable names + * @return Map of retrieved features for each rowKey + */ + @Override + public Map getFeaturesFromSSTable( + String tableName, List rowKeys, List columnFamilies) { + Query rowQuery = Query.create(tableName); + Filters.InterleaveFilter familyFilter = Filters.FILTERS.interleave(); + columnFamilies.forEach(cf -> familyFilter.filter(Filters.FILTERS.family().exactMatch(cf))); + + for (ByteString rowKey : rowKeys) { + rowQuery.rowKey(rowKey); + } + + return StreamSupport.stream(client.readRows(rowQuery).spliterator(), false) + .collect(Collectors.toMap(Row::getKey, Function.identity())); + } + /** + * AvroRuntimeException is thrown if feature name does not exist in avro schema. Empty Object is + * returned when null is retrieved from BigTable RowCell. + * + * @param tableName Name of BigTable table + * @param value Value of BigTable cell where first 4 bytes represent the schema reference and + * remaining bytes represent avro-serialized features + * @param featureReferences List of feature references + * @param timestamp Timestamp of rowCell + * @return @NativeFeature with retrieved value stored in BigTable RowCell + * @throws IOException + */ + private List decodeFeatures( + String tableName, + ByteString value, + List featureReferences, + BinaryDecoder reusedDecoder, + long timestamp) + throws IOException { + ByteString schemaReferenceBytes = value.substring(0, 4); + byte[] featureValueBytes = value.substring(4).toByteArray(); + + BigTableSchemaRegistry.SchemaReference schemaReference = + new BigTableSchemaRegistry.SchemaReference(tableName, schemaReferenceBytes); + + GenericDatumReader reader = schemaRegistry.getReader(schemaReference); + + reusedDecoder = DecoderFactory.get().binaryDecoder(featureValueBytes, reusedDecoder); + GenericRecord record = reader.read(null, reusedDecoder); + + return featureReferences.stream() + .map( + featureReference -> { + Object featureValue; + try { + featureValue = record.get(featureReference.getName()); + } catch (AvroRuntimeException e) { + // Feature is not found in schema + return null; + } + if (featureValue != null) { + return new NativeFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + featureValue); + } + return new NativeFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + new Object()); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + } diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java index bd646c7..8211bb4 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -22,10 +22,10 @@ import com.datastax.oss.driver.api.querybuilder.QueryBuilder; import com.datastax.oss.driver.api.querybuilder.select.Select; import com.google.protobuf.Timestamp; -import feast.proto.serving.ServingAPIProto; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; import feast.storage.api.retriever.Feature; import feast.storage.api.retriever.NativeFeature; -import feast.storage.api.retriever.OnlineRetrieverV2; import feast.storage.connectors.sstable.retriever.SSTableOnlineRetriever; import java.io.IOException; import java.nio.ByteBuffer; @@ -39,7 +39,7 @@ import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.DecoderFactory; -public class CassandraOnlineRetriever implements SSTableOnlineRetriever, OnlineRetrieverV2 { +public class CassandraOnlineRetriever implements SSTableOnlineRetriever { private final CqlSession session; private final CassandraSchemaRegistry schemaRegistry; @@ -60,8 +60,8 @@ public CassandraOnlineRetriever(CqlSession session) { * @param entityNames List of entities related to feature references in retrieval call * @return Cassandra key for retrieval */ - private ByteBuffer convertEntityValueToKey( - ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow entityRow, List entityNames) { + @Override + public ByteBuffer convertEntityValueToKey(EntityRow entityRow, List entityNames) { return ByteBuffer.wrap( entityNames.stream() .map(entity -> entityRow.getFieldsMap().get(entity)) @@ -70,111 +70,20 @@ private ByteBuffer convertEntityValueToKey( .getBytes()); } - private List decodeFeatures( - ByteBuffer schemaRefKey, - ByteBuffer value, - List featureReferences, - BinaryDecoder reusedDecoder, - long timestamp) - throws IOException { - - CassandraSchemaRegistry.SchemaReference schemaReference = - new CassandraSchemaRegistry.SchemaReference(schemaRefKey); - - // Convert ByteBuffer to ByteArray - byte[] bytesArray = new byte[value.remaining()]; - value.get(bytesArray, 0, bytesArray.length); - GenericDatumReader reader = schemaRegistry.getReader(schemaReference); - reusedDecoder = DecoderFactory.get().binaryDecoder(bytesArray, reusedDecoder); - GenericRecord record = reader.read(null, reusedDecoder); - - return featureReferences.stream() - .map( - featureReference -> { - Object featureValue; - try { - featureValue = record.get(featureReference.getName()); - } catch (AvroRuntimeException e) { - // Feature is not found in schema - return null; - } - if (featureValue != null) { - return new NativeFeature( - featureReference, - Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), - featureValue); - } - return new NativeFeature( - featureReference, - Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), - new Object()); - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - - @Override - public List> getOnlineFeatures( - String project, - List entityRows, - List featureReferences, - List entityNames) { - - List columnFamilies = getColumns(featureReferences); - String tableName = getTableName(project, entityNames); - - List rowKeys = - entityRows.stream() - .map(row -> convertEntityValueToKey(row, entityNames)) - .collect(Collectors.toList()); - - Map rowsFromCassandra = - getFeaturesFromCassandra(tableName, rowKeys, columnFamilies); - - return convertRowToFeature(rowKeys, rowsFromCassandra, featureReferences); - } - - /** - * Retrieve rows for each row entity key by generating Cassandra Query with filters based on - * columns. - * - * @param tableName Name of Cassandra table - * @param rowKeys List of keys of rows to retrieve - * @param featureTables List of FeatureTable names - * @return Map of retrieved features for each rowKey - */ - private Map getFeaturesFromCassandra( - String tableName, List rowKeys, List featureTables) { - List schemaRefColumns = - featureTables.stream().map(c -> c + SCHEMA_REF_SUFFIX).collect(Collectors.toList()); - Select query = - QueryBuilder.selectFrom(tableName) - .columns(featureTables) - .columns(schemaRefColumns) - .column(ENTITY_KEY); - for (String featureTable : featureTables) { - query = query.writeTime(featureTable).as(featureTable + EVENT_TIMESTAMP_SUFFIX); - } - query = query.whereColumn(ENTITY_KEY).in(QueryBuilder.bindMarker()); - - BoundStatement statement = session.prepare(query.build()).bind(rowKeys); - - return StreamSupport.stream(session.execute(statement).spliterator(), false) - .collect(Collectors.toMap((Row row) -> row.getByteBuffer(ENTITY_KEY), Function.identity())); - } - /** - * Converts rowCell feature value into @NativeFeature type. + * Converts Cassandra rows into @NativeFeature type. * * @param rowKeys List of keys of rows to retrieve * @param rows Map of rowKey to Row related to it * @param featureReferences List of feature references * @return List of List of Features associated with respective rowKey */ - private List> convertRowToFeature( + @Override + public List> convertRowToFeature( + String tableName, List rowKeys, Map rows, - List featureReferences) { + List featureReferences) { BinaryDecoder reusedDecoder = DecoderFactory.get().binaryDecoder(new byte[0], null); @@ -186,7 +95,7 @@ private List> convertRowToFeature( } else { Row row = rows.get(rowKey); return featureReferences.stream() - .map(ServingAPIProto.FeatureReferenceV2::getFeatureTable) + .map(FeatureReferenceV2::getFeatureTable) .distinct() .flatMap( featureTableColumn -> { @@ -195,7 +104,7 @@ private List> convertRowToFeature( row.getByteBuffer(featureTableColumn + SCHEMA_REF_SUFFIX); // Prevent retrieval of features from incorrect FeatureTable - List localFeatureReferences = + List localFeatureReferences = featureReferences.stream() .filter( featureReference -> @@ -224,4 +133,77 @@ private List> convertRowToFeature( }) .collect(Collectors.toList()); } + + /** + * Retrieve rows for each row entity key by generating Cassandra Query with filters based on + * columns. + * + * @param tableName Name of Cassandra table + * @param rowKeys List of keys of rows to retrieve + * @param columnFamilies List of FeatureTable names + * @return Map of retrieved features for each rowKey + */ + @Override + public Map getFeaturesFromSSTable( + String tableName, List rowKeys, List columnFamilies) { + List schemaRefColumns = + columnFamilies.stream().map(c -> c + SCHEMA_REF_SUFFIX).collect(Collectors.toList()); + Select query = + QueryBuilder.selectFrom(tableName) + .columns(columnFamilies) + .columns(schemaRefColumns) + .column(ENTITY_KEY); + for (String columnFamily : columnFamilies) { + query = query.writeTime(columnFamily).as(columnFamily + EVENT_TIMESTAMP_SUFFIX); + } + query = query.whereColumn(ENTITY_KEY).in(QueryBuilder.bindMarker()); + + BoundStatement statement = session.prepare(query.build()).bind(rowKeys); + + return StreamSupport.stream(session.execute(statement).spliterator(), false) + .collect(Collectors.toMap((Row row) -> row.getByteBuffer(ENTITY_KEY), Function.identity())); + } + + private List decodeFeatures( + ByteBuffer schemaRefKey, + ByteBuffer value, + List featureReferences, + BinaryDecoder reusedDecoder, + long timestamp) + throws IOException { + + CassandraSchemaRegistry.SchemaReference schemaReference = + new CassandraSchemaRegistry.SchemaReference(schemaRefKey); + + // Convert ByteBuffer to ByteArray + byte[] bytesArray = new byte[value.remaining()]; + value.get(bytesArray, 0, bytesArray.length); + GenericDatumReader reader = schemaRegistry.getReader(schemaReference); + reusedDecoder = DecoderFactory.get().binaryDecoder(bytesArray, reusedDecoder); + GenericRecord record = reader.read(null, reusedDecoder); + + return featureReferences.stream() + .map( + featureReference -> { + Object featureValue; + try { + featureValue = record.get(featureReference.getName()); + } catch (AvroRuntimeException e) { + // Feature is not found in schema + return null; + } + if (featureValue != null) { + return new NativeFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + featureValue); + } + return new NativeFeature( + featureReference, + Timestamp.newBuilder().setSeconds(timestamp / 1000).build(), + new Object()); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } } diff --git a/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java b/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java index 0d3fe53..fbfff84 100644 --- a/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java +++ b/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java @@ -16,12 +16,75 @@ */ package feast.storage.connectors.sstable.retriever; -import feast.proto.serving.ServingAPIProto; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; import feast.proto.types.ValueProto; +import feast.storage.api.retriever.Feature; +import feast.storage.api.retriever.OnlineRetrieverV2; + import java.util.List; +import java.util.Map; import java.util.stream.Collectors; -public interface SSTableOnlineRetriever { +/** + * @param Decoded value type of the partition key + * @param Type of the SSTable row + */ +public interface SSTableOnlineRetriever extends OnlineRetrieverV2 { + + @Override + default List> getOnlineFeatures( + String project, + List entityRows, + List featureReferences, + List entityNames) { + + List columnFamilies = getSSTableColumns(featureReferences); + String tableName = getSSTable(project, entityNames); + + List rowKeys = + entityRows.stream() + .map(row -> convertEntityValueToKey(row, entityNames)) + .collect(Collectors.toList()); + + Map rowsFromSSTable = getFeaturesFromSSTable(tableName, rowKeys, columnFamilies); + + return convertRowToFeature(tableName, rowKeys, rowsFromSSTable, featureReferences); + } + + /** + * Generate SSTable key. + * + * @param entityRow Single EntityRow representation in feature retrieval call + * @param entityNames List of entities related to feature references in retrieval call + * @return SSTable key for retrieval + */ + K convertEntityValueToKey(EntityRow entityRow, List entityNames); + + /** + * Converts SSTable rows into @NativeFeature type. + * + * @param tableName Name of SSTable + * @param rowKeys List of keys of rows to retrieve + * @param rows Map of rowKey to Row related to it + * @param featureReferences List of feature references + * @return List of List of Features associated with respective rowKey + */ + List> convertRowToFeature( + String tableName, + List rowKeys, + Map rows, + List featureReferences); + + /** + * Retrieve rows for each row entity key. + * + * @param tableName Name of SSTable + * @param rowKeys List of keys of rows to retrieve + * @param columnFamilies List of column names + * @return Map of retrieved features for each rowKey + */ + Map getFeaturesFromSSTable(String tableName, List rowKeys, List columnFamilies); /** * Retrieve name of SSTable corresponding to entities in retrieval call @@ -30,7 +93,7 @@ public interface SSTableOnlineRetriever { * @param entityNames List of entities used in retrieval call * @return Name of Cassandra table */ - default String getTableName(String project, List entityNames) { + default String getSSTable(String project, List entityNames) { return String.format("%s__%s", project, String.join("__", entityNames)); } @@ -69,10 +132,11 @@ default String valueToString(ValueProto.Value v) { * @param featureReferences List of feature references in retrieval call * @return List of String of column names */ - default List getColumns(List featureReferences) { + default List getSSTableColumns(List featureReferences) { return featureReferences.stream() - .map(ServingAPIProto.FeatureReferenceV2::getFeatureTable) + .map(FeatureReferenceV2::getFeatureTable) .distinct() .collect(Collectors.toList()); } + } From ae2c48c3094365a4a39184996c152295a51f1d93 Mon Sep 17 00:00:00 2001 From: Terence Lim Date: Thu, 8 Apr 2021 16:59:41 +0800 Subject: [PATCH 20/21] Fix formatting Signed-off-by: Terence Lim --- .../bigtable/retriever/BigTableOnlineRetriever.java | 3 ++- .../retriever/CassandraOnlineRetriever.java | 12 ++++++++++++ .../sstable/retriever/SSTableOnlineRetriever.java | 2 -- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java index d7d5945..cf82c14 100644 --- a/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java +++ b/storage/connectors/bigtable/src/main/java/feast/storage/connectors/bigtable/retriever/BigTableOnlineRetriever.java @@ -152,6 +152,7 @@ public Map getFeaturesFromSSTable( return StreamSupport.stream(client.readRows(rowQuery).spliterator(), false) .collect(Collectors.toMap(Row::getKey, Function.identity())); } + /** * AvroRuntimeException is thrown if feature name does not exist in avro schema. Empty Object is * returned when null is retrieved from BigTable RowCell. @@ -160,6 +161,7 @@ public Map getFeaturesFromSSTable( * @param value Value of BigTable cell where first 4 bytes represent the schema reference and * remaining bytes represent avro-serialized features * @param featureReferences List of feature references + * @param reusedDecoder Decoder for decoding feature values * @param timestamp Timestamp of rowCell * @return @NativeFeature with retrieved value stored in BigTable RowCell * @throws IOException @@ -206,5 +208,4 @@ private List decodeFeatures( .filter(Objects::nonNull) .collect(Collectors.toList()); } - } diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java index 8211bb4..508cd08 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -73,6 +73,7 @@ public ByteBuffer convertEntityValueToKey(EntityRow entityRow, List enti /** * Converts Cassandra rows into @NativeFeature type. * + * @param tableName Name of Cassandra table * @param rowKeys List of keys of rows to retrieve * @param rows Map of rowKey to Row related to it * @param featureReferences List of feature references @@ -164,6 +165,17 @@ public Map getFeaturesFromSSTable( .collect(Collectors.toMap((Row row) -> row.getByteBuffer(ENTITY_KEY), Function.identity())); } + /** + * AvroRuntimeException is thrown if feature name does not exist in avro schema. + * + * @param schemaRefKey Schema reference key + * @param value Value of Cassandra cell where bytes represent avro-serialized features + * @param featureReferences List of feature references + * @param reusedDecoder Decoder for decoding feature values + * @param timestamp Timestamp of rowCell + * @return @NativeFeature with retrieved value stored in Cassandra cell + * @throws IOException + */ private List decodeFeatures( ByteBuffer schemaRefKey, ByteBuffer value, diff --git a/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java b/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java index fbfff84..957f0d3 100644 --- a/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java +++ b/storage/connectors/sstable/src/main/java/feast/storage/connectors/sstable/retriever/SSTableOnlineRetriever.java @@ -21,7 +21,6 @@ import feast.proto.types.ValueProto; import feast.storage.api.retriever.Feature; import feast.storage.api.retriever.OnlineRetrieverV2; - import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -138,5 +137,4 @@ default List getSSTableColumns(List featureReference .distinct() .collect(Collectors.toList()); } - } From 0539fb4a7ae94b703e4af6f92ffd8c2c279a1612 Mon Sep 17 00:00:00 2001 From: Khor Shu Heng Date: Fri, 9 Apr 2021 00:18:30 +0800 Subject: [PATCH 21/21] Ignore features with null values Signed-off-by: Khor Shu Heng --- .../cassandra/retriever/CassandraOnlineRetriever.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java index 508cd08..55198e0 100644 --- a/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java +++ b/storage/connectors/cassandra/src/main/java/feast/storage/connectors/cassandra/retriever/CassandraOnlineRetriever.java @@ -184,6 +184,10 @@ private List decodeFeatures( long timestamp) throws IOException { + if (value == null || schemaRefKey == null) { + return Collections.emptyList(); + } + CassandraSchemaRegistry.SchemaReference schemaReference = new CassandraSchemaRegistry.SchemaReference(schemaRefKey);