Skip to content

Commit b90ac39

Browse files
terryyylimterryyylim
committed
Partial implementation of Bigtable retriever
Signed-off-by: Terence Lim <terencelimxp@gmail.com>
1 parent 0453a94 commit b90ac39

13 files changed

Lines changed: 1030 additions & 4 deletions

File tree

‎serving/pom.xml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,11 +84,27 @@
8484
<version>${project.version}</version>
8585
</dependency>
8686

87+
<dependency>
88+
<groupId>dev.feast</groupId>
89+
<artifactId>feast-storage-connector-bigtable</artifactId>
90+
<version>${project.version}</version>
91+
</dependency>
92+
8793
<dependency>
8894
<groupId>dev.feast</groupId>
8995
<artifactId>feast-common</artifactId>
9096
<version>${project.version}</version>
9197
</dependency>
98+
99+
<dependency>
100+
<groupId>com.google.cloud</groupId>
101+
<artifactId>google-cloud-bigtable</artifactId>
102+
</dependency>
103+
104+
<dependency>
105+
<groupId>com.google.cloud</groupId>
106+
<artifactId>google-cloud-bigtable-emulator</artifactId>
107+
</dependency>
92108

93109
<!-- TODO: SLF4J is being used via Lombok, but also jog4j - pick one -->
94110
<dependency>
@@ -129,6 +145,14 @@
129145
<artifactId>spring-boot-starter-actuator</artifactId>
130146
</dependency>
131147

148+
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-test -->
149+
<dependency>
150+
<groupId>org.springframework.boot</groupId>
151+
<artifactId>spring-boot-test</artifactId>
152+
<version>2.3.1.RELEASE</version>
153+
<scope>test</scope>
154+
</dependency>
155+
132156
<!--compile "io.grpc:grpc-services:${grpc.version}"-->
133157
<dependency>
134158
<groupId>io.grpc</groupId>
@@ -234,6 +258,13 @@
234258
<scope>test</scope>
235259
</dependency>
236260

261+
<!-- https://mvnrepository.com/artifact/org.apache.avro/avro -->
262+
<dependency>
263+
<groupId>org.apache.avro</groupId>
264+
<artifactId>avro</artifactId>
265+
<version>1.10.2</version>
266+
</dependency>
267+
237268
<!-- Utilities -->
238269
<dependency>
239270
<groupId>com.fasterxml.jackson.dataformat</groupId>
@@ -307,6 +338,12 @@
307338
<version>1.15.1</version>
308339
<scope>test</scope>
309340
</dependency>
341+
<dependency>
342+
<groupId>org.testcontainers</groupId>
343+
<artifactId>gcloud</artifactId>
344+
<version>1.15.2</version>
345+
<scope>test</scope>
346+
</dependency>
310347
<dependency>
311348
<groupId>org.awaitility</groupId>
312349
<artifactId>awaitility</artifactId>

‎serving/src/main/java/feast/serving/config/FeastProperties.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import feast.common.auth.config.SecurityProperties.AuthorizationProperties;
2727
import feast.common.auth.credentials.CoreAuthenticationProperties;
2828
import feast.common.logging.config.LoggingProperties;
29+
import feast.storage.connectors.bigtable.retriever.BigTableStoreConfig;
2930
import feast.storage.connectors.redis.retriever.RedisClusterStoreConfig;
3031
import feast.storage.connectors.redis.retriever.RedisStoreConfig;
3132
import io.lettuce.core.ReadFrom;
@@ -269,7 +270,7 @@ public void setName(String name) {
269270
}
270271

271272
/**
272-
* Gets the store type. Example are REDIS or REDIS_CLUSTER
273+
* Gets the store type. Example are REDIS, REDIS_CLUSTER or BIGTABLE
273274
*
274275
* @return the store type as a String.
275276
*/
@@ -311,6 +312,10 @@ public RedisStoreConfig getRedisConfig() {
311312
Boolean.valueOf(this.config.getOrDefault("ssl", "false")));
312313
}
313314

315+
public BigTableStoreConfig getBigtableConfig() {
316+
return new BigTableStoreConfig(this.config.get("project_id"), this.config.get("instance_id"));
317+
}
318+
314319
/**
315320
* Sets the store config. Please protos/feast/core/Store.proto for the specific options for each
316321
* store.
@@ -323,6 +328,7 @@ public void setConfig(Map<String, String> config) {
323328
}
324329

325330
public enum StoreType {
331+
BIGTABLE,
326332
REDIS,
327333
REDIS_CLUSTER;
328334
}

‎serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,17 @@
1616
*/
1717
package feast.serving.config;
1818

19-
import com.fasterxml.jackson.core.JsonProcessingException;
20-
import com.google.protobuf.InvalidProtocolBufferException;
19+
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
20+
import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
2121
import feast.serving.service.OnlineServingServiceV2;
2222
import feast.serving.service.ServingServiceV2;
2323
import feast.serving.specs.CachedSpecService;
2424
import feast.storage.api.retriever.OnlineRetrieverV2;
25+
import feast.storage.connectors.bigtable.retriever.BigTableOnlineRetriever;
26+
import feast.storage.connectors.bigtable.retriever.BigTableStoreConfig;
2527
import feast.storage.connectors.redis.retriever.*;
2628
import io.opentracing.Tracer;
29+
import java.io.IOException;
2730
import org.slf4j.Logger;
2831
import org.springframework.context.annotation.Bean;
2932
import org.springframework.context.annotation.Configuration;
@@ -32,10 +35,22 @@
3235
public class ServingServiceConfigV2 {
3336
private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceConfigV2.class);
3437

38+
String projectId;
39+
String instanceId;
40+
41+
@Bean
42+
public BigtableDataClient bigtableClient() throws IOException {
43+
return BigtableDataClient.create(
44+
BigtableDataSettings.newBuilder()
45+
.setProjectId(projectId)
46+
.setInstanceId(instanceId)
47+
.build());
48+
}
49+
3550
@Bean
3651
public ServingServiceV2 servingServiceV2(
3752
FeastProperties feastProperties, CachedSpecService specService, Tracer tracer)
38-
throws InvalidProtocolBufferException, JsonProcessingException {
53+
throws IOException {
3954
ServingServiceV2 servingService = null;
4055
FeastProperties.Store store = feastProperties.getActiveStore();
4156

@@ -51,6 +66,15 @@ public ServingServiceV2 servingServiceV2(
5166
OnlineRetrieverV2 redisRetriever = new OnlineRetriever(redisClient);
5267
servingService = new OnlineServingServiceV2(redisRetriever, specService, tracer);
5368
break;
69+
case BIGTABLE:
70+
BigTableStoreConfig config = store.getBigtableConfig();
71+
projectId = config.getProjectId();
72+
instanceId = config.getInstanceId();
73+
74+
BigtableDataClient bigtableClient = bigtableClient();
75+
OnlineRetrieverV2 bigtableRetriever = new BigTableOnlineRetriever(bigtableClient);
76+
servingService = new OnlineServingServiceV2(bigtableRetriever, specService, tracer);
77+
break;
5478
}
5579

5680
return servingService;

‎serving/src/main/resources/application.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,11 @@ feast:
5454
read_from: MASTER
5555
# Redis operation timeout in ISO-8601 format
5656
timeout: PT0.5S
57+
- name: bigtable
58+
type: BIGTABLE
59+
config:
60+
project_id: <gcp_project>
61+
instance_id: <gcp_bigtable_instance>
5762
tracing:
5863
# If true, Feast will provide tracing data (using OpenTracing API) for various RPC method calls
5964
# which can be useful to debug performance issues and perform benchmarking

‎serving/src/test/java/feast/serving/it/BaseAuthBigtableIT.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2021 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.serving.it;
18+
19+
import org.springframework.boot.test.context.SpringBootTest;
20+
import org.springframework.test.context.ActiveProfiles;
21+
import org.springframework.test.context.DynamicPropertyRegistry;
22+
import org.springframework.test.context.DynamicPropertySource;
23+
24+
@ActiveProfiles("it")
25+
@SpringBootTest
26+
public class BaseAuthBigtableIT {
27+
28+
static final int SERVICE_START_MAX_WAIT_TIME_IN_MINUTES = 3;
29+
static final String CLIENT_ID = "client_id";
30+
static final String CLIENT_SECRET = "client_secret";
31+
static final String TOKEN_URL = "http://localhost:4444/oauth2/token";
32+
static final String JWK_URI = "http://localhost:4444/.well-known/jwks.json";
33+
34+
static final String GRANT_TYPE = "client_credentials";
35+
36+
static final String AUDIENCE = "https://localhost";
37+
38+
static final String BIGTABLE = "bigtable_1";
39+
static final int BIGTABLE_PORT = 8086;
40+
41+
static final String CORE = "core_1";
42+
static final int FEAST_CORE_PORT = 6565;
43+
44+
@DynamicPropertySource
45+
static void properties(DynamicPropertyRegistry registry) {
46+
registry.add("feast.active_store", () -> "bigtable");
47+
registry.add("feast.stores[0].name", () -> "bigtable");
48+
registry.add("feast.stores[0].type", () -> "BIGTABLE");
49+
// Redis needs to accessible by both core and serving, hence using host address
50+
registry.add("feast.stores[0].config.project_id", () -> "test-project");
51+
registry.add("feast.stores[0].config.instance_id", () -> "test-instance");
52+
registry.add("feast.stores[0].subscriptions[0].name", () -> "*");
53+
registry.add("feast.stores[0].subscriptions[0].project", () -> "*");
54+
55+
registry.add("feast.core-authentication.options.oauth_url", () -> TOKEN_URL);
56+
registry.add("feast.core-authentication.options.grant_type", () -> GRANT_TYPE);
57+
registry.add("feast.core-authentication.options.client_id", () -> CLIENT_ID);
58+
registry.add("feast.core-authentication.options.client_secret", () -> CLIENT_SECRET);
59+
registry.add("feast.core-authentication.options.audience", () -> AUDIENCE);
60+
registry.add("feast.core-authentication.options.jwkEndpointURI", () -> JWK_URI);
61+
registry.add("feast.security.authentication.options.jwkEndpointURI", () -> JWK_URI);
62+
}
63+
}

0 commit comments

Comments
 (0)