Skip to content

feat: Elasticsearch vector database #4188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
HaoXuAI merged 18 commits into from
May 13, 2024
Merged

feat: Elasticsearch vector database #4188

Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
e33aa20
add elasticsearch as online store
HaoXuAI May 9, 2024
091e838
add elasticsearch as online store
HaoXuAI May 9, 2024
de668fb
format
HaoXuAI May 9, 2024
d3619f6
format
HaoXuAI May 9, 2024
7a09138
format
HaoXuAI May 9, 2024
f33e769
format
HaoXuAI May 9, 2024
9153983
fix search
HaoXuAI May 12, 2024
5a212ba
format
HaoXuAI May 12, 2024
0689021
format
HaoXuAI May 12, 2024
1dc08ba
format
HaoXuAI May 12, 2024
c689a04
format
HaoXuAI May 12, 2024
999cba1
add test
HaoXuAI May 12, 2024
e93956d
add test
HaoXuAI May 12, 2024
369cfd4
fix e2e test
HaoXuAI May 12, 2024
ff96123
format
HaoXuAI May 12, 2024
d76bce6
fix test
HaoXuAI May 13, 2024
48a3e75
Merge branch 'master' into hao-xu-elasticsearch-vector
HaoXuAI May 13, 2024
bdb973e
fix test
HaoXuAI May 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
format 
Signed-off-by: cmuhao <sduxuhao@gmail.com>
  • Loading branch information
@HaoXuAI
HaoXuAI committed May 9, 2024
commit f33e7698464b33298a43318a7eb5e353b497fdd1
60 changes: 30 additions & 30 deletions sdk/python/feast/infra/online_stores/contrib/elastichsearch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,13 @@ def _bulk_batch_actions(self, batch):
}

def online_write_batch(
self,
config: RepoConfig,
table: FeatureView,
data: List[
Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]]
],
progress: Optional[Callable[[int], Any]],
self,
config: RepoConfig,
table: FeatureView,
data: List[
Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]]
],
progress: Optional[Callable[[int], Any]],
) -> None:
insert_values = []
for entity_key, values, timestamp, created_ts in data:
Expand All @@ -96,16 +96,16 @@ def online_write_batch(

batch_size = config.online_config.batch_size
for i in range(0, len(insert_values), batch_size):
batch = insert_values[i: i + batch_size]
batch = insert_values[i : i + batch_size]
actions = self._bulk_batch_actions(batch)
helpers.bulk(self._get_client(config), actions)

def online_read(
self,
config: RepoConfig,
table: FeatureView,
entity_keys: List[EntityKeyProto],
requested_features: Optional[List[str]] = None,
self,
config: RepoConfig,
table: FeatureView,
entity_keys: List[EntityKeyProto],
requested_features: Optional[List[str]] = None,
) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]:
if not requested_features:
body = {
Expand Down Expand Up @@ -136,13 +136,13 @@ def online_read(
return results

def update(
self,
config: RepoConfig,
tables_to_delete: Sequence[FeatureView],
tables_to_keep: Sequence[FeatureView],
entities_to_delete: Sequence[Entity],
entities_to_keep: Sequence[Entity],
partial: bool,
self,
config: RepoConfig,
tables_to_delete: Sequence[FeatureView],
tables_to_keep: Sequence[FeatureView],
entities_to_delete: Sequence[Entity],
entities_to_keep: Sequence[Entity],
partial: bool,
):
# implement the update method
for table in tables_to_delete:
Expand All @@ -151,20 +151,20 @@ def update(
self.create_index(config, table)

def teardown(
self,
config: RepoConfig,
tables: Sequence[FeatureView],
entities: Sequence[Entity],
self,
config: RepoConfig,
tables: Sequence[FeatureView],
entities: Sequence[Entity],
):
pass

def retrieve_online_documents(
self,
config: RepoConfig,
table: FeatureView,
requested_feature: str,
embedding: List[float],
top_k: int,
self,
config: RepoConfig,
table: FeatureView,
requested_feature: str,
embedding: List[float],
top_k: int,
) -> List[
Tuple[
Optional[datetime],
Expand Down