forked from feldera/feldera
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathoutput_handler.py
More file actions
41 lines (31 loc) · 1.2 KB
/
Copy pathoutput_handler.py
File metadata and controls
41 lines (31 loc) · 1.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
import pandas as pd
from queue import Queue
from feldera import FelderaClient
from feldera._callback_runner import CallbackRunner
class OutputHandler:
def __init__(self, client: FelderaClient, pipeline_name: str, view_name: str, queue: Queue):
"""
Initializes the output handler, but doesn't start it.
To start the output handler, call the `.OutputHandler.start` method.
"""
self.client: FelderaClient = client
self.pipeline_name: str = pipeline_name
self.view_name: str = view_name
self.queue: Queue = queue
self.buffer: list[pd.DataFrame] = []
# the callback that is passed to the `CallbackRunner`
def callback(df: pd.DataFrame, _: int):
self.buffer.append(df)
# sets up the callback runner
self.handler = CallbackRunner(self.client, self.pipeline_name, self.view_name, callback, queue)
def start(self):
"""
Starts the output handler in a separate thread
"""
self.handler.start()
def to_pandas(self):
"""
Returns the output of the pipeline as a pandas DataFrame
"""
self.handler.join()
return pd.concat(self.buffer)