Skip to content

Commit 45add7c

Browse files
abhizerryzhyk
abhizer
and
ryzhyk
authored
ci: run python tests (feldera#1873)
* ci: run python tests Signed-off-by: Abhinav Gyawali <22275402+abhizer@users.noreply.github.com> * ci: install feldera-py, pytest in install-python-deps Signed-off-by: Abhinav Gyawali <22275402+abhizer@users.noreply.github.com> * ci: python: copy python dir before installing Signed-off-by: Abhinav Gyawali <22275402+abhizer@users.noreply.github.com> * ci: py: copy python/tests to pythontests Signed-off-by: Abhinav Gyawali <22275402+abhizer@users.noreply.github.com> * ci: run python tests Signed-off-by: Abhinav Gyawali <22275402+abhizer@users.noreply.github.com> * ci: py: don't run kafka tests Signed-off-by: Abhinav Gyawali <22275402+abhizer@users.noreply.github.com> --------- Signed-off-by: Abhinav Gyawali <22275402+abhizer@users.noreply.github.com> Co-authored-by: Leonid Ryzhyk <ryzhyk@gmail.com>
1 parent cd23f86 commit 45add7c

4 files changed

Lines changed: 30 additions & 3 deletions

File tree

‎Earthfile‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,14 +98,22 @@ install-python-deps:
9898
FROM +install-deps
9999
RUN pip install wheel
100100
COPY demo/demo_notebooks/requirements.txt requirements.txt
101+
COPY --dir python ./
102+
RUN pip install --upgrade pip
101103
RUN pip wheel -r requirements.txt --wheel-dir=wheels
104+
RUN pip wheel -r python/tests/requirements.txt --wheel-dir=wheels
105+
RUN pip wheel python/ --wheel-dir=wheels
102106
SAVE ARTIFACT wheels /wheels
103107

104108
install-python:
105109
FROM +install-deps
106110
COPY +install-python-deps/wheels wheels
107111
COPY demo/demo_notebooks/requirements.txt requirements.txt
112+
COPY --dir python ./
113+
RUN pip install --upgrade pip # remove after upgrading to ubuntu 24
108114
RUN pip install --user -v --no-index --find-links=wheels -r requirements.txt
115+
RUN pip install --user -v --no-index --find-links=wheels -r python/tests/requirements.txt
116+
RUN pip install --user -v --no-index --find-links=wheels feldera
109117
SAVE ARTIFACT /root/.local/lib/python3.10
110118
SAVE ARTIFACT /root/.local/bin
111119

@@ -258,6 +266,7 @@ test-python:
258266

259267
COPY demo/demo_notebooks demo/demo_notebooks
260268
COPY demo/simple-join demo/simple-join
269+
COPY python/tests tests
261270

262271
# Reuse `Cargo.lock` to ensure consistent crate versions.
263272
RUN mkdir -p /working-dir/cargo_workspace
@@ -270,11 +279,13 @@ test-python:
270279
ENV RUST_LOG=error
271280
ENV WITH_POSTGRES=1
272281
ENV IN_CI=1
282+
ENV KAFKA_URL="localhost:9092"
273283
WITH DOCKER --pull postgres
274284
RUN docker run --shm-size=512MB -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -e PGDATA=/dev/shm -d postgres && \
275285
sleep 10 && \
276286
(./pipeline-manager --bind-address=0.0.0.0 --api-server-working-directory=/working-dir --compiler-working-directory=/working-dir --runner-working-directory=/working-dir --sql-compiler-home=/dbsp/sql-to-dbsp-compiler --dbsp-override-path=/dbsp --db-connection-string=postgresql://postgres:postgres@localhost:5432 --compilation-profile=unoptimized &) && \
277287
sleep 5 && \
288+
cd tests && python3 -m pytest . && cd .. && \
278289
python3 demo/simple-join/run.py --api-url http://localhost:8080 && \
279290
cd demo/demo_notebooks && jupyter execute fraud_detection.ipynb --JupyterApp.log_level='DEBUG'
280291
END

‎python/pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ dependencies = [
2525
"requests",
2626
"pandas",
2727
"typing-extensions",
28+
"numpy<2",
2829
]
2930
[project.urls]
3031
Homepage = "https://www.feldera.com"

‎python/tests/requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
kafka-python==2.0.2
2+
pytest

‎python/tests/test_wireframes.py‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import os
12
import unittest
23
import pandas as pd
34
from kafka import KafkaProducer, KafkaConsumer
@@ -184,6 +185,13 @@ def test_kafka(self):
184185
import json
185186

186187
KAFKA_SERVER = "localhost:19092"
188+
PIPELINE_TO_KAFKA_SERVER = "redpanda:9092"
189+
190+
in_ci = os.environ.get("IN_CI")
191+
192+
if in_ci == "1":
193+
# if running in CI, skip the test
194+
return
187195

188196
print("(Re-)creating topics...")
189197
admin_client = KafkaAdminClient(
@@ -224,7 +232,6 @@ def test_kafka(self):
224232
sql.register_table(TABLE_NAME, SQLSchema({"id": "INT NOT NULL PRIMARY KEY"}))
225233
sql.register_view(VIEW_NAME, f"SELECT COUNT(*) as num_rows FROM {TABLE_NAME}")
226234

227-
PIPELINE_TO_KAFKA_SERVER = "redpanda:9092"
228235

229236
source_config = {
230237
"topics": [INPUT_TOPIC],
@@ -281,10 +288,16 @@ def test_http_get(self):
281288
def test_avro_format(self):
282289
from feldera.formats import AvroFormat
283290

284-
KAFKA_URL_FROM_PIPELINE = "redpanda:9092"
291+
PIPELINE_TO_KAFKA_SERVER = "redpanda:9092"
285292
KAFKA_SERVER = "localhost:19092"
286293
TOPIC = "test_avro_format"
287294

295+
in_ci = os.environ.get("IN_CI")
296+
297+
if in_ci == "1":
298+
# if running in CI, skip the test
299+
return
300+
288301
admin_client = KafkaAdminClient(
289302
bootstrap_servers=KAFKA_SERVER,
290303
client_id="test_client"
@@ -305,7 +318,7 @@ def test_avro_format(self):
305318

306319
sink_config = {
307320
"topic": TOPIC,
308-
"bootstrap.servers": KAFKA_URL_FROM_PIPELINE,
321+
"bootstrap.servers": PIPELINE_TO_KAFKA_SERVER,
309322
"auto.offset.reset": "earliest",
310323
}
311324

0 commit comments

Comments
 (0)