-
Notifications
You must be signed in to change notification settings - Fork 144
Expand file tree
/
Copy pathdocker-compose.yml
More file actions
187 lines (180 loc) · 6.08 KB
/
Copy pathdocker-compose.yml
File metadata and controls
187 lines (180 loc) · 6.08 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
name: feldera
services:
pipeline-manager:
tty: true
image: ghcr.io/feldera/pipeline-manager:${FELDERA_VERSION:-0.17.0}
ports:
# If you change the host side of the port mapping here, don't forget to
# also add a corresponding --allowed-origins argument to the pipeline
# manager command below (e.g., --allowed-origins "http://localhost:xyz").
# Otherwise, browsers will complain about CORS errors
- "8080:8080"
stop_grace_period: 0s
environment:
- RUST_BACKTRACE=1
- REDPANDA_BROKERS=redpanda:9092
- RUST_LOG=info,actix_web=error,tokio_postgres=info
- AUTH_CLIENT_ID
- AUTH_ISSUER
security_opt:
# The default seccomp profile disables the `io_uring_*` syscalls that we need.
# Ideally, we would minimally relax the default profile, which is usually in
# /usr/share/container/seccomp.json, but we often start via `curl` of just this
# `docker-compose.yml`, so there's no way to grab the specific file at the
# same time, so instead we just use `seccomp:unconfined`.
#
#- seccomp:seccomp.json
- seccomp:unconfined
healthcheck:
# TODO: add `/status` endpoint.
test:
[
"CMD-SHELL",
"curl --fail --request GET --url http://localhost:8080/healthz"
]
interval: 10s
timeout: 5s
retries: 5
redpanda:
profiles: [ "demo", "debezium", "snowflake", "demo-service-related" ]
command:
- redpanda
- start
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
# Address the broker advertises to clients that connect to the Kafka API.
# Use the internal addresses to connect to the Redpanda brokers'
# from inside the same Docker network.
# Use the external addresses to connect to the Redpanda brokers'
# from outside the Docker network.
- --advertise-kafka-addr internal://redpanda:9092,external://localhost:19092
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
# Address the broker advertises to clients that connect to the HTTP Proxy.
- --advertise-pandaproxy-addr internal://redpanda:8082,external://localhost:18082
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
# Redpanda brokers use the RPC API to communicate with eachother internally.
- --rpc-addr redpanda:33145
- --advertise-rpc-addr redpanda:33145
# Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
- --smp 1
# The amount of memory to make available to Redpanda.
- --memory 2G
# Mode dev-container uses well-known configuration properties for development in containers.
- --mode dev-container
# enable logs for debugging.
- --default-log-level=error
image: docker.redpanda.com/vectorized/redpanda:v23.2.3
ports:
- 18081:18081
- 18082:18082
- 19092:19092
- 19644:9644
healthcheck:
test:
[
"CMD-SHELL",
"rpk cluster health | grep -E 'Healthy:.+true' || exit 1"
]
interval: 10s
timeout: 3s
retries: 5
start_period: 5s
# Kafka connect.
# TODO: add a healthcheck to this container.
connect:
profiles: [ "debezium", "snowflake" ]
build:
context: ../
dockerfile: deploy/Dockerfile
target: kafka-connect
image: ghcr.io/feldera/kafka-connect:${FELDERA_VERSION:-0.17.0}
depends_on:
redpanda:
condition: service_healthy
ports:
- "8083:8083"
environment:
BOOTSTRAP_SERVERS: "redpanda:9092"
GROUP_ID: "1"
CONFIG_STORAGE_TOPIC: "inventory.configs"
OFFSET_STORAGE_TOPIC: "inventory.offset"
STATUS_STORAGE_TOPIC: "inventory.status"
LOG_LEVEL: "ERROR"
healthcheck:
test:
[
"CMD-SHELL",
"curl --fail --request GET --url http://localhost:8083/"
]
interval: 10s
timeout: 5s
retries: 5
demo:
profiles: [ "demo" ]
depends_on:
pipeline-manager:
condition: service_healthy
redpanda:
condition: service_healthy
image: ghcr.io/feldera/demo-container:${FELDERA_VERSION:-0.17.0}
environment:
- RUST_BACKTRACE=1
- REDPANDA_BROKERS=redpanda:9092
- SECOPS_DEMO_ARGS
- RUST_LOG=info
# Run the demos after giving the service some time to initialize
command:
- bash
- -c
- |
sleep 1
demo/demo-container.sh
snowflake-demo:
profiles: [ "snowflake" ]
depends_on:
pipeline-manager:
condition: service_healthy
connect:
condition: service_healthy
image: ghcr.io/feldera/demo-container:${FELDERA_VERSION:-0.17.0}
environment:
- RUST_BACKTRACE=1
- REDPANDA_BROKERS=redpanda:9092
- RUST_LOG=info
- KAFKA_CONNECT_SERVER=http://connect:8083
- SNOWFLAKE_CI_USER_PRIVATE_KEY_PASSPHRASE
- SNOWFLAKE_CI_USER_PRIVATE_KEY
- SNOWFLAKE_CI_USER_PASSWORD
command:
- bash
- -c
- "sleep 1 && env && cd demo/project_demo07-SnowflakeSink/ && python3 run.py --api-url http://pipeline-manager:8080"
s3-demo:
profiles: [ "s3" ]
depends_on:
pipeline-manager:
condition: service_healthy
image: ghcr.io/feldera/demo-container:${FELDERA_VERSION:-0.10.0}
environment:
- RUST_BACKTRACE=1
- RUST_LOG=info
- CI_S3_AWS_ACCESS_KEY
- CI_S3_AWS_SECRET
command:
- bash
- -c
- "sleep 1 && env && cd demo/project_demo09-S3/ && python3 run.py --api-url http://pipeline-manager:8080"
demo-service-related:
profiles: [ "demo-service-related" ]
depends_on:
pipeline-manager:
condition: service_healthy
redpanda:
condition: service_healthy
image: ghcr.io/feldera/demo-container:${FELDERA_VERSION:-0.17.0}
environment:
- RUST_BACKTRACE=1
- RUST_LOG=info
command:
- bash
- -c
- "sleep 1 && cd demo/service-probe && python3 run.py --api-url http://pipeline-manager:8080 && sleep 1 && cd ../simple-count && python3 run.py --api-url http://pipeline-manager:8080"