Skip to content

Commit 035c405

Browse files
snithishsnithish
committed
Compose file from confluent example and custom connect for REST Source
0 parents  commit 035c405

6 files changed

+180
-0
lines changed

‎docker-compose.yml

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
---
2+
version: '2'
3+
services:
4+
zookeeper:
5+
image: confluentinc/cp-zookeeper:5.3.1
6+
hostname: zookeeper
7+
container_name: zookeeper
8+
ports:
9+
- "2181:2181"
10+
environment:
11+
ZOOKEEPER_CLIENT_PORT: 2181
12+
ZOOKEEPER_TICK_TIME: 2000
13+
14+
broker:
15+
image: confluentinc/cp-enterprise-kafka:5.3.1
16+
hostname: broker
17+
container_name: broker
18+
depends_on:
19+
- zookeeper
20+
ports:
21+
- "9092:9092"
22+
environment:
23+
KAFKA_BROKER_ID: 1
24+
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
25+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
26+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
27+
KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
28+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
29+
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
30+
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092
31+
CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
32+
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
33+
CONFLUENT_METRICS_ENABLE: 'true'
34+
CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
35+
36+
schema-registry:
37+
image: confluentinc/cp-schema-registry:5.3.1
38+
hostname: schema-registry
39+
container_name: schema-registry
40+
depends_on:
41+
- zookeeper
42+
- broker
43+
ports:
44+
- "8081:8081"
45+
environment:
46+
SCHEMA_REGISTRY_HOST_NAME: schema-registry
47+
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
48+
49+
connect:
50+
image: kafka-connect
51+
hostname: connect
52+
container_name: connect
53+
depends_on:
54+
- zookeeper
55+
- broker
56+
- schema-registry
57+
ports:
58+
- "8083:8083"
59+
environment:
60+
CONNECT_BOOTSTRAP_SERVERS: 'broker:29092'
61+
CONNECT_REST_ADVERTISED_HOST_NAME: connect
62+
CONNECT_REST_PORT: 8083
63+
CONNECT_GROUP_ID: compose-connect-group
64+
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
65+
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
66+
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
67+
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
68+
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
69+
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
70+
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
71+
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
72+
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
73+
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
74+
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
75+
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
76+
CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'
77+
# CLASSPATH required due to CC-2422
78+
CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-5.3.1.jar
79+
CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
80+
CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
81+
CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
82+
CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
83+
84+
control-center:
85+
image: confluentinc/cp-enterprise-control-center:5.3.1
86+
hostname: control-center
87+
container_name: control-center
88+
depends_on:
89+
- zookeeper
90+
- broker
91+
- schema-registry
92+
- connect
93+
- ksql-server
94+
ports:
95+
- "9021:9021"
96+
environment:
97+
CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092'
98+
CONTROL_CENTER_ZOOKEEPER_CONNECT: 'zookeeper:2181'
99+
CONTROL_CENTER_CONNECT_CLUSTER: 'connect:8083'
100+
CONTROL_CENTER_KSQL_URL: "http://ksql-server:8088"
101+
CONTROL_CENTER_KSQL_ADVERTISED_URL: "http://localhost:8088"
102+
CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
103+
CONTROL_CENTER_REPLICATION_FACTOR: 1
104+
CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
105+
CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
106+
CONFLUENT_METRICS_TOPIC_REPLICATION: 1
107+
PORT: 9021
108+
109+
ksql-server:
110+
image: confluentinc/cp-ksql-server:5.3.1
111+
hostname: ksql-server
112+
container_name: ksql-server
113+
depends_on:
114+
- broker
115+
- connect
116+
ports:
117+
- "8088:8088"
118+
environment:
119+
KSQL_CONFIG_DIR: "/etc/ksql"
120+
KSQL_LOG4J_OPTS: "-Dlog4j.configuration=file:/etc/ksql/log4j-rolling.properties"
121+
KSQL_BOOTSTRAP_SERVERS: "broker:29092"
122+
KSQL_HOST_NAME: ksql-server
123+
KSQL_LISTENERS: "http://0.0.0.0:8088"
124+
KSQL_CACHE_MAX_BYTES_BUFFERING: 0
125+
KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
126+
KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
127+
KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
128+
129+
ksql-cli:
130+
image: confluentinc/cp-ksql-cli:5.3.1
131+
container_name: ksql-cli
132+
depends_on:
133+
- broker
134+
- connect
135+
- ksql-server
136+
entrypoint: /bin/sh
137+
tty: true
138+
139+
ksql-datagen:
140+
# Downrev ksql-examples to 5.1.2 due to DEVX-798 (work around issues in 5.2.0)
141+
image: confluentinc/ksql-examples:5.3.1
142+
hostname: ksql-datagen
143+
container_name: ksql-datagen
144+
depends_on:
145+
- ksql-server
146+
- broker
147+
- schema-registry
148+
- connect
149+
command: "bash -c 'echo Waiting for Kafka to be ready... && \
150+
cub kafka-ready -b broker:29092 1 40 && \
151+
echo Waiting for Confluent Schema Registry to be ready... && \
152+
cub sr-ready schema-registry 8081 40 && \
153+
echo Waiting a few seconds for topic creation to finish... && \
154+
sleep 11 && \
155+
tail -f /dev/null'"
156+
environment:
157+
KSQL_CONFIG_DIR: "/etc/ksql"
158+
KSQL_LOG4J_OPTS: "-Dlog4j.configuration=file:/etc/ksql/log4j-rolling.properties"
159+
STREAMS_BOOTSTRAP_SERVERS: broker:29092
160+
STREAMS_SCHEMA_REGISTRY_HOST: schema-registry
161+
STREAMS_SCHEMA_REGISTRY_PORT: 8081
162+
163+
rest-proxy:
164+
image: confluentinc/cp-kafka-rest:5.3.1
165+
depends_on:
166+
- zookeeper
167+
- broker
168+
- schema-registry
169+
ports:
170+
- 8082:8082
171+
hostname: rest-proxy
172+
container_name: rest-proxy
173+
environment:
174+
KAFKA_REST_HOST_NAME: rest-proxy
175+
KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092'
176+
KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
177+
KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'

‎kafka-connect/Dockerfile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
FROM confluentinc/cp-kafka-connect-base:5.3.1
2+
RUN mkdir /usr/share/java/kafka-connect-rest
3+
COPY jars/* /usr/share/java/kafka-connect-rest/

‎kafka-connect/jars/kafka-connect-rest-plugin-1.0.3-shaded.jar

1.63 MB
Binary file not shown.

‎kafka-connect/jars/kafka-connect-transform-add-headers-1.0.3-shaded.jar

4.55 KB
Binary file not shown.

‎kafka-connect/jars/kafka-connect-transform-from-json-plugin-1.0.3-shaded.jar

3.88 MB
Binary file not shown.

‎kafka-connect/jars/kafka-connect-transform-velocity-eval-1.0.3-shaded.jar

980 KB
Binary file not shown.

0 commit comments

Comments
 (0)