Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added production setup architecture #638

Merged
merged 3 commits into from
Jun 27, 2024
Merged

Added production setup architecture #638

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
050532e
Added production setup architecture
subkanthi Jun 11, 2024
22cba5c
Added README files for documenting production setup
subkanthi Jun 12, 2024
0c2c492
Updated production setup image
subkanthi Jun 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 94 additions & 0 deletions doc/img/production_setup.drawio
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
<mxfile host="app.diagrams.net" modified="2024-06-12T21:03:50.053Z" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" etag="s984ve9IqoB3ItVRo93g" version="24.5.3" type="device">
<diagram name="Page-1" id="NtnH0vJFmS5YRrL-5Pee">
<mxGraphModel dx="3188" dy="865" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
<mxCell id="TG6KOx0psHuuXlbvLAoE-1" value="binlogs" style="outlineConnect=0;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;shape=mxgraph.aws3.mysql_db_instance_2;fillColor=#2E73B8;gradientColor=none;" vertex="1" parent="1">
<mxGeometry x="-1690" y="293.5" width="60" height="63" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-2" value="" style="rounded=1;whiteSpace=wrap;html=1;dashed=1;" vertex="1" parent="1">
<mxGeometry x="-1600" y="180" width="920" height="640" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-4" value="&lt;b&gt;Debezium&lt;/b&gt;&lt;div&gt;&lt;b&gt;Queue&lt;/b&gt;&lt;/div&gt;" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;align=center;strokeColor=none;fillColor=#00BEF2;shape=mxgraph.azure.queue_generic;pointerEvents=1;" vertex="1" parent="1">
<mxGeometry x="-1570" y="300" width="180" height="50" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-14" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="TG6KOx0psHuuXlbvLAoE-6" target="TG6KOx0psHuuXlbvLAoE-13">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-6" value="&lt;b&gt;Sink&amp;nbsp;&lt;/b&gt;&lt;div&gt;&lt;b&gt;Connector Queue&lt;/b&gt;&lt;/div&gt;" style="verticalLabelPosition=bottom;html=1;verticalAlign=top;align=center;strokeColor=#d6b656;fillColor=#fff2cc;shape=mxgraph.azure.queue_generic;pointerEvents=1;" vertex="1" parent="1">
<mxGeometry x="-1230" y="300" width="180" height="50" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-7" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" source="TG6KOx0psHuuXlbvLAoE-1" target="TG6KOx0psHuuXlbvLAoE-4">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-8" value="ClickHouse" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="1">
<mxGeometry x="-620" y="440" width="60" height="80" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-10" value="&lt;b&gt;Main&lt;br&gt;Thread&lt;/b&gt;" style="whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="-1335" y="190" width="60" height="270" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-11" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=-0.017;entryY=0.53;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" source="TG6KOx0psHuuXlbvLAoE-4" target="TG6KOx0psHuuXlbvLAoE-10">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-12" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" source="TG6KOx0psHuuXlbvLAoE-10" target="TG6KOx0psHuuXlbvLAoE-6">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-13" value="&lt;b&gt;Thread&lt;/b&gt;&lt;div&gt;&lt;b&gt;Pool&lt;/b&gt;&lt;/div&gt;" style="whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="-990" y="190" width="140" height="580" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-16" value="Thread-1" style="whiteSpace=wrap;html=1;aspect=fixed;" vertex="1" parent="1">
<mxGeometry x="-960" y="220" width="80" height="80" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-17" value="Thread-2" style="whiteSpace=wrap;html=1;aspect=fixed;" vertex="1" parent="1">
<mxGeometry x="-960" y="330" width="80" height="80" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-18" value="Thread-3" style="whiteSpace=wrap;html=1;aspect=fixed;" vertex="1" parent="1">
<mxGeometry x="-960" y="510" width="80" height="80" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-19" value="Thread-4" style="whiteSpace=wrap;html=1;aspect=fixed;" vertex="1" parent="1">
<mxGeometry x="-960" y="630" width="80" height="80" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-20" value="&lt;b&gt;Sink Connector Lightweight&lt;/b&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="-1235" y="830" width="190" height="30" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-22" value="" style="rounded=1;whiteSpace=wrap;html=1;strokeWidth=2;fillWeight=4;hachureGap=8;hachureAngle=45;fillColor=#1ba1e2;sketch=1;" vertex="1" parent="1">
<mxGeometry x="-960" y="270" width="80" height="30" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-32" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="1" source="TG6KOx0psHuuXlbvLAoE-23" target="TG6KOx0psHuuXlbvLAoE-8">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-23" value="&lt;b&gt;Buffer&lt;/b&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="-890" y="270" width="60" height="30" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-24" value="" style="rounded=1;whiteSpace=wrap;html=1;strokeWidth=2;fillWeight=4;hachureGap=8;hachureAngle=45;fillColor=#1ba1e2;sketch=1;" vertex="1" parent="1">
<mxGeometry x="-960" y="380" width="80" height="30" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-25" value="" style="rounded=1;whiteSpace=wrap;html=1;strokeWidth=2;fillWeight=4;hachureGap=8;hachureAngle=45;fillColor=#1ba1e2;sketch=1;" vertex="1" parent="1">
<mxGeometry x="-960" y="560" width="80" height="30" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-26" value="" style="rounded=1;whiteSpace=wrap;html=1;strokeWidth=2;fillWeight=4;hachureGap=8;hachureAngle=45;fillColor=#1ba1e2;sketch=1;" vertex="1" parent="1">
<mxGeometry x="-960" y="680" width="80" height="30" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-27" value="&lt;b&gt;Buffer&lt;/b&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="-950" y="410" width="60" height="30" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-29" value="&lt;b&gt;Buffer&lt;/b&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="-950" y="590" width="60" height="30" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-31" value="&lt;b&gt;Buffer&lt;/b&gt;" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="-950" y="710" width="60" height="30" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-33" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0;entryDx=0;entryDy=15;entryPerimeter=0;" edge="1" parent="1" source="TG6KOx0psHuuXlbvLAoE-24" target="TG6KOx0psHuuXlbvLAoE-8">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-34" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" source="TG6KOx0psHuuXlbvLAoE-25" target="TG6KOx0psHuuXlbvLAoE-8">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="TG6KOx0psHuuXlbvLAoE-35" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" source="TG6KOx0psHuuXlbvLAoE-26" target="TG6KOx0psHuuXlbvLAoE-8">
<mxGeometry relative="1" as="geometry" />
</mxCell>
</root>
</mxGraphModel>
</diagram>
</mxfile>
Binary file added doc/img/production_setup.jpg
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
44 changes: 44 additions & 0 deletions doc/production_setup.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
## Production setup
![](img/production_setup.jpg)


### Improving throughput and/or Memory usage.

As detailed in the diagram above, there are components that store the messages and
can be configured to improve throughput and/or memory usage.

1. **Debezium Queue**:

The following configuration parameters are used to configure the size of the debezium queue
in terms of number of elements the queue can hold and the maximum size of the queue in bytes.

```
#Positive integer value that specifies the maximum size of each batch of events that should be processed during each iteration of this connector. Defaults to 2048.
max.batch.size: 20000

#Positive integer value that specifies the maximum number of records that the blocking queue can hold.
max.queue.size: 100000

# A long integer value that specifies the maximum volume of the blocking queue in bytes.
max.queue.size.in.bytes: 1000000000
```

2. **Sink connector Queue**:

```
# The maximum number of records that should be loaded into memory while streaming data from MySQL to ClickHouse.
sink.connector.max.queue.size: "100000"

```

3. **Thread Pool**:
```
# Maximum number of threads in the thread pool for processing CDC records.
thread.pool.size: 10

# Max number of records for the flush buffer.
buffer.max.records: "1000000"

Flush time of the buffer in milliseconds. The buffer that is stored in memory before being flushed to ClickHouse.
buffer.flush.time.ms: "1000"
```
3 changes: 3 additions & 0 deletions sink-connector-lightweight/docker/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,9 @@ restart.event.loop: "true"
#restart.event.loop.timeout.period.secs: Defines the restart timeout period.
restart.event.loop.timeout.period.secs: "3000"

# Flush time of the buffer in milliseconds. The buffer that is stored in memory before being flushed to ClickHouse.
#buffer.flush.time.ms: "1000"

# Max number of records for the flush buffer.
#buffer.max.records: "10000"

Expand Down
Loading