From fc63466ffc7f199962c958ab3e8f752c2954ea79 Mon Sep 17 00:00:00 2001
From: wangxiaoyou1993 <80284865+wangxiaoyou1993@users.noreply.github.com>
Date: Fri, 17 Mar 2023 03:10:24 -0700
Subject: [PATCH] [xy] Add contributing guide for streaming pipeline. (#2222)

---
 docs/guides/streaming/contributing.mdx | 50 ++++++++++++++++++++++++++
 docs/guides/streaming/overview.mdx     |  5 +++
 docs/mint.json                         |  3 +-
 3 files changed, 57 insertions(+), 1 deletion(-)
 create mode 100644 docs/guides/streaming/contributing.mdx

diff --git a/docs/guides/streaming/contributing.mdx b/docs/guides/streaming/contributing.mdx
new file mode 100644
index 000000000000..92e76bb34138
--- /dev/null
+++ b/docs/guides/streaming/contributing.mdx
@@ -0,0 +1,50 @@
+---
+title: "Contributing"
+---
+
+This doc talks about how to add a new source or destination to the streaming pipeline.
+
+## Prerequisites
+Follow this [doc](https://docs.mage.ai/community/contributing) to set up the development environment for Mage.
+
+## Add a new source
+
+Example PR for adding a source: https://github.com/mage-ai/mage-ai/pull/1953
+
+* Add the new source type to [mage_ai/streaming/constants.py](https://github.com/mage-ai/mage-ai/blob/master/mage_ai/streaming/constants.py).
+* Add the source file to the folder https://github.com/mage-ai/mage-ai/tree/master/mage_ai/streaming/sources
+    * Define the source class which inherits from the `BaseSource`.
+    * Define the source config class.
+    * Implement the methods:
+        * `init_client`: Initialize the client used to connect to the source.
+        * `batch_read`: Batch read messages from the source and use `handler` method to process the messages.
+* Add the new source to the [SourceFactory](https://github.com/mage-ai/mage-ai/blob/master/mage_ai/streaming/sources/source_factory.py).
+* Add a template file with example config to the folder: https://github.com/mage-ai/mage-ai/tree/master/mage_ai/data_preparation/templates/data_loaders/streaming
+* Add new source type to the frontend code.
+    * https://github.com/mage-ai/mage-ai/blob/master/mage_ai/frontend/components/PipelineDetail/AddNewBlocks/utils.tsx#L31
+    * https://github.com/mage-ai/mage-ai/blob/master/mage_ai/frontend/interfaces/DataSourceType.ts#L3
+    * https://github.com/mage-ai/mage-ai/blob/master/mage_ai/frontend/interfaces/DataSourceType.ts#L22
+* Add unit tests: https://github.com/mage-ai/mage-ai/tree/master/mage_ai/tests/streaming/sources
+* Add doc: https://github.com/mage-ai/mage-ai/tree/master/docs/guides/streaming/sources
+
+---
+
+## Add a new destination (sink)
+
+Example PR for adding a sink: https://github.com/mage-ai/mage-ai/pull/2121
+
+* Add the new sink type to [mage_ai/streaming/constants.py](https://github.com/mage-ai/mage-ai/blob/master/mage_ai/streaming/constants.py).
+* Add the sink file to the folder https://github.com/mage-ai/mage-ai/tree/master/mage_ai/streaming/sinks
+    * Define the sink class which inherits from the `BaseSink`.
+    * Define the sink config class.
+    * Implement the methods:
+        * `init_client`: Initialize the client used to connect to the destination.
+        * `batch_write`: Batch write messages to destination.
+* Add the new sink to the [SinkFactory](https://github.com/mage-ai/mage-ai/blob/master/mage_ai/streaming/sinks/sink_factory.py).
+* Add a template file with example config to the folder: https://github.com/mage-ai/mage-ai/tree/master/mage_ai/data_preparation/templates/data_exporters/streaming
+* Add new source type to the frontend code.
+    * https://github.com/mage-ai/mage-ai/blob/master/mage_ai/frontend/components/PipelineDetail/AddNewBlocks/utils.tsx#L37
+    * https://github.com/mage-ai/mage-ai/blob/master/mage_ai/frontend/interfaces/DataSourceType.ts#L3
+    * https://github.com/mage-ai/mage-ai/blob/master/mage_ai/frontend/interfaces/DataSourceType.ts#L22
+* Add unit tests: https://github.com/mage-ai/mage-ai/tree/master/mage_ai/streaming/sinks
+* Add doc: https://github.com/mage-ai/mage-ai/tree/master/docs/guides/streaming/destinations
diff --git a/docs/guides/streaming/overview.mdx b/docs/guides/streaming/overview.mdx
index e17a61d384cf..5e3457574a72 100644
--- a/docs/guides/streaming/overview.mdx
+++ b/docs/guides/streaming/overview.mdx
@@ -28,3 +28,8 @@ Check out this [tutorial](https://docs.mage.ai/guides/streaming-pipeline#create-
 
 ## Test pipeline execution
 After finishing configuring the streaming pipeline, you can click the button `Execution pipeline` to test streaming pipeline execution.
+
+
+## Contributing guide
+
+Follow this [doc](contributing) to add a new source or destination (sink) to Mage streaming pipeline.
diff --git a/docs/mint.json b/docs/mint.json
index 0729921129af..c6889b5e5a57 100644
--- a/docs/mint.json
+++ b/docs/mint.json
@@ -202,7 +202,8 @@
                 "guides/streaming/destinations/opensearch",
                 "guides/streaming/destinations/redshift"
               ]
-            }
+            },
+            "guides/streaming/contributing"
           ]
         },
         "production/configuring-production-settings/runtime-variable",