feat(repo): add connectors runtime

Author: spetz

Cargo.lock

@@ -30,6 +30,12 @@ members = [
     "core/binary_protocol",
     "core/cli",
     "core/common",
+    "core/connectors/data_producer",
+    "core/connectors/runtime",
+    "core/connectors/sdk",
+    "core/connectors/sinks/quickwit_sink",
+    "core/connectors/sinks/stdout_sink",
+    "core/connectors/sources/test_source",
     "core/examples",
     "core/integration",
     "core/sdk",
@@ -68,16 +74,18 @@ byte-unit = { version = "5.1.6", default-features = false, features = [
 ] }
 bytes = "1.10.1"
 charming = "0.4.0"
-chrono = "0.4.41"
+chrono = { version = "0.4.41", features = ["serde"] }
 clap = { version = "4.5.37", features = ["derive"] }
+config = { version = "0.15.11" }
 comfy-table = "7.1.4"
 crc32fast = "1.4.2"
 crossbeam = "0.8.4"
 dashmap = "6.1.0"
+derive_builder = "0.20.2"
 derive_more = { version = "2.0.1", features = ["full"] }
 derive-new = "0.7.0"
 dirs = "6.0.0"
-derive_builder = "0.20.2"
+dlopen2 = "0.7.0"
 enum_dispatch = "0.3.13"
 figlet-rs = "0.1.5"
 flume = "0.11.1"
@@ -87,9 +95,11 @@ human-repr = "1.1.0"
 humantime = "2.2.0"
 keyring = { version = "3.6.2", features = ["sync-secret-service", "vendored"] }
 nonzero_lit = "0.1.2"
+once_cell = "1.21.3"
 openssl = { version = "0.10.72", features = ["vendored"] }
 passterm = "=2.0.1"
 quinn = "0.11.8"
+postcard = { version = "1.1.1", features = ["alloc"] }
 rand = "0.9.1"
 reqwest = { version = "0.12.15", default-features = false, features = [
     "json",
@@ -101,7 +111,9 @@ rustls = { version = "0.23.27", features = ["ring"] }
 serde = { version = "1.0.219", features = ["derive", "rc"] }
 serde_json = "1.0.140"
 serde_with = { version = "3.12.0", features = ["base64", "macros"] }
+serde_yml = "0.0.12"
 serial_test = "3.2.0"
+simd-json = { version = "0.15.1", features = ["serde_impl"] }
 sysinfo = "0.35.0"
 tempfile = "3.19.1"
 thiserror = "2.0.12"
@@ -116,13 +128,15 @@ tracing-subscriber = { version = "0.3.19", default-features = false, features =
     "ansi",
 ] }
 uuid = { version = "1.16.0", features = [
+    "v4",
     "v7",
     "fast-rng",
     "serde",
     "zerocopy",
 ] }
 rust-s3 = { version = "0.35.1", features = ["default"] }
 strum = { version = "0.27.1", features = ["derive"] }
+strum_macros = "0.27.1"
 aes-gcm = "0.10.3"
 base64 = "0.22.1"
 twox-hash = { version = "2.1.0", features = ["xxhash32"] }
@@ -151,8 +165,9 @@ mimalloc = "0.1"
 console-subscriber = "0.4.1"
 
 # Path dependencies
-iggy_common = { path = "core/common", version = "0.7.0" }
 iggy_binary_protocol = { path = "core/binary_protocol", version = "0.7.0" }
+iggy_common = { path = "core/common", version = "0.7.0" }
+iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.1.0" }
 iggy = { path = "core/sdk", version = "0.7.0" }
 server = { path = "core/server" }
 integration = { path = "core/integration" }

Cargo.toml

@@ -32,6 +32,7 @@ anyhow: 1.0.98, "Apache-2.0 OR MIT",
 anymap2: 0.13.0, "Apache-2.0 OR MIT",
 arbitrary: 1.4.1, "Apache-2.0 OR MIT",
 arc-swap: 1.7.1, "Apache-2.0 OR MIT",
+arraydeque: 0.5.1, "Apache-2.0 OR MIT",
 arrayref: 0.3.9, "BSD-2-Clause",
 arrayvec: 0.7.6, "Apache-2.0 OR MIT",
 assert_cmd: 2.0.17, "Apache-2.0 OR MIT",
@@ -47,6 +48,7 @@ async-stream-impl: 0.3.6, "MIT",
 async-trait: 0.1.88, "Apache-2.0 OR MIT",
 async_zip: 0.0.17, "MIT",
 atomic: 0.6.0, "Apache-2.0 OR MIT",
+atomic-polyfill: 1.0.3, "Apache-2.0 OR MIT",
 atomic-waker: 1.1.2, "Apache-2.0 OR MIT",
 attohttpc: 0.28.5, "MPL-2.0",
 autocfg: 1.4.0, "Apache-2.0 OR MIT",
@@ -115,11 +117,13 @@ clap_complete: 4.5.50, "Apache-2.0 OR MIT",
 clap_derive: 4.5.32, "Apache-2.0 OR MIT",
 clap_lex: 0.7.4, "Apache-2.0 OR MIT",
 cmake: 0.1.54, "Apache-2.0 OR MIT",
+cobs: 0.2.3, "Apache-2.0 OR MIT",
 colorchoice: 1.0.3, "Apache-2.0 OR MIT",
 colored: 3.0.0, "MPL-2.0",
 combine: 4.6.7, "MIT",
 comfy-table: 7.1.4, "MIT",
 concurrent-queue: 2.5.0, "Apache-2.0 OR MIT",
+config: 0.15.11, "Apache-2.0 OR MIT",
 console-api: 0.8.1, "MIT",
 console-subscriber: 0.4.1, "MIT",
 console_error_panic_hook: 0.1.7, "Apache-2.0 OR MIT",
@@ -137,6 +141,7 @@ cpufeatures: 0.2.17, "Apache-2.0 OR MIT",
 crc: 3.3.0, "Apache-2.0 OR MIT",
 crc-catalog: 2.4.0, "Apache-2.0 OR MIT",
 crc32fast: 1.4.2, "Apache-2.0 OR MIT",
+critical-section: 1.2.0, "Apache-2.0 OR MIT",
 crossbeam: 0.8.4, "Apache-2.0 OR MIT",
 crossbeam-channel: 0.5.15, "Apache-2.0 OR MIT",
 crossbeam-deque: 0.8.6, "Apache-2.0 OR MIT",
@@ -172,6 +177,8 @@ dircpy: 0.3.19, "MIT",
 dirs: 6.0.0, "Apache-2.0 OR MIT",
 dirs-sys: 0.5.0, "Apache-2.0 OR MIT",
 displaydoc: 0.2.5, "Apache-2.0 OR MIT",
+dlopen2: 0.7.0, "MIT",
+dlopen2_derive: 0.4.1, "Custom License File",
 dlv-list: 0.5.2, "Apache-2.0 OR MIT",
 doc-comment: 0.3.3, "MIT",
 dotenvy: 0.15.7, "MIT",
@@ -181,6 +188,8 @@ dtor: 0.0.6, "Apache-2.0 OR MIT",
 dtor-proc-macro: 0.0.5, "Apache-2.0 OR MIT",
 dunce: 1.0.5, "Apache-2.0 OR CC0-1.0 OR MIT-0",
 either: 1.15.0, "Apache-2.0 OR MIT",
+embedded-io: 0.4.0, "Apache-2.0 OR MIT",
+embedded-io: 0.6.1, "Apache-2.0 OR MIT",
 encoding_rs: 0.8.35, "(Apache-2.0 OR MIT) AND BSD-3-Clause",
 enum_dispatch: 0.3.13, "Apache-2.0 OR MIT",
 env_filter: 0.1.3, "Apache-2.0 OR MIT",
@@ -265,11 +274,15 @@ gloo-worker-macros: 0.1.0, "Apache-2.0 OR MIT",
 governor: 0.10.0, "MIT",
 h2: 0.3.26, "MIT",
 h2: 0.4.10, "MIT",
+halfbrown: 0.3.0, "Apache-2.0 OR MIT",
 handlebars: 4.5.0, "MIT",
+hash32: 0.2.1, "Apache-2.0 OR MIT",
 hashbrown: 0.12.3, "Apache-2.0 OR MIT",
 hashbrown: 0.14.5, "Apache-2.0 OR MIT",
 hashbrown: 0.15.3, "Apache-2.0 OR MIT",
+hashlink: 0.10.0, "Apache-2.0 OR MIT",
 hdrhistogram: 7.5.4, "Apache-2.0 OR MIT",
+heapless: 0.7.17, "Apache-2.0 OR MIT",
 heck: 0.5.0, "Apache-2.0 OR MIT",
 hermit-abi: 0.3.9, "Apache-2.0 OR MIT",
 hex: 0.4.3, "Apache-2.0 OR MIT",
@@ -308,6 +321,12 @@ iggy: 0.7.0, "Apache-2.0",
 iggy-cli: 0.9.0, "Apache-2.0",
 iggy_binary_protocol: 0.7.0, "Apache-2.0",
 iggy_common: 0.7.0, "Apache-2.0",
+iggy_connector_data_producer: 0.1.0, "Apache-2.0",
+iggy_connector_quickwit_sink: 0.1.0, "Apache-2.0",
+iggy_connector_runtime: 0.1.0, "Apache-2.0",
+iggy_connector_sdk: 0.1.0, "Apache-2.0",
+iggy_connector_stdout_sink: 0.1.0, "Apache-2.0",
+iggy_connector_test_source: 0.1.0, "Apache-2.0",
 iggy_examples: 0.0.5, "Apache-2.0",
 impl-more: 0.1.9, "Apache-2.0 OR MIT",
 implicit-clone: 0.4.9, "Apache-2.0 OR MIT",
@@ -333,6 +352,7 @@ jni: 0.21.1, "Apache-2.0 OR MIT",
 jni-sys: 0.3.0, "Apache-2.0 OR MIT",
 jobserver: 0.1.33, "Apache-2.0 OR MIT",
 js-sys: 0.3.77, "Apache-2.0 OR MIT",
+json5: 0.4.1, "ISC",
 jsonwebtoken: 9.3.1, "MIT",
 jwalk: 0.8.1, "MIT",
 keyring: 3.6.2, "Apache-2.0 OR MIT",
@@ -350,6 +370,7 @@ libloading: 0.8.6, "ISC",
 libmimalloc-sys: 0.1.42, "MIT",
 libredox: 0.1.3, "MIT",
 libssh2-sys: 0.3.1, "Apache-2.0 OR MIT",
+libyml: 0.0.5, "MIT",
 libz-sys: 1.1.22, "Apache-2.0 OR MIT",
 linux-raw-sys: 0.4.15, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT",
 linux-raw-sys: 0.9.4, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT",
@@ -432,6 +453,7 @@ parking_lot_core: 0.8.6, "Apache-2.0 OR MIT",
 parking_lot_core: 0.9.10, "Apache-2.0 OR MIT",
 passterm: 2.0.1, "BSD-3-Clause",
 paste: 1.0.15, "Apache-2.0 OR MIT",
+pathdiff: 0.2.3, "Apache-2.0 OR MIT",
 pbkdf2: 0.12.2, "Apache-2.0 OR MIT",
 pear: 0.2.9, "Apache-2.0 OR MIT",
 pear_codegen: 0.2.9, "Apache-2.0 OR MIT",
@@ -451,6 +473,7 @@ polonius-the-crab: 0.2.1, "Apache-2.0 OR MIT OR Zlib",
 polyval: 0.6.2, "Apache-2.0 OR MIT",
 portable-atomic: 1.11.0, "Apache-2.0 OR MIT",
 portable-atomic-util: 0.2.4, "Apache-2.0 OR MIT",
+postcard: 1.1.1, "Apache-2.0 OR MIT",
 potential_utf: 0.1.2, "Unicode-3.0",
 powerfmt: 0.2.0, "Apache-2.0 OR MIT",
 ppv-lite86: 0.2.21, "Apache-2.0 OR MIT",
@@ -493,6 +516,8 @@ rcgen: 0.13.2, "Apache-2.0 OR MIT",
 redox_syscall: 0.2.16, "MIT",
 redox_syscall: 0.5.12, "MIT",
 redox_users: 0.5.0, "MIT",
+ref-cast: 1.0.24, "Apache-2.0 OR MIT",
+ref-cast-impl: 1.0.24, "Apache-2.0 OR MIT",
 regex: 1.11.1, "Apache-2.0 OR MIT",
 regex-automata: 0.1.10, "MIT OR Unlicense",
 regex-automata: 0.4.9, "Apache-2.0 OR MIT",
@@ -507,6 +532,7 @@ retry-policies: 0.4.0, "Apache-2.0 OR MIT",
 ring: 0.17.14, "Apache-2.0 AND ISC",
 rkyv: 0.7.45, "MIT",
 rkyv_derive: 0.7.45, "MIT",
+ron: 0.8.1, "Apache-2.0 OR MIT",
 route-recognizer: 0.3.1, "MIT",
 rust-ini: 0.21.1, "MIT",
 rust-s3: 0.35.1, "MIT",
@@ -551,6 +577,7 @@ serde_spanned: 0.6.8, "Apache-2.0 OR MIT",
 serde_urlencoded: 0.7.1, "Apache-2.0 OR MIT",
 serde_with: 3.12.0, "Apache-2.0 OR MIT",
 serde_with_macros: 3.12.0, "Apache-2.0 OR MIT",
+serde_yml: 0.0.12, "Apache-2.0 OR MIT",
 serial_test: 3.2.0, "MIT",
 serial_test_derive: 3.2.0, "MIT",
 server: 0.5.0, "Apache-2.0",
@@ -560,6 +587,7 @@ sharded-slab: 0.1.7, "MIT",
 shlex: 1.3.0, "Apache-2.0 OR MIT",
 signal-hook-registry: 1.4.5, "Apache-2.0 OR MIT",
 simd-adler32: 0.3.7, "MIT",
+simd-json: 0.15.1, "Apache-2.0 OR MIT",
 simdutf8: 0.1.5, "Apache-2.0 OR MIT",
 simple_asn1: 0.6.3, "ISC",
 siphasher: 1.0.1, "Apache-2.0 OR MIT",
@@ -652,6 +680,7 @@ utf8parse: 0.2.2, "Apache-2.0 OR MIT",
 uuid: 1.16.0, "Apache-2.0 OR MIT",
 v_htmlescape: 0.15.8, "Apache-2.0 OR MIT",
 valuable: 0.1.1, "MIT",
+value-trait: 0.11.0, "Apache-2.0 OR MIT",
 vcpkg: 0.2.15, "Apache-2.0 OR MIT",
 vergen: 9.0.6, "Apache-2.0 OR MIT",
 vergen-git2: 1.0.7, "Apache-2.0 OR MIT",
@@ -749,6 +778,7 @@ wit-bindgen-rt: 0.39.0, "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT",
 writeable: 0.6.1, "Unicode-3.0",
 wyz: 0.5.1, "MIT",
 xz2: 0.1.7, "Apache-2.0 OR MIT",
+yaml-rust2: 0.10.2, "(MIT AND Apache-2.0) OR MIT",
 yansi: 1.0.1, "Apache-2.0 OR MIT",
 yasna: 0.5.2, "Apache-2.0 OR MIT",
 yew: 0.21.0, "Apache-2.0 OR MIT",

DEPENDENCIES.md

@@ -65,6 +65,7 @@ The name is an abbreviation for the Italian Greyhound - small yet extremely fast
 - Additional features such as **server side message deduplication**
 - **Multi-tenant** support via abstraction of **streams** whch group **topics**
 - **TLS** support for all transport protocols (TCP, QUIC, HTTPS)
+- **[Connectors](https://github.com/apache/iggy/tree/master/core/connectors)** - sinks, sources and data transformations based on the **custom Rust plugins**
 - Optional server-side as well as client-side **data encryption** using AES-256-GCM
 - Optional metadata support in the form of **message headers**
 - Optional **data backups & archivization** on disk and/or the **S3** compatible cloud storage (e.g. AWS S3)
@@ -78,6 +79,7 @@ The name is an abbreviation for the Italian Greyhound - small yet extremely fast
 
 ![files structure](assets/files_structure.png)
 
+
 ---
 
 ## Architecture
@@ -86,30 +88,36 @@ This is the high-level architecture of the Iggy message streaming server, where
 
 ![server](assets/iggy_architecture.png)
 
+---
+
+## Version
+
+The latest released version is `0.4.300` for Iggy server, which is compatible with `0.6` Rust client SDK and the others.
+
+The recent improvements based on the zero-copy (de)serialization, along with updated SDKs etc. will be available in the upcoming release with Iggy server `0.5.0`, Rust SDK `0.7` and all the other SDKs.
 
 ---
 
 ## Roadmap
 
 - **Shared-nothing** design and **io_uring** support (PoC on experimental branch, WiP on the main branch)
 - **Clustering** & data replication based on **[VSR](https://pmg.csail.mit.edu/papers/vr-revisited.pdf)** (on sandbox project using Raft, will be implemented after shared-nothing design is completed)
-- Plugins & extensions support (design and PoC as discussed [here](https://github.com/apache/iggy/discussions/1670), WiP)
-
 
 ---
 
 ## Supported languages SDK (work in progress)
 
-We're in the process of migrating all the remaining SDKs and other tooling from [iggy-rs](https://github.com/iggy-rs/) organization to this monorepo (WiP).
+- Rust
+- C#
+- Java
+- Go
+- Python
+- Node
+- C++
+- Elixir
+
 
-- [Rust](https://crates.io/crates/iggy)
-- [C#](https://github.com/iggy-rs/iggy-dotnet-client)
-- [Go](https://github.com/iggy-rs/iggy-go-client)
-- [Node](https://github.com/iggy-rs/iggy-node-client)
-- [Python](https://github.com/iggy-rs/iggy-python-client)
-- [Java](https://github.com/iggy-rs/iggy-java-client)
-- [C++](https://github.com/iggy-rs/iggy-cpp-client)
-- [Elixir](https://github.com/iggy-rs/iggy-elixir-client)
+Almost all the tooling has been migrated from [iggy-rs](https://github.com/iggy-rs/) organization to this monorepo. There's no an ongoing work to support the zero-copy (de)serialization for all the SDKs.
 
 ---