fix: separate some modules from common_types (#1084)

## Rationale Some modules should not be set on the `common_types`. ## Detailed Changes - Move some modules from the `common_types` crate - Remove unnecessary features of `common_types` crate - Remove `test` feature and `tests` sub-module from `test_util` crate ## Test Plan Existing tests.
apache · Jul 19, 2023 · ec47df5 · ec47df5
1 parent a26988b
commit ec47df5
Show file tree

Hide file tree

Showing 81 changed files with 452 additions and 492 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -24,6 +24,7 @@ members = [
     "components/bytes_ext",
     "components/codec",
     "components/future_cancel",
+    "components/hash_ext",
     "components/id_allocator",
     "components/logger",
     "components/macros",
@@ -69,6 +70,7 @@ name = "ceresdb-server"
 path = "src/bin/ceresdb-server.rs"
 
 [workspace.dependencies]
+alloc_tracker = { path = "components/alloc_tracker" }
 arrow = { version = "38.0.0", features = ["prettyprint"] }
 arrow_ipc = { version = "38.0.0" }
 arrow_ext = { path = "components/arrow_ext" }
@@ -82,6 +84,7 @@ bytes_ext = { path = "components/bytes_ext" }
 catalog = { path = "catalog" }
 catalog_impls = { path = "catalog_impls" }
 ceresdbproto = "1.0"
+codec = { path = "components/codec" }
 chrono = "0.4"
 clap = "3.0"
 clru = "0.6.1"
@@ -91,23 +94,31 @@ common_types = { path = "common_types" }
 datafusion = { git = "https://github.com/ceresdb/arrow-datafusion.git", rev = "acb5d97a8a8de5296989740f97db3773fe3aa45a" }
 datafusion-proto = { git = "https://github.com/ceresdb/arrow-datafusion.git", rev = "acb5d97a8a8de5296989740f97db3773fe3aa45a" }
 df_operator = { path = "df_operator" }
+future_cancel = { path = "components/future_cancel" }
 etcd-client = "0.10.3"
 env_logger = "0.6"
 futures = "0.3"
-xorfilter-rs = { git = "https://github.com/CeresDB/xorfilter", rev = "ac8ef01" }
+generic_error = { path = "components/generic_error" }
+hash_ext = { path = "components/hash_ext" }
+hex = "0.4.3"
 lazy_static = "1.4.0"
 log = "0.4"
 logger = { path = "components/logger" }
 lru = "0.7.6"
+id_allocator = { path = "components/id_allocator" }
 influxql-logical-planner = { git = "https://github.com/CeresDB/influxql", rev = "b65a125b9cdfa3121a3c8843bc48441b91049e31", package = "iox_query_influxql" }
 influxql-parser = { git = "https://github.com/CeresDB/influxql", rev = "b65a125b9cdfa3121a3c8843bc48441b91049e31", package = "influxdb_influxql_parser" }
 influxql-query = { git = "https://github.com/CeresDB/influxql", rev = "b65a125b9cdfa3121a3c8843bc48441b91049e31", package = "iox_query" }
 influxql-schema = { git = "https://github.com/CeresDB/influxql", rev = "b65a125b9cdfa3121a3c8843bc48441b91049e31", package = "schema" }
 interpreters = { path = "interpreters" }
 itertools = "0.10.5"
 macros = { path = "components/macros" }
+message_queue = { path = "components/message_queue" }
 meta_client = { path = "meta_client" }
+metric_ext = { path = "components/metric_ext" }
 object_store = { path = "components/object_store" }
+panic_ext = { path = "components/panic_ext" }
+partitioned_lock = { path = "components/partitioned_lock" }
 partition_table_engine = { path = "partition_table_engine" }
 parquet_ext = { path = "components/parquet_ext" }
 parquet = { version = "38.0.0" }
@@ -139,29 +150,20 @@ system_catalog = { path = "system_catalog" }
 table_engine = { path = "table_engine" }
 table_kv = { path = "components/table_kv" }
 tempfile = "3.1.0"
-test_util = { path = "components/test_util", features = ["test"] }
+test_util = { path = "components/test_util" }
 time_ext = { path = "components/time_ext" }
 toml = "0.7"
 toml_ext = { path = "components/toml_ext" }
-generic_error = { path = "components/generic_error" }
-partitioned_lock = { path = "components/partitioned_lock" }
+timed_task = { path = "components/timed_task" }
 tracing_util = { path = "components/tracing_util" }
 trace_metric = { path = "components/trace_metric" }
 trace_metric_derive = { path = "components/trace_metric_derive" }
 trace_metric_derive_tests = { path = "components/trace_metric_derive_tests" }
 tonic = "0.8.1"
 tokio = { version = "1.25", features = ["full"] }
 wal = { path = "wal" }
-message_queue = { path = "components/message_queue" }
+xorfilter-rs = { git = "https://github.com/CeresDB/xorfilter", rev = "ac8ef01" }
 zstd = { version = "0.12", default-features = false }
-id_allocator = { path = "components/id_allocator" }
-panic_ext = { path = "components/panic_ext" }
-timed_task = { path = "components/timed_task" }
-future_cancel = { path = "components/future_cancel" }
-alloc_tracker = { path = "components/alloc_tracker" }
-metric_ext = { path = "components/metric_ext" }
-codec = { path = "components/codec" }
-hex = "0.4.3"
 
 [dependencies]
 analytic_engine = { workspace = true }

diff --git a/analytic_engine/Cargo.toml b/analytic_engine/Cargo.toml
@@ -21,7 +21,7 @@ arrow = { workspace = true }
 async-stream = { workspace = true }
 async-trait = { workspace = true }
 base64 = { workspace = true }
-bytes = { workspace = true }
+bytes_ext = { workspace = true }
 ceresdbproto = { workspace = true }
 codec = { workspace = true }
 common_types = { workspace = true }

diff --git a/analytic_engine/src/compaction/picker.rs b/analytic_engine/src/compaction/picker.rs
@@ -684,8 +684,8 @@ impl LevelPicker for TimeWindowPicker {
 mod tests {
     use std::time::Duration;
 
+    use bytes_ext::Bytes;
     use common_types::{
-        bytes::Bytes,
         tests::build_schema,
         time::{TimeRange, Timestamp},
     };

diff --git a/analytic_engine/src/instance/write.rs b/analytic_engine/src/instance/write.rs
@@ -2,10 +2,10 @@
 
 //! Write logic of instance
 
+use bytes_ext::ByteVec;
 use ceresdbproto::{schema as schema_pb, table_requests};
 use codec::row;
 use common_types::{
-    bytes::ByteVec,
     row::{RowGroup, RowGroupSlicer},
     schema::{IndexInWriterSchema, Schema},
 };

diff --git a/analytic_engine/src/manifest/details.rs b/analytic_engine/src/manifest/details.rs
@@ -706,7 +706,6 @@ mod tests {
     use object_store::LocalFileSystem;
     use runtime::Runtime;
     use table_engine::table::{SchemaId, TableId, TableSeqGenerator};
-    use test_util::tests::init_log_for_test;
     use wal::rocks_impl::manager::Builder as WalBuilder;
 
     use super::*;
@@ -824,7 +823,7 @@ mod tests {
 
     impl TestContext {
         fn new(prefix: &str, schema_id: SchemaId) -> Self {
-            init_log_for_test();
+            test_util::init_log_for_test();
             let dir = tempfile::Builder::new().prefix(prefix).tempdir().unwrap();
             let runtime = build_runtime(2);
 

diff --git a/analytic_engine/src/manifest/meta_edit.rs b/analytic_engine/src/manifest/meta_edit.rs
@@ -4,7 +4,7 @@
 
 use std::convert::TryFrom;
 
-use bytes::{Buf, BufMut};
+use bytes_ext::{Buf, BufMut};
 use ceresdbproto::{manifest as manifest_pb, schema as schema_pb};
 use common_types::{
     schema::{Schema, Version},

diff --git a/analytic_engine/src/memtable/key.rs b/analytic_engine/src/memtable/key.rs
@@ -12,14 +12,9 @@
 
 use std::mem;
 
-use bytes::BufMut;
+use bytes_ext::{BufMut, BytesMut, SafeBuf, SafeBufMut};
 use codec::{memcomparable::MemComparable, Decoder, Encoder};
-use common_types::{
-    bytes::{BytesMut, SafeBuf, SafeBufMut},
-    row::Row,
-    schema::Schema,
-    SequenceNumber,
-};
+use common_types::{row::Row, schema::Schema, SequenceNumber};
 use macros::define_result;
 use snafu::{ensure, Backtrace, ResultExt, Snafu};
 
@@ -29,19 +24,19 @@ pub enum Error {
     EncodeKeyDatum { source: codec::memcomparable::Error },
 
     #[snafu(display("Failed to encode sequence, err:{}", source))]
-    EncodeSequence { source: common_types::bytes::Error },
+    EncodeSequence { source: bytes_ext::Error },
 
     #[snafu(display("Failed to encode row index, err:{}", source))]
-    EncodeIndex { source: common_types::bytes::Error },
+    EncodeIndex { source: bytes_ext::Error },
 
     #[snafu(display("Failed to decode sequence, err:{}", source))]
-    DecodeSequence { source: common_types::bytes::Error },
+    DecodeSequence { source: bytes_ext::Error },
 
     #[snafu(display("Failed to decode row index, err:{}", source))]
-    DecodeIndex { source: common_types::bytes::Error },
+    DecodeIndex { source: bytes_ext::Error },
 
     #[snafu(display(
-        "Insufficent internal key length, len:{}.\nBacktrace:\n{}",
+        "Insufficient internal key length, len:{}.\nBacktrace:\n{}",
         len,
         backtrace
     ))]

diff --git a/analytic_engine/src/memtable/mod.rs b/analytic_engine/src/memtable/mod.rs
@@ -8,8 +8,8 @@ pub mod skiplist;
 
 use std::{ops::Bound, sync::Arc, time::Instant};
 
+use bytes_ext::{ByteVec, Bytes};
 use common_types::{
-    bytes::{ByteVec, Bytes},
     projected_schema::ProjectedSchema,
     record_batch::RecordBatchWithKey,
     row::Row,

diff --git a/analytic_engine/src/memtable/skiplist/iter.rs b/analytic_engine/src/memtable/skiplist/iter.rs
@@ -5,9 +5,9 @@
 use std::{cmp::Ordering, iter::Rev, ops::Bound, time::Instant};
 
 use arena::{Arena, BasicStats};
+use bytes_ext::{Bytes, BytesMut};
 use codec::row;
 use common_types::{
-    bytes::{Bytes, BytesMut},
     projected_schema::{ProjectedSchema, RowProjector},
     record_batch::{RecordBatchWithKey, RecordBatchWithKeyBuilder},
     row::contiguous::{ContiguousRowReader, ProjectedContiguousRow},