From 5b2f9b7b176c99633561532a5a8c60047759b98e Mon Sep 17 00:00:00 2001 From: Jiacai Liu Date: Wed, 7 Jun 2023 09:34:36 +0800 Subject: [PATCH] chore: bump xor8 (#972) ## Rationale Related with #967, reduce CPU consumption. ## Detailed Changes - https://github.com/CeresDB/xorfilter/pull/1 - https://github.com/CeresDB/xorfilter/pull/2 ## Test Plan @zouxiang1993 will benchmark this in his test env. --- Cargo.lock | 82 ++------------------ Cargo.toml | 4 +- analytic_engine/src/sst/parquet/meta_data.rs | 2 +- 3 files changed, 7 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8bb9862e43..1dfc250c11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,12 +144,6 @@ dependencies = [ "backtrace", ] -[[package]] -name = "arbitrary" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d098ff73c1ca148721f37baad5ea6a465a13f9573aba8641fbbbae8164a54e" - [[package]] name = "arc-swap" version = "0.4.8" @@ -1053,31 +1047,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "cbordata" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d1b01dea75baf7521dfb06c2462d8339d71f4320a2244d1f8f6b94265ded858" -dependencies = [ - "cbordata-derive", - "num-bigint 0.4.3", - "num-traits", -] - -[[package]] -name = "cbordata-derive" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9fd16dfe515476789addde9311da025a1f0c45ddd81b7b0315c9d426577dcb" -dependencies = [ - "heck 0.3.3", - "lazy_static", - "proc-macro-error 0.4.12", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "cc" version = "1.0.79" @@ -1279,7 +1248,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" dependencies = [ "heck 0.4.1", - "proc-macro-error 1.0.4", + "proc-macro-error", "proc-macro2", "quote", "syn 1.0.109", @@ -3881,7 +3850,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" dependencies = [ - "arbitrary", "autocfg", "num-integer", "num-traits", @@ -4557,45 +4525,19 @@ dependencies = [ "toml 0.5.11", ] -[[package]] -name = "proc-macro-error" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" -dependencies = [ - "proc-macro-error-attr 0.4.12", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - [[package]] name = "proc-macro-error" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ - "proc-macro-error-attr 1.0.4", + "proc-macro-error-attr", "proc-macro2", "quote", "syn 1.0.109", "version_check", ] -[[package]] -name = "proc-macro-error-attr" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", - "syn-mid", - "version_check", -] - [[package]] name = "proc-macro-error-attr" version = "1.0.4" @@ -6150,7 +6092,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" dependencies = [ "heck 0.3.3", - "proc-macro-error 1.0.4", + "proc-macro-error", "proc-macro2", "quote", "syn 1.0.109", @@ -6259,17 +6201,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn-mid" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baa8e7560a164edb1621a55d18a0c59abf49d360f47aa7b821061dd7eea7fac9" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "sync_wrapper" version = "0.1.2" @@ -7532,11 +7463,8 @@ dependencies = [ [[package]] name = "xorfilter-rs" -version = "0.5.2" -source = "git+https://github.com/datafuse-extras/xorfilter?tag=databend-alpha.4#3bd8eb6b5b5e134703cc3b2807dfe43a763aa886" -dependencies = [ - "cbordata", -] +version = "0.6.0" +source = "git+https://github.com/CeresDB/xorfilter?rev=ac8ef01#ac8ef01a800b84b568bb55ec7d84fdf410616929" [[package]] name = "xz2" diff --git a/Cargo.toml b/Cargo.toml index 054ba9cf78..2f2036df2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,9 +82,7 @@ df_operator = { path = "df_operator" } etcd-client = "0.10.3" env_logger = "0.6" futures = "0.3" -xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", features = [ - "cbordata", -], tag = "databend-alpha.4" } +xorfilter-rs = { git = "https://github.com/CeresDB/xorfilter", rev = "ac8ef01" } lazy_static = "1.4.0" log = "0.4" logger = { path = "components/logger" } diff --git a/analytic_engine/src/sst/parquet/meta_data.rs b/analytic_engine/src/sst/parquet/meta_data.rs index fe1f65cd58..4a6778b7f3 100644 --- a/analytic_engine/src/sst/parquet/meta_data.rs +++ b/analytic_engine/src/sst/parquet/meta_data.rs @@ -14,7 +14,7 @@ use common_types::{ }; use common_util::define_result; use snafu::{Backtrace, OptionExt, ResultExt, Snafu}; -use xorfilter::{Xor8, Xor8Builder}; +use xorfilter::xor8::{Xor8, Xor8Builder}; use crate::sst::writer::MetaData;