Skip to content

Commit 420b7fa

Browse files
KKouldKKould
authored
perf: stream analyze statistics with KLL sketch (#354)
1 parent b5965dc commit 420b7fa

20 files changed

Lines changed: 969 additions & 360 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 46 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -15,67 +15,71 @@ jobs:
1515
# 1
1616
check:
1717
name: Rust project check
18-
runs-on: [self-hosted, Linux, X64]
18+
runs-on: ubuntu-latest
1919
steps:
20-
- uses: actions/checkout@v2
21-
- name: Use preinstalled tools
22-
run: |
23-
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
24-
echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
20+
- uses: actions/checkout@v4
21+
- uses: dtolnay/rust-toolchain@stable
22+
- uses: Swatinem/rust-cache@v2
2523

2624
- name: Run cargo test
2725
run: make test
2826
# 2
2927
fmt:
3028
name: Rust fmt
31-
runs-on: [self-hosted, Linux, X64]
29+
runs-on: ubuntu-latest
3230
steps:
33-
- uses: actions/checkout@v2
34-
- name: Use preinstalled tools
35-
run: |
36-
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
37-
echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
31+
- uses: actions/checkout@v4
32+
- uses: dtolnay/rust-toolchain@stable
33+
with:
34+
components: rustfmt
35+
- uses: Swatinem/rust-cache@v2
3836

3937
- name: Run cargo fmt
4038
run: make fmt
4139
# 3
4240
e2e:
4341
name: Rust e2e sqllogictest
44-
runs-on: [self-hosted, Linux, X64]
42+
runs-on: ubuntu-latest
4543
steps:
46-
- uses: actions/checkout@v2
47-
- name: Use preinstalled tools
48-
run: |
49-
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
50-
echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
44+
- uses: actions/checkout@v4
45+
- uses: dtolnay/rust-toolchain@stable
46+
- uses: Swatinem/rust-cache@v2
5147

5248
- name: Run sqllogictest suite
5349
run: make test-slt
5450
# 4
5551
wasm-tests:
5652
name: Wasm cargo tests
57-
runs-on: [self-hosted, Linux, X64]
53+
runs-on: ubuntu-latest
5854
steps:
59-
- uses: actions/checkout@v2
55+
- uses: actions/checkout@v4
56+
- uses: dtolnay/rust-toolchain@stable
57+
with:
58+
targets: wasm32-unknown-unknown
59+
- uses: actions/setup-node@v4
60+
with:
61+
node-version: 20
62+
- uses: Swatinem/rust-cache@v2
6063

61-
- name: Use preinstalled tools
62-
run: |
63-
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
64-
echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
64+
- uses: taiki-e/install-action@wasm-pack
6565

6666
- name: Run wasm-bindgen tests (wasm32 target)
6767
run: make test-wasm
6868
# 5
6969
wasm-examples:
7070
name: Wasm examples (nodejs)
71-
runs-on: [self-hosted, Linux, X64]
71+
runs-on: ubuntu-latest
7272
steps:
73-
- uses: actions/checkout@v2
73+
- uses: actions/checkout@v4
74+
- uses: dtolnay/rust-toolchain@stable
75+
with:
76+
targets: wasm32-unknown-unknown
77+
- uses: actions/setup-node@v4
78+
with:
79+
node-version: 20
80+
- uses: Swatinem/rust-cache@v2
7481

75-
- name: Use preinstalled tools
76-
run: |
77-
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
78-
echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
82+
- uses: taiki-e/install-action@wasm-pack
7983

8084
- name: Build wasm package
8185
run: make wasm-build
@@ -85,28 +89,25 @@ jobs:
8589
# 6
8690
native-examples:
8791
name: Native examples
88-
runs-on: [self-hosted, Linux, X64]
92+
runs-on: ubuntu-latest
8993
steps:
90-
- uses: actions/checkout@v2
91-
92-
- name: Use preinstalled tools
93-
run: |
94-
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
95-
echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
94+
- uses: actions/checkout@v4
95+
- uses: dtolnay/rust-toolchain@stable
96+
- uses: Swatinem/rust-cache@v2
9697

9798
- name: Run native examples
9899
run: make native-examples
99100
# 7
100101
python-tests:
101102
name: Python bindings tests
102-
runs-on: [self-hosted, Linux, X64]
103+
runs-on: ubuntu-latest
103104
steps:
104-
- uses: actions/checkout@v2
105-
106-
- name: Use preinstalled tools
107-
run: |
108-
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
109-
echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
105+
- uses: actions/checkout@v4
106+
- uses: dtolnay/rust-toolchain@stable
107+
- uses: actions/setup-python@v5
108+
with:
109+
python-version: "3.12"
110+
- uses: Swatinem/rust-cache@v2
110111

111112
- name: Run python binding tests
112-
run: make test-python
113+
run: PYO3_PYTHON="$(which python)" make test-python

‎Cargo.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
[package]
44
name = "kite_sql"
5-
version = "0.3.0"
5+
version = "0.3.1"
66
edition = "2021"
77
build = "build.rs"
88
authors = ["Kould <kould2333@gmail.com>", "Xwg <loloxwg@gmail.com>"]

‎src/execution/ddl/create_index.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CreateIndex {
140140
with_projection_tmp_value(arena, None, &column_exprs, |arena, value| {
141141
let mut state = arena.local_state(plan_arena);
142142
let (transaction, table_codec) = state.transaction_codec_mut();
143-
let index = Index::new(index_id, value, ty);
143+
let index = Index::new(index_id, &value, ty);
144144
transaction.add_index(table_codec, table_name.as_ref(), index, &tuple_pk)
145145
})?;
146146
}

‎src/execution/dml/analyze.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use crate::execution::{
2020
};
2121
use crate::expression::ScalarExpression;
2222
use crate::iter_ext::Itertools;
23-
use crate::optimizer::core::histogram::HistogramBuilder;
23+
use crate::optimizer::core::histogram::{HistogramBuilder, ANALYZE_STATISTICS_RELATIVE_ERROR};
2424
use crate::optimizer::core::statistics_meta::StatisticsMeta;
2525
use crate::planner::operator::analyze::AnalyzeOperator;
2626
use crate::planner::LogicalPlan;
@@ -105,7 +105,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Analyze {
105105
Ok(State {
106106
index_id: index.id,
107107
exprs: index.column_exprs(table, plan_arena)?,
108-
builder: HistogramBuilder::new(index, None),
108+
builder: HistogramBuilder::new(index, ANALYZE_STATISTICS_RELATIVE_ERROR)?,
109109
histogram_buckets: self.histogram_buckets,
110110
})
111111
})

‎src/execution/dml/delete.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Delete {
102102
transaction.del_index(
103103
table_codec,
104104
&self.table_name,
105-
&Index::new(*index_id, value, *index_ty),
105+
&Index::new(*index_id, &value, *index_ty),
106106
tuple_id,
107107
)
108108
})?;

‎src/execution/dml/insert.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Insert {
170170
with_projection_tmp_value(arena, Some(&tuple), exprs, |arena, value| {
171171
let mut state = arena.local_state(plan_arena);
172172
let (transaction, table_codec) = state.transaction_codec_mut();
173-
let index = Index::new(index_meta.id, value, index_meta.ty);
173+
let index = Index::new(index_meta.id, &value, index_meta.ty);
174174
transaction.add_index(table_codec, &self.table_name, index, tuple_id)
175175
})?;
176176
}

‎src/execution/dml/update.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Update {
124124
with_projection_tmp_value(arena, Some(&tuple), exprs, |arena, value| {
125125
let mut state = arena.local_state(plan_arena);
126126
let (transaction, table_codec) = state.transaction_codec_mut();
127-
let index = Index::new(index_meta.id, value, index_meta.ty);
127+
let index = Index::new(index_meta.id, &value, index_meta.ty);
128128
transaction.del_index(table_codec, &self.table_name, &index, &old_pk)
129129
})?;
130130
}
@@ -152,7 +152,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Update {
152152
with_projection_tmp_value(arena, Some(&tuple), exprs, |arena, value| {
153153
let mut state = arena.local_state(plan_arena);
154154
let (transaction, table_codec) = state.transaction_codec_mut();
155-
let index = Index::new(index_meta.id, value, index_meta.ty);
155+
let index = Index::new(index_meta.id, &value, index_meta.ty);
156156
transaction.add_index(table_codec, &self.table_name, index, new_pk)
157157
})?;
158158
}

‎src/execution/dql/sort.rs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,11 +65,6 @@ impl<'a, T> NullableVec<'a, T> {
6565
unsafe { self.0[offset].assume_init_read() }
6666
}
6767

68-
#[inline]
69-
pub(crate) fn get(&self, offset: usize) -> &T {
70-
unsafe { self.0[offset].assume_init_ref() }
71-
}
72-
7368
#[inline]
7469
pub(crate) fn len(&self) -> usize {
7570
self.0.len()

‎src/execution/mod.rs‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -413,7 +413,7 @@ pub(crate) fn with_projection_tmp_value<'a, T: Transaction + 'a>(
413413
arena: &mut ExecArena<'a, T>,
414414
tuple: Option<&dyn TupleLike>,
415415
exprs: &[ScalarExpression],
416-
f: impl FnOnce(&mut ExecArena<'a, T>, &DataValue) -> Result<(), DatabaseError>,
416+
f: impl FnOnce(&mut ExecArena<'a, T>, DataValue) -> Result<(), DatabaseError>,
417417
) -> Result<(), DatabaseError> {
418418
arena.with_projection_tmp(|arena, projection_tmp| {
419419
{
@@ -424,16 +424,16 @@ pub(crate) fn with_projection_tmp_value<'a, T: Transaction + 'a>(
424424
}
425425
}
426426

427-
if projection_tmp.len() > 1 {
428-
let value = DataValue::Tuple(std::mem::take(projection_tmp), false);
429-
let ret = f(arena, &value);
430-
let DataValue::Tuple(values, _) = value else {
431-
unreachable!()
432-
};
433-
*projection_tmp = values;
434-
ret?;
435-
} else if let Some(value) = projection_tmp.first() {
436-
f(arena, value)?;
427+
match projection_tmp.len() {
428+
0 => {}
429+
1 => {
430+
let value = projection_tmp.pop().expect("projection has one value");
431+
f(arena, value)?;
432+
}
433+
_ => {
434+
let value = DataValue::Tuple(std::mem::take(projection_tmp), false);
435+
f(arena, value)?;
436+
}
437437
}
438438
Ok(())
439439
})

0 commit comments

Comments
 (0)