Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature] support Quantile Sketche #54569

Open
wants to merge 53 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
53 commits
Select commit Hold shift + click to select a range
d4fa445
support dataSketches include Quantile Sketches、Theta Sketch、Frequency…
chenminghua8 Oct 22, 2024
cf082a9
fix format exception.
chenminghua8 Oct 23, 2024
2540ae9
fix format exception.
chenminghua8 Oct 24, 2024
02dc2b0
fix format exception.
chenminghua8 Oct 24, 2024
358b652
fix format exception.
chenminghua8 Oct 24, 2024
60f073e
Merge branch 'StarRocks:main' into support_dataSketches
chenminghua8 Oct 24, 2024
01a85d0
Merge branch 'StarRocks:main' into support_dataSketches
chenminghua8 Nov 24, 2024
6aff6a7
fix AggStateCombinatorTest fault
chenminghua8 Nov 24, 2024
c573071
fix AggStateCombinatorTest fault
chenminghua8 Nov 25, 2024
51e8596
Merge branch 'StarRocks:main' into support_dataSketches
chenminghua8 Nov 25, 2024
accf967
Merge branch 'StarRocks:main' into support_dataSketches
chenminghua8 Nov 26, 2024
e0257e9
Merge branch 'StarRocks:main' into support_dataSketches
chenminghua8 Dec 2, 2024
38b153e
add tests
chenminghua8 Dec 2, 2024
19d9389
add tests
chenminghua8 Dec 4, 2024
793cc6c
Merge branch 'StarRocks:main' into support_dataSketches
chenminghua8 Dec 4, 2024
8d337aa
Add test cases to improve code coverage
chenminghua8 Dec 4, 2024
643e868
Add test cases to improve code coverage
chenminghua8 Dec 4, 2024
6e0516b
Add test cases to improve code coverage
chenminghua8 Dec 4, 2024
707b4f1
support dataSketches include Quantile Sketches、Theta Sketch、Frequency…
chenminghua8 Oct 22, 2024
f8276df
fix format exception.
chenminghua8 Oct 23, 2024
7bde2bc
fix format exception.
chenminghua8 Oct 24, 2024
573ee8c
fix format exception.
chenminghua8 Oct 24, 2024
fab9f3f
fix format exception.
chenminghua8 Oct 24, 2024
cd3cba0
fix AggStateCombinatorTest fault
chenminghua8 Nov 24, 2024
b00773a
fix AggStateCombinatorTest fault
chenminghua8 Nov 25, 2024
aaae8aa
add tests
chenminghua8 Dec 2, 2024
2f092b1
add tests
chenminghua8 Dec 4, 2024
e4db576
Add test cases to improve code coverage
chenminghua8 Dec 4, 2024
cc875af
Add test cases to improve code coverage
chenminghua8 Dec 4, 2024
139ac86
Add test cases to improve code coverage
chenminghua8 Dec 4, 2024
ebaca8d
Fix aggregate_test test error due to using unused AnyValUtils
chenminghua8 Dec 5, 2024
ca4e313
Fix aggregate_test test error due to using unused AnyValUtils
chenminghua8 Dec 5, 2024
260d88b
Fix aggregate_test test error due to using unused AnyValUtils
chenminghua8 Dec 5, 2024
6d2dcf2
Fix other sketch nullptr and add ds_theta_count_distinct syntax
chenminghua8 Dec 8, 2024
499533f
Fix Clang format failling
chenminghua8 Dec 8, 2024
bbe0ec7
Fix Clang format failling
chenminghua8 Dec 8, 2024
aaa6acc
Fix Clang format failling
chenminghua8 Dec 8, 2024
cd04f84
Merge branch 'StarRocks:main' into support_dataSketches
chenminghua8 Dec 30, 2024
026988f
Refactor to support DataSketches Quantile、Theta、Frequency Sketches
chenminghua8 Dec 31, 2024
6495748
Refactor to support DataSketches Quantile、Theta、Frequency Sketches
chenminghua8 Dec 31, 2024
78a11af
Refactor to support DataSketches Quantile、Theta、Frequency Sketches
chenminghua8 Dec 31, 2024
40589e6
Merge branch 'StarRocks:main' into support_dataSketches
chenminghua8 Dec 31, 2024
f95f6d4
Merge branch 'support_dataSketches' of github.com:chenminghua8/starro…
chenminghua8 Dec 31, 2024
df3b57b
Refactor to support DataSketches Quantile、Theta、Frequency Sketches
chenminghua8 Dec 31, 2024
35b7ef4
Refactor to support DataSketches Quantile、Theta、Frequency Sketches
chenminghua8 Dec 31, 2024
8cc2d01
Refactor to support DataSketches Quantile、Theta、Frequency Sketches
chenminghua8 Dec 31, 2024
ecac63f
support DataSketches quantile sketche
chenminghua8 Dec 31, 2024
8b1b598
support DataSketches quantile sketche
chenminghua8 Jan 2, 2025
8d08860
Merge branch 'StarRocks:main' into support_quantile_sketche
chenminghua8 Jan 6, 2025
5adb7aa
support DataSketches quantile sketche
chenminghua8 Jan 6, 2025
d3f49cf
Merge branch 'StarRocks:main' into support_quantile_sketche
chenminghua8 Jan 7, 2025
983ea74
support DataSketches quantile sketche
chenminghua8 Jan 7, 2025
ed11a11
support DataSketches quantile sketche
chenminghua8 Jan 7, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix format exception.
Signed-off-by: chenminghua8 <cmptmn@126.com>
  • Loading branch information
chenminghua8 authored and LiShuMing committed Dec 5, 2024
commit f8276dfe91b9c1c408a1b6fc868ed2415e39e8f6
90 changes: 30 additions & 60 deletions be/src/exprs/agg/ds_agg.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,11 @@ enum SketchType {
HLL = 0,
QUANTILE = 1,
FREQUENT = 2,
THETA =3,
THETA = 3,
};

template <LogicalType LT, SketchType ST>
struct DSSketchState {
};
struct DSSketchState {};

template <LogicalType LT>
struct DSSketchState<LT, HLL> {
Expand All @@ -48,16 +47,15 @@ struct DSSketchState<LT, HLL> {
ds_sketch_wrapper = std::make_unique<DataSketchesHll>(log_k, tgt_type, &memory_usage);
}

bool is_inited() const {
return ds_sketch_wrapper != nullptr;
}
bool is_inited() const { return ds_sketch_wrapper != nullptr; }

void merge(const BinaryColumn* sketch_data_column, size_t row_num) {
DSSketchState<LT, HLL> other_state;
other_state.deserialize(sketch_data_column->get(row_num).get_slice(), &memory_usage);
if (UNLIKELY(!is_inited())) {
ds_sketch_wrapper = std::make_unique<DataSketchesHll>(
other_state.ds_sketch_wrapper->get_lg_config_k(), other_state.ds_sketch_wrapper->get_target_type(), &memory_usage);
ds_sketch_wrapper =
std::make_unique<DataSketchesHll>(other_state.ds_sketch_wrapper->get_lg_config_k(),
other_state.ds_sketch_wrapper->get_target_type(), &memory_usage);
}
ds_sketch_wrapper->merge(*other_state.ds_sketch_wrapper);
}
Expand Down Expand Up @@ -101,13 +99,9 @@ struct DSSketchState<LT, HLL> {
}
}

size_t serialize(uint8_t* dst) const {
return ds_sketch_wrapper->serialize(dst);
}
size_t serialize(uint8_t* dst) const { return ds_sketch_wrapper->serialize(dst); }

size_t serialize_size() const {
return ds_sketch_wrapper->serialize_size();
}
size_t serialize_size() const { return ds_sketch_wrapper->serialize_size(); }

void deserialize(const Slice& slice, int64_t* memory_usage) {
ds_sketch_wrapper = std::make_unique<DataSketchesHll>(slice, memory_usage);
Expand All @@ -124,9 +118,7 @@ struct DSSketchState<LT, HLL> {
}
}

static std::string getFunName() {
return "ds_hll_count_distinct";
}
static std::string getFunName() { return "ds_hll_count_distinct"; }

private:
// parse log_k and target type from args
Expand Down Expand Up @@ -162,7 +154,6 @@ struct DSSketchState<LT, QUANTILE> {
std::unique_ptr<SketchWarapperType> ds_sketch_wrapper = nullptr;
int64_t memory_usage = 0;


void init(FunctionContext* ctx) {
DatumArray datum_array;
uint16_t k;
Expand All @@ -186,9 +177,7 @@ struct DSSketchState<LT, QUANTILE> {
ds_sketch_wrapper = std::make_unique<SketchWarapperType>(k, &memory_usage);
}

bool is_inited() const {
return ds_sketch_wrapper != nullptr;
}
bool is_inited() const { return ds_sketch_wrapper != nullptr; }

void update(const Column* data_column, size_t row_num) const {
const ColumnType* column = down_cast<const ColumnType*>(data_column);
Expand All @@ -215,8 +204,8 @@ struct DSSketchState<LT, QUANTILE> {
*ranks_prt = other_state.ranks.get()[i];
ranks_prt++;
}
ds_sketch_wrapper = std::make_unique<SketchWarapperType>(
other_state.ds_sketch_wrapper->get_k(), &memory_usage);
ds_sketch_wrapper =
std::make_unique<SketchWarapperType>(other_state.ds_sketch_wrapper->get_k(), &memory_usage);
}
ds_sketch_wrapper->merge(*other_state.ds_sketch_wrapper);
}
Expand All @@ -228,7 +217,7 @@ struct DSSketchState<LT, QUANTILE> {
memcpy(dst + offset, ranks.get(), ranks_size * sizeof(double));
offset = offset + ranks_size * sizeof(double);
size_t ser_sketch_size = ds_sketch_wrapper->serialize(dst + offset);
return offset + ser_sketch_size;
return offset + ser_sketch_size;
}

size_t serialize_size() const {
Expand All @@ -244,11 +233,10 @@ struct DSSketchState<LT, QUANTILE> {
}
offset = offset + sizeof(uint32_t);
ranks = std::make_unique<double[]>(ranks_size);
memcpy(ranks.get(), ptr + offset, ranks_size * sizeof(double));
memcpy(ranks.get(), ptr + offset, ranks_size * sizeof(double));
offset = offset + ranks_size * sizeof(double);
const Slice sketch_data_slice = Slice(slice.get_data() + offset, slice.size - offset);
ds_sketch_wrapper = std::make_unique<SketchWarapperType>(sketch_data_slice, memory_usage);

}

void get_values(Column* dst, size_t start, size_t end) const {
Expand All @@ -263,7 +251,7 @@ struct DSSketchState<LT, QUANTILE> {
result = ds_sketch_wrapper->get_quantiles(ranks.get(), ranks_size);
}

uint32_t index =0;
uint32_t index = 0;
for (size_t row = start; row < end; row++) {
for (CppType result_data : result) {
result_column->append(result_data);
Expand All @@ -274,9 +262,7 @@ struct DSSketchState<LT, QUANTILE> {
}
}

static std::string getFunName() {
return "ds_quantile";
}
static std::string getFunName() { return "ds_quantile"; }

private:
// parse k and rank_arr from args
Expand Down Expand Up @@ -320,7 +306,7 @@ struct SpecialCppType<TYPE_BINARY> {
};
template <>
struct SpecialCppType<TYPE_VARBINARY> {
using CppType = std::string;
using CppType = std::string;
};
template <>
struct SpecialCppType<TYPE_CHAR> {
Expand Down Expand Up @@ -348,13 +334,11 @@ struct DSSketchState<LT, FREQUENT> {
ds_sketch_wrapper = std::make_unique<SketchWarapperType>(lg_max_map_size, lg_start_map_size, &memory_usage);
}

bool is_inited() const {
return ds_sketch_wrapper != nullptr;
}
bool is_inited() const { return ds_sketch_wrapper != nullptr; }

void update(const Column* data_column, size_t row_num) const {
if constexpr (!IsSlice<OriginalCppType>) {
const ColumnType* column = down_cast<const ColumnType *>(data_column);
const ColumnType* column = down_cast<const ColumnType*>(data_column);
const auto& values = column->get_data();
ds_sketch_wrapper->update(values[row_num]);
} else {
Expand All @@ -366,7 +350,7 @@ struct DSSketchState<LT, FREQUENT> {

void update_batch_single_state_with_frame(const Column* data_column, int64_t frame_start, int64_t frame_end) const {
if constexpr (!IsSlice<OriginalCppType>) {
const ColumnType* column = down_cast<const ColumnType *>(data_column);
const ColumnType* column = down_cast<const ColumnType*>(data_column);
const auto& values = column->get_data();
for (size_t i = frame_start; i < frame_end; ++i) {
ds_sketch_wrapper->update(values[i]);
Expand All @@ -377,7 +361,6 @@ struct DSSketchState<LT, FREQUENT> {
const Slice data = column->get_slice(i);
ds_sketch_wrapper->update(std::string(data.get_data(), data.size));
}

}
}

Expand All @@ -402,7 +385,7 @@ struct DSSketchState<LT, FREQUENT> {
memcpy(dst + offset, &lg_start_map_size, sizeof(uint8_t));
offset = offset + sizeof(uint8_t);
size_t ser_sketch_size = ds_sketch_wrapper->serialize(dst + offset);
return offset + ser_sketch_size;
return offset + ser_sketch_size;
}

size_t serialize_size() const {
Expand All @@ -419,8 +402,8 @@ struct DSSketchState<LT, FREQUENT> {
memcpy(&lg_start_map_size, ptr + offset, sizeof(uint8_t));
offset = offset + sizeof(uint8_t);
const Slice sketch_data_slice = Slice(slice.get_data() + offset, slice.size - offset);
ds_sketch_wrapper = std::make_unique<SketchWarapperType>(sketch_data_slice, lg_max_map_size,
lg_start_map_size, memory_usage);
ds_sketch_wrapper = std::make_unique<SketchWarapperType>(sketch_data_slice, lg_max_map_size, lg_start_map_size,
memory_usage);
}

void get_values(Column* dst, size_t start, size_t end) const {
Expand All @@ -439,7 +422,7 @@ struct DSSketchState<LT, FREQUENT> {
if (LIKELY(ds_sketch_wrapper != nullptr)) {
result = ds_sketch_wrapper->get_frequent_items(0);
}
uint32_t index =0;
uint32_t index = 0;
for (size_t row = start; row < end; row++) {
uint32_t counter_num_index = 0;
for (FrequentRow<CppType> frequentRow : result) {
Expand All @@ -466,9 +449,7 @@ struct DSSketchState<LT, FREQUENT> {
}
}

static std::string getFunName() {
return "ds_frequent";
}
static std::string getFunName() { return "ds_frequent"; }

private:
// parse threshold lg_max_map_size and lg_start_map_size from args
Expand All @@ -489,7 +470,6 @@ struct DSSketchState<LT, FREQUENT> {
lg_max_map_size = lg_start_map_size;
}
return {counter_num, lg_max_map_size, lg_start_map_size};

}
};

Expand All @@ -502,13 +482,9 @@ struct DSSketchState<LT, THETA> {
std::unique_ptr<SketchWarapperType> ds_sketch_wrapper = nullptr;
int64_t memory_usage = 0;

void init(FunctionContext* ctx) {
ds_sketch_wrapper = std::make_unique<SketchWarapperType>(&memory_usage);
}
void init(FunctionContext* ctx) { ds_sketch_wrapper = std::make_unique<SketchWarapperType>(&memory_usage); }

bool is_inited() const {
return ds_sketch_wrapper != nullptr;
}
bool is_inited() const { return ds_sketch_wrapper != nullptr; }

void merge(const BinaryColumn* sketch_data_column, size_t row_num) {
DSSketchState<LT, THETA> other_state;
Expand Down Expand Up @@ -558,13 +534,9 @@ struct DSSketchState<LT, THETA> {
}
}

size_t serialize(uint8_t* dst) const {
return ds_sketch_wrapper->serialize(dst);
}
size_t serialize(uint8_t* dst) const { return ds_sketch_wrapper->serialize(dst); }

size_t serialize_size() const {
return ds_sketch_wrapper->serialize_size();
}
size_t serialize_size() const { return ds_sketch_wrapper->serialize_size(); }

void deserialize(const Slice& slice, int64_t* memory_usage) {
ds_sketch_wrapper = std::make_unique<SketchWarapperType>(slice, memory_usage);
Expand All @@ -581,9 +553,7 @@ struct DSSketchState<LT, THETA> {
}
}

static std::string getFunName() {
return "ds_theta";
}
static std::string getFunName() { return "ds_theta"; }
};

template <LogicalType LT, SketchType ST, typename StateType = DSSketchState<LT, ST>, typename T = RunTimeCppType<LT>>
Expand Down
3 changes: 1 addition & 2 deletions be/src/exprs/agg/factory/aggregate_resolver_approx.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,7 @@ struct DataSketchesBuilder {
"ds_hll_count_distinct", false,
AggregateFactory::MakeDataSketchesAggregateFunction<lt, SketchType::HLL>());
resolver->add_aggregate_mapping<lt, TYPE_BIGINT, DSSketchState<lt, SketchType::THETA>>(
"ds_theta", false,
AggregateFactory::MakeDataSketchesAggregateFunction<lt, SketchType::THETA>());
"ds_theta", false, AggregateFactory::MakeDataSketchesAggregateFunction<lt, SketchType::THETA>());
}
if constexpr (lt_is_integer<lt> || lt_is_float<lt>) {
resolver->add_aggregate_mapping<lt, TYPE_ARRAY, DSSketchState<lt, SketchType::QUANTILE>>(
Expand Down
Loading