Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor the filter rewrite optimization #14464

Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
3f43898
Refactor
bowenlan-amzn Jun 19, 2024
7d9d57e
Refactor
bowenlan-amzn Jun 19, 2024
1a067ba
Refactor
bowenlan-amzn Jun 19, 2024
e8e9ad3
Refactor
bowenlan-amzn Jun 20, 2024
7c491b9
Refactor
bowenlan-amzn Jun 20, 2024
a158f78
Refactor
bowenlan-amzn Jun 21, 2024
8f10faf
Refactor
bowenlan-amzn Jun 21, 2024
6c097f3
Fix a bug
bowenlan-amzn Jun 21, 2024
40c42fa
address comment
bowenlan-amzn Jul 2, 2024
5fde041
prepareFromSegment now doesn't return Ranges
bowenlan-amzn Jul 2, 2024
510a052
how it looks like when introduce interfaces
bowenlan-amzn Jul 2, 2024
4a333af
remove interface, clean up
bowenlan-amzn Jul 2, 2024
9325061
improve doc
bowenlan-amzn Jul 2, 2024
bf958ea
Merge branch 'main' into 14435-refactor-range-agg-optimization
bowenlan-amzn Jul 2, 2024
5439401
Merge branch 'main' into 14435-refactor-range-agg-optimization
bowenlan-amzn Jul 10, 2024
3a60a81
move multirangetraversal logic to helper
bowenlan-amzn Jul 10, 2024
094c25b
improve the refactor
bowenlan-amzn Jul 17, 2024
85009ad
Merge branch 'main' into 14435-refactor-range-agg-optimization
bowenlan-amzn Aug 1, 2024
afc9bbd
Merge branch 'main' into 14435-refactor-range-agg-optimization
bowenlan-amzn Aug 5, 2024
778f1ce
Address Marc's comments
bowenlan-amzn Aug 7, 2024
234eb44
Address concurrent segment search concern
bowenlan-amzn Aug 7, 2024
e896927
remove circular dependency
bowenlan-amzn Aug 7, 2024
8962ee3
Address comment
bowenlan-amzn Aug 8, 2024
86cacab
Merge branch 'main' into 14435-refactor-range-agg-optimization
bowenlan-amzn Aug 8, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
prepareFromSegment now doesn't return Ranges
Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>
  • Loading branch information
bowenlan-amzn committed Jul 2, 2024
commit 5fde0414fa6575c318f73e630932317c66579534
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ void setOptimizationContext(OptimizationContext optimizationContext) {

protected abstract void prepare() throws IOException;

abstract Ranges prepare(LeafReaderContext leaf) throws IOException;
abstract void prepareFromSegment(LeafReaderContext leaf) throws IOException;

abstract void tryOptimize(PointValues values, BiConsumer<Long, Long> incrementDocCount, Ranges ranges) throws IOException;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,13 @@ protected boolean canOptimize(CompositeValuesSourceConfig[] sourceConfigs) {

protected void buildRanges(SearchContext context) throws IOException {
long[] bounds = Helper.getDateHistoAggBounds(context, fieldType.name());
this.optimizationContext.setRanges(buildRanges(bounds));
optimizationContext.setRanges(buildRanges(bounds));
}

@Override
OptimizationContext.Ranges prepare(LeafReaderContext leaf) throws IOException {
void prepareFromSegment(LeafReaderContext leaf) throws IOException {
long[] bounds = Helper.getSegmentBounds(leaf, fieldType.name());
return buildRanges(bounds);
optimizationContext.setRangesFromSegment(buildRanges(bounds));
}

private OptimizationContext.Ranges buildRanges(long[] bounds) {
Expand All @@ -97,7 +97,7 @@ private OptimizationContext.Ranges buildRanges(long[] bounds) {
getRoundingPrepared(),
bounds[0],
bounds[1],
this.optimizationContext.maxAggRewriteFilters
optimizationContext.maxAggRewriteFilters
);
}

Expand Down Expand Up @@ -153,7 +153,7 @@ final void tryOptimize(PointValues values, BiConsumer<Long, Long> incrementDocCo
incrementDocCount.accept(ord, (long) docCount);
};

this.optimizationContext.consumeDebugInfo(multiRangesTraverse(values.getPointTree(), ranges, incrementFunc, size));
optimizationContext.consumeDebugInfo(multiRangesTraverse(values.getPointTree(), ranges, incrementFunc, size));
}

private static long getBucketOrd(long bucketOrd) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ public final class OptimizationContext {
String shardId;

private Ranges ranges;
private Ranges rangesFromSegment;

// debug info related fields
private int leaf;
Expand Down Expand Up @@ -75,20 +76,24 @@ public boolean canOptimize(final Object parent, final int subAggLength, SearchCo

public void prepare() throws IOException {
assert ranges == null : "Ranges should only be built once at shard level, but they are already built";
this.aggregatorBridge.prepare();
aggregatorBridge.prepare();
if (ranges != null) {
preparedAtShardLevel = true;
}
}

private Ranges prepare(LeafReaderContext leaf) throws IOException {
return this.aggregatorBridge.prepare(leaf);
private void prepareFromSegment(LeafReaderContext leaf) throws IOException {
aggregatorBridge.prepareFromSegment(leaf);
}

void setRanges(Ranges ranges) {
this.ranges = ranges;
}

void setRangesFromSegment(Ranges ranges) {
this.rangesFromSegment = ranges;
}

/**
* Try to populate the bucket doc counts for aggregation
* <p>
Expand Down Expand Up @@ -119,7 +124,7 @@ public boolean tryOptimize(final LeafReaderContext leafCtx, final BiConsumer<Lon
return false;
}

Ranges ranges = prepareFromSegment(leafCtx);
Ranges ranges = prepare(leafCtx);
if (ranges == null) return false;

aggregatorBridge.tryOptimize(values, incrementDocCount, ranges);
Expand All @@ -134,15 +139,15 @@ public boolean tryOptimize(final LeafReaderContext leafCtx, final BiConsumer<Lon
* Even when ranges cannot be built at shard level, we can still build ranges
* at segment level when it's functionally match-all at segment level
*/
private Ranges prepareFromSegment(LeafReaderContext leafCtx) throws IOException {
private Ranges prepare(LeafReaderContext leafCtx) throws IOException {
if (!preparedAtShardLevel && !aggregatorBridge.segmentMatchAll(leafCtx)) {
return null;
}

Ranges ranges = this.ranges;
if (ranges == null) { // not built at shard level but segment match all
logger.debug("Shard {} segment {} functionally match all documents. Build the fast filter", shardId, leafCtx.ord);
ranges = prepare(leafCtx);
prepareFromSegment(leafCtx);
return rangesFromSegment;
}
return ranges;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ protected void buildRanges(RangeAggregator.Range[] ranges) {
}

@Override
OptimizationContext.Ranges prepare(LeafReaderContext leaf) {
void prepareFromSegment(LeafReaderContext leaf) {
throw new UnsupportedOperationException("Range aggregation should not build ranges at segment level");
}

Expand Down