generated from amazon-archives/__template_Custom
-
Notifications
You must be signed in to change notification settings - Fork 181
Add time window and window assigner #950
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
dai-chen
merged 3 commits into
opensearch-project:feature/maximus-m1
from
dai-chen:maximus-m1/add-window-assigner
Oct 27, 2022
Merged
Changes from all commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
28 changes: 28 additions & 0 deletions
28
core/src/main/java/org/opensearch/sql/planner/streaming/windowing/Window.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.planner.streaming.windowing; | ||
|
|
||
| import lombok.Data; | ||
|
|
||
| /** | ||
| * A time window is a window of time interval with inclusive start time and exclusive end time. | ||
| */ | ||
| @Data | ||
| public class Window { | ||
|
|
||
| /** Start timestamp (inclusive) of the time window. */ | ||
| private final long startTime; | ||
|
|
||
| /** End timestamp (exclusive) of the time window. */ | ||
| private final long endTime; | ||
|
|
||
| /** | ||
| * Return the maximum timestamp (inclusive) of the window. | ||
| */ | ||
| public long maxTimestamp() { | ||
| return endTime - 1; | ||
| } | ||
| } |
57 changes: 57 additions & 0 deletions
57
...n/java/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssigner.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.planner.streaming.windowing.assigner; | ||
|
|
||
| import com.google.common.base.Preconditions; | ||
| import java.util.LinkedList; | ||
| import java.util.List; | ||
| import org.opensearch.sql.planner.streaming.windowing.Window; | ||
| import org.opensearch.sql.utils.DateTimeUtils; | ||
|
|
||
| /** | ||
| * A sliding window assigner assigns multiple overlapped window per event timestamp. | ||
| * The overlap size is determined by the given slide interval. | ||
| */ | ||
| public class SlidingWindowAssigner implements WindowAssigner { | ||
|
|
||
| /** Window size in millisecond. */ | ||
| private final long windowSize; | ||
|
|
||
| /** Slide size in millisecond. */ | ||
| private final long slideSize; | ||
|
|
||
| /** | ||
| * Create sliding window assigner with the given window and slide size in millisecond. | ||
| * | ||
| * @param windowSize window size in millisecond | ||
| * @param slideSize slide size in millisecond | ||
| */ | ||
| public SlidingWindowAssigner(long windowSize, long slideSize) { | ||
| Preconditions.checkArgument(windowSize > 0, | ||
| "Window size [%s] must be positive number", windowSize); | ||
| Preconditions.checkArgument(slideSize > 0, | ||
| "Slide size [%s] must be positive number", slideSize); | ||
| this.windowSize = windowSize; | ||
| this.slideSize = slideSize; | ||
| } | ||
|
|
||
| @Override | ||
| public List<Window> assign(long timestamp) { | ||
| LinkedList<Window> windows = new LinkedList<>(); | ||
|
|
||
| // Assign window from the last start time to the first until timestamp outside current window | ||
| long startTime = DateTimeUtils.getWindowStartTime(timestamp, slideSize); | ||
| for (Window win = window(startTime); win.maxTimestamp() >= timestamp; win = window(startTime)) { | ||
| windows.addFirst(win); | ||
| startTime -= slideSize; | ||
| } | ||
| return windows; | ||
| } | ||
|
|
||
| private Window window(long startTime) { | ||
| return new Window(startTime, startTime + windowSize); | ||
| } | ||
| } |
38 changes: 38 additions & 0 deletions
38
.../java/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssigner.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.planner.streaming.windowing.assigner; | ||
|
|
||
| import com.google.common.base.Preconditions; | ||
| import java.util.Collections; | ||
| import java.util.List; | ||
| import org.opensearch.sql.planner.streaming.windowing.Window; | ||
| import org.opensearch.sql.utils.DateTimeUtils; | ||
|
|
||
| /** | ||
| * A tumbling window assigner assigns a single window per event timestamp without overlap. | ||
| */ | ||
| public class TumblingWindowAssigner implements WindowAssigner { | ||
|
|
||
| /** Window size in millisecond. */ | ||
| private final long windowSize; | ||
|
|
||
| /** | ||
| * Create tumbling window assigner with the given window size. | ||
| * | ||
| * @param windowSize window size in millisecond | ||
| */ | ||
| public TumblingWindowAssigner(long windowSize) { | ||
| Preconditions.checkArgument(windowSize > 0, | ||
| "Window size [%s] must be positive number", windowSize); | ||
| this.windowSize = windowSize; | ||
| } | ||
|
|
||
| @Override | ||
| public List<Window> assign(long timestamp) { | ||
| long startTime = DateTimeUtils.getWindowStartTime(timestamp, windowSize); | ||
| return Collections.singletonList(new Window(startTime, startTime + windowSize)); | ||
| } | ||
| } | ||
24 changes: 24 additions & 0 deletions
24
...src/main/java/org/opensearch/sql/planner/streaming/windowing/assigner/WindowAssigner.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.planner.streaming.windowing.assigner; | ||
|
|
||
| import java.util.List; | ||
| import org.opensearch.sql.planner.streaming.windowing.Window; | ||
|
|
||
| /** | ||
| * A window assigner assigns zero or more window to an event timestamp | ||
| * based on different windowing approach. | ||
| */ | ||
| public interface WindowAssigner { | ||
|
|
||
| /** | ||
| * Return window(s) assigned to the timestamp. | ||
| * @param timestamp given event timestamp | ||
| * @return windows assigned | ||
| */ | ||
| List<Window> assign(long timestamp); | ||
|
|
||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
21 changes: 21 additions & 0 deletions
21
core/src/test/java/org/opensearch/sql/planner/streaming/windowing/WindowTest.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.planner.streaming.windowing; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.assertEquals; | ||
|
|
||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| class WindowTest { | ||
|
|
||
| @Test | ||
| void test() { | ||
| Window window = new Window(1000, 2000); | ||
| assertEquals(1000, window.getStartTime()); | ||
| assertEquals(2000, window.getEndTime()); | ||
| assertEquals(1999, window.maxTimestamp()); | ||
| } | ||
| } |
52 changes: 52 additions & 0 deletions
52
...va/org/opensearch/sql/planner/streaming/windowing/assigner/SlidingWindowAssignerTest.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.planner.streaming.windowing.assigner; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.assertEquals; | ||
| import static org.junit.jupiter.api.Assertions.assertThrows; | ||
|
|
||
| import java.util.List; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.opensearch.sql.planner.streaming.windowing.Window; | ||
|
|
||
| class SlidingWindowAssignerTest { | ||
|
|
||
| @Test | ||
| void testAssignWindows() { | ||
| long windowSize = 1000; | ||
| long slideSize = 500; | ||
| SlidingWindowAssigner assigner = new SlidingWindowAssigner(windowSize, slideSize); | ||
|
|
||
| assertEquals( | ||
| List.of( | ||
| new Window(0, 1000), | ||
| new Window(500, 1500)), | ||
| assigner.assign(500)); | ||
|
|
||
| assertEquals( | ||
| List.of( | ||
| new Window(0, 1000), | ||
| new Window(500, 1500)), | ||
| assigner.assign(999)); | ||
|
|
||
| assertEquals( | ||
| List.of( | ||
| new Window(500, 1500), | ||
| new Window(1000, 2000)), | ||
| assigner.assign(1000)); | ||
| } | ||
|
|
||
| @Test | ||
| void testConstructWithIllegalArguments() { | ||
| IllegalArgumentException error1 = assertThrows(IllegalArgumentException.class, | ||
| () -> new SlidingWindowAssigner(-1, 100)); | ||
| assertEquals("Window size [-1] must be positive number", error1.getMessage()); | ||
|
|
||
| IllegalArgumentException error2 = assertThrows(IllegalArgumentException.class, | ||
| () -> new SlidingWindowAssigner(1000, 0)); | ||
| assertEquals("Slide size [0] must be positive number", error2.getMessage()); | ||
| } | ||
| } |
39 changes: 39 additions & 0 deletions
39
...a/org/opensearch/sql/planner/streaming/windowing/assigner/TumblingWindowAssignerTest.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.planner.streaming.windowing.assigner; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.assertEquals; | ||
| import static org.junit.jupiter.api.Assertions.assertThrows; | ||
|
|
||
| import java.util.Collections; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.opensearch.sql.planner.streaming.windowing.Window; | ||
|
|
||
| class TumblingWindowAssignerTest { | ||
|
|
||
| @Test | ||
| void testAssignWindow() { | ||
| long windowSize = 1000; | ||
| TumblingWindowAssigner assigner = new TumblingWindowAssigner(windowSize); | ||
|
|
||
| assertEquals( | ||
| Collections.singletonList(new Window(0, 1000)), | ||
| assigner.assign(500)); | ||
| assertEquals( | ||
| Collections.singletonList(new Window(1000, 2000)), | ||
| assigner.assign(1999)); | ||
| assertEquals( | ||
| Collections.singletonList(new Window(2000, 3000)), | ||
| assigner.assign(2000)); | ||
| } | ||
|
|
||
| @Test | ||
| void testConstructWithIllegalWindowSize() { | ||
| IllegalArgumentException error = assertThrows(IllegalArgumentException.class, | ||
| () -> new TumblingWindowAssigner(-1)); | ||
| assertEquals("Window size [-1] must be positive number", error.getMessage()); | ||
| } | ||
| } |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.