From de6f4f2d0a5d2a70c9c45d9f706044acd0a219d9 Mon Sep 17 00:00:00 2001 From: xuchuan Date: Mon, 5 Sep 2022 13:53:01 +0800 Subject: [PATCH] enhance(datastore): make scan atomic (#1119) --- .../starwhale/mlops/datastore/DataStore.java | 233 ++- .../mlops/datastore/MemoryTable.java | 22 +- .../mlops/datastore/TableScanIterator.java | 30 - .../mlops/datastore/TableSchema.java | 2 +- .../mlops/datastore/impl/MemoryTableImpl.java | 206 +-- .../mlops/datastore/DataStoreTest.java | 304 +++- .../datastore/impl/MemoryTableImplTest.java | 1525 ++++++++--------- 7 files changed, 1293 insertions(+), 1029 deletions(-) delete mode 100644 server/controller/src/main/java/ai/starwhale/mlops/datastore/TableScanIterator.java diff --git a/server/controller/src/main/java/ai/starwhale/mlops/datastore/DataStore.java b/server/controller/src/main/java/ai/starwhale/mlops/datastore/DataStore.java index d417065e2d..a5e9ea9a90 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/datastore/DataStore.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/datastore/DataStore.java @@ -19,12 +19,15 @@ import ai.starwhale.mlops.exception.SWValidationException; import org.springframework.stereotype.Component; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.stream.Collectors; @Component public class DataStore { @@ -51,69 +54,180 @@ public void update(String tableName, TableSchemaDesc schema, List> records) { var table = this.tables.computeIfAbsent(tableName, k -> new MemoryTableImpl(tableName, this.walManager)); - table.update(schema, records); + table.lock(); + try { + table.update(schema, records); + } finally { + table.unlock(); + } } public RecordList query(DataStoreQueryRequest req) { var table = this.getTable(req.getTableName()); - var columns = req.getColumns(); - - return table.query(columns, - req.getOrderBy(), - req.getFilter(), - req.getStart(), - req.getLimit(), - req.isKeepNone(), - req.isRawResult()); - } - - public RecordList scan(DataStoreScanRequest req) { - List iters = new ArrayList<>(); - for (var info : req.getTables()) { - var table = this.getTable(info.getTableName()); - var iter = table.scan(info.getColumns(), + table.lock(); + try { + var schema = table.getSchema(); + var columns = this.getColumnAliases(schema, req.getColumns()); + var columnTypeMap = schema.getColumnTypeMapping(columns); + var results = table.query( + columns, + req.getOrderBy(), + req.getFilter(), req.getStart(), - req.isStartInclusive(), - req.getEnd(), - req.isEndInclusive(), - info.isKeepNone(), + req.getLimit(), + req.isKeepNone(), req.isRawResult()); - iter.next(); - if (iter.getRecord() != null) { - iters.add(iter); + String lastKey; + if (results.isEmpty()) { + lastKey = null; + } else { + lastKey = schema.getKeyColumnType().encode(results.get(results.size() - 1).getKey(), req.isRawResult()); } + var records = results.stream() + .map(r -> this.encodeRecord(columnTypeMap, r.getValues(), req.isRawResult())) + .collect(Collectors.toList()); + return new RecordList(columnTypeMap, records, lastKey); + } finally { + table.unlock(); } - if (iters.isEmpty()) { - return new RecordList(null, null, null); + } + + public RecordList scan(DataStoreScanRequest req) { + var limit = req.getLimit(); + if (limit > 1000) { + throw new SWValidationException(SWValidationException.ValidSubject.DATASTORE, + "limit must be less or equal to 1000. request=" + req); } - var columnTypeMap = new HashMap(); - for (var it : iters) { - columnTypeMap.putAll(it.getColumnTypeMapping()); + if (limit < 0) { + limit = 1000; } - var keyColumnType = iters.get(0).getKeyColumnType(); - Object lastKey = null; - List> ret = new ArrayList<>(); - while (!iters.isEmpty() && (req.getLimit() < 0 || ret.size() < req.getLimit())) { - lastKey = Collections.min(iters, (a, b) -> { - @SuppressWarnings("rawtypes") var x = (Comparable) a.getKey(); - @SuppressWarnings("rawtypes") var y = (Comparable) b.getKey(); - //noinspection unchecked - return x.compareTo(y); - }).getKey(); - var record = new HashMap(); - for (var iter : iters) { - if (iter.getKey().equals(lastKey)) { - record.putAll(iter.getRecord()); - iter.next(); + + var tablesToLock = + req.getTables() + .stream() + .map(DataStoreScanRequest.TableInfo::getTableName) + .sorted() // prevent deadlock + .map(this::getTable) + .collect(Collectors.toList()); + + for (var table : tablesToLock) { + table.lock(); + } + try { + class TableMeta { + String tableName; + MemoryTable table; + TableSchema schema; + Map columns; + Map columnTypeMap; + boolean keepNone; + } + + var tables = req.getTables().stream().map(info -> { + var ret = new TableMeta(); + ret.tableName = info.getTableName(); + ret.table = this.getTable(info.getTableName()); + ret.schema = ret.table.getSchema(); + ret.columns = this.getColumnAliases(ret.schema, info.getColumns()); + ret.columnTypeMap = ret.schema.getColumnTypeMapping(ret.columns); + ret.keepNone = info.isKeepNone(); + return ret; + }).collect(Collectors.toList()); + + var columnTypeMap = new HashMap(); + for (var table : tables) { + if (table.schema.getKeyColumnType() != tables.get(0).schema.getKeyColumnType()) { + throw new SWValidationException(SWValidationException.ValidSubject.DATASTORE, + MessageFormat.format( + "conflicting key column type. {0}: key={1}, type={2}, {3}: key={4}, type={5}", + tables.get(0).tableName, + tables.get(0).schema.getKeyColumn(), + tables.get(0).schema.getKeyColumnType(), + table.tableName, + table.schema.getKeyColumn(), + table.schema.getKeyColumnType())); + } + for (var entry : table.columnTypeMap.entrySet()) { + var columnName = entry.getKey(); + var columnType = entry.getValue(); + var old = columnTypeMap.putIfAbsent(columnName, columnType); + if (old != null && old != columnType) { + for (var t : tables) { + if (t.columnTypeMap.get(columnName) != null) { + throw new SWValidationException(SWValidationException.ValidSubject.DATASTORE, + MessageFormat.format( + "conflicting column type. {0}: column={1}, alias={2}, type={3}, " + + "{4}: column={5}, alias={6}, type={7}", + t.tableName, + columnName, + t.columns.get(columnName), + t.columnTypeMap.get(columnName), + table.tableName, + columnName, + table.columns.get(columnName), + table.columnTypeMap.get(columnName))); + } + } + } + } + } + class TableRecords { + TableMeta meta; + List records; + int index; + + public MemoryTable.RecordResult getRecord() { + return this.records.get(this.index); + } + + public Object getKey() { + return this.getRecord().getKey(); } } - if (!req.isKeepNone()) { - record.entrySet().removeIf(x -> x.getValue() == null); + var records = new ArrayList(); + for (var table : tables) { + var r = new TableRecords(); + r.meta = table; + r.records = table.table.scan(table.columns, + req.getStart(), + req.isStartInclusive(), + req.getEnd(), + req.isEndInclusive(), + limit, + table.keepNone); + if (!r.records.isEmpty()) { + records.add(r); + } + } + var keyColumnType = tables.get(0).schema.getKeyColumnType(); + Object lastKey = null; + List> ret = new ArrayList<>(); + while (!records.isEmpty() && ret.size() < limit) { + lastKey = Collections.min(records, (a, b) -> { + @SuppressWarnings("rawtypes") var x = (Comparable) a.getKey(); + @SuppressWarnings("rawtypes") var y = (Comparable) b.getKey(); + //noinspection unchecked + return x.compareTo(y); + }).getKey(); + var record = new HashMap(); + for (var r : records) { + if (r.getKey().equals(lastKey)) { + record.putAll(this.encodeRecord(r.meta.columnTypeMap, r.getRecord().values, req.isRawResult())); + ++r.index; + } + } + if (!req.isKeepNone()) { + record.entrySet().removeIf(x -> x.getValue() == null); + } + ret.add(record); + records.removeIf(r -> r.index == r.records.size()); + } + return new RecordList(columnTypeMap, ret, keyColumnType.encode(lastKey, false)); + } finally { + for (var table : tablesToLock) { + table.unlock(); } - ret.add(record); - iters.removeIf(x -> x.getRecord() == null); } - return new RecordList(columnTypeMap, ret, keyColumnType.encode(lastKey, false)); } private MemoryTable getTable(String tableName) { @@ -124,4 +238,25 @@ private MemoryTable getTable(String tableName) { } return table; } + + private Map getColumnAliases(TableSchema schema, Map columns) { + if (columns == null || columns.isEmpty()) { + return schema.getColumnSchemas().stream() + .map(ColumnSchema::getName) + .collect(Collectors.toMap(Function.identity(), Function.identity())); + } + return columns; + } + + private Map encodeRecord(Map columnTypeMap, + Map values, + boolean rawResult) { + var ret = new HashMap(); + for (var entry : values.entrySet()) { + var columnName = entry.getKey(); + var columnValue = entry.getValue(); + ret.put(columnName, columnTypeMap.get(columnName).encode(columnValue, rawResult)); + } + return ret; + } } diff --git a/server/controller/src/main/java/ai/starwhale/mlops/datastore/MemoryTable.java b/server/controller/src/main/java/ai/starwhale/mlops/datastore/MemoryTable.java index 701476b32c..66522e6a41 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/datastore/MemoryTable.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/datastore/MemoryTable.java @@ -15,6 +15,9 @@ */ package ai.starwhale.mlops.datastore; +import lombok.AllArgsConstructor; +import lombok.Data; + import java.util.List; import java.util.Map; @@ -25,7 +28,14 @@ public interface MemoryTable { void update(TableSchemaDesc schema, List> records); - RecordList query(Map columns, + @Data + @AllArgsConstructor + class RecordResult { + Object key; + Map values; + } + + List query(Map columns, List orderBy, TableQueryFilter filter, int start, @@ -33,12 +43,16 @@ RecordList query(Map columns, boolean keepNone, boolean rawResult); - TableScanIterator scan( + List scan( Map columns, String start, boolean startInclusive, String end, boolean endInclusive, - boolean keepNone, - boolean rawResult); + int limit, + boolean keepNone); + + void lock(); + + void unlock(); } diff --git a/server/controller/src/main/java/ai/starwhale/mlops/datastore/TableScanIterator.java b/server/controller/src/main/java/ai/starwhale/mlops/datastore/TableScanIterator.java deleted file mode 100644 index 21738e5dc9..0000000000 --- a/server/controller/src/main/java/ai/starwhale/mlops/datastore/TableScanIterator.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2022 Starwhale, Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ai.starwhale.mlops.datastore; - -import java.util.Map; - -public interface TableScanIterator { - ColumnType getKeyColumnType(); - - Map getColumnTypeMapping(); - - Object getKey(); - - Map getRecord(); - - void next(); -} diff --git a/server/controller/src/main/java/ai/starwhale/mlops/datastore/TableSchema.java b/server/controller/src/main/java/ai/starwhale/mlops/datastore/TableSchema.java index ffe7bf0e9f..68fe0bb423 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/datastore/TableSchema.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/datastore/TableSchema.java @@ -128,7 +128,7 @@ public Map getColumnTypeMapping(@NonNull Map for (var entry : columnAliases.entrySet()) { var columnSchema = this.columnSchemaMap.get(entry.getKey()); if (columnSchema == null) { - throw new SWValidationException(SWValidationException.ValidSubject.DATASTORE).tip( + throw new SWValidationException(SWValidationException.ValidSubject.DATASTORE, "column name " + entry.getKey() + " not found"); } ret.put(entry.getValue(), columnSchema.getType()); diff --git a/server/controller/src/main/java/ai/starwhale/mlops/datastore/impl/MemoryTableImpl.java b/server/controller/src/main/java/ai/starwhale/mlops/datastore/impl/MemoryTableImpl.java index 03224133ed..fac35bef99 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/datastore/impl/MemoryTableImpl.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/datastore/impl/MemoryTableImpl.java @@ -20,29 +20,26 @@ import ai.starwhale.mlops.datastore.ColumnType; import ai.starwhale.mlops.datastore.MemoryTable; import ai.starwhale.mlops.datastore.OrderByDesc; -import ai.starwhale.mlops.datastore.RecordList; import ai.starwhale.mlops.datastore.TableQueryFilter; -import ai.starwhale.mlops.datastore.TableScanIterator; import ai.starwhale.mlops.datastore.TableSchema; import ai.starwhale.mlops.datastore.TableSchemaDesc; import ai.starwhale.mlops.datastore.Wal; import ai.starwhale.mlops.datastore.WalManager; import ai.starwhale.mlops.exception.SWValidationException; import com.google.protobuf.ByteString; -import lombok.Getter; +import lombok.NonNull; import java.nio.ByteBuffer; import java.text.MessageFormat; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; -import java.util.ConcurrentModificationException; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Objects; import java.util.TreeMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; public class MemoryTableImpl implements MemoryTable { @@ -57,11 +54,21 @@ public class MemoryTableImpl implements MemoryTable { // used only for initialization from WAL private final Map indexMap = new HashMap<>(); + private final Lock lock = new ReentrantLock(); + public MemoryTableImpl(String tableName, WalManager walManager) { this.tableName = tableName; this.walManager = walManager; } + public void lock() { + this.lock.lock(); + } + + public void unlock() { + this.lock.unlock(); + } + public TableSchema getSchema() { return this.schema == null ? null : new TableSchema(this.schema); } @@ -145,7 +152,7 @@ private Object parseValue(ColumnSchema columnSchema, Wal.Column col) { @Override - synchronized public void update(TableSchemaDesc schema, List> records) { + public void update(TableSchemaDesc schema, List> records) { var logEntryBuilder = Wal.WalEntry.newBuilder() .setEntryType(Wal.WalEntry.Type.UPDATE) .setTableName(this.tableName); @@ -269,8 +276,8 @@ private static Wal.Column.Builder writeColumn(TableSchema schema, String name, O } @Override - synchronized public RecordList query( - Map columns, + public List query( + @NonNull Map columns, List orderBy, TableQueryFilter filter, int start, @@ -278,13 +285,9 @@ synchronized public RecordList query( boolean keepNone, boolean rawResult) { if (this.schema == null) { - return new RecordList(null, List.of(), null); + return Collections.emptyList(); } - if (columns == null) { - columns = this.schema.getColumnSchemas().stream() - .collect(Collectors.toMap(ColumnSchema::getName, ColumnSchema::getName)); - } - var columnTypeMapping = this.schema.getColumnTypeMapping(columns); + this.schema.getColumnTypeMapping(columns); // check if all column names are valid if (orderBy != null) { for (var col : orderBy) { if (col == null) { @@ -319,7 +322,8 @@ synchronized public RecordList query( if (orderBy != null) { results.sort((a, b) -> { for (var col : orderBy) { - var result = MemoryTableImpl.sortCompare(a.get(col.getColumnName()), b.get(col.getColumnName())); + var result = MemoryTableImpl.sortCompare(a.get(col.getColumnName()), + b.get(col.getColumnName())); if (result != 0) { if (col.isDescending()) { return -result; @@ -331,127 +335,71 @@ synchronized public RecordList query( }); } - var finalColumns = columns; - return new RecordList(columnTypeMapping, - results.subList(start, end).stream().map(record -> { - var r = new HashMap(); - for (var entry : finalColumns.entrySet()) { - var value = record.get(entry.getKey()); - if (keepNone || value != null) { - r.put(entry.getValue(), columnTypeMapping.get(entry.getValue()).encode(value, rawResult)); - } - } - return r; - }).collect(Collectors.toList()), - null); - } - - private class InternalIterator implements TableScanIterator { - @Getter - private Map columnTypeMapping; - private Map columns; - private Object endKey; - private boolean keepNone; - private boolean rawResult; - private Iterator>> iterator; - @Getter - private Object key; - @Getter - private ColumnType keyColumnType; - @Getter - private Map record; - - public InternalIterator( - Map columns, - String start, - boolean startInclusive, - String end, - boolean endInclusive, - boolean keepNone, - boolean rawResult) { - if (MemoryTableImpl.this.schema == null) { - return; - } - this.columns = Objects.requireNonNullElseGet(columns, - () -> MemoryTableImpl.this.schema.getColumnSchemas().stream() - .collect(Collectors.toMap(ColumnSchema::getName, ColumnSchema::getName))); - this.keepNone = keepNone; - this.rawResult = rawResult; - this.columnTypeMapping = MemoryTableImpl.this.schema.getColumnTypeMapping(this.columns); - Object startKey = MemoryTableImpl.this.schema.getKeyColumnType().decode(start); - Object endKey = MemoryTableImpl.this.schema.getKeyColumnType().decode(end); - if (startKey == null) { - if (!MemoryTableImpl.this.recordMap.isEmpty()) { - startKey = MemoryTableImpl.this.recordMap.firstKey(); - } - } else if (startInclusive) { - startKey = MemoryTableImpl.this.recordMap.ceilingKey(startKey); - } else { - startKey = MemoryTableImpl.this.recordMap.higherKey(startKey); - } - if (endKey == null) { - if (!MemoryTableImpl.this.recordMap.isEmpty()) { - this.endKey = MemoryTableImpl.this.recordMap.lastKey(); - } - } else if (endInclusive) { - this.endKey = MemoryTableImpl.this.recordMap.floorKey(endKey); - } else { - this.endKey = MemoryTableImpl.this.recordMap.lowerKey(endKey); - } - //noinspection rawtypes,unchecked - if (startKey != null && this.endKey != null && ((Comparable) startKey).compareTo(this.endKey) <= 0) { - this.iterator = MemoryTableImpl.this.recordMap.subMap(startKey, true, this.endKey, true) - .entrySet() - .iterator(); - } - this.keyColumnType = MemoryTableImpl.this.schema.getKeyColumnType(); - } - - public void next() { - if (this.iterator == null) { - return; - } - synchronized (MemoryTableImpl.this) { - Map r; - for (; ; ) { - try { - r = this.iterator.next().getValue(); - break; - } catch (ConcurrentModificationException e) { - this.iterator = MemoryTableImpl.this.recordMap.subMap(this.key, false, this.endKey, true) - .entrySet() - .iterator(); - } catch (NoSuchElementException e) { - this.iterator = null; - this.record = null; - return; - } - } - this.record = new HashMap<>(); - this.key = r.get(MemoryTableImpl.this.schema.getKeyColumn()); - for (var entry : this.columns.entrySet()) { - var value = r.get(entry.getKey()); - if (value != null) { - this.record.put(entry.getValue(), - this.columnTypeMapping.get(entry.getValue()).encode(value, rawResult)); - } else if (this.keepNone) { - this.record.put(entry.getValue(), null); - } + return results.subList(start, end).stream().map(record -> { + var r = new HashMap(); + for (var entry : columns.entrySet()) { + var value = record.get(entry.getKey()); + if (keepNone || value != null) { + r.put(entry.getValue(), value); } } - } + return new RecordResult(record.get(this.schema.getKeyColumn()), r); + }).collect(Collectors.toList()); } + @Override - synchronized public TableScanIterator scan( - Map columns, + public List scan( + @NonNull Map columns, String start, boolean startInclusive, String end, boolean endInclusive, - boolean keepNone, - boolean rawResult) { - return new InternalIterator(columns, start, startInclusive, end, endInclusive, keepNone, rawResult); + int limit, + boolean keepNone) { + if (this.schema == null) { + return Collections.emptyList(); + } + if (this.recordMap.isEmpty() || limit == 0) { + return Collections.emptyList(); + } + + var startKey = MemoryTableImpl.this.schema.getKeyColumnType().decode(start); + var endKey = MemoryTableImpl.this.schema.getKeyColumnType().decode(end); + if (startKey == null) { + startKey = MemoryTableImpl.this.recordMap.firstKey(); + startInclusive = true; + } + if (endKey == null) { + endKey = MemoryTableImpl.this.recordMap.lastKey(); + endInclusive = true; + } + //noinspection rawtypes,unchecked + if (((Comparable) startKey).compareTo(endKey) > 0) { + return Collections.emptyList(); + } + var keyColumn = this.schema.getKeyColumn(); + var records = new ArrayList(); + for (var record : MemoryTableImpl.this.recordMap.subMap(startKey, startInclusive, endKey, endInclusive).values()) { + var values = new HashMap(); + for (var entry : columns.entrySet()) { + var columnName = entry.getKey(); + var alias = entry.getValue(); + var value = record.get(columnName); + if (keepNone || value != null) { + values.put(alias, value); + } + } + records.add(new RecordResult(record.get(keyColumn), values)); + if (records.size() == limit) { + break; + } + } + if (records.isEmpty()) { + return Collections.emptyList(); + } else { + return records; + } } private static int sortCompare(Object a, Object b) { diff --git a/server/controller/src/test/java/ai/starwhale/mlops/datastore/DataStoreTest.java b/server/controller/src/test/java/ai/starwhale/mlops/datastore/DataStoreTest.java index 8f2bc631d3..701606b1fb 100644 --- a/server/controller/src/test/java/ai/starwhale/mlops/datastore/DataStoreTest.java +++ b/server/controller/src/test/java/ai/starwhale/mlops/datastore/DataStoreTest.java @@ -15,21 +15,30 @@ */ package ai.starwhale.mlops.datastore; +import ai.starwhale.mlops.exception.SWValidationException; import ai.starwhale.mlops.memory.impl.SwByteBufferManager; import ai.starwhale.mlops.objectstore.impl.FileSystemObjectStore; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.io.File; import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertThrows; public class DataStoreTest { @TempDir @@ -47,10 +56,15 @@ public class DataStoreTest { public void setUp() throws IOException { this.bufferManager = new SwByteBufferManager(); this.objectStore = new FileSystemObjectStore(bufferManager, this.rootDir.getAbsolutePath()); - this.walManager = new WalManager(objectStore, bufferManager, 256, 4096, "test/", 10); + this.walManager = new WalManager(this.objectStore, this.bufferManager, 256, 4096, "test/", 10); this.dataStore = new DataStore(this.walManager); } + @AfterEach + public void tearDown() { + this.dataStore.terminate(); + } + @Test public void testUpdate() throws IOException { this.dataStore.update("t1", @@ -67,17 +81,17 @@ public void testUpdate() throws IOException { new ColumnSchemaDesc("x", "INT32"))), List.of(Map.of("k", "3", "x", "2"))); this.dataStore.update("t1", - null, - List.of(Map.of("k", "0", "a", "5"), Map.of("k", "4", "-", "1"))); + null, + List.of(Map.of("k", "0", "a", "5"), Map.of("k", "4", "-", "1"))); assertThat("t1", - this.dataStore.scan(DataStoreScanRequest.builder() - .tables(List.of(DataStoreScanRequest.TableInfo.builder() - .tableName("t1") - .columns(Map.of("k", "k", "a", "a")) - .build())) - .build()) - .getRecords(), - is(List.of(Map.of("k", "0", "a", "5"),Map.of("k", "1", "a", "2")))); + this.dataStore.scan(DataStoreScanRequest.builder() + .tables(List.of(DataStoreScanRequest.TableInfo.builder() + .tableName("t1") + .columns(Map.of("k", "k", "a", "a")) + .build())) + .build()) + .getRecords(), + is(List.of(Map.of("k", "0", "a", "5"), Map.of("k", "1", "a", "2")))); this.dataStore.update("t1", null, List.of(Map.of("k", "0", "-", "anyString"), Map.of("k", "4", "-", "1"))); @@ -150,6 +164,24 @@ public void testQuery() { recordList.getRecords(), is(List.of(Map.of("a", "3"), Map.of("a", "4")))); + + recordList = this.dataStore.query(DataStoreQueryRequest.builder() + .tableName("t1") + .filter(TableQueryFilter.builder() + .operator(TableQueryFilter.Operator.GREATER) + .operands(List.of(new TableQueryFilter.Column("a"), 1)) + .build()) + .orderBy(List.of(new OrderByDesc("a"))) + .start(1) + .limit(2) + .build()); + assertThat("all columns", + recordList.getColumnTypeMap(), + is(Map.of("k", ColumnType.STRING, "a", ColumnType.INT32))); + assertThat("all columns", + recordList.getRecords(), + is(List.of(Map.of("k", "2", "a", "3"), + Map.of("k", "1", "a", "4")))); } @Test @@ -206,6 +238,42 @@ public void testScanOneTable() { recordList.getRecords(), is(List.of(Map.of("a", "4"), Map.of()))); assertThat("test", recordList.getLastKey(), is("2")); + + recordList = this.dataStore.scan(DataStoreScanRequest.builder() + .tables(List.of(DataStoreScanRequest.TableInfo.builder() + .tableName("t1") + .keepNone(true) + .build())) + .start("1") + .startInclusive(true) + .end("3") + .endInclusive(true) + .limit(2) + .build()); + assertThat("all columns", + recordList.getColumnTypeMap(), + is(Map.of("k", ColumnType.STRING, "a", ColumnType.INT32))); + assertThat("all columns", + recordList.getRecords(), + is(List.of(Map.of("k", "1", "a", "4"), Map.of("k", "2")))); + assertThat("all columns", recordList.getLastKey(), is("2")); + + recordList = this.dataStore.scan(DataStoreScanRequest.builder() + .tables(List.of(DataStoreScanRequest.TableInfo.builder() + .tableName("t1") + .keepNone(true) + .build())) + .limit(0) + .build()); + assertThat("schema only", + recordList.getColumnTypeMap(), + is(Map.of("k", ColumnType.STRING, "a", ColumnType.INT32))); + assertThat("schema only", recordList.getRecords(), empty()); + + assertThrows(SWValidationException.class, () -> this.dataStore.scan(DataStoreScanRequest.builder() + .tables(List.of(DataStoreScanRequest.TableInfo.builder().tableName("t1").build())) + .limit(1001) + .build())); } @Test @@ -231,6 +299,11 @@ public void testScanMultipleTables() { List.of(new ColumnSchemaDesc("k", "STRING"), new ColumnSchemaDesc("a", "INT32"))), List.of(Map.of("k", "2"))); + this.dataStore.update("t4", + new TableSchemaDesc("k", + List.of(new ColumnSchemaDesc("k", "INT32"), + new ColumnSchemaDesc("a", "INT32"))), + List.of(Map.of("k", "2"))); var recordList = this.dataStore.scan(DataStoreScanRequest.builder() .tables(List.of(DataStoreScanRequest.TableInfo.builder() .tableName("t1") @@ -267,6 +340,37 @@ public void testScanMultipleTables() { Map.of("k", "4", "a", "1", "b", "11")))); assertThat("test", recordList.getLastKey(), is("4")); + recordList = this.dataStore.scan(DataStoreScanRequest.builder() + .tables(List.of(DataStoreScanRequest.TableInfo.builder() + .tableName("t1") + .keepNone(true) + .build(), + DataStoreScanRequest.TableInfo.builder() + .tableName("t2") + .keepNone(true) + .build(), + DataStoreScanRequest.TableInfo.builder() + .tableName("t3") + .keepNone(true) + .build())) + .build()); + assertThat("test", + recordList.getColumnTypeMap(), + is(Map.of("k", + ColumnType.STRING, + "a", + ColumnType.INT32, + "b", + ColumnType.INT32))); + assertThat("test", + recordList.getRecords(), + is(List.of(Map.of("k", "0", "a", "5", "b", "15"), + Map.of("k", "1", "a", "4"), + Map.of("k", "2", "b", "13"), + Map.of("k", "3", "a", "2"), + Map.of("k", "4", "a", "1", "b", "11")))); + assertThat("test", recordList.getLastKey(), is("4")); + recordList = this.dataStore.scan(DataStoreScanRequest.builder() .tables(List.of(DataStoreScanRequest.TableInfo.builder() .tableName("t1") @@ -282,9 +386,183 @@ public void testScanMultipleTables() { .build())) .start("7") .build()); - assertThat("empty", recordList.getColumnTypeMap(), nullValue()); - assertThat("empty", recordList.getRecords(), nullValue()); + assertThat("empty", + recordList.getColumnTypeMap(), + is(Map.of("a", ColumnType.INT32, "b", ColumnType.INT32, "k", ColumnType.STRING))); + assertThat("empty", recordList.getRecords(), empty()); assertThat("empty", recordList.getLastKey(), nullValue()); + + recordList = this.dataStore.scan(DataStoreScanRequest.builder() + .tables(List.of(DataStoreScanRequest.TableInfo.builder() + .tableName("t1") + .keepNone(true) + .build(), + DataStoreScanRequest.TableInfo.builder() + .tableName("t2") + .columns(Map.of("b", "a")) + .keepNone(true) + .build(), + DataStoreScanRequest.TableInfo.builder() + .tableName("t3") + .keepNone(true) + .build())) + .keepNone(true) + .build()); + assertThat("alias", + recordList.getColumnTypeMap(), + is(Map.of("k", ColumnType.STRING, "a", ColumnType.INT32))); + assertThat("alias", + recordList.getRecords(), + is(List.of(Map.of("k", "0", "a", "15"), + Map.of("k", "1", "a", "4"), + new HashMap<>() {{ + put("k", "2"); + put("a", null); + }}, + Map.of("k", "3", "a", "2"), + Map.of("k", "4", "a", "11")))); + assertThat("alias", recordList.getLastKey(), is("4")); + + assertThrows(SWValidationException.class, () -> this.dataStore.scan(DataStoreScanRequest.builder() + .tables(List.of(DataStoreScanRequest.TableInfo.builder().tableName("t1").build(), + DataStoreScanRequest.TableInfo.builder().tableName("t4").build())) + .build())); + assertThrows(SWValidationException.class, () -> this.dataStore.scan(DataStoreScanRequest.builder() + .tables(List.of(DataStoreScanRequest.TableInfo.builder().tableName("t1").build(), + DataStoreScanRequest.TableInfo.builder() + .tableName("t2") + .columns(Map.of("k", "a")) + .build())) + .build())); } + @Test + public void testMultiThreads() throws Throwable { + this.dataStore.terminate(); + this.walManager = new WalManager(this.objectStore, this.bufferManager, 65536, 65536 * 1024, "test/", 1000); + this.dataStore = new DataStore(this.walManager); + abstract class TestThread extends Thread { + protected final Random random = new Random(); + protected final SimpleDateFormat dateFormat = new SimpleDateFormat("hh:mm:ss.SSS"); + private Throwable throwable; + + public void run() { + try { + this.execute(); + } catch (Throwable t) { + t.printStackTrace(); + this.throwable = t; + } + } + + abstract void execute(); + + public void checkException() throws Throwable { + if (this.throwable != null) { + throw this.throwable; + } + } + } + var threads = new ArrayList(); + for (int i = 0; i < 20; ++i) { + // update + var index = i; + var tableName = "t" + i % 4; + dataStore.update(tableName, + new TableSchemaDesc("k", + List.of(new ColumnSchemaDesc("k", "STRING"), + new ColumnSchemaDesc("a", "INT32"))), + null); + threads.add(new TestThread() { + public void execute() { + var columnName = Integer.toString(index); + for (int j = 0; j < 100000; ++j) { + dataStore.update(tableName, + new TableSchemaDesc("k", + List.of(new ColumnSchemaDesc("k", "STRING"), + new ColumnSchemaDesc("a", "INT32"), + new ColumnSchemaDesc(columnName, "INT32"))), + List.of(Map.of("k", + String.format("%06d", j), + "a", + Integer.toHexString(index * 10000 + j), + columnName, + Integer.toHexString(index)))); + Thread.yield(); + } + System.out.printf("%s update %d done\n", this.dateFormat.format(new Date()), index); + } + }); + } + for (int i = 0; i < 200; ++i) { + // scan + var index = i; + var tableName1 = "t" + i % 4; + var tableName2 = "t" + (i % 4 + 1) % 4; + threads.add(new TestThread() { + public void execute() { + for (int j = 0; j < 50; ++j) { + var records = dataStore.scan(DataStoreScanRequest.builder() + .tables(List.of(DataStoreScanRequest.TableInfo.builder() + .tableName(tableName1) + .build(), + DataStoreScanRequest.TableInfo.builder() + .tableName(tableName2) + .build())) + .limit(100) + .rawResult(true) + .keepNone(true) + .build()).getRecords(); + if (records != null) { + for (var record : records) { + var k = Integer.parseInt(record.get("k")); + var a = Integer.parseInt(record.get("a")); + assertThat(a % 10000, is(k)); + assertThat(record.toString(), record.get(Integer.toString(a / 10000)), notNullValue()); + } + } + try { + Thread.sleep(this.random.nextInt(5)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + System.out.printf("%s scan %d done\n", dateFormat.format(new Date()), index); + } + }); + } + for (int i = 0; i < 200; ++i) { + // query + var index = i; + var tableName = "t" + i % 4; + threads.add(new TestThread() { + public void execute() { + for (int j = 0; j < 100; ++j) { + var records = dataStore.query(DataStoreQueryRequest.builder() + .tableName(tableName) + .limit(10) + .rawResult(true) + .build()).getRecords(); + try { + Thread.sleep(this.random.nextInt(5)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + System.out.printf("%s query %d done\n", dateFormat.format(new Date()), index); + } + }); + } + for (var thread : threads) { + thread.start(); + } + for (var thread : threads) { + try { + thread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + thread.checkException(); + } + } } diff --git a/server/controller/src/test/java/ai/starwhale/mlops/datastore/impl/MemoryTableImplTest.java b/server/controller/src/test/java/ai/starwhale/mlops/datastore/impl/MemoryTableImplTest.java index 4331be7e8a..fe13d1d568 100644 --- a/server/controller/src/test/java/ai/starwhale/mlops/datastore/impl/MemoryTableImplTest.java +++ b/server/controller/src/test/java/ai/starwhale/mlops/datastore/impl/MemoryTableImplTest.java @@ -15,13 +15,11 @@ */ package ai.starwhale.mlops.datastore.impl; -import ai.starwhale.mlops.datastore.ColumnSchema; import ai.starwhale.mlops.datastore.ColumnSchemaDesc; import ai.starwhale.mlops.datastore.ColumnType; import ai.starwhale.mlops.datastore.MemoryTable; import ai.starwhale.mlops.datastore.OrderByDesc; import ai.starwhale.mlops.datastore.TableQueryFilter; -import ai.starwhale.mlops.datastore.TableScanIterator; import ai.starwhale.mlops.datastore.TableSchema; import ai.starwhale.mlops.datastore.TableSchemaDesc; import ai.starwhale.mlops.datastore.WalManager; @@ -29,6 +27,8 @@ import ai.starwhale.mlops.memory.SwBufferManager; import ai.starwhale.mlops.memory.impl.SwByteBufferManager; import ai.starwhale.mlops.objectstore.impl.FileSystemObjectStore; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.PrefixFileFilter; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; @@ -36,6 +36,7 @@ import org.junit.jupiter.api.io.TempDir; import java.io.File; +import java.io.FileFilter; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -44,6 +45,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -56,34 +59,16 @@ public class MemoryTableImplTest { - private static List> getRecords(TableScanIterator it) { - var ret = new ArrayList>(); - for (; ; ) { - it.next(); - var record = it.getRecord(); - if (record == null) { - break; - } - ret.add(it.getRecord()); - } - return ret; - } - - private static List> scanAll(MemoryTable memoryTable, boolean keepNone) { - return MemoryTableImplTest.getRecords(memoryTable.scan(null, null, true, null, false, keepNone, false)); - } - - private static List> decodeRecords(Map columnTypeMap, - List> records) { - return records.stream() - .map(r -> { - var record = new HashMap(); - for (var entry : r.entrySet()) { - record.put(entry.getKey(), columnTypeMap.get(entry.getKey()).decode(entry.getValue())); - } - return record; - }) - .collect(Collectors.toList()); + private static List scanAll(MemoryTable memoryTable, + List columns, + boolean keepNone) { + return memoryTable.scan(columns.stream().collect(Collectors.toMap(Function.identity(), Function.identity())), + null, + true, + null, + false, + 1000, + keepNone); } @TempDir @@ -99,8 +84,15 @@ public void setUp() throws IOException { } @AfterEach - public void afterEach() { + public void tearDown() throws IOException { walManager.terminate(); + for (var f : Objects.requireNonNull(new File(System.getProperty("java.io.tmpdir")) + .listFiles((FileFilter) new PrefixFileFilter("junit")))) { + if (!rootDir.equals(f)) { + System.out.println("deleting" + f); + FileUtils.deleteDirectory(f); + } + } } @Nested @@ -119,43 +111,45 @@ public void testUpdateCommon() { new ColumnSchemaDesc("k", "STRING"), new ColumnSchemaDesc("a", "INT32"))), List.of(Map.of("k", "0", "a", "a"))); - assertThat("init", scanAll(this.memoryTable, false), contains(Map.of("k", "0", "a", "a"))); + assertThat("init", + scanAll(this.memoryTable, List.of("k", "a"), false), + contains(new MemoryTable.RecordResult("0", Map.of("k", "0", "a", 10)))); this.memoryTable.update(null, List.of(Map.of("k", "1", "a", "b"))); - assertThat("insert", scanAll(this.memoryTable, false), contains( - Map.of("k", "0", "a", "a"), - Map.of("k", "1", "a", "b"))); + assertThat("insert", scanAll(this.memoryTable, List.of("k", "a"), false), + contains(new MemoryTable.RecordResult("0", Map.of("k", "0", "a", 10)), + new MemoryTable.RecordResult("1", Map.of("k", "1", "a", 11)))); this.memoryTable.update( null, List.of(Map.of("k", "2", "a", "c"), Map.of("k", "3", "a", "d"))); - assertThat("insert multiple", scanAll(this.memoryTable, false), contains( - Map.of("k", "0", "a", "a"), - Map.of("k", "1", "a", "b"), - Map.of("k", "2", "a", "c"), - Map.of("k", "3", "a", "d"))); + assertThat("insert multiple", scanAll(this.memoryTable, List.of("k", "a"), false), + contains(new MemoryTable.RecordResult("0", Map.of("k", "0", "a", 10)), + new MemoryTable.RecordResult("1", Map.of("k", "1", "a", 11)), + new MemoryTable.RecordResult("2", Map.of("k", "2", "a", 12)), + new MemoryTable.RecordResult("3", Map.of("k", "3", "a", 13)))); this.memoryTable.update(null, List.of(Map.of("k", "1", "a", "c"))); - assertThat("overwrite", scanAll(this.memoryTable, false), contains( - Map.of("k", "0", "a", "a"), - Map.of("k", "1", "a", "c"), - Map.of("k", "2", "a", "c"), - Map.of("k", "3", "a", "d"))); + assertThat("overwrite", scanAll(this.memoryTable, List.of("k", "a"), false), + contains(new MemoryTable.RecordResult("0", Map.of("k", "0", "a", 10)), + new MemoryTable.RecordResult("1", Map.of("k", "1", "a", 12)), + new MemoryTable.RecordResult("2", Map.of("k", "2", "a", 12)), + new MemoryTable.RecordResult("3", Map.of("k", "3", "a", 13)))); this.memoryTable.update(null, List.of(Map.of("k", "2", "-", "1"))); - assertThat("delete", scanAll(this.memoryTable, false), contains( - Map.of("k", "0", "a", "a"), - Map.of("k", "1", "a", "c"), - Map.of("k", "3", "a", "d"))); + assertThat("delete", scanAll(this.memoryTable, List.of("k", "a"), false), + contains(new MemoryTable.RecordResult("0", Map.of("k", "0", "a", 10)), + new MemoryTable.RecordResult("1", Map.of("k", "1", "a", 12)), + new MemoryTable.RecordResult("3", Map.of("k", "3", "a", 13)))); this.memoryTable.update( new TableSchemaDesc(null, List.of(new ColumnSchemaDesc("b", "INT32"))), List.of(Map.of("k", "1", "b", "0"))); - assertThat("new column", scanAll(this.memoryTable, false), contains( - Map.of("k", "0", "a", "a"), - Map.of("k", "1", "a", "c", "b", "0"), - Map.of("k", "3", "a", "d"))); + assertThat("new column", scanAll(this.memoryTable, List.of("k", "a", "b"), false), + contains(new MemoryTable.RecordResult("0", Map.of("k", "0", "a", 10)), + new MemoryTable.RecordResult("1", Map.of("k", "1", "a", 12, "b", 0)), + new MemoryTable.RecordResult("3", Map.of("k", "3", "a", 13)))); this.memoryTable.update( null, @@ -171,11 +165,11 @@ public void testUpdateCommon() { put("k", "3"); put("b", null); }})); - assertThat("null value", scanAll(this.memoryTable, false), contains( - Map.of("k", "0", "a", "a"), - Map.of("k", "1", "b", "0"), - Map.of("k", "2"), - Map.of("k", "3", "a", "d"))); + assertThat("null value", scanAll(this.memoryTable, List.of("k", "a", "b"), false), + contains(new MemoryTable.RecordResult("0", Map.of("k", "0", "a", 10)), + new MemoryTable.RecordResult("1", Map.of("k", "1", "b", 0)), + new MemoryTable.RecordResult("2", Map.of("k", "2")), + new MemoryTable.RecordResult("3", Map.of("k", "3", "a", 13)))); this.memoryTable.update( new TableSchemaDesc(null, List.of(new ColumnSchemaDesc("c", "INT32"))), @@ -190,19 +184,19 @@ public void testUpdateCommon() { }}, Map.of("k", "0", "-", "1"), Map.of("k", "2", "-", "1"))); - assertThat("mixed", scanAll(this.memoryTable, false), contains( - Map.of("k", "1", "c", "1"), - Map.of("k", "3", "a", "0"), - Map.of("k", "4", "c", "0"))); + assertThat("mixed", scanAll(this.memoryTable, List.of("k", "a", "b", "c"), false), + contains(new MemoryTable.RecordResult("1", Map.of("k", "1", "c", 1)), + new MemoryTable.RecordResult("3", Map.of("k", "3", "a", 0)), + new MemoryTable.RecordResult("4", Map.of("k", "4", "c", 0)))); this.memoryTable.update( new TableSchemaDesc(null, List.of(new ColumnSchemaDesc("a-b/c/d:e_f", "INT32"))), List.of(Map.of("k", "0", "a-b/c/d:e_f", "0"))); - assertThat("complex name", scanAll(this.memoryTable, false), contains( - Map.of("k", "0", "a-b/c/d:e_f", "0"), - Map.of("k", "1", "c", "1"), - Map.of("k", "3", "a", "0"), - Map.of("k", "4", "c", "0"))); + assertThat("complex name", scanAll(this.memoryTable, List.of("k", "a", "b", "c", "a-b/c/d:e_f"), false), + contains(new MemoryTable.RecordResult("0", Map.of("k", "0", "a-b/c/d:e_f", 0)), + new MemoryTable.RecordResult("1", Map.of("k", "1", "c", 1)), + new MemoryTable.RecordResult("3", Map.of("k", "3", "a", 0)), + new MemoryTable.RecordResult("4", Map.of("k", "4", "c", 0)))); this.memoryTable.update( new TableSchemaDesc(null, List.of(new ColumnSchemaDesc("x", "UNKNOWN"))), @@ -213,11 +207,11 @@ public void testUpdateCommon() { assertThat("unknown", this.memoryTable.getSchema().getColumnSchemaByName("x").getType(), is(ColumnType.UNKNOWN)); - assertThat("unknown", scanAll(this.memoryTable, false), contains( - Map.of("k", "0", "a-b/c/d:e_f", "0"), - Map.of("k", "1", "c", "1"), - Map.of("k", "3", "a", "0"), - Map.of("k", "4", "c", "0"))); + assertThat("unknown", scanAll(this.memoryTable, List.of("k", "a", "b", "c", "a-b/c/d:e_f"), false), + contains(new MemoryTable.RecordResult("0", Map.of("k", "0", "a-b/c/d:e_f", 0)), + new MemoryTable.RecordResult("1", Map.of("k", "1", "c", 1)), + new MemoryTable.RecordResult("3", Map.of("k", "3", "a", 0)), + new MemoryTable.RecordResult("4", Map.of("k", "4", "c", 0)))); this.memoryTable.update( new TableSchemaDesc(null, List.of(new ColumnSchemaDesc("x", "INT32"))), @@ -225,11 +219,12 @@ public void testUpdateCommon() { assertThat("update unknown", this.memoryTable.getSchema().getColumnSchemaByName("x").getType(), is(ColumnType.INT32)); - assertThat("update unknown", scanAll(this.memoryTable, false), contains( - Map.of("k", "0", "a-b/c/d:e_f", "0"), - Map.of("k", "1", "c", "1", "x", "1"), - Map.of("k", "3", "a", "0"), - Map.of("k", "4", "c", "0"))); + assertThat("update unknown", + scanAll(this.memoryTable, List.of("k", "a", "b", "c", "a-b/c/d:e_f", "x"), false), + contains(new MemoryTable.RecordResult("0", Map.of("k", "0", "a-b/c/d:e_f", 0)), + new MemoryTable.RecordResult("1", Map.of("k", "1", "c", 1, "x", 1)), + new MemoryTable.RecordResult("3", Map.of("k", "3", "a", 0)), + new MemoryTable.RecordResult("4", Map.of("k", "4", "c", 0)))); this.memoryTable.update( new TableSchemaDesc(null, List.of(new ColumnSchemaDesc("x", "UNKNOWN"))), @@ -240,11 +235,12 @@ public void testUpdateCommon() { assertThat("unknown again", this.memoryTable.getSchema().getColumnSchemaByName("x").getType(), is(ColumnType.INT32)); - assertThat("unknown again", scanAll(this.memoryTable, false), contains( - Map.of("k", "0", "a-b/c/d:e_f", "0"), - Map.of("k", "1", "c", "1"), - Map.of("k", "3", "a", "0"), - Map.of("k", "4", "c", "0"))); + assertThat("unknown again", + scanAll(this.memoryTable, List.of("k", "a", "b", "c", "a-b/c/d:e_f", "x"), false), + contains(new MemoryTable.RecordResult("0", Map.of("k", "0", "a-b/c/d:e_f", 0)), + new MemoryTable.RecordResult("1", Map.of("k", "1", "c", 1)), + new MemoryTable.RecordResult("3", Map.of("k", "3", "a", 0)), + new MemoryTable.RecordResult("4", Map.of("k", "4", "c", 0)))); } @Test @@ -273,18 +269,20 @@ public void testUpdateAllColumnTypes() { put("h", Base64.getEncoder().encodeToString("test".getBytes(StandardCharsets.UTF_8))); put("i", null); }})); - assertThat("all types", scanAll(this.memoryTable, false), contains( - new HashMap<>() {{ - put("key", "x"); - put("a", "1"); - put("b", "10"); - put("c", "1000"); - put("d", "100000"); - put("e", "10000000"); - put("f", Integer.toHexString(Float.floatToIntBits(1.1f))); - put("g", Long.toHexString(Double.doubleToLongBits(1.1))); - put("h", Base64.getEncoder().encodeToString("test".getBytes(StandardCharsets.UTF_8))); - }})); + assertThat("all types", + scanAll(this.memoryTable, List.of("key", "a", "b", "c", "d", "e", "f", "g", "h", "i"), false), + contains(new MemoryTable.RecordResult("x", + new HashMap<>() {{ + put("key", "x"); + put("a", true); + put("b", (byte) 16); + put("c", Short.parseShort("1000", 16)); + put("d", Integer.parseInt("100000", 16)); + put("e", Long.parseLong("10000000", 16)); + put("f", 1.1f); + put("g", 1.1); + put("h", ByteBuffer.wrap("test".getBytes(StandardCharsets.UTF_8))); + }}))); } @Test @@ -293,52 +291,68 @@ public void testUpdateAllKeyColumnTypes() { this.memoryTable.update( new TableSchemaDesc("k", List.of(new ColumnSchemaDesc("k", "BOOL"))), List.of(Map.of("k", "1"))); - assertThat("bool", scanAll(this.memoryTable, false), contains(Map.of("k", "1"))); + assertThat("bool", + scanAll(this.memoryTable, List.of("k"), false), + contains(new MemoryTable.RecordResult(true, Map.of("k", true)))); this.memoryTable = new MemoryTableImpl("test", MemoryTableImplTest.this.walManager); this.memoryTable.update( new TableSchemaDesc("k", List.of(new ColumnSchemaDesc("k", "INT8"))), List.of(Map.of("k", "10"))); - assertThat("int8", scanAll(this.memoryTable, false), contains(Map.of("k", "10"))); + assertThat("int8", + scanAll(this.memoryTable, List.of("k"), false), + contains(new MemoryTable.RecordResult((byte) 16, Map.of("k", (byte) 16)))); this.memoryTable = new MemoryTableImpl("test", MemoryTableImplTest.this.walManager); this.memoryTable.update( new TableSchemaDesc("k", List.of(new ColumnSchemaDesc("k", "INT16"))), List.of(Map.of("k", "1000"))); - assertThat("int16", scanAll(this.memoryTable, false), contains(Map.of("k", "1000"))); + assertThat("int16", + scanAll(this.memoryTable, List.of("k"), false), + contains(new MemoryTable.RecordResult(Short.parseShort("1000", 16), + Map.of("k", Short.parseShort("1000", 16))))); this.memoryTable = new MemoryTableImpl("test", MemoryTableImplTest.this.walManager); this.memoryTable.update( new TableSchemaDesc("k", List.of(new ColumnSchemaDesc("k", "INT32"))), List.of(Map.of("k", "100000"))); - assertThat("int32", scanAll(this.memoryTable, false), contains(Map.of("k", "100000"))); + assertThat("int32", + scanAll(this.memoryTable, List.of("k"), false), + contains(new MemoryTable.RecordResult(Integer.parseInt("100000", 16), + Map.of("k", Integer.parseInt("100000", 16))))); this.memoryTable = new MemoryTableImpl("", MemoryTableImplTest.this.walManager); this.memoryTable.update( new TableSchemaDesc("k", List.of(new ColumnSchemaDesc("k", "INT64"))), List.of(Map.of("k", "10000000"))); - assertThat("int64", scanAll(this.memoryTable, false), contains(Map.of("k", "10000000"))); + assertThat("int64", + scanAll(this.memoryTable, List.of("k"), false), + contains(new MemoryTable.RecordResult(Long.parseLong("10000000", 16), + Map.of("k", Long.parseLong("10000000", 16))))); this.memoryTable = new MemoryTableImpl("", MemoryTableImplTest.this.walManager); this.memoryTable.update( new TableSchemaDesc("k", List.of(new ColumnSchemaDesc("k", "FLOAT32"))), List.of(Map.of("k", Integer.toHexString(Float.floatToIntBits(1.1f))))); - assertThat("float32", scanAll(this.memoryTable, false), contains( - Map.of("k", Integer.toHexString(Float.floatToIntBits(1.1f))))); + assertThat("float32", scanAll(this.memoryTable, List.of("k"), false), + contains(new MemoryTable.RecordResult(1.1f, + Map.of("k", 1.1f)))); this.memoryTable = new MemoryTableImpl("", MemoryTableImplTest.this.walManager); this.memoryTable.update( new TableSchemaDesc("k", List.of(new ColumnSchemaDesc("k", "FLOAT64"))), List.of(Map.of("k", Long.toHexString(Double.doubleToLongBits(1.1))))); - assertThat("float64", scanAll(this.memoryTable, false), contains( - Map.of("k", Long.toHexString(Double.doubleToLongBits(1.1))))); + assertThat("float64", scanAll(this.memoryTable, List.of("k"), false), + contains(new MemoryTable.RecordResult(1.1, + Map.of("k", 1.1)))); this.memoryTable = new MemoryTableImpl("", MemoryTableImplTest.this.walManager); this.memoryTable.update( new TableSchemaDesc("k", List.of(new ColumnSchemaDesc("k", "BYTES"))), List.of(Map.of("k", Base64.getEncoder().encodeToString("test".getBytes(StandardCharsets.UTF_8))))); - assertThat("bytes", scanAll(this.memoryTable, false), contains( - Map.of("k", Base64.getEncoder().encodeToString("test".getBytes(StandardCharsets.UTF_8))))); + assertThat("bytes", scanAll(this.memoryTable, List.of("k"), false), + contains(new MemoryTable.RecordResult(ByteBuffer.wrap("test".getBytes(StandardCharsets.UTF_8)), + Map.of("k", ByteBuffer.wrap("test".getBytes(StandardCharsets.UTF_8)))))); } @Test @@ -352,14 +366,14 @@ public void testUpdateExceptions() { new TableSchemaDesc("k", List.of(new ColumnSchemaDesc("a", "INT32"))), List.of(Map.of("k", "0"), Map.of("k", "1"))), "no key column schema"); - assertThat("no key column schema", scanAll(this.memoryTable, false), empty()); + assertThat("no key column schema", scanAll(this.memoryTable, List.of("k", "a"), false), empty()); assertThrows(SWValidationException.class, () -> this.memoryTable.update( new TableSchemaDesc("k", List.of(new ColumnSchemaDesc("k", "STRING"))), List.of(Map.of("k", "0"), Map.of("k", "1", "a", "1"))), "extra column data"); - assertThat("extra column data", scanAll(this.memoryTable, false), empty()); + assertThat("extra column data", scanAll(this.memoryTable, List.of("k", "a"), false), empty()); assertThrows(SWValidationException.class, () -> this.memoryTable.update( @@ -368,7 +382,7 @@ public void testUpdateExceptions() { new ColumnSchemaDesc("a", "INT32"))), List.of(Map.of("k", "0"), Map.of("k", "1", "a", "h"))), "fail to decode"); - assertThat("fail to decode", scanAll(this.memoryTable, false), empty()); + assertThat("fail to decode", scanAll(this.memoryTable, List.of("k", "a"), false), empty()); } @Test @@ -387,7 +401,7 @@ public void testUpdateWalError() { List.of(Map.of("k", "a".repeat(5000)))), "huge entry"); assertThat("schema", this.memoryTable.getSchema(), is(new TableSchema(schema))); - assertThat("records", scanAll(this.memoryTable, false), empty()); + assertThat("records", scanAll(this.memoryTable, List.of("k"), false), empty()); } @Test @@ -433,13 +447,28 @@ public void testUpdateFromWal() throws IOException { while (it.hasNext()) { this.memoryTable.updateFromWal(it.next()); } - assertThat(scanAll(this.memoryTable, true), is(records)); + assertThat(scanAll(this.memoryTable, List.of("key", "a", "b", "c", "d", "e", "f", "g", "h", "i"), true), + is(IntStream.range(0, 100) + .mapToObj(index -> new MemoryTable.RecordResult( + String.format("%03d", index), + new HashMap<>() {{ + put("key", String.format("%03d", index)); + put("a", index % 2 == 1); + put("b", (byte) (index + 10)); + put("c", (short) (index + 1000)); + put("d", index + 100000); + put("e", index + 10000000L); + put("f", index + 0.1f); + put("g", index + 0.1); + put("h", ByteBuffer.wrap(("test" + index).getBytes(StandardCharsets.UTF_8))); + put("i", null); + }})) + .collect(Collectors.toList()))); } } @Nested public class QueryScanTest { - private List> records; private MemoryTableImpl memoryTable; @BeforeEach @@ -454,40 +483,38 @@ public void setUp() { {5.0, 6.0, 7.0, 8.0, null, 0.0, 1.0, 2.0, 3.0, 4.0}, {"6", "7", "8", null, "0", "1", "2", "3", "4", "5"}, {"7", "8", null, "0", "1", "2", "3", "4", "5", "6"}}; - this.records = IntStream.rangeClosed(0, 9).mapToObj( + var records = IntStream.rangeClosed(0, 9).mapToObj( i -> { - Map ret = new HashMap<>(); - ret.put("key", Integer.toHexString(i)); + Map values = new HashMap<>(); + values.put("key", Integer.toHexString(i)); if (data[0][i] != null) { - ret.put("a", ColumnType.BOOL.encode(data[0][i], false)); + values.put("a", data[0][i]); } if (data[1][i] != null) { - ret.put("b", ColumnType.INT8.encode(data[1][i], false)); + values.put("b", data[1][i]); } if (data[2][i] != null) { - ret.put("c", ColumnType.INT16.encode(data[2][i], false)); + values.put("c", data[2][i]); } if (data[3][i] != null) { - ret.put("d", ColumnType.INT32.encode(data[3][i], false)); + values.put("d", data[3][i]); } if (data[4][i] != null) { - ret.put("e", ColumnType.INT64.encode(data[4][i], false)); + values.put("e", data[4][i]); } if (data[5][i] != null) { - ret.put("f", ColumnType.FLOAT32.encode(data[5][i], false)); + values.put("f", data[5][i]); } if (data[6][i] != null) { - ret.put("g", ColumnType.FLOAT64.encode(data[6][i], false)); + values.put("g", data[6][i]); } if (data[7][i] != null) { - ret.put("h", (String) data[7][i]); + values.put("h", data[7][i]); } if (data[8][i] != null) { - ret.put("i", ColumnType.BYTES.encode( - ByteBuffer.wrap(((String) data[8][i]).getBytes(StandardCharsets.UTF_8)), - false)); + values.put("i", ByteBuffer.wrap(((String) data[8][i]).getBytes(StandardCharsets.UTF_8))); } - return ret; + return new MemoryTable.RecordResult(Integer.toHexString(i), values); }) .collect(Collectors.toList()); var schema = new TableSchemaDesc("key", List.of( @@ -503,15 +530,19 @@ public void setUp() { new ColumnSchemaDesc("i", "BYTES"), new ColumnSchemaDesc("z", "UNKNOWN"))); this.memoryTable = new MemoryTableImpl("test", MemoryTableImplTest.this.walManager); - this.memoryTable.update(schema, records); + this.memoryTable.update(schema, + records.stream() + .map(r -> r.getValues().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, + entry -> ColumnType.getColumnType(entry.getValue()) + .encode(entry.getValue(), false)))) + .collect(Collectors.toList())); } @Test public void testQueryInitialEmptyTable() { this.memoryTable = new MemoryTableImpl("test", MemoryTableImplTest.this.walManager); - var recordList = this.memoryTable.query(null, null, null, -1, -1, false, false); - assertThat("empty", recordList.getColumnTypeMap(), nullValue()); - assertThat("empty", recordList.getRecords(), empty()); + var results = this.memoryTable.query(Map.of("a", "a"), null, null, -1, -1, false, false); + assertThat(results, empty()); } @Test @@ -519,40 +550,24 @@ public void testQueryEmptyTableWithSchema() { this.memoryTable = new MemoryTableImpl("test", MemoryTableImplTest.this.walManager); this.memoryTable.update(new TableSchemaDesc("k", List.of(new ColumnSchemaDesc("k", "STRING"))), List.of(Map.of("k", "0", "-", "1"))); - var recordList = this.memoryTable.query(null, null, null, -1, -1, false, false); - assertThat("empty", recordList.getColumnTypeMap(), is(Map.of("k", ColumnType.STRING))); - assertThat("empty", recordList.getRecords(), empty()); - } - - @Test - public void testQueryAll() { - var recordList = this.memoryTable.query(null, null, null, -1, -1, false, false); - assertThat("all", - recordList.getColumnTypeMap(), - is(this.memoryTable.getSchema().getColumnSchemas().stream() - .collect(Collectors.toMap(ColumnSchema::getName, ColumnSchema::getType)))); - assertThat("all", recordList.getRecords(), is(this.records)); + var results = this.memoryTable.query(Map.of("k", "k"), null, null, -1, -1, false, false); + assertThat(results, empty()); } @Test public void testQueryColumnAliases() { - var recordList = this.memoryTable.query(Map.of("a", "x", "d", "y"), null, null, -1, -1, false, false); - assertThat("columns", - recordList.getColumnTypeMap(), - is(Map.of("x", ColumnType.BOOL, "y", ColumnType.INT32))); - assertThat("columns", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(Map.of("a", "x", "d", "y")), - recordList.getRecords()), - is(List.of(Map.of("y", 2), - Map.of("x", false, "y", 3), - Map.of("x", true, "y", 4), - Map.of("x", false, "y", 5), - Map.of("x", true, "y", 6), - Map.of("x", false, "y", 7), - Map.of("x", true, "y", 8), - Map.of("x", false), - Map.of("x", true, "y", 0), - Map.of("x", false, "y", 1)))); + var results = this.memoryTable.query(Map.of("a", "x", "d", "y"), null, null, -1, -1, false, false); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("y", 2)), + new MemoryTable.RecordResult(1, Map.of("x", false, "y", 3)), + new MemoryTable.RecordResult(2, Map.of("x", true, "y", 4)), + new MemoryTable.RecordResult(3, Map.of("x", false, "y", 5)), + new MemoryTable.RecordResult(4, Map.of("x", true, "y", 6)), + new MemoryTable.RecordResult(5, Map.of("x", false, "y", 7)), + new MemoryTable.RecordResult(6, Map.of("x", true, "y", 8)), + new MemoryTable.RecordResult(7, Map.of("x", false)), + new MemoryTable.RecordResult(8, Map.of("x", true, "y", 0)), + new MemoryTable.RecordResult(9, Map.of("x", false, "y", 1))))); } @Test @@ -564,7 +579,7 @@ public void testQueryColumnAliasesInvalid() { @Test public void testQueryOrderBySingle() { - var recordList = this.memoryTable.query( + var results = this.memoryTable.query( Map.of("a", "a", "b", "b"), List.of(new OrderByDesc("a")), null, @@ -572,26 +587,22 @@ public void testQueryOrderBySingle() { -1, false, false); - assertThat("order by a", - recordList.getColumnTypeMap(), - is(Map.of("a", ColumnType.BOOL, "b", ColumnType.INT8))); - assertThat("order by a", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(), recordList.getRecords()), - is(List.of(Map.of("b", (byte) 0), - Map.of("a", false, "b", (byte) 1), - Map.of("a", false, "b", (byte) 3), - Map.of("a", false, "b", (byte) 5), - Map.of("a", false, "b", (byte) 7), - Map.of("a", false), - Map.of("a", true, "b", (byte) 2), - Map.of("a", true, "b", (byte) 4), - Map.of("a", true, "b", (byte) 6), - Map.of("a", true, "b", (byte) 8)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("b", (byte) 0)), + new MemoryTable.RecordResult(1, Map.of("a", false, "b", (byte) 1)), + new MemoryTable.RecordResult(3, Map.of("a", false, "b", (byte) 3)), + new MemoryTable.RecordResult(5, Map.of("a", false, "b", (byte) 5)), + new MemoryTable.RecordResult(7, Map.of("a", false, "b", (byte) 7)), + new MemoryTable.RecordResult(9, Map.of("a", false)), + new MemoryTable.RecordResult(2, Map.of("a", true, "b", (byte) 2)), + new MemoryTable.RecordResult(4, Map.of("a", true, "b", (byte) 4)), + new MemoryTable.RecordResult(6, Map.of("a", true, "b", (byte) 6)), + new MemoryTable.RecordResult(8, Map.of("a", true, "b", (byte) 8))))); } @Test public void testQueryOrderByDescending() { - var recordList = this.memoryTable.query( + var results = this.memoryTable.query( Map.of("a", "a", "b", "b"), List.of(new OrderByDesc("a"), new OrderByDesc("b", true)), null, @@ -599,26 +610,22 @@ public void testQueryOrderByDescending() { -1, false, false); - assertThat("", - recordList.getColumnTypeMap(), - is(Map.of("a", ColumnType.BOOL, "b", ColumnType.INT8))); - assertThat("", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(), recordList.getRecords()), - is(List.of(Map.of("b", (byte) 0), - Map.of("a", false, "b", (byte) 7), - Map.of("a", false, "b", (byte) 5), - Map.of("a", false, "b", (byte) 3), - Map.of("a", false, "b", (byte) 1), - Map.of("a", false), - Map.of("a", true, "b", (byte) 8), - Map.of("a", true, "b", (byte) 6), - Map.of("a", true, "b", (byte) 4), - Map.of("a", true, "b", (byte) 2)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("b", (byte) 0)), + new MemoryTable.RecordResult(7, Map.of("a", false, "b", (byte) 7)), + new MemoryTable.RecordResult(5, Map.of("a", false, "b", (byte) 5)), + new MemoryTable.RecordResult(3, Map.of("a", false, "b", (byte) 3)), + new MemoryTable.RecordResult(1, Map.of("a", false, "b", (byte) 1)), + new MemoryTable.RecordResult(9, Map.of("a", false)), + new MemoryTable.RecordResult(8, Map.of("a", true, "b", (byte) 8)), + new MemoryTable.RecordResult(6, Map.of("a", true, "b", (byte) 6)), + new MemoryTable.RecordResult(4, Map.of("a", true, "b", (byte) 4)), + new MemoryTable.RecordResult(2, Map.of("a", true, "b", (byte) 2))))); } @Test public void testQueryOrderByInt8() { - var recordList = this.memoryTable.query( + var results = this.memoryTable.query( Map.of("a", "a", "b", "b"), List.of(new OrderByDesc("a"), new OrderByDesc("b")), null, @@ -626,26 +633,22 @@ public void testQueryOrderByInt8() { -1, false, false); - assertThat("order by a,b", - recordList.getColumnTypeMap(), - is(Map.of("a", ColumnType.BOOL, "b", ColumnType.INT8))); - assertThat("order by a,b", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(), recordList.getRecords()), - is(List.of(Map.of("b", (byte) 0), - Map.of("a", false), - Map.of("a", false, "b", (byte) 1), - Map.of("a", false, "b", (byte) 3), - Map.of("a", false, "b", (byte) 5), - Map.of("a", false, "b", (byte) 7), - Map.of("a", true, "b", (byte) 2), - Map.of("a", true, "b", (byte) 4), - Map.of("a", true, "b", (byte) 6), - Map.of("a", true, "b", (byte) 8)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("b", (byte) 0)), + new MemoryTable.RecordResult(9, Map.of("a", false)), + new MemoryTable.RecordResult(1, Map.of("a", false, "b", (byte) 1)), + new MemoryTable.RecordResult(3, Map.of("a", false, "b", (byte) 3)), + new MemoryTable.RecordResult(5, Map.of("a", false, "b", (byte) 5)), + new MemoryTable.RecordResult(7, Map.of("a", false, "b", (byte) 7)), + new MemoryTable.RecordResult(2, Map.of("a", true, "b", (byte) 2)), + new MemoryTable.RecordResult(4, Map.of("a", true, "b", (byte) 4)), + new MemoryTable.RecordResult(6, Map.of("a", true, "b", (byte) 6)), + new MemoryTable.RecordResult(8, Map.of("a", true, "b", (byte) 8))))); } @Test public void testQueryOrderByInt16() { - var recordList = this.memoryTable.query( + var results = this.memoryTable.query( Map.of("a", "a", "c", "c"), List.of(new OrderByDesc("a"), new OrderByDesc("c")), null, @@ -653,26 +656,22 @@ public void testQueryOrderByInt16() { -1, false, false); - assertThat("order by a,c", - recordList.getColumnTypeMap(), - is(Map.of("a", ColumnType.BOOL, "c", ColumnType.INT16))); - assertThat("order by a,c", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(), recordList.getRecords()), - is(List.of(Map.of("c", (short) 1), - Map.of("a", false, "c", (short) 0), - Map.of("a", false, "c", (short) 2), - Map.of("a", false, "c", (short) 4), - Map.of("a", false, "c", (short) 6), - Map.of("a", false, "c", (short) 8), - Map.of("a", true), - Map.of("a", true, "c", (short) 3), - Map.of("a", true, "c", (short) 5), - Map.of("a", true, "c", (short) 7)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("c", (short) 1)), + new MemoryTable.RecordResult(9, Map.of("a", false, "c", (short) 0)), + new MemoryTable.RecordResult(1, Map.of("a", false, "c", (short) 2)), + new MemoryTable.RecordResult(3, Map.of("a", false, "c", (short) 4)), + new MemoryTable.RecordResult(5, Map.of("a", false, "c", (short) 6)), + new MemoryTable.RecordResult(7, Map.of("a", false, "c", (short) 8)), + new MemoryTable.RecordResult(8, Map.of("a", true)), + new MemoryTable.RecordResult(2, Map.of("a", true, "c", (short) 3)), + new MemoryTable.RecordResult(4, Map.of("a", true, "c", (short) 5)), + new MemoryTable.RecordResult(6, Map.of("a", true, "c", (short) 7))))); } @Test public void testQueryOrderByInt32() { - var recordList = this.memoryTable.query( + var results = this.memoryTable.query( Map.of("a", "a", "d", "d"), List.of(new OrderByDesc("a"), new OrderByDesc("d")), null, @@ -680,26 +679,22 @@ public void testQueryOrderByInt32() { -1, false, false); - assertThat("order by a,d", - recordList.getColumnTypeMap(), - is(Map.of("a", ColumnType.BOOL, "d", ColumnType.INT32))); - assertThat("order by a,d", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(), recordList.getRecords()), - is(List.of(Map.of("d", 2), - Map.of("a", false), - Map.of("a", false, "d", 1), - Map.of("a", false, "d", 3), - Map.of("a", false, "d", 5), - Map.of("a", false, "d", 7), - Map.of("a", true, "d", 0), - Map.of("a", true, "d", 4), - Map.of("a", true, "d", 6), - Map.of("a", true, "d", 8)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("d", 2)), + new MemoryTable.RecordResult(7, Map.of("a", false)), + new MemoryTable.RecordResult(9, Map.of("a", false, "d", 1)), + new MemoryTable.RecordResult(1, Map.of("a", false, "d", 3)), + new MemoryTable.RecordResult(3, Map.of("a", false, "d", 5)), + new MemoryTable.RecordResult(5, Map.of("a", false, "d", 7)), + new MemoryTable.RecordResult(8, Map.of("a", true, "d", 0)), + new MemoryTable.RecordResult(2, Map.of("a", true, "d", 4)), + new MemoryTable.RecordResult(4, Map.of("a", true, "d", 6)), + new MemoryTable.RecordResult(6, Map.of("a", true, "d", 8))))); } @Test public void testQueryOrderByInt64() { - var recordList = this.memoryTable.query( + var results = this.memoryTable.query( Map.of("a", "a", "e", "e"), List.of(new OrderByDesc("a"), new OrderByDesc("e")), null, @@ -707,26 +702,22 @@ public void testQueryOrderByInt64() { -1, false, false); - assertThat("order by a,e", - recordList.getColumnTypeMap(), - is(Map.of("a", ColumnType.BOOL, "e", ColumnType.INT64))); - assertThat("order by a,e", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(), recordList.getRecords()), - is(List.of(Map.of("e", 3L), - Map.of("a", false, "e", 0L), - Map.of("a", false, "e", 2L), - Map.of("a", false, "e", 4L), - Map.of("a", false, "e", 6L), - Map.of("a", false, "e", 8L), - Map.of("a", true), - Map.of("a", true, "e", 1L), - Map.of("a", true, "e", 5L), - Map.of("a", true, "e", 7L)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("e", 3L)), + new MemoryTable.RecordResult(7, Map.of("a", false, "e", 0L)), + new MemoryTable.RecordResult(9, Map.of("a", false, "e", 2L)), + new MemoryTable.RecordResult(1, Map.of("a", false, "e", 4L)), + new MemoryTable.RecordResult(3, Map.of("a", false, "e", 6L)), + new MemoryTable.RecordResult(5, Map.of("a", false, "e", 8L)), + new MemoryTable.RecordResult(6, Map.of("a", true)), + new MemoryTable.RecordResult(8, Map.of("a", true, "e", 1L)), + new MemoryTable.RecordResult(2, Map.of("a", true, "e", 5L)), + new MemoryTable.RecordResult(4, Map.of("a", true, "e", 7L))))); } @Test public void testQueryOrderByFloat32() { - var recordList = this.memoryTable.query( + var results = this.memoryTable.query( Map.of("a", "a", "f", "f"), List.of(new OrderByDesc("a"), new OrderByDesc("f")), null, @@ -734,26 +725,22 @@ public void testQueryOrderByFloat32() { -1, false, false); - assertThat("order by a,f", - recordList.getColumnTypeMap(), - is(Map.of("a", ColumnType.BOOL, "f", ColumnType.FLOAT32))); - assertThat("order by a,f", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(), recordList.getRecords()), - is(List.of(Map.of("f", 4.f), - Map.of("a", false), - Map.of("a", false, "f", 1.f), - Map.of("a", false, "f", 3.f), - Map.of("a", false, "f", 5.f), - Map.of("a", false, "f", 7.f), - Map.of("a", true, "f", 0.f), - Map.of("a", true, "f", 2.f), - Map.of("a", true, "f", 6.f), - Map.of("a", true, "f", 8.f)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("f", 4.f)), + new MemoryTable.RecordResult(5, Map.of("a", false)), + new MemoryTable.RecordResult(7, Map.of("a", false, "f", 1.f)), + new MemoryTable.RecordResult(9, Map.of("a", false, "f", 3.f)), + new MemoryTable.RecordResult(1, Map.of("a", false, "f", 5.f)), + new MemoryTable.RecordResult(3, Map.of("a", false, "f", 7.f)), + new MemoryTable.RecordResult(6, Map.of("a", true, "f", 0.f)), + new MemoryTable.RecordResult(8, Map.of("a", true, "f", 2.f)), + new MemoryTable.RecordResult(2, Map.of("a", true, "f", 6.f)), + new MemoryTable.RecordResult(4, Map.of("a", true, "f", 8.f))))); } @Test public void testQueryOrderByFloat64() { - var recordList = this.memoryTable.query( + var results = this.memoryTable.query( Map.of("a", "a", "g", "g"), List.of(new OrderByDesc("a"), new OrderByDesc("g")), null, @@ -761,26 +748,22 @@ public void testQueryOrderByFloat64() { -1, false, false); - assertThat("order by a,g", - recordList.getColumnTypeMap(), - is(Map.of("a", ColumnType.BOOL, "g", ColumnType.FLOAT64))); - assertThat("order by a,g", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(), recordList.getRecords()), - is(List.of(Map.of("g", 5.), - Map.of("a", false, "g", 0.), - Map.of("a", false, "g", 2.), - Map.of("a", false, "g", 4.), - Map.of("a", false, "g", 6.), - Map.of("a", false, "g", 8.), - Map.of("a", true), - Map.of("a", true, "g", 1.), - Map.of("a", true, "g", 3.), - Map.of("a", true, "g", 7.)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("g", 5.)), + new MemoryTable.RecordResult(5, Map.of("a", false, "g", 0.)), + new MemoryTable.RecordResult(7, Map.of("a", false, "g", 2.)), + new MemoryTable.RecordResult(9, Map.of("a", false, "g", 4.)), + new MemoryTable.RecordResult(1, Map.of("a", false, "g", 6.)), + new MemoryTable.RecordResult(3, Map.of("a", false, "g", 8.)), + new MemoryTable.RecordResult(4, Map.of("a", true)), + new MemoryTable.RecordResult(6, Map.of("a", true, "g", 1.)), + new MemoryTable.RecordResult(8, Map.of("a", true, "g", 3.)), + new MemoryTable.RecordResult(2, Map.of("a", true, "g", 7.))))); } @Test public void testQueryOrderByString() { - var recordList = this.memoryTable.query( + var results = this.memoryTable.query( Map.of("a", "a", "h", "h"), List.of(new OrderByDesc("a"), new OrderByDesc("h")), null, @@ -788,26 +771,22 @@ public void testQueryOrderByString() { -1, false, false); - assertThat("order by a,h", - recordList.getColumnTypeMap(), - is(Map.of("a", ColumnType.BOOL, "h", ColumnType.STRING))); - assertThat("order by a,h", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(), recordList.getRecords()), - is(List.of(Map.of("h", "6"), - Map.of("a", false), - Map.of("a", false, "h", "1"), - Map.of("a", false, "h", "3"), - Map.of("a", false, "h", "5"), - Map.of("a", false, "h", "7"), - Map.of("a", true, "h", "0"), - Map.of("a", true, "h", "2"), - Map.of("a", true, "h", "4"), - Map.of("a", true, "h", "8")))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("h", "6")), + new MemoryTable.RecordResult(3, Map.of("a", false)), + new MemoryTable.RecordResult(5, Map.of("a", false, "h", "1")), + new MemoryTable.RecordResult(7, Map.of("a", false, "h", "3")), + new MemoryTable.RecordResult(9, Map.of("a", false, "h", "5")), + new MemoryTable.RecordResult(1, Map.of("a", false, "h", "7")), + new MemoryTable.RecordResult(4, Map.of("a", true, "h", "0")), + new MemoryTable.RecordResult(6, Map.of("a", true, "h", "2")), + new MemoryTable.RecordResult(8, Map.of("a", true, "h", "4")), + new MemoryTable.RecordResult(2, Map.of("a", true, "h", "8"))))); } @Test public void testQueryOrderByBytes() { - var recordList = this.memoryTable.query( + var results = this.memoryTable.query( Map.of("a", "a", "i", "i"), List.of(new OrderByDesc("a"), new OrderByDesc("i")), null, @@ -815,26 +794,31 @@ public void testQueryOrderByBytes() { -1, false, false); - assertThat("order by a,i", - recordList.getColumnTypeMap(), - is(Map.of("a", ColumnType.BOOL, "i", ColumnType.BYTES))); - assertThat("order by a,i", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(), recordList.getRecords()), - is(List.of(Map.of("i", ByteBuffer.wrap("7".getBytes(StandardCharsets.UTF_8))), - Map.of("a", false, "i", ByteBuffer.wrap("0".getBytes(StandardCharsets.UTF_8))), - Map.of("a", false, "i", ByteBuffer.wrap("2".getBytes(StandardCharsets.UTF_8))), - Map.of("a", false, "i", ByteBuffer.wrap("4".getBytes(StandardCharsets.UTF_8))), - Map.of("a", false, "i", ByteBuffer.wrap("6".getBytes(StandardCharsets.UTF_8))), - Map.of("a", false, "i", ByteBuffer.wrap("8".getBytes(StandardCharsets.UTF_8))), - Map.of("a", true), - Map.of("a", true, "i", ByteBuffer.wrap("1".getBytes(StandardCharsets.UTF_8))), - Map.of("a", true, "i", ByteBuffer.wrap("3".getBytes(StandardCharsets.UTF_8))), - Map.of("a", true, "i", ByteBuffer.wrap("5".getBytes(StandardCharsets.UTF_8)))))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, + Map.of("i", ByteBuffer.wrap("7".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(3, + Map.of("a", false, "i", ByteBuffer.wrap("0".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(5, + Map.of("a", false, "i", ByteBuffer.wrap("2".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(7, + Map.of("a", false, "i", ByteBuffer.wrap("4".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(9, + Map.of("a", false, "i", ByteBuffer.wrap("6".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(1, + Map.of("a", false, "i", ByteBuffer.wrap("8".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(2, Map.of("a", true)), + new MemoryTable.RecordResult(4, + Map.of("a", true, "i", ByteBuffer.wrap("1".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(6, + Map.of("a", true, "i", ByteBuffer.wrap("3".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(8, + Map.of("a", true, "i", ByteBuffer.wrap("5".getBytes(StandardCharsets.UTF_8))))))); } @Test public void testQueryOrderByMixed() { - var recordList = this.memoryTable.query( + var results = this.memoryTable.query( Map.of("d", "x"), List.of(new OrderByDesc("a"), new OrderByDesc("d")), null, @@ -842,35 +826,36 @@ public void testQueryOrderByMixed() { -1, false, false); - assertThat("aliases and order by", - recordList.getColumnTypeMap(), - is(Map.of("x", ColumnType.INT32))); - assertThat("aliases and order by", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(Map.of("d", "x")), - recordList.getRecords()), - is(List.of(Map.of("x", 2), - Map.of(), - Map.of("x", 1), - Map.of("x", 3), - Map.of("x", 5), - Map.of("x", 7), - Map.of("x", 0), - Map.of("x", 4), - Map.of("x", 6), - Map.of("x", 8)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("x", 2)), + new MemoryTable.RecordResult(7, Map.of()), + new MemoryTable.RecordResult(9, Map.of("x", 1)), + new MemoryTable.RecordResult(1, Map.of("x", 3)), + new MemoryTable.RecordResult(3, Map.of("x", 5)), + new MemoryTable.RecordResult(5, Map.of("x", 7)), + new MemoryTable.RecordResult(8, Map.of("x", 0)), + new MemoryTable.RecordResult(2, Map.of("x", 4)), + new MemoryTable.RecordResult(4, Map.of("x", 6)), + new MemoryTable.RecordResult(6, Map.of("x", 8))))); } @Test public void testQueryOrderByInvalid() { assertThrows(SWValidationException.class, - () -> this.memoryTable.query(null, List.of(new OrderByDesc("x")), null, -1, -1, false, false), + () -> this.memoryTable.query(Map.of("a", "a"), + List.of(new OrderByDesc("x")), + null, + -1, + -1, + false, + false), "invalid order by column"); } @Test public void testQueryFilterNullEqual() { var columns = Map.of("a", "a", "d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, null, TableQueryFilter.builder() @@ -883,16 +868,15 @@ public void testQueryFilterNullEqual() { -1, -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("d", 2)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("d", 2))))); } @Test public void testQueryFilterBoolEqual() { var columns = Map.of("a", "a", "d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, null, TableQueryFilter.builder() @@ -902,19 +886,18 @@ public void testQueryFilterBoolEqual() { -1, -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), + assertThat(results, is(List.of( - Map.of("a", true, "d", 4), - Map.of("a", true, "d", 6), - Map.of("a", true, "d", 8), - Map.of("a", true, "d", 0)))); + new MemoryTable.RecordResult(2, Map.of("a", true, "d", 4)), + new MemoryTable.RecordResult(4, Map.of("a", true, "d", 6)), + new MemoryTable.RecordResult(6, Map.of("a", true, "d", 8)), + new MemoryTable.RecordResult(8, Map.of("a", true, "d", 0))))); } @Test public void testQueryFilterInt8Equal() { var columns = Map.of("b", "b"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, null, TableQueryFilter.builder() @@ -925,15 +908,13 @@ public void testQueryFilterInt8Equal() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("b", (byte) 5)))); + assertThat(results, is(List.of(new MemoryTable.RecordResult(5, Map.of("b", (byte) 5))))); } @Test public void testQueryFilterInt16Equal() { var columns = Map.of("c", "c"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, null, TableQueryFilter.builder() @@ -944,15 +925,13 @@ public void testQueryFilterInt16Equal() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("c", (short) 5)))); + assertThat(results, is(List.of(new MemoryTable.RecordResult(4, Map.of("c", (short) 5))))); } @Test public void testQueryFilterInt32Equal() { var columns = Map.of("d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, null, TableQueryFilter.builder() @@ -963,15 +942,13 @@ public void testQueryFilterInt32Equal() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("d", 5)))); + assertThat(results, is(List.of(new MemoryTable.RecordResult(3, Map.of("d", 5))))); } @Test public void testQueryFilterInt64Equal() { var columns = Map.of("e", "e"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, null, TableQueryFilter.builder() @@ -982,15 +959,13 @@ public void testQueryFilterInt64Equal() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("e", 5L)))); + assertThat(results, is(List.of(new MemoryTable.RecordResult(2, Map.of("e", 5L))))); } @Test public void testQueryFilterFloat32Equal() { var columns = Map.of("f", "f"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, null, TableQueryFilter.builder() @@ -1001,15 +976,13 @@ public void testQueryFilterFloat32Equal() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("f", 5.f)))); + assertThat(results, is(List.of(new MemoryTable.RecordResult(1, Map.of("f", 5.f))))); } @Test public void testQueryFilterFloat64Equal() { var columns = Map.of("g", "g"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, null, TableQueryFilter.builder() @@ -1020,15 +993,13 @@ public void testQueryFilterFloat64Equal() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("g", 5.)))); + assertThat(results, is(List.of(new MemoryTable.RecordResult(0, Map.of("g", 5.))))); } @Test public void testQueryFilterStringEqual() { var columns = Map.of("h", "h"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, null, TableQueryFilter.builder() @@ -1039,15 +1010,13 @@ public void testQueryFilterStringEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("h", "5")))); + assertThat(results, is(List.of(new MemoryTable.RecordResult(9, Map.of("h", "5"))))); } @Test public void testQueryFilterBytesEqual() { var columns = Map.of("i", "i"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, null, TableQueryFilter.builder() @@ -1060,15 +1029,15 @@ public void testQueryFilterBytesEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("i", ByteBuffer.wrap("5".getBytes(StandardCharsets.UTF_8)))))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(8, + Map.of("i", ByteBuffer.wrap("5".getBytes(StandardCharsets.UTF_8))))))); } @Test public void testQueryFilterBoolLess() { var columns = Map.of("a", "a", "d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("d")), TableQueryFilter.builder() @@ -1079,20 +1048,19 @@ public void testQueryFilterBoolLess() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), + assertThat(results, is(List.of( - Map.of("a", false), - Map.of("a", false, "d", 1), - Map.of("a", false, "d", 3), - Map.of("a", false, "d", 5), - Map.of("a", false, "d", 7)))); + new MemoryTable.RecordResult(7, Map.of("a", false)), + new MemoryTable.RecordResult(9, Map.of("a", false, "d", 1)), + new MemoryTable.RecordResult(1, Map.of("a", false, "d", 3)), + new MemoryTable.RecordResult(3, Map.of("a", false, "d", 5)), + new MemoryTable.RecordResult(5, Map.of("a", false, "d", 7))))); } @Test public void testQueryFilterInt8Less() { var columns = Map.of("b", "b"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("b")), TableQueryFilter.builder() @@ -1103,19 +1071,18 @@ public void testQueryFilterInt8Less() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("b", (byte) 0), - Map.of("b", (byte) 1), - Map.of("b", (byte) 2), - Map.of("b", (byte) 3), - Map.of("b", (byte) 4)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("b", (byte) 0)), + new MemoryTable.RecordResult(1, Map.of("b", (byte) 1)), + new MemoryTable.RecordResult(2, Map.of("b", (byte) 2)), + new MemoryTable.RecordResult(3, Map.of("b", (byte) 3)), + new MemoryTable.RecordResult(4, Map.of("b", (byte) 4))))); } @Test public void testQueryFilterInt16Less() { var columns = Map.of("c", "c"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("c")), TableQueryFilter.builder() @@ -1126,19 +1093,18 @@ public void testQueryFilterInt16Less() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("c", (short) 0), - Map.of("c", (short) 1), - Map.of("c", (short) 2), - Map.of("c", (short) 3), - Map.of("c", (short) 4)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(9, Map.of("c", (short) 0)), + new MemoryTable.RecordResult(0, Map.of("c", (short) 1)), + new MemoryTable.RecordResult(1, Map.of("c", (short) 2)), + new MemoryTable.RecordResult(2, Map.of("c", (short) 3)), + new MemoryTable.RecordResult(3, Map.of("c", (short) 4))))); } @Test public void testQueryFilterInt32Less() { var columns = Map.of("d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("d")), TableQueryFilter.builder() @@ -1149,19 +1115,18 @@ public void testQueryFilterInt32Less() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("d", 0), - Map.of("d", 1), - Map.of("d", 2), - Map.of("d", 3), - Map.of("d", 4)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(8, Map.of("d", 0)), + new MemoryTable.RecordResult(9, Map.of("d", 1)), + new MemoryTable.RecordResult(0, Map.of("d", 2)), + new MemoryTable.RecordResult(1, Map.of("d", 3)), + new MemoryTable.RecordResult(2, Map.of("d", 4))))); } @Test public void testQueryFilterInt64Less() { var columns = Map.of("e", "e"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("e")), TableQueryFilter.builder() @@ -1172,19 +1137,18 @@ public void testQueryFilterInt64Less() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("e", 0L), - Map.of("e", 1L), - Map.of("e", 2L), - Map.of("e", 3L), - Map.of("e", 4L)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(7, Map.of("e", 0L)), + new MemoryTable.RecordResult(8, Map.of("e", 1L)), + new MemoryTable.RecordResult(9, Map.of("e", 2L)), + new MemoryTable.RecordResult(0, Map.of("e", 3L)), + new MemoryTable.RecordResult(1, Map.of("e", 4L))))); } @Test public void testQueryFilterFloat32Less() { var columns = Map.of("f", "f"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("f")), TableQueryFilter.builder() @@ -1195,19 +1159,18 @@ public void testQueryFilterFloat32Less() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("f", 0.f), - Map.of("f", 1.f), - Map.of("f", 2.f), - Map.of("f", 3.f), - Map.of("f", 4.f)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(6, Map.of("f", 0.f)), + new MemoryTable.RecordResult(7, Map.of("f", 1.f)), + new MemoryTable.RecordResult(8, Map.of("f", 2.f)), + new MemoryTable.RecordResult(9, Map.of("f", 3.f)), + new MemoryTable.RecordResult(0, Map.of("f", 4.f))))); } @Test public void testQueryFilterFloat64Less() { var columns = Map.of("g", "g"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("g")), TableQueryFilter.builder() @@ -1218,19 +1181,18 @@ public void testQueryFilterFloat64Less() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("g", 0.), - Map.of("g", 1.), - Map.of("g", 2.), - Map.of("g", 3.), - Map.of("g", 4.)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(5, Map.of("g", 0.)), + new MemoryTable.RecordResult(6, Map.of("g", 1.)), + new MemoryTable.RecordResult(7, Map.of("g", 2.)), + new MemoryTable.RecordResult(8, Map.of("g", 3.)), + new MemoryTable.RecordResult(9, Map.of("g", 4.))))); } @Test public void testQueryFilterStringLess() { var columns = Map.of("h", "h"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("g")), TableQueryFilter.builder() @@ -1241,19 +1203,18 @@ public void testQueryFilterStringLess() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("h", "0"), - Map.of("h", "1"), - Map.of("h", "2"), - Map.of("h", "3"), - Map.of("h", "4")))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(4, Map.of("h", "0")), + new MemoryTable.RecordResult(5, Map.of("h", "1")), + new MemoryTable.RecordResult(6, Map.of("h", "2")), + new MemoryTable.RecordResult(7, Map.of("h", "3")), + new MemoryTable.RecordResult(8, Map.of("h", "4"))))); } @Test public void testQueryFilterBytesLess() { var columns = Map.of("i", "i"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("i")), TableQueryFilter.builder() @@ -1266,19 +1227,23 @@ public void testQueryFilterBytesLess() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("i", ByteBuffer.wrap("0".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("1".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("2".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("3".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("4".getBytes(StandardCharsets.UTF_8)))))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(3, + Map.of("i", ByteBuffer.wrap("0".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(4, + Map.of("i", ByteBuffer.wrap("1".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(5, + Map.of("i", ByteBuffer.wrap("2".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(6, + Map.of("i", ByteBuffer.wrap("3".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(7, + Map.of("i", ByteBuffer.wrap("4".getBytes(StandardCharsets.UTF_8))))))); } @Test public void testQueryFilterBoolLessEqual() { var columns = Map.of("a", "a", "d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("d")), TableQueryFilter.builder() @@ -1289,24 +1254,23 @@ public void testQueryFilterBoolLessEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), + assertThat(results, is(List.of( - Map.of("a", false), - Map.of("a", true, "d", 0), - Map.of("a", false, "d", 1), - Map.of("a", false, "d", 3), - Map.of("a", true, "d", 4), - Map.of("a", false, "d", 5), - Map.of("a", true, "d", 6), - Map.of("a", false, "d", 7), - Map.of("a", true, "d", 8)))); + new MemoryTable.RecordResult(7, Map.of("a", false)), + new MemoryTable.RecordResult(8, Map.of("a", true, "d", 0)), + new MemoryTable.RecordResult(9, Map.of("a", false, "d", 1)), + new MemoryTable.RecordResult(1, Map.of("a", false, "d", 3)), + new MemoryTable.RecordResult(2, Map.of("a", true, "d", 4)), + new MemoryTable.RecordResult(3, Map.of("a", false, "d", 5)), + new MemoryTable.RecordResult(4, Map.of("a", true, "d", 6)), + new MemoryTable.RecordResult(5, Map.of("a", false, "d", 7)), + new MemoryTable.RecordResult(6, Map.of("a", true, "d", 8))))); } @Test public void testQueryFilterInt8LessEqual() { var columns = Map.of("b", "b"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("b")), TableQueryFilter.builder() @@ -1317,20 +1281,19 @@ public void testQueryFilterInt8LessEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("b", (byte) 0), - Map.of("b", (byte) 1), - Map.of("b", (byte) 2), - Map.of("b", (byte) 3), - Map.of("b", (byte) 4), - Map.of("b", (byte) 5)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("b", (byte) 0)), + new MemoryTable.RecordResult(1, Map.of("b", (byte) 1)), + new MemoryTable.RecordResult(2, Map.of("b", (byte) 2)), + new MemoryTable.RecordResult(3, Map.of("b", (byte) 3)), + new MemoryTable.RecordResult(4, Map.of("b", (byte) 4)), + new MemoryTable.RecordResult(5, Map.of("b", (byte) 5))))); } @Test public void testQueryFilterInt16LessEqual() { var columns = Map.of("c", "c"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("c")), TableQueryFilter.builder() @@ -1341,20 +1304,19 @@ public void testQueryFilterInt16LessEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("c", (short) 0), - Map.of("c", (short) 1), - Map.of("c", (short) 2), - Map.of("c", (short) 3), - Map.of("c", (short) 4), - Map.of("c", (short) 5)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(9, Map.of("c", (short) 0)), + new MemoryTable.RecordResult(0, Map.of("c", (short) 1)), + new MemoryTable.RecordResult(1, Map.of("c", (short) 2)), + new MemoryTable.RecordResult(2, Map.of("c", (short) 3)), + new MemoryTable.RecordResult(3, Map.of("c", (short) 4)), + new MemoryTable.RecordResult(4, Map.of("c", (short) 5))))); } @Test public void testQueryFilterInt32LessEqual() { var columns = Map.of("d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("d")), TableQueryFilter.builder() @@ -1365,20 +1327,19 @@ public void testQueryFilterInt32LessEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("d", 0), - Map.of("d", 1), - Map.of("d", 2), - Map.of("d", 3), - Map.of("d", 4), - Map.of("d", 5)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(8, Map.of("d", 0)), + new MemoryTable.RecordResult(9, Map.of("d", 1)), + new MemoryTable.RecordResult(0, Map.of("d", 2)), + new MemoryTable.RecordResult(1, Map.of("d", 3)), + new MemoryTable.RecordResult(2, Map.of("d", 4)), + new MemoryTable.RecordResult(3, Map.of("d", 5))))); } @Test public void testQueryFilterInt64LessEqual() { var columns = Map.of("e", "e"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("e")), TableQueryFilter.builder() @@ -1389,20 +1350,19 @@ public void testQueryFilterInt64LessEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("e", 0L), - Map.of("e", 1L), - Map.of("e", 2L), - Map.of("e", 3L), - Map.of("e", 4L), - Map.of("e", 5L)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(7, Map.of("e", 0L)), + new MemoryTable.RecordResult(8, Map.of("e", 1L)), + new MemoryTable.RecordResult(9, Map.of("e", 2L)), + new MemoryTable.RecordResult(0, Map.of("e", 3L)), + new MemoryTable.RecordResult(1, Map.of("e", 4L)), + new MemoryTable.RecordResult(2, Map.of("e", 5L))))); } @Test public void testQueryFilterFloat32LessEqual() { var columns = Map.of("f", "f"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("f")), TableQueryFilter.builder() @@ -1413,20 +1373,19 @@ public void testQueryFilterFloat32LessEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("f", 0.f), - Map.of("f", 1.f), - Map.of("f", 2.f), - Map.of("f", 3.f), - Map.of("f", 4.f), - Map.of("f", 5.f)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(6, Map.of("f", 0.f)), + new MemoryTable.RecordResult(7, Map.of("f", 1.f)), + new MemoryTable.RecordResult(8, Map.of("f", 2.f)), + new MemoryTable.RecordResult(9, Map.of("f", 3.f)), + new MemoryTable.RecordResult(0, Map.of("f", 4.f)), + new MemoryTable.RecordResult(1, Map.of("f", 5.f))))); } @Test public void testQueryFilterFloat64LessEqual() { var columns = Map.of("g", "g"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("g")), TableQueryFilter.builder() @@ -1437,20 +1396,19 @@ public void testQueryFilterFloat64LessEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("g", 0.), - Map.of("g", 1.), - Map.of("g", 2.), - Map.of("g", 3.), - Map.of("g", 4.), - Map.of("g", 5.)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(5, Map.of("g", 0.)), + new MemoryTable.RecordResult(6, Map.of("g", 1.)), + new MemoryTable.RecordResult(7, Map.of("g", 2.)), + new MemoryTable.RecordResult(8, Map.of("g", 3.)), + new MemoryTable.RecordResult(9, Map.of("g", 4.)), + new MemoryTable.RecordResult(0, Map.of("g", 5.))))); } @Test public void testQueryFilterStringLessEqual() { var columns = Map.of("h", "h"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("g")), TableQueryFilter.builder() @@ -1461,20 +1419,19 @@ public void testQueryFilterStringLessEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("h", "0"), - Map.of("h", "1"), - Map.of("h", "2"), - Map.of("h", "3"), - Map.of("h", "4"), - Map.of("h", "5")))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(4, Map.of("h", "0")), + new MemoryTable.RecordResult(5, Map.of("h", "1")), + new MemoryTable.RecordResult(6, Map.of("h", "2")), + new MemoryTable.RecordResult(7, Map.of("h", "3")), + new MemoryTable.RecordResult(8, Map.of("h", "4")), + new MemoryTable.RecordResult(9, Map.of("h", "5"))))); } @Test public void testQueryFilterBytesLessEqual() { var columns = Map.of("i", "i"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("i")), TableQueryFilter.builder() @@ -1487,20 +1444,25 @@ public void testQueryFilterBytesLessEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("i", ByteBuffer.wrap("0".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("1".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("2".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("3".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("4".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("5".getBytes(StandardCharsets.UTF_8)))))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(3, + Map.of("i", ByteBuffer.wrap("0".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(4, + Map.of("i", ByteBuffer.wrap("1".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(5, + Map.of("i", ByteBuffer.wrap("2".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(6, + Map.of("i", ByteBuffer.wrap("3".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(7, + Map.of("i", ByteBuffer.wrap("4".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(8, + Map.of("i", ByteBuffer.wrap("5".getBytes(StandardCharsets.UTF_8))))))); } @Test public void testQueryFilterBoolGreater() { var columns = Map.of("a", "a", "d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("d")), TableQueryFilter.builder() @@ -1511,19 +1473,18 @@ public void testQueryFilterBoolGreater() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), + assertThat(results, is(List.of( - Map.of("a", true, "d", 0), - Map.of("a", true, "d", 4), - Map.of("a", true, "d", 6), - Map.of("a", true, "d", 8)))); + new MemoryTable.RecordResult(8, Map.of("a", true, "d", 0)), + new MemoryTable.RecordResult(2, Map.of("a", true, "d", 4)), + new MemoryTable.RecordResult(4, Map.of("a", true, "d", 6)), + new MemoryTable.RecordResult(6, Map.of("a", true, "d", 8))))); } @Test public void testQueryFilterInt8Greater() { var columns = Map.of("b", "b"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("b")), TableQueryFilter.builder() @@ -1534,17 +1495,16 @@ public void testQueryFilterInt8Greater() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("b", (byte) 6), - Map.of("b", (byte) 7), - Map.of("b", (byte) 8)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(6, Map.of("b", (byte) 6)), + new MemoryTable.RecordResult(7, Map.of("b", (byte) 7)), + new MemoryTable.RecordResult(8, Map.of("b", (byte) 8))))); } @Test public void testQueryFilterInt16Greater() { var columns = Map.of("c", "c"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("c")), TableQueryFilter.builder() @@ -1555,17 +1515,16 @@ public void testQueryFilterInt16Greater() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("c", (short) 6), - Map.of("c", (short) 7), - Map.of("c", (short) 8)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(5, Map.of("c", (short) 6)), + new MemoryTable.RecordResult(6, Map.of("c", (short) 7)), + new MemoryTable.RecordResult(7, Map.of("c", (short) 8))))); } @Test public void testQueryFilterInt32Greater() { var columns = Map.of("d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("d")), TableQueryFilter.builder() @@ -1576,17 +1535,16 @@ public void testQueryFilterInt32Greater() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("d", 6), - Map.of("d", 7), - Map.of("d", 8)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(4, Map.of("d", 6)), + new MemoryTable.RecordResult(5, Map.of("d", 7)), + new MemoryTable.RecordResult(6, Map.of("d", 8))))); } @Test public void testQueryFilterInt64Greater() { var columns = Map.of("e", "e"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("e")), TableQueryFilter.builder() @@ -1597,17 +1555,16 @@ public void testQueryFilterInt64Greater() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("e", 6L), - Map.of("e", 7L), - Map.of("e", 8L)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(3, Map.of("e", 6L)), + new MemoryTable.RecordResult(4, Map.of("e", 7L)), + new MemoryTable.RecordResult(5, Map.of("e", 8L))))); } @Test public void testQueryFilterFloat32Greater() { var columns = Map.of("f", "f"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("f")), TableQueryFilter.builder() @@ -1618,17 +1575,16 @@ public void testQueryFilterFloat32Greater() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("f", 6.f), - Map.of("f", 7.f), - Map.of("f", 8.f)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(2, Map.of("f", 6.f)), + new MemoryTable.RecordResult(3, Map.of("f", 7.f)), + new MemoryTable.RecordResult(4, Map.of("f", 8.f))))); } @Test public void testQueryFilterFloat64Greater() { var columns = Map.of("g", "g"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("g")), TableQueryFilter.builder() @@ -1639,17 +1595,16 @@ public void testQueryFilterFloat64Greater() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("g", 6.), - Map.of("g", 7.), - Map.of("g", 8.)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(1, Map.of("g", 6.)), + new MemoryTable.RecordResult(2, Map.of("g", 7.)), + new MemoryTable.RecordResult(3, Map.of("g", 8.))))); } @Test public void testQueryFilterStringGreater() { var columns = Map.of("h", "h"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("g")), TableQueryFilter.builder() @@ -1660,17 +1615,16 @@ public void testQueryFilterStringGreater() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("h", "6"), - Map.of("h", "7"), - Map.of("h", "8")))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("h", "6")), + new MemoryTable.RecordResult(1, Map.of("h", "7")), + new MemoryTable.RecordResult(2, Map.of("h", "8"))))); } @Test public void testQueryFilterBytesGreater() { var columns = Map.of("i", "i"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("i")), TableQueryFilter.builder() @@ -1683,17 +1637,19 @@ public void testQueryFilterBytesGreater() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("i", ByteBuffer.wrap("6".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("7".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("8".getBytes(StandardCharsets.UTF_8)))))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(9, + Map.of("i", ByteBuffer.wrap("6".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(0, + Map.of("i", ByteBuffer.wrap("7".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(1, + Map.of("i", ByteBuffer.wrap("8".getBytes(StandardCharsets.UTF_8))))))); } @Test public void testQueryFilterBoolGreaterEqual() { var columns = Map.of("a", "a", "d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("d")), TableQueryFilter.builder() @@ -1704,24 +1660,23 @@ public void testQueryFilterBoolGreaterEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), + assertThat(results, is(List.of( - Map.of("a", false), - Map.of("a", true, "d", 0), - Map.of("a", false, "d", 1), - Map.of("a", false, "d", 3), - Map.of("a", true, "d", 4), - Map.of("a", false, "d", 5), - Map.of("a", true, "d", 6), - Map.of("a", false, "d", 7), - Map.of("a", true, "d", 8)))); + new MemoryTable.RecordResult(7, Map.of("a", false)), + new MemoryTable.RecordResult(8, Map.of("a", true, "d", 0)), + new MemoryTable.RecordResult(9, Map.of("a", false, "d", 1)), + new MemoryTable.RecordResult(1, Map.of("a", false, "d", 3)), + new MemoryTable.RecordResult(2, Map.of("a", true, "d", 4)), + new MemoryTable.RecordResult(3, Map.of("a", false, "d", 5)), + new MemoryTable.RecordResult(4, Map.of("a", true, "d", 6)), + new MemoryTable.RecordResult(5, Map.of("a", false, "d", 7)), + new MemoryTable.RecordResult(6, Map.of("a", true, "d", 8))))); } @Test public void testQueryFilterInt8GreaterEqual() { var columns = Map.of("b", "b"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("b")), TableQueryFilter.builder() @@ -1732,18 +1687,17 @@ public void testQueryFilterInt8GreaterEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("b", (byte) 5), - Map.of("b", (byte) 6), - Map.of("b", (byte) 7), - Map.of("b", (byte) 8)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(5, Map.of("b", (byte) 5)), + new MemoryTable.RecordResult(6, Map.of("b", (byte) 6)), + new MemoryTable.RecordResult(7, Map.of("b", (byte) 7)), + new MemoryTable.RecordResult(8, Map.of("b", (byte) 8))))); } @Test public void testQueryFilterInt16GreaterEqual() { var columns = Map.of("c", "c"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("c")), TableQueryFilter.builder() @@ -1754,18 +1708,17 @@ public void testQueryFilterInt16GreaterEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("c", (short) 5), - Map.of("c", (short) 6), - Map.of("c", (short) 7), - Map.of("c", (short) 8)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(4, Map.of("c", (short) 5)), + new MemoryTable.RecordResult(5, Map.of("c", (short) 6)), + new MemoryTable.RecordResult(6, Map.of("c", (short) 7)), + new MemoryTable.RecordResult(7, Map.of("c", (short) 8))))); } @Test public void testQueryFilterInt32GreaterEqual() { var columns = Map.of("d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("d")), TableQueryFilter.builder() @@ -1776,18 +1729,17 @@ public void testQueryFilterInt32GreaterEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("d", 5), - Map.of("d", 6), - Map.of("d", 7), - Map.of("d", 8)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(3, Map.of("d", 5)), + new MemoryTable.RecordResult(4, Map.of("d", 6)), + new MemoryTable.RecordResult(5, Map.of("d", 7)), + new MemoryTable.RecordResult(6, Map.of("d", 8))))); } @Test public void testQueryFilterInt64GreaterEqual() { var columns = Map.of("e", "e"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("e")), TableQueryFilter.builder() @@ -1798,18 +1750,17 @@ public void testQueryFilterInt64GreaterEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("e", 5L), - Map.of("e", 6L), - Map.of("e", 7L), - Map.of("e", 8L)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(2, Map.of("e", 5L)), + new MemoryTable.RecordResult(3, Map.of("e", 6L)), + new MemoryTable.RecordResult(4, Map.of("e", 7L)), + new MemoryTable.RecordResult(5, Map.of("e", 8L))))); } @Test public void testQueryFilterFloat32GreaterEqual() { var columns = Map.of("f", "f"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("f")), TableQueryFilter.builder() @@ -1820,18 +1771,17 @@ public void testQueryFilterFloat32GreaterEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("f", 5.f), - Map.of("f", 6.f), - Map.of("f", 7.f), - Map.of("f", 8.f)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(1, Map.of("f", 5.f)), + new MemoryTable.RecordResult(2, Map.of("f", 6.f)), + new MemoryTable.RecordResult(3, Map.of("f", 7.f)), + new MemoryTable.RecordResult(4, Map.of("f", 8.f))))); } @Test public void testQueryFilterFloat64GreaterEqual() { var columns = Map.of("g", "g"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("g")), TableQueryFilter.builder() @@ -1842,18 +1792,17 @@ public void testQueryFilterFloat64GreaterEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("g", 5.), - Map.of("g", 6.), - Map.of("g", 7.), - Map.of("g", 8.)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("g", 5.)), + new MemoryTable.RecordResult(1, Map.of("g", 6.)), + new MemoryTable.RecordResult(2, Map.of("g", 7.)), + new MemoryTable.RecordResult(3, Map.of("g", 8.))))); } @Test public void testQueryFilterStringGreaterEqual() { var columns = Map.of("h", "h"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("g")), TableQueryFilter.builder() @@ -1864,18 +1813,17 @@ public void testQueryFilterStringGreaterEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("h", "5"), - Map.of("h", "6"), - Map.of("h", "7"), - Map.of("h", "8")))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(9, Map.of("h", "5")), + new MemoryTable.RecordResult(0, Map.of("h", "6")), + new MemoryTable.RecordResult(1, Map.of("h", "7")), + new MemoryTable.RecordResult(2, Map.of("h", "8"))))); } @Test public void testQueryFilterBytesGreaterEqual() { var columns = Map.of("i", "i"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("i")), TableQueryFilter.builder() @@ -1888,18 +1836,21 @@ public void testQueryFilterBytesGreaterEqual() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("i", ByteBuffer.wrap("5".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("6".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("7".getBytes(StandardCharsets.UTF_8))), - Map.of("i", ByteBuffer.wrap("8".getBytes(StandardCharsets.UTF_8)))))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(8, + Map.of("i", ByteBuffer.wrap("5".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(9, + Map.of("i", ByteBuffer.wrap("6".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(0, + Map.of("i", ByteBuffer.wrap("7".getBytes(StandardCharsets.UTF_8)))), + new MemoryTable.RecordResult(1, + Map.of("i", ByteBuffer.wrap("8".getBytes(StandardCharsets.UTF_8))))))); } @Test public void testQueryFilterNot() { var columns = Map.of("d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("d")), TableQueryFilter.builder() @@ -1916,23 +1867,22 @@ public void testQueryFilterNot() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of(), - Map.of("d", 0), - Map.of("d", 1), - Map.of("d", 2), - Map.of("d", 3), - Map.of("d", 4), - Map.of("d", 6), - Map.of("d", 7), - Map.of("d", 8)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(7, Map.of()), + new MemoryTable.RecordResult(8, Map.of("d", 0)), + new MemoryTable.RecordResult(9, Map.of("d", 1)), + new MemoryTable.RecordResult(0, Map.of("d", 2)), + new MemoryTable.RecordResult(1, Map.of("d", 3)), + new MemoryTable.RecordResult(2, Map.of("d", 4)), + new MemoryTable.RecordResult(4, Map.of("d", 6)), + new MemoryTable.RecordResult(5, Map.of("d", 7)), + new MemoryTable.RecordResult(6, Map.of("d", 8))))); } @Test public void testQueryFilterAnd() { var columns = Map.of("a", "a", "d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("a"), new OrderByDesc("d")), TableQueryFilter.builder() @@ -1955,16 +1905,15 @@ public void testQueryFilterAnd() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("a", true, "d", 0), - Map.of("a", true, "d", 4)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(8, Map.of("a", true, "d", 0)), + new MemoryTable.RecordResult(2, Map.of("a", true, "d", 4))))); } @Test public void testQueryFilterOr() { var columns = Map.of("a", "a", "d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("a"), new OrderByDesc("d")), TableQueryFilter.builder() @@ -1987,21 +1936,20 @@ public void testQueryFilterOr() { -1, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("d", 2), - Map.of("a", false, "d", 1), - Map.of("a", false, "d", 3), - Map.of("a", true, "d", 0), - Map.of("a", true, "d", 4), - Map.of("a", true, "d", 6), - Map.of("a", true, "d", 8)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("d", 2)), + new MemoryTable.RecordResult(9, Map.of("a", false, "d", 1)), + new MemoryTable.RecordResult(1, Map.of("a", false, "d", 3)), + new MemoryTable.RecordResult(8, Map.of("a", true, "d", 0)), + new MemoryTable.RecordResult(2, Map.of("a", true, "d", 4)), + new MemoryTable.RecordResult(4, Map.of("a", true, "d", 6)), + new MemoryTable.RecordResult(6, Map.of("a", true, "d", 8))))); } @Test public void testQueryStartLimit() { var columns = Map.of("d", "d"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("d")), null, @@ -2009,16 +1957,15 @@ public void testQueryStartLimit() { 2, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of("d", 4), - Map.of("d", 5)))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(2, Map.of("d", 4)), + new MemoryTable.RecordResult(3, Map.of("d", 5))))); } @Test public void testQueryUnknown() { var columns = Map.of("z", "z"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, List.of(new OrderByDesc("z")), null, @@ -2026,15 +1973,15 @@ public void testQueryUnknown() { 2, false, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(Map.of(), Map.of()))); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(5, Map.of()), + new MemoryTable.RecordResult(6, Map.of())))); } @Test public void testQueryKeepNone() { var columns = Map.of("a", "a"); - var records = this.memoryTable.query( + var results = this.memoryTable.query( columns, null, TableQueryFilter.builder() @@ -2045,20 +1992,17 @@ public void testQueryKeepNone() { -1, true, false); - assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(columns), records.getRecords()), - is(List.of(new HashMap() {{ + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, new HashMap<>() {{ put("a", null); - }}))); + }})))); } @Test public void testScanInitialEmptyTable() { this.memoryTable = new MemoryTableImpl("test", MemoryTableImplTest.this.walManager); - var it = this.memoryTable.scan(null, null, false, null, false, false, false); - var recordList = MemoryTableImplTest.getRecords(it); - assertThat("empty", it.getColumnTypeMapping(), nullValue()); - assertThat("empty", recordList, empty()); + var results = this.memoryTable.scan(Map.of("a", "a"), null, false, null, false, 1000, false); + assertThat("empty", results, empty()); } @Test @@ -2066,131 +2010,106 @@ public void testScanEmptyTableWithSchema() { this.memoryTable = new MemoryTableImpl("test", MemoryTableImplTest.this.walManager); this.memoryTable.update(new TableSchemaDesc("k", List.of(new ColumnSchemaDesc("k", "STRING"))), List.of(Map.of("k", "0", "-", "1"))); - var it = this.memoryTable.scan(null, null, false, null, false, false, false); - var recordList = MemoryTableImplTest.getRecords(it); - assertThat("empty", it.getColumnTypeMapping(), is(Map.of("k", ColumnType.STRING))); - assertThat("empty", recordList, empty()); - } - - @Test - public void testScanAll() { - var it = this.memoryTable.scan(null, null, false, null, false, false, false); - var recordList = MemoryTableImplTest.getRecords(it); - assertThat("all", - it.getColumnTypeMapping(), - is(this.memoryTable.getSchema().getColumnSchemas().stream() - .collect(Collectors.toMap(ColumnSchema::getName, ColumnSchema::getType)))); - assertThat("all", recordList, is(this.records)); + var results = this.memoryTable.scan(Map.of("a", "a"), null, false, null, false, 1000, false); + assertThat("empty", results, empty()); } @Test public void testScanColumnAliases() { - var it = this.memoryTable.scan(Map.of("a", "x", "d", "y"), null, false, null, false, false, false); - var recordList = MemoryTableImplTest.getRecords(it); - assertThat("columns", - it.getColumnTypeMapping(), - is(Map.of("x", ColumnType.BOOL, "y", ColumnType.INT32))); - assertThat("columns", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(Map.of("a", "x", "d", "y")), - recordList), - is(List.of(Map.of("y", 2), - Map.of("x", false, "y", 3), - Map.of("x", true, "y", 4), - Map.of("x", false, "y", 5), - Map.of("x", true, "y", 6), - Map.of("x", false, "y", 7), - Map.of("x", true, "y", 8), - Map.of("x", false), - Map.of("x", true, "y", 0), - Map.of("x", false, "y", 1)))); + var results = this.memoryTable.scan(Map.of("a", "x", "d", "y"), null, false, null, false, 1000, false); + assertThat(results, + is(List.of(new MemoryTable.RecordResult(0, Map.of("y", 2)), + new MemoryTable.RecordResult(1, Map.of("x", false, "y", 3)), + new MemoryTable.RecordResult(2, Map.of("x", true, "y", 4)), + new MemoryTable.RecordResult(3, Map.of("x", false, "y", 5)), + new MemoryTable.RecordResult(4, Map.of("x", true, "y", 6)), + new MemoryTable.RecordResult(5, Map.of("x", false, "y", 7)), + new MemoryTable.RecordResult(6, Map.of("x", true, "y", 8)), + new MemoryTable.RecordResult(7, Map.of("x", false)), + new MemoryTable.RecordResult(8, Map.of("x", true, "y", 0)), + new MemoryTable.RecordResult(9, Map.of("x", false, "y", 1))))); } @Test public void testScanStartEnd() { assertThat("start,non-inclusive", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(Map.of("d", "d")), - MemoryTableImplTest.getRecords( - this.memoryTable.scan(Map.of("d", "d"), "5", false, null, false, false, false))), - is(List.of(Map.of("d", 8), - Map.of(), - Map.of("d", 0), - Map.of("d", 1)))); + this.memoryTable.scan(Map.of("d", "d"), "5", false, null, false, 1000, false), + is(List.of(new MemoryTable.RecordResult(6, Map.of("d", 8)), + new MemoryTable.RecordResult(7, Map.of()), + new MemoryTable.RecordResult(8, Map.of("d", 0)), + new MemoryTable.RecordResult(9, Map.of("d", 1))))); assertThat("start,inclusive", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(Map.of("d", "d")), - MemoryTableImplTest.getRecords( - this.memoryTable.scan(Map.of("d", "d"), "5", true, null, false, false, false))), - is(List.of(Map.of("d", 7), - Map.of("d", 8), - Map.of(), - Map.of("d", 0), - Map.of("d", 1)))); + this.memoryTable.scan(Map.of("d", "d"), "5", true, null, false, 1000, false), + is(List.of(new MemoryTable.RecordResult(5, Map.of("d", 7)), + new MemoryTable.RecordResult(6, Map.of("d", 8)), + new MemoryTable.RecordResult(7, Map.of()), + new MemoryTable.RecordResult(8, Map.of("d", 0)), + new MemoryTable.RecordResult(9, Map.of("d", 1))))); assertThat("end,non-inclusive", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(Map.of("d", "d")), - MemoryTableImplTest.getRecords( - this.memoryTable.scan(Map.of("d", "d"), null, false, "5", false, false, false))), - is(List.of(Map.of("d", 2), - Map.of("d", 3), - Map.of("d", 4), - Map.of("d", 5), - Map.of("d", 6)))); + this.memoryTable.scan(Map.of("d", "d"), null, false, "5", false, 1000, false), + is(List.of(new MemoryTable.RecordResult(0, Map.of("d", 2)), + new MemoryTable.RecordResult(1, Map.of("d", 3)), + new MemoryTable.RecordResult(2, Map.of("d", 4)), + new MemoryTable.RecordResult(3, Map.of("d", 5)), + new MemoryTable.RecordResult(4, Map.of("d", 6))))); assertThat("end,inclusive", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(Map.of("d", "d")), - MemoryTableImplTest.getRecords( - this.memoryTable.scan(Map.of("d", "d"), null, false, "5", true, false, false))), - is(List.of(Map.of("d", 2), - Map.of("d", 3), - Map.of("d", 4), - Map.of("d", 5), - Map.of("d", 6), - Map.of("d", 7)))); + this.memoryTable.scan(Map.of("d", "d"), null, false, "5", true, 1000, false), + is(List.of(new MemoryTable.RecordResult(0, Map.of("d", 2)), + new MemoryTable.RecordResult(1, Map.of("d", 3)), + new MemoryTable.RecordResult(2, Map.of("d", 4)), + new MemoryTable.RecordResult(3, Map.of("d", 5)), + new MemoryTable.RecordResult(4, Map.of("d", 6)), + new MemoryTable.RecordResult(5, Map.of("d", 7))))); assertThat("start+end", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(Map.of("d", "d")), - MemoryTableImplTest.getRecords( - this.memoryTable.scan(Map.of("d", "d"), "2", true, "5", false, false, false))), - is(List.of(Map.of("d", 4), - Map.of("d", 5), - Map.of("d", 6)))); + this.memoryTable.scan(Map.of("d", "d"), "2", true, "5", false, 1000, false), + is(List.of(new MemoryTable.RecordResult(2, Map.of("d", 4)), + new MemoryTable.RecordResult(3, Map.of("d", 5)), + new MemoryTable.RecordResult(4, Map.of("d", 6))))); + } + + @Test + public void testScanLimit() { + assertThat("test", + this.memoryTable.scan(Map.of("d", "d"), null, false, null, false, 2, true), + is(List.of(new MemoryTable.RecordResult(0, Map.of("d", 2)), + new MemoryTable.RecordResult(1, Map.of("d", 3))))); } @Test public void testScanKeepNone() { assertThat("test", - decodeRecords(this.memoryTable.getSchema().getColumnTypeMapping(Map.of("d", "d")), - MemoryTableImplTest.getRecords( - this.memoryTable.scan(Map.of("d", "d"), null, false, null, false, true, false))), - is(List.of(Map.of("d", 2), - Map.of("d", 3), - Map.of("d", 4), - Map.of("d", 5), - Map.of("d", 6), - Map.of("d", 7), - Map.of("d", 8), - new HashMap() {{ + this.memoryTable.scan(Map.of("d", "d"), null, false, null, false, 1000, true), + is(List.of(new MemoryTable.RecordResult(0, Map.of("d", 2)), + new MemoryTable.RecordResult(1, Map.of("d", 3)), + new MemoryTable.RecordResult(2, Map.of("d", 4)), + new MemoryTable.RecordResult(3, Map.of("d", 5)), + new MemoryTable.RecordResult(4, Map.of("d", 6)), + new MemoryTable.RecordResult(5, Map.of("d", 7)), + new MemoryTable.RecordResult(6, Map.of("d", 8)), + new MemoryTable.RecordResult(7, new HashMap<>() {{ put("d", null); - }}, - Map.of("d", 0), - Map.of("d", 1)))); + }}), + new MemoryTable.RecordResult(8, Map.of("d", 0)), + new MemoryTable.RecordResult(9, Map.of("d", 1))))); } @Test public void testScanUnknown() { - assertThat("test", - MemoryTableImplTest.getRecords( - this.memoryTable.scan(Map.of("z", "z"), null, false, null, false, false, false)), - is(List.of(Map.of(), - Map.of(), - Map.of(), - Map.of(), - Map.of(), - Map.of(), - Map.of(), - Map.of(), - Map.of(), - Map.of()))); + assertThat(this.memoryTable.scan(Map.of("z", "z"), null, false, null, false, 1000, false), + is(List.of(new MemoryTable.RecordResult(0, Map.of()), + new MemoryTable.RecordResult(1, Map.of()), + new MemoryTable.RecordResult(2, Map.of()), + new MemoryTable.RecordResult(3, Map.of()), + new MemoryTable.RecordResult(4, Map.of()), + new MemoryTable.RecordResult(5, Map.of()), + new MemoryTable.RecordResult(6, Map.of()), + new MemoryTable.RecordResult(7, Map.of()), + new MemoryTable.RecordResult(8, Map.of()), + new MemoryTable.RecordResult(9, Map.of())))); } } }