Improved entity serialization

Author: Intelli

src/main/java/net/coreprotect/database/DuckDBDatabase.java

@@ -85,8 +85,8 @@ static void createTables(String prefix, Connection forceConnection, boolean purg
                 statement.executeUpdate("CREATE TABLE IF NOT EXISTS " + prefix + "entity_interaction (" + rowId(prefix, "entity_interaction") + ", time INTEGER, \"user\" INTEGER, entity_spawn_rowid INTEGER NOT NULL, wid INTEGER, x INTEGER, y INTEGER, z INTEGER, type INTEGER, action TINYINT, metadata BLOB, rolled_back TINYINT)");
                 statement.executeUpdate("CREATE TABLE IF NOT EXISTS " + prefix + "item (" + rowId(prefix, "item") + ", time INTEGER, \"user\" INTEGER, wid INTEGER, x INTEGER, y INTEGER, z INTEGER, type INTEGER, data BLOB, amount INTEGER, action TINYINT, rolled_back TINYINT)");
                 statement.executeUpdate("CREATE TABLE IF NOT EXISTS " + prefix + "database_lock (rowid INTEGER PRIMARY KEY, status TINYINT, time INTEGER)");
-                statement.executeUpdate("CREATE TABLE IF NOT EXISTS " + prefix + "entity (" + rowId(prefix, "entity") + ", time INTEGER, data VARCHAR)");
-                statement.executeUpdate("CREATE TABLE IF NOT EXISTS " + prefix + "entity_spawn (" + rowId(prefix, "entity_spawn") + ", time INTEGER, block_rowid BIGINT, kill_rowid INTEGER, uuid VARCHAR UNIQUE, wid INTEGER, current_wid INTEGER, origin_x DOUBLE, origin_y DOUBLE, origin_z DOUBLE, x DOUBLE, y DOUBLE, z DOUBLE, yaw FLOAT, pitch FLOAT, data VARCHAR, removed TINYINT, UNIQUE(kill_rowid))");
+                statement.executeUpdate("CREATE TABLE IF NOT EXISTS " + prefix + "entity (" + rowId(prefix, "entity") + ", time INTEGER, data BLOB)");
+                statement.executeUpdate("CREATE TABLE IF NOT EXISTS " + prefix + "entity_spawn (" + rowId(prefix, "entity_spawn") + ", time INTEGER, block_rowid BIGINT, kill_rowid INTEGER, uuid VARCHAR UNIQUE, wid INTEGER, current_wid INTEGER, origin_x DOUBLE, origin_y DOUBLE, origin_z DOUBLE, x DOUBLE, y DOUBLE, z DOUBLE, yaw FLOAT, pitch FLOAT, data BLOB, removed TINYINT, UNIQUE(kill_rowid))");
                 statement.executeUpdate("CREATE TABLE IF NOT EXISTS " + prefix + "entity_map (" + rowId(prefix, "entity_map") + ", id INTEGER, entity VARCHAR)");
                 statement.executeUpdate("CREATE TABLE IF NOT EXISTS " + prefix + "material_map (" + rowId(prefix, "material_map") + ", id INTEGER, material VARCHAR)");
                 statement.executeUpdate("CREATE TABLE IF NOT EXISTS " + prefix + "blockdata_map (" + rowId(prefix, "blockdata_map") + ", id INTEGER, data VARCHAR)");
@@ -129,8 +129,8 @@ private static void validateEntityDataColumns(Connection connection, String pref
         }
         for (String table : new String[] { entityTable, entitySpawnTable }) {
             String type = types.get(table);
-            if (!"VARCHAR".equalsIgnoreCase(type)) {
-                throw new SQLException("Unsupported DuckDB " + table + ".data format: " + (type == null ? "missing" : type) + " (expected VARCHAR)");
+            if (!"BLOB".equalsIgnoreCase(type)) {
+                throw new SQLException("Unsupported DuckDB " + table + ".data format: " + (type == null ? "missing" : type) + " (expected BLOB)");
             }
         }
     }

src/main/java/net/coreprotect/database/RelationalConsumerWriteBatch.java

@@ -18,7 +18,6 @@
 import net.coreprotect.config.ConfigHandler;
 import net.coreprotect.database.statement.EntitySpawnStatement;
 import net.coreprotect.utility.ErrorReporter;
-import net.coreprotect.utility.serialize.EntityDataCodec;
 
 public final class RelationalConsumerWriteBatch implements ConsumerWriteBatch {
 
@@ -612,10 +611,10 @@ private PreparedStatement entitySpawnStatement() throws SQLException {
 
     private static void setDuckDBEntityData(PreparedStatement statement, int index, byte[] data) throws SQLException {
         if (data == null) {
-            statement.setNull(index, Types.VARCHAR);
+            statement.setNull(index, Types.BLOB);
         }
         else {
-            statement.setString(index, EntityDataCodec.toText(data));
+            statement.setBytes(index, data);
         }
     }
 

src/main/java/net/coreprotect/database/clickhouse/ClickHouseEntitySpawnUpdates.java

@@ -26,15 +26,15 @@
 import net.coreprotect.model.entity.EntityContainerRollbackUpdate;
 import net.coreprotect.model.entity.EntitySpawnData;
 import net.coreprotect.model.entity.EntitySpawnIdentity;
+import net.coreprotect.utility.DatabaseUtils;
 import net.coreprotect.utility.ErrorReporter;
 import net.coreprotect.utility.WorldUtils;
-import net.coreprotect.utility.serialize.EntityDataCodec;
 
 final class ClickHouseEntitySpawnUpdates implements ConsumerEntitySpawnUpdates {
 
     private static final int SELECT_BATCH_SIZE = 500;
     private static final String COLUMNS = "rowid,if(block_rowid_present=1,block_rowid,NULL) AS block_rowid,if(kill_rowid_present=1,kill_rowid,NULL) AS kill_rowid,uuid,current_wid,current_x,current_y,current_z,yaw,pitch,"
-            + "if(entity_data_present=1,entity_data,NULL) AS data"
+            + ClickHouseSchema.binary("if(entity_data_present=1,entity_data,NULL)", "data")
             + ",removed,producer_id,producer_sequence,batch_ordinal,time,wid,x AS key_x,z AS key_z";
 
     private final ClickHouseConsumerWriteBatch owner;
@@ -535,8 +535,7 @@ private static void rejectDifferentOwner(ClickHouseEntityState owner, Integer al
     private ClickHouseEntityState readState(ResultSet resultSet) throws Exception {
         int rowId = resultSet.getInt("rowid");
         ClickHouseEventPointer pointer = new ClickHouseEventPointer(datasetId, ClickHouseFamily.ENTITY_SPAWN, UUID.fromString(resultSet.getString("producer_id")), resultSet.getLong("producer_sequence"), resultSet.getInt("batch_ordinal"), rowId, resultSet.getInt("time"), resultSet.getInt("wid"), resultSet.getInt("key_x"), resultSet.getInt("key_z"));
-        String text = resultSet.getString("data");
-        byte[] data = text == null ? null : EntityDataCodec.fromText(text);
+        byte[] data = DatabaseUtils.getBytes(resultSet, "data");
         return new ClickHouseEntityState(pointer, nullableLong(resultSet, "block_rowid"), nullableInteger(resultSet, "kill_rowid"), UUID.fromString(resultSet.getString("uuid")), resultSet.getInt("current_wid"), resultSet.getDouble("current_x"), resultSet.getDouble("current_y"), resultSet.getDouble("current_z"), resultSet.getFloat("yaw"), resultSet.getFloat("pitch"), data, resultSet.getInt("removed") == 1);
     }
 

src/main/java/net/coreprotect/database/clickhouse/ClickHouseEventBatch.java

@@ -106,7 +106,7 @@ public long addDatabaseLockVersion(long rowId, int time, int status) throws SQLE
 
     public long addEntity(int time, byte[] data) throws SQLException {
         long rowId = beginRow(ClickHouseFamily.ENTITY, time);
-        setEntityText("payload", data);
+        setEntityData("payload", data);
         commitRow(ClickHouseFamily.ENTITY, rowId);
         return rowId;
     }
@@ -130,7 +130,7 @@ public long addEntitySpawn(int time, Long blockRowId, Integer killRowId, UUID uu
         set("current_z", currentZ);
         set("yaw", yaw);
         set("pitch", pitch);
-        setEntityText("entity_data", data);
+        setEntityData("entity_data", data);
         set("entity_data_present", data == null ? 0 : 1);
         set("removed", removed);
         commitRow(ClickHouseFamily.ENTITY_SPAWN, rowId);
@@ -249,7 +249,7 @@ public void addCompatibilityRow(ClickHouseFamily family, long rowId, Map<String,
             }
             Object value = entry.getValue();
             if (canonicalColumn.equals("data") && (family == ClickHouseFamily.ENTITY || family == ClickHouseFamily.ENTITY_SPAWN)) {
-                value = entityText(value);
+                value = entityData(value);
             }
             String physicalColumn = compatibilityColumn(family, canonicalColumn);
             set(physicalColumn, value);
@@ -400,18 +400,24 @@ private void setBinary(String column, byte[] value) {
         set(column, value);
     }
 
-    private void setEntityText(String column, byte[] value) {
-        set(column, value == null ? null : EntityDataCodec.toText(value));
+    private void setEntityData(String column, byte[] value) {
+        if (value != null && !EntityDataCodec.isEncoded(value)) {
+            throw new IllegalArgumentException("Entity data does not use the CoreProtect binary format");
+        }
+        set(column, value);
     }
 
-    private static Object entityText(Object value) {
-        if (value instanceof byte[]) {
-            return EntityDataCodec.toText((byte[]) value);
+    private static Object entityData(Object value) {
+        if (value == null) {
+            return null;
         }
-        if (value instanceof String) {
-            EntityDataCodec.fromText((String) value);
+        if (value instanceof byte[]) {
+            if (!EntityDataCodec.isEncoded((byte[]) value)) {
+                throw new IllegalArgumentException("Entity data does not use the CoreProtect binary format");
+            }
+            return value;
         }
-        return value;
+        throw new IllegalArgumentException("ClickHouse entity data must be binary");
     }
 
     private void set(String column, Object value) {

src/main/java/net/coreprotect/database/clickhouse/ClickHouseSchema.java

@@ -285,7 +285,7 @@ private static void addCompatibilityViews(List<String> statements, Names names)
         statements.add(view(names, ClickHouseFamily.ENTITY_INTERACTION, "e.rowid AS rowid,e.time AS time,e.user_id AS `user`,e.entity_spawn_rowid AS entity_spawn_rowid," + location("wid") + "," + location("x") + ",e.y AS y," + location("z") + ",e.type AS type,e.action AS action," + binary("e.metadata", "metadata") + ",e.rolled_back AS rolled_back"));
         statements.add(rollbackView(names, ClickHouseFamily.ITEM, "e.rowid AS rowid,e.time AS time,e.user_id AS `user`," + location("wid") + "," + location("x") + ",e.y AS y," + location("z") + ",e.type AS type," + binary("e.payload", "data") + ",e.amount AS amount,e.action AS action"));
         statements.add(currentView(names, ClickHouseFamily.DATABASE_LOCK, "e.rowid AS rowid,e.status AS status,e.database_lock_time AS time"));
-        statements.add(view(names, ClickHouseFamily.ENTITY, "e.rowid AS rowid,e.time AS time,e.payload AS data"));
+        statements.add(view(names, ClickHouseFamily.ENTITY, "e.rowid AS rowid,e.time AS time," + binary("e.payload", "data")));
         statements.add(entitySpawnView(names));
         statements.add(view(names, ClickHouseFamily.ENTITY_MAP, "e.rowid AS rowid,e.id AS id,e.name AS entity"));
         statements.add(view(names, ClickHouseFamily.MATERIAL_MAP, "e.rowid AS rowid,e.id AS id,e.name AS material"));
@@ -323,7 +323,7 @@ private static String entitySpawnView(Names names) {
                 + ",e.uuid AS uuid," + location("wid") + ",e.current_wid AS current_wid"
                 + ",e.origin_x AS origin_x,e.origin_y AS origin_y,e.origin_z AS origin_z"
                 + ",e.current_x AS x,e.current_y AS y,e.current_z AS z"
-                + ",e.yaw AS yaw,e.pitch AS pitch,if(e.entity_data_present=1,e.entity_data,NULL) AS data,e.removed AS removed"
+                + ",e.yaw AS yaw,e.pitch AS pitch," + binary("if(e.entity_data_present=1,e.entity_data,NULL)", "data") + ",e.removed AS removed"
                 + " FROM " + currentEvents(names, ClickHouseFamily.ENTITY_SPAWN) + " AS e";
     }
 

src/main/java/net/coreprotect/database/clickhouse/ClickHouseStateBatch.java

@@ -6,8 +6,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import net.coreprotect.utility.serialize.EntityDataCodec;
-
 final class ClickHouseStateBatch implements AutoCloseable {
 
     private final ClickHouseBatchIdentity identity;
@@ -77,7 +75,7 @@ void appendTo(ClickHouseRowBinaryBuffer rows, int firstOrdinal) throws SQLExcept
             rows.set("yaw", state.getYaw());
             rows.set("pitch", state.getPitch());
             byte[] data = state.getData();
-            rows.set("entity_data", data == null ? null : EntityDataCodec.toText(data));
+            rows.set("entity_data", data);
             rows.set("entity_data_present", data == null ? 0 : 1);
             rows.set("removed", state.isRemoved() ? 1 : 0);
             rows.commitRow("entity state update");

src/main/java/net/coreprotect/database/statement/EntitySpawnStatement.java

@@ -36,7 +36,6 @@
 import net.coreprotect.utility.ErrorReporter;
 import net.coreprotect.utility.EntitySpawnTracking;
 import net.coreprotect.utility.WorldUtils;
-import net.coreprotect.utility.serialize.EntityDataCodec;
 import net.coreprotect.utility.serialize.EntityDataCodec.Kind;
 
 public final class EntitySpawnStatement {
@@ -672,10 +671,7 @@ private void setLocation(PreparedStatement statement, Location value, int offset
 
         private void setNullableData(PreparedStatement statement, int index, byte[] value) throws Exception {
             if (value == null) {
-                statement.setNull(index, databaseType.isDuckDB() ? Types.VARCHAR : Types.BLOB);
-            }
-            else if (databaseType.isDuckDB()) {
-                statement.setString(index, EntityDataCodec.toText(value));
+                statement.setNull(index, Types.BLOB);
             }
             else {
                 statement.setBytes(index, value);

src/main/java/net/coreprotect/database/statement/EntityStatement.java

@@ -208,15 +208,7 @@ public static List<Object> deserializeData(byte[] data, Kind kind) {
     }
 
     public static List<Object> readData(ResultSet resultSet, String column, Kind kind) throws SQLException {
-        byte[] data;
-        if (ConfigHandler.databaseType.isColumnar()) {
-            String text = resultSet.getString(column);
-            data = text == null ? null : EntityDataCodec.fromText(text);
-        }
-        else {
-            data = DatabaseUtils.getBytes(resultSet, column);
-        }
-        return deserializeData(data, kind);
+        return deserializeData(DatabaseUtils.getBytes(resultSet, column), kind);
     }
 
     private static List<Object> deserializeDataStrict(byte[] data, Kind kind) throws Exception {