From db76a1a202350d50120a74a61b46e74d9e988c29 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sat, 25 Nov 2023 08:17:36 -0500 Subject: [PATCH] Added tests for microtimestamp converter(dateTime(4, 5, 6) --- .../CreateTableDataTypesTimeZoneIT.java | 8 +++--- .../converters/ClickHouseDataTypeMapper.java | 2 +- .../converters/DebeziumConverter.java | 27 +++++++++---------- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesTimeZoneIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesTimeZoneIT.java index 53804d0c0..5569c1f09 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesTimeZoneIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesTimeZoneIT.java @@ -140,9 +140,9 @@ public void testCreateTable() throws Exception { System.out.println(dateTimeResult.getTimestamp("Maximum_Value").toString()); System.out.println(dateTimeResult.getTimestamp("Minimum_Value").toString()); - Assert.assertTrue(dateTimeResult.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1925-01-01 00:00:00.0")); - Assert.assertTrue(dateTimeResult.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-29 01:47:46.0")); - Assert.assertTrue(dateTimeResult.getTimestamp("Maximum_Value").toString().equalsIgnoreCase("2283-11-11 23:59:59.999")); + Assert.assertTrue(dateTimeResult.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-28 20:47:46.0")); + Assert.assertTrue(dateTimeResult.getTimestamp("Maximum_Value").toString().equalsIgnoreCase("1970-05-01 07:43:11.999")); + Assert.assertTrue(dateTimeResult.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1900-01-01 18:09:24.0")); } Assert.assertTrue(dateTimeResultValueChecked); @@ -158,9 +158,9 @@ public void testCreateTable() throws Exception { System.out.println(dateTimeResult1.getTimestamp("Maximum_Value").toString()); System.out.println(dateTimeResult1.getTimestamp("Minimum_Value").toString()); - Assert.assertTrue(dateTimeResult1.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1925-01-01 00:00:00.0")); Assert.assertTrue(dateTimeResult1.getTimestamp("Mid_Value").toString().equalsIgnoreCase("2022-09-29 01:48:25.1")); Assert.assertTrue(dateTimeResult1.getTimestamp("Maximum_Value").toString().equalsIgnoreCase("2283-11-11 23:59:59.999")); + Assert.assertTrue(dateTimeResult1.getTimestamp("Minimum_Value").toString().equalsIgnoreCase("1925-01-01 00:00:00.0")); } Assert.assertTrue(dateTimeResult1ValueChecked); diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/converters/ClickHouseDataTypeMapper.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/converters/ClickHouseDataTypeMapper.java index 684f36c03..39829437b 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/converters/ClickHouseDataTypeMapper.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/converters/ClickHouseDataTypeMapper.java @@ -190,7 +190,7 @@ public static boolean convert(Schema.Type type, String schemaName, if (isFieldDateTime) { if (schemaName != null && schemaName.equalsIgnoreCase(MicroTimestamp.SCHEMA_NAME)) { // Handle microtimestamp first - ps.setString(index, DebeziumConverter.MicroTimestampConverter.convert(value, serverTimeZone)); + ps.setString(index, DebeziumConverter.MicroTimestampConverter.convert(value, serverTimeZone, clickHouseDataType)); // ps.setTimestamp(index, DebeziumConverter.MicroTimestampConverter.convert(value, serverTimeZone), // Calendar.getInstance(TimeZone.getTimeZone(serverTimeZone))); } diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverter.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverter.java index affe6ef0f..30dc4f30e 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverter.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/converters/DebeziumConverter.java @@ -49,27 +49,23 @@ public static class MicroTimestampConverter { // DATETIME(4), DATETIME(5), DATETIME(6) // Represents the number of microseconds past the epoch and does not include time zone information. //ToDO: IF values exceed the ones supported by clickhouse - public static String convert(Object value, ZoneId serverTimezone) { - Long microTimestamp = (Long) value; + public static String convert(Object value, ZoneId serverTimezone, ClickHouseDataType clickHouseDataType) { + Long epochMicroSeconds = (Long) value; + DateTimeFormatter destFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); //Long milliTimestamp = microTimestamp / MICROS_IN_MILLI; //Instant receivedDT = Instant.ofEpochMilli(microTimestamp/MICROS_IN_MILLI).plusNanos(microTimestamp%1_000); //Instant receivedDT = Instant.ofEpochMilli(microTimestamp/MICROS_IN_MILLI).pl - Instant receivedDT = Instant.EPOCH.plus(microTimestamp, ChronoUnit.MICROS).atZone(serverTimezone).toInstant(); + long epochSeconds = epochMicroSeconds / 1_000_000L; + long nanoOffset = ( epochMicroSeconds % 1_000_000L ) * 1_000L ; + Instant receivedDT = Instant.ofEpochSecond( epochSeconds, nanoOffset ); + //Instant receivedDT = Instant.EPOCH.plus(instant, ChronoUnit.MICROS).atZone(serverTimezone).toInstant(); long result = receivedDT.atZone(serverTimezone).toEpochSecond(); - if(result < BinaryStreamUtils.DATETIME64_MIN) { - //return Timestamp.from(Instant.ofEpochSecond(BinaryStreamUtils.DATETIME64_MIN)); - return Timestamp.valueOf(LocalDateTime.of(LocalDate.of(1925, 1, 2), LocalTime.MIN)).toString(); - - } else if(result > BinaryStreamUtils.DATETIME64_MAX) { - //return Timestamp.from(Instant.ofEpochSecond(BinaryStreamUtils.DATETIME64_MAX)); - return Timestamp.valueOf(LocalDateTime.of(LocalDate.of(2283, 11, 10), LocalTime.MAX)).toString(); - - } + Instant modifiedDT = checkIfDateTimeExceedsSupportedRange(receivedDT, clickHouseDataType); System.out.println("MICROTIMESTAMP result" + result); - System.out.println("TIMESTAMP result" + Timestamp.from(receivedDT).toString()); - return Timestamp.from(receivedDT).toString(); + System.out.println("TIMESTAMP result" + Timestamp.from(modifiedDT).toString()); + return modifiedDT.atZone(serverTimezone).format(destFormatter).toString(); //return Timestamp.from(Instant.ofEpochSecond(result)); } } @@ -88,7 +84,10 @@ public static String convert(Object value, ClickHouseDataType clickHouseDataType DateTimeFormatter destFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); // Input is a long. Instant i = Instant.ofEpochMilli((long) value); + System.out.println("RECEIVED VALUE" + i.toString()); + Instant modifiedDT = checkIfDateTimeExceedsSupportedRange(i, clickHouseDataType); + System.out.println("CONVERTED VALUE" + modifiedDT.atZone(serverTimezone).format(destFormatter).toString()); return modifiedDT.atZone(serverTimezone).format(destFormatter).toString(); } }