Skip to content

Adding missing col combination assert for empty #161

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
taherkl merged 9 commits into from
Mar 24, 2025
Merged

Adding missing col combination assert for empty #161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
59ad876
ADDED Missing Combination
pawankashyapollion Mar 24, 2025
e2b43e9
Address the assert the same
pawankashyapollion Mar 24, 2025
95812a8
Merge branch 'acceptance_test_fix' into Adding_missing_col_combination
pawankashyapollion Mar 24, 2025
ace034f
Schema Fixes
pawankashyapollion Mar 24, 2025
d3961e3
Schema FIxes
pawankashyapollion Mar 24, 2025
64bb152
Added SCHEMA FIXES
pawankashyapollion Mar 24, 2025
e49d2b0
Schema Correction
pawankashyapollion Mar 24, 2025
96aea48
Added Fixes
pawankashyapollion Mar 24, 2025
0b693ea
Added Schema Fixes
pawankashyapollion Mar 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
175 changes: 131 additions & 44 deletions ...db/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToCassandraSourceDbIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
Expand Down Expand Up @@ -189,7 +188,7 @@ public void spannerToCasandraSourceDbBasic() throws InterruptedException, IOExce
*/
@Test
public void spannerToCasandraSourceDbJSONEmptyOperation()
throws InterruptedException, IOException {
throws InterruptedException, IOException, MultipleFailureException {
assertThatPipeline(jobInfo).isRunning();
writeJSONEmptyInSpanner();
assertJSONEmptyRowInCassandraDB();
Expand All @@ -206,125 +205,125 @@ private void writeJSONEmptyInSpanner() {
.set("double_float_map_col")
.to(Value.json("{}"))
.set("decimal_set_col")
.to(Value.numericArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("date_double_map_col")
.to(Value.json("{}"))
.set("uuid_ascii_map_col")
.to(Value.json("{}"))
.set("ascii_text_map_col")
.to(Value.json("{}"))
.set("timestamp_list_col")
.to(Value.timestampArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("int_set_col")
.to(Value.int64Array(Collections.emptyList()))
.to(Value.json("[]"))
.set("smallint_set_col")
.to(Value.int64Array(Collections.emptyList()))
.to(Value.json("[]"))
.set("varchar_list_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("inet_list_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("bigint_list_col")
.to(Value.int64Array(Collections.emptyList()))
.to(Value.json("[]"))
.set("tinyint_varint_map_col")
.to(Value.json("{}"))
.set("text_set_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("double_set_col")
.to(Value.float64Array(Collections.emptyList()))
.to(Value.json("[]"))
.set("time_list_col")
.to(Value.timestampArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("frozen_ascii_list_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("int_list_col")
.to(Value.int64Array(Collections.emptyList()))
.to(Value.json("[]"))
.set("ascii_list_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("date_set_col")
.to(Value.dateArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("double_inet_map_col")
.to(Value.json("{}"))
.set("timestamp_set_col")
.to(Value.timestampArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("time_tinyint_map_col")
.to(Value.json("{}"))
.set("bigint_set_col")
.to(Value.int64Array(Collections.emptyList()))
.to(Value.json("[]"))
.set("varchar_set_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("tinyint_set_col")
.to(Value.int64Array(Collections.emptyList()))
.to(Value.json("[]"))
.set("bigint_boolean_map_col")
.to(Value.json("{}"))
.set("text_list_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("boolean_list_col")
.to(Value.boolArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("blob_list_col")
.to(Value.bytesArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("timeuuid_set_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("int_time_map_col")
.to(Value.json("{}"))
.set("time_set_col")
.to(Value.timestampArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("boolean_set_col")
.to(Value.boolArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("float_set_col")
.to(Value.float64Array(Collections.emptyList()))
.to(Value.json("[]"))
.set("ascii_set_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("uuid_list_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("varchar_bigint_map_col")
.to(Value.json("{}"))
.set("blob_int_map_col")
.to(Value.json("{}"))
.set("varint_blob_map_col")
.to(Value.json("{}"))
.set("double_list_col")
.to(Value.float64Array(Collections.emptyList()))
.to(Value.json("[]"))
.set("float_list_col")
.to(Value.float64Array(Collections.emptyList()))
.to(Value.json("[]"))
.set("smallint_list_col")
.to(Value.int64Array(Collections.emptyList()))
.to(Value.json("[]"))
.set("varint_list_col")
.to(Value.numericArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("float_smallint_map_col")
.to(Value.json("{}"))
.set("smallint_timestamp_map_col")
.to(Value.json("{}"))
.set("text_timeuuid_map_col")
.to(Value.json("{}"))
.set("timeuuid_list_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("date_list_col")
.to(Value.dateArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("uuid_set_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("boolean_decimal_map_col")
.to(Value.json("{}"))
.set("blob_set_col")
.to(Value.bytesArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("inet_text_map_col")
.to(Value.json("{}"))
.set("varint_set_col")
.to(Value.numericArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("tinyint_list_col")
.to(Value.int64Array(Collections.emptyList()))
.to(Value.json("[]"))
.set("timestamp_uuid_map_col")
.to(Value.json("{}"))
.set("decimal_duration_map_col")
.to(Value.json("{}"))
.set("decimal_list_col")
.to(Value.numericArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("inet_set_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("timeuuid_varchar_map_col")
.to(Value.json("{}"))
.set("duration_list_col")
.to(Value.stringArray(Collections.emptyList()))
.to(Value.json("[]"))
.set("frozen_ascii_set_col")
.to(Value.stringArray(Collections.emptyList()));
.to(Value.json("[]"));

Mutation mutation = mutationBuilder.build();
spannerResourceManager.write(mutation);
Expand All @@ -337,13 +336,101 @@ private void writeJSONEmptyInSpanner() {
* condition.
* @throws RuntimeException if reading from the Cassandra table fails.
*/
private void assertJSONEmptyRowInCassandraDB() throws InterruptedException {
private void assertJSONEmptyRowInCassandraDB()
throws InterruptedException, MultipleFailureException {
PipelineOperator.Result result =
pipelineOperator()
.waitForCondition(
createConfig(jobInfo, Duration.ofMinutes(10)),
() -> getRowCount(EMPTY_STRING_JSON_TABLE) == 1);
assertThatResult(result).meetsConditions();

Iterable<Row> rows;
try {
rows = cassandraResourceManager.readTable(EMPTY_STRING_JSON_TABLE);
} catch (Exception e) {
throw new RuntimeException(
"Failed to read from Cassandra table: " + EMPTY_STRING_JSON_TABLE, e);
}

assertThat(rows).hasSize(1);
Row row = rows.iterator().next();
LOG.info("[AssertJSONEmptyRowInCassandraDB] row: {}", row.getFormattedContents());

assertAll(
() -> assertThat(row.getString("varchar_column")).isEqualTo("SampleVarchar"),
() -> assertThat(row.getString("empty_column")).isEmpty(),
() -> assertThat(row.getMap("double_float_map_col", Double.class, Float.class)).isEmpty(),
() ->
assertThat(row.getMap("date_double_map_col", LocalDate.class, Double.class)).isEmpty(),
() -> assertThat(row.getMap("uuid_ascii_map_col", UUID.class, String.class)).isEmpty(),
() -> assertThat(row.getMap("ascii_text_map_col", String.class, String.class)).isEmpty(),
() ->
assertThat(row.getMap("tinyint_varint_map_col", String.class, String.class)).isEmpty(),
() -> assertThat(row.getMap("time_tinyint_map_col", String.class, Byte.class)).isEmpty(),
() -> assertThat(row.getMap("bigint_boolean_map_col", Long.class, Boolean.class)).isEmpty(),
() -> assertThat(row.getMap("varchar_bigint_map_col", String.class, Long.class)).isEmpty(),
() -> assertThat(row.getMap("blob_int_map_col", ByteBuffer.class, Integer.class)).isEmpty(),
() ->
assertThat(row.getMap("varint_blob_map_col", String.class, ByteBuffer.class)).isEmpty(),
() -> assertThat(row.getMap("float_smallint_map_col", Float.class, Short.class)).isEmpty(),
() ->
assertThat(row.getMap("smallint_timestamp_map_col", Short.class, Instant.class))
.isEmpty(),
() -> assertThat(row.getMap("text_timeuuid_map_col", String.class, UUID.class)).isEmpty(),
() ->
assertThat(row.getMap("inet_text_map_col", InetAddress.class, String.class)).isEmpty(),
() -> assertThat(row.getMap("timestamp_uuid_map_col", String.class, UUID.class)).isEmpty(),
() ->
assertThat(row.getMap("boolean_decimal_map_col", Boolean.class, BigDecimal.class))
.isEmpty(),
() ->
assertThat(row.getMap("decimal_duration_map_col", BigDecimal.class, String.class))
.isEmpty(),
() ->
assertThat(row.getMap("double_inet_map_col", Double.class, InetAddress.class))
.isEmpty(),
() ->
assertThat(row.getMap("timeuuid_varchar_map_col", UUID.class, String.class)).isEmpty(),
() -> assertThat(row.getMap("int_time_map_col", Integer.class, String.class)).isEmpty(),
() -> assertThat(row.getList("timestamp_list_col", Instant.class)).isEmpty(),
() -> assertThat(row.getList("varchar_list_col", String.class)).isEmpty(),
() -> assertThat(row.getList("inet_list_col", InetAddress.class)).isEmpty(),
() -> assertThat(row.getList("bigint_list_col", Long.class)).isEmpty(),
() -> assertThat(row.getList("time_list_col", String.class)).isEmpty(),
() -> assertThat(row.getList("frozen_ascii_list_col", String.class)).isEmpty(),
() -> assertThat(row.getList("int_list_col", Integer.class)).isEmpty(),
() -> assertThat(row.getList("ascii_list_col", String.class)).isEmpty(),
() -> assertThat(row.getList("date_list_col", LocalDate.class)).isEmpty(),
() -> assertThat(row.getList("double_list_col", Double.class)).isEmpty(),
() -> assertThat(row.getList("float_list_col", Float.class)).isEmpty(),
() -> assertThat(row.getList("smallint_list_col", Short.class)).isEmpty(),
() -> assertThat(row.getList("varint_list_col", String.class)).isEmpty(),
() -> assertThat(row.getList("text_list_col", String.class)).isEmpty(),
() -> assertThat(row.getList("boolean_list_col", Boolean.class)).isEmpty(),
() -> assertThat(row.getList("blob_list_col", ByteBuffer.class)).isEmpty(),
() -> assertThat(row.getList("timeuuid_list_col", UUID.class)).isEmpty(),
() -> assertThat(row.getList("duration_list_col", String.class)).isEmpty(),
() -> assertThat(row.getList("decimal_list_col", BigDecimal.class)).isEmpty(),
() -> assertThat(row.getList("tinyint_list_col", Byte.class)).isEmpty(),
() -> assertThat(row.getSet("decimal_set_col", BigDecimal.class)).isEmpty(),
() -> assertThat(row.getSet("int_set_col", Integer.class)).isEmpty(),
() -> assertThat(row.getSet("smallint_set_col", Short.class)).isEmpty(),
() -> assertThat(row.getSet("text_set_col", String.class)).isEmpty(),
() -> assertThat(row.getSet("double_set_col", Double.class)).isEmpty(),
() -> assertThat(row.getSet("date_set_col", LocalDate.class)).isEmpty(),
() -> assertThat(row.getSet("timestamp_set_col", String.class)).isEmpty(),
() -> assertThat(row.getSet("bigint_set_col", Long.class)).isEmpty(),
() -> assertThat(row.getSet("varchar_set_col", String.class)).isEmpty(),
() -> assertThat(row.getSet("tinyint_set_col", Byte.class)).isEmpty(),
() -> assertThat(row.getSet("boolean_set_col", Boolean.class)).isEmpty(),
() -> assertThat(row.getSet("float_set_col", Float.class)).isEmpty(),
() -> assertThat(row.getSet("ascii_set_col", String.class)).isEmpty(),
() -> assertThat(row.getSet("uuid_set_col", UUID.class)).isEmpty(),
() -> assertThat(row.getSet("varint_set_col", String.class)).isEmpty(),
() -> assertThat(row.getSet("blob_set_col", ByteBuffer.class)).isEmpty(),
() -> assertThat(row.getSet("inet_set_col", InetAddress.class)).isEmpty(),
() -> assertThat(row.getSet("frozen_ascii_set_col", String.class)).isEmpty());
}

/**
Expand Down
40 changes: 20 additions & 20 deletions v2/spanner-to-sourcedb/src/test/resources/SpannerToCassandraSourceIT/cassandra-schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,51 +139,51 @@ CREATE TABLE BoundaryConversionTestTable (
);

CREATE TABLE EmptyStringJsonTable (
varchar_column text PRIMARY KEY,
empty_column text,
varchar_column TEXT PRIMARY KEY,
empty_column TEXT,
double_float_map_col MAP<DOUBLE, FLOAT>,
decimal_set_col SET<DECIMAL>,
date_double_map_col MAP<DATE, DOUBLE>,
uuid_ascii_map_col MAP<UUID, ASCII>,
ascii_text_map_col MAP<ASCII, TEXT>,
uuid_ascii_map_col MAP<UUID, TEXT>,
ascii_text_map_col MAP<TEXT, TEXT>,
timestamp_list_col LIST<TIMESTAMP>,
int_set_col SET<INT>,
smallint_set_col SET<SMALLINT>,
varchar_list_col LIST<VARCHAR>,
inet_list_col LIST<INET>,
bigint_list_col LIST<BIGINT>,
tinyint_varint_map_col MAP<TINYINT, VARINT>,
tinyint_varint_map_col MAP<TEXT, TEXT>,
text_set_col SET<TEXT>,
double_set_col SET<DOUBLE>,
time_list_col LIST<TIME>,
frozen_ascii_list_col LIST<FROZEN<ASCII>>,
time_list_col LIST<TEXT>,
frozen_ascii_list_col frozen<LIST<ASCII>>,
int_list_col LIST<INT>,
ascii_list_col LIST<ASCII>,
date_set_col SET<DATE>,
double_inet_map_col MAP<DOUBLE, INET>,
timestamp_set_col SET<TIMESTAMP>,
time_tinyint_map_col MAP<TIME, TINYINT>,
timestamp_set_col SET<TEXT>,
time_tinyint_map_col MAP<TEXT, TINYINT>,
bigint_set_col SET<BIGINT>,
varchar_set_col SET<VARCHAR>,
varchar_set_col SET<TEXT>,
tinyint_set_col SET<TINYINT>,
bigint_boolean_map_col MAP<BIGINT, BOOLEAN>,
text_list_col LIST<TEXT>,
boolean_list_col LIST<BOOLEAN>,
blob_list_col LIST<BLOB>,
timeuuid_set_col SET<TIMEUUID>,
int_time_map_col MAP<INT, TIME>,
time_set_col SET<TIME>,
int_time_map_col MAP<INT, TEXT>,
time_set_col SET<TEXT>,
boolean_set_col SET<BOOLEAN>,
float_set_col SET<FLOAT>,
ascii_set_col SET<ASCII>,
uuid_list_col LIST<UUID>,
varchar_bigint_map_col MAP<VARCHAR, BIGINT>,
varchar_bigint_map_col MAP<TEXT, BIGINT>,
blob_int_map_col MAP<BLOB, INT>,
varint_blob_map_col MAP<VARINT, BLOB>,
varint_blob_map_col MAP<TEXT, BLOB>,
double_list_col LIST<DOUBLE>,
float_list_col LIST<FLOAT>,
smallint_list_col LIST<SMALLINT>,
varint_list_col LIST<VARINT>,
varint_list_col LIST<TEXT>,
float_smallint_map_col MAP<FLOAT, SMALLINT>,
smallint_timestamp_map_col MAP<SMALLINT, TIMESTAMP>,
text_timeuuid_map_col MAP<TEXT, TIMEUUID>,
Expand All @@ -193,13 +193,13 @@ CREATE TABLE EmptyStringJsonTable (
boolean_decimal_map_col MAP<BOOLEAN, DECIMAL>,
blob_set_col SET<BLOB>,
inet_text_map_col MAP<INET, TEXT>,
varint_set_col SET<VARINT>,
varint_set_col SET<TEXT>,
tinyint_list_col LIST<TINYINT>,
timestamp_uuid_map_col MAP<TIMESTAMP, UUID>,
decimal_duration_map_col MAP<DECIMAL, DURATION>,
timestamp_uuid_map_col MAP<TEXT, UUID>,
decimal_duration_map_col MAP<DECIMAL, TEXT>,
decimal_list_col LIST<DECIMAL>,
inet_set_col SET<INET>,
timeuuid_varchar_map_col MAP<TIMEUUID, VARCHAR>,
duration_list_col LIST<DURATION>,
frozen_ascii_set_col SET<FROZEN<ASCII>>
duration_list_col LIST<TEXT>,
frozen_ascii_set_col frozen<SET<ASCII>>
);
Loading
Loading