Skip to content

Commit 1016f27

Browse files
szehon-hoHyukjinKwon
szehon-ho
authored and
HyukjinKwon
committed
[SPARK-51906][SQL][FOLLOW-UP] Enforce ANSI mode in previous tests
### What changes were proposed in this pull request? Limit the tests added in #50701 and #50593 only for ANSI_ENABLED mode. ### Why are the changes needed? These tests fail in non-ANSI mode. The reason is that https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtils.scala only converts majority of catalyst => V2 expressions in ANSI mode. So , we do not get any V2Expression in non-ANSI case. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Existing unit test run with SPARK_ANSI_SQL_MODE=false ### Was this patch authored or co-authored using generative AI tooling? No Closes #50851 from szehon-ho/add_column_default_val_follow. Authored-by: Szehon Ho <szehon.apache@gmail.com> Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
1 parent 4788bae commit 1016f27

File tree

1 file changed

+122
-118
lines changed

1 file changed

+122
-118
lines changed

‎sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala

Lines changed: 122 additions & 118 deletions
Original file line numberDiff line numberDiff line change
@@ -346,137 +346,141 @@ class DataSourceV2DataFrameSuite
346346
test("create/replace table with complex foldable default values") {
347347
val tableName = "testcat.ns1.ns2.tbl"
348348
withTable(tableName) {
349-
val createExec = executeAndKeepPhysicalPlan[CreateTableExec] {
350-
sql(
351-
s"""
352-
|CREATE TABLE $tableName (
353-
| id INT,
354-
| salary INT DEFAULT (100 + 23),
355-
| dep STRING DEFAULT ('h' || 'r'),
356-
| active BOOLEAN DEFAULT CAST(1 AS BOOLEAN)
357-
|) USING foo
358-
|""".stripMargin)
349+
withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
350+
val createExec = executeAndKeepPhysicalPlan[CreateTableExec] {
351+
sql(
352+
s"""
353+
|CREATE TABLE $tableName (
354+
| id INT,
355+
| salary INT DEFAULT (100 + 23),
356+
| dep STRING DEFAULT ('h' || 'r'),
357+
| active BOOLEAN DEFAULT CAST(1 AS BOOLEAN)
358+
|) USING foo
359+
|""".stripMargin)
360+
}
361+
362+
checkDefaultValues(
363+
createExec.columns,
364+
Array(
365+
null,
366+
new ColumnDefaultValue(
367+
"(100 + 23)",
368+
new GeneralScalarExpression(
369+
"+",
370+
Array(LiteralValue(100, IntegerType), LiteralValue(23, IntegerType))),
371+
LiteralValue(123, IntegerType)),
372+
new ColumnDefaultValue(
373+
"('h' || 'r')",
374+
new GeneralScalarExpression(
375+
"CONCAT",
376+
Array(
377+
LiteralValue(UTF8String.fromString("h"), StringType),
378+
LiteralValue(UTF8String.fromString("r"), StringType))),
379+
LiteralValue(UTF8String.fromString("hr"), StringType)),
380+
new ColumnDefaultValue(
381+
"CAST(1 AS BOOLEAN)",
382+
new V2Cast(LiteralValue(1, IntegerType), IntegerType, BooleanType),
383+
LiteralValue(true, BooleanType))))
384+
385+
val df1 = Seq(1).toDF("id")
386+
df1.writeTo(tableName).append()
387+
388+
sql(s"ALTER TABLE $tableName ALTER COLUMN dep SET DEFAULT ('i' || 't')")
389+
390+
val df2 = Seq(2).toDF("id")
391+
df2.writeTo(tableName).append()
392+
393+
checkAnswer(
394+
sql(s"SELECT * FROM $tableName"),
395+
Seq(
396+
Row(1, 123, "hr", true),
397+
Row(2, 123, "it", true)))
398+
399+
val replaceExec = executeAndKeepPhysicalPlan[ReplaceTableExec] {
400+
sql(
401+
s"""
402+
|REPLACE TABLE $tableName (
403+
| id INT,
404+
| salary INT DEFAULT (50 * 2),
405+
| dep STRING DEFAULT ('un' || 'known'),
406+
| active BOOLEAN DEFAULT CAST(0 AS BOOLEAN)
407+
|) USING foo
408+
|""".stripMargin)
409+
}
410+
411+
checkDefaultValues(
412+
replaceExec.columns,
413+
Array(
414+
null,
415+
new ColumnDefaultValue(
416+
"(50 * 2)",
417+
new GeneralScalarExpression(
418+
"*",
419+
Array(LiteralValue(50, IntegerType), LiteralValue(2, IntegerType))),
420+
LiteralValue(100, IntegerType)),
421+
new ColumnDefaultValue(
422+
"('un' || 'known')",
423+
new GeneralScalarExpression(
424+
"CONCAT",
425+
Array(
426+
LiteralValue(UTF8String.fromString("un"), StringType),
427+
LiteralValue(UTF8String.fromString("known"), StringType))),
428+
LiteralValue(UTF8String.fromString("unknown"), StringType)),
429+
new ColumnDefaultValue(
430+
"CAST(0 AS BOOLEAN)",
431+
new V2Cast(LiteralValue(0, IntegerType), IntegerType, BooleanType),
432+
LiteralValue(false, BooleanType))))
433+
434+
val df3 = Seq(1).toDF("id")
435+
df3.writeTo(tableName).append()
436+
437+
checkAnswer(
438+
sql(s"SELECT * FROM $tableName"),
439+
Seq(Row(1, 100, "unknown", false)))
359440
}
360-
361-
checkDefaultValues(
362-
createExec.columns,
363-
Array(
364-
null,
365-
new ColumnDefaultValue(
366-
"(100 + 23)",
367-
new GeneralScalarExpression(
368-
"+",
369-
Array(LiteralValue(100, IntegerType), LiteralValue(23, IntegerType))),
370-
LiteralValue(123, IntegerType)),
371-
new ColumnDefaultValue(
372-
"('h' || 'r')",
373-
new GeneralScalarExpression(
374-
"CONCAT",
375-
Array(
376-
LiteralValue(UTF8String.fromString("h"), StringType),
377-
LiteralValue(UTF8String.fromString("r"), StringType))),
378-
LiteralValue(UTF8String.fromString("hr"), StringType)),
379-
new ColumnDefaultValue(
380-
"CAST(1 AS BOOLEAN)",
381-
new V2Cast(LiteralValue(1, IntegerType), IntegerType, BooleanType),
382-
LiteralValue(true, BooleanType))))
383-
384-
val df1 = Seq(1).toDF("id")
385-
df1.writeTo(tableName).append()
386-
387-
sql(s"ALTER TABLE $tableName ALTER COLUMN dep SET DEFAULT ('i' || 't')")
388-
389-
val df2 = Seq(2).toDF("id")
390-
df2.writeTo(tableName).append()
391-
392-
checkAnswer(
393-
sql(s"SELECT * FROM $tableName"),
394-
Seq(
395-
Row(1, 123, "hr", true),
396-
Row(2, 123, "it", true)))
397-
398-
val replaceExec = executeAndKeepPhysicalPlan[ReplaceTableExec] {
399-
sql(
400-
s"""
401-
|REPLACE TABLE $tableName (
402-
| id INT,
403-
| salary INT DEFAULT (50 * 2),
404-
| dep STRING DEFAULT ('un' || 'known'),
405-
| active BOOLEAN DEFAULT CAST(0 AS BOOLEAN)
406-
|) USING foo
407-
|""".stripMargin)
408-
}
409-
410-
checkDefaultValues(
411-
replaceExec.columns,
412-
Array(
413-
null,
414-
new ColumnDefaultValue(
415-
"(50 * 2)",
416-
new GeneralScalarExpression(
417-
"*",
418-
Array(LiteralValue(50, IntegerType), LiteralValue(2, IntegerType))),
419-
LiteralValue(100, IntegerType)),
420-
new ColumnDefaultValue(
421-
"('un' || 'known')",
422-
new GeneralScalarExpression(
423-
"CONCAT",
424-
Array(
425-
LiteralValue(UTF8String.fromString("un"), StringType),
426-
LiteralValue(UTF8String.fromString("known"), StringType))),
427-
LiteralValue(UTF8String.fromString("unknown"), StringType)),
428-
new ColumnDefaultValue(
429-
"CAST(0 AS BOOLEAN)",
430-
new V2Cast(LiteralValue(0, IntegerType), IntegerType, BooleanType),
431-
LiteralValue(false, BooleanType))))
432-
433-
val df3 = Seq(1).toDF("id")
434-
df3.writeTo(tableName).append()
435-
436-
checkAnswer(
437-
sql(s"SELECT * FROM $tableName"),
438-
Seq(Row(1, 100, "unknown", false)))
439441
}
440442
}
441443

442444
test("alter table with complex foldable default values") {
443445
val tableName = "testcat.ns1.ns2.tbl"
444-
withTable(tableName) {
445-
sql(
446+
withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
447+
withTable(tableName) {
448+
sql(
446449
s"""
447450
|CREATE TABLE $tableName (
448451
| dummy INT
449452
|) USING foo
450453
|""".stripMargin)
451454

452-
val alterExec = executeAndKeepPhysicalPlan[AlterTableExec] {
453-
sql(s"ALTER TABLE $tableName ADD COLUMNS (" +
454-
s"salary INT DEFAULT (100 + 23), " +
455-
s"dep STRING DEFAULT ('h' || 'r'), " +
456-
s"active BOOLEAN DEFAULT CAST(1 AS BOOLEAN))")
455+
val alterExec = executeAndKeepPhysicalPlan[AlterTableExec] {
456+
sql(s"ALTER TABLE $tableName ADD COLUMNS (" +
457+
s"salary INT DEFAULT (100 + 23), " +
458+
s"dep STRING DEFAULT ('h' || 'r'), " +
459+
s"active BOOLEAN DEFAULT CAST(1 AS BOOLEAN))")
460+
}
461+
462+
checkDefaultValues(
463+
alterExec.changes.map(_.asInstanceOf[AddColumn]).toArray,
464+
Array(
465+
new ColumnDefaultValue(
466+
"(100 + 23)",
467+
new GeneralScalarExpression(
468+
"+",
469+
Array(LiteralValue(100, IntegerType), LiteralValue(23, IntegerType))),
470+
LiteralValue(123, IntegerType)),
471+
new ColumnDefaultValue(
472+
"('h' || 'r')",
473+
new GeneralScalarExpression(
474+
"CONCAT",
475+
Array(
476+
LiteralValue(UTF8String.fromString("h"), StringType),
477+
LiteralValue(UTF8String.fromString("r"), StringType))),
478+
LiteralValue(UTF8String.fromString("hr"), StringType)),
479+
new ColumnDefaultValue(
480+
"CAST(1 AS BOOLEAN)",
481+
new V2Cast(LiteralValue(1, IntegerType), IntegerType, BooleanType),
482+
LiteralValue(true, BooleanType))))
457483
}
458-
459-
checkDefaultValues(
460-
alterExec.changes.map(_.asInstanceOf[AddColumn]).toArray,
461-
Array(
462-
new ColumnDefaultValue(
463-
"(100 + 23)",
464-
new GeneralScalarExpression(
465-
"+",
466-
Array(LiteralValue(100, IntegerType), LiteralValue(23, IntegerType))),
467-
LiteralValue(123, IntegerType)),
468-
new ColumnDefaultValue(
469-
"('h' || 'r')",
470-
new GeneralScalarExpression(
471-
"CONCAT",
472-
Array(
473-
LiteralValue(UTF8String.fromString("h"), StringType),
474-
LiteralValue(UTF8String.fromString("r"), StringType))),
475-
LiteralValue(UTF8String.fromString("hr"), StringType)),
476-
new ColumnDefaultValue(
477-
"CAST(1 AS BOOLEAN)",
478-
new V2Cast(LiteralValue(1, IntegerType), IntegerType, BooleanType),
479-
LiteralValue(true, BooleanType))))
480484
}
481485
}
482486

0 commit comments

Comments
 (0)