-
Notifications
You must be signed in to change notification settings - Fork 28.3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[SPARK-33468][SQL] ParseUrl in ANSI mode should fail if input string is not a valid url #30399
Changes from 1 commit
62bf87f
5aec76a
a408734
2202ca4
497015c
ecd2143
d7e437b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -1404,7 +1404,10 @@ case class ParseUrl(children: Seq[Expression]) | |
try { | ||
new URI(url.toString) | ||
} catch { | ||
case e: URISyntaxException => null | ||
// We fail on error if in ansi mode. | ||
case e: URISyntaxException if SQLConf.get.ansiEnabled => | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please update the ANSI doc, too. https://github.com/apache/spark/blame/master/docs/sql-ref-ansi-compliance.md#L110 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Missed here, updated ! There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we follow other expressions and add a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. get it ! |
||
throw new IllegalArgumentException(s"Find an invaild url string ${url.toString}", e) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. To clarify, the raw exception message is something like |
||
case _: URISyntaxException => null | ||
} | ||
} | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -943,6 +943,21 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { | |
GenerateUnsafeProjection.generate(ParseUrl(Seq(Literal("\"quote"), Literal("\"quote"))) :: Nil) | ||
} | ||
|
||
test("SPARK-33468: ParseUrl should fail if input string is not a valid url") { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
// fail on error if in ansi mode. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We can remove this. |
||
withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { | ||
val msg = intercept[IllegalArgumentException] { | ||
evaluateWithoutCodegen( | ||
ParseUrl(Seq("https://a.b.c/index.php?params1=a|b¶ms2=x", "HOST"))) | ||
}.getMessage | ||
assert(msg.contains("Find an invaild url string")) | ||
} | ||
withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { | ||
checkEvaluation( | ||
ParseUrl(Seq("https://a.b.c/index.php?params1=a|b¶ms2=x", "HOST")), null) | ||
} | ||
} | ||
|
||
test("Sentences") { | ||
val nullString = Literal.create(null, StringType) | ||
checkEvaluation(Sentences(nullString, nullString, nullString), null) | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We can remove this comment.