Skip to content

Commit 9c92a3b

Browse files
sarutakdongjoon-hyun
authored andcommitted
[SPARK-34225][CORE][FOLLOWUP] Replace Hadoop's Path with Utils.resolveURI to make the way to get URI simple
### What changes were proposed in this pull request? This PR proposes to replace Hadoop's `Path` with `Utils.resolveURI` to make the way to get URI simple in `SparkContext`. ### Why are the changes needed? Keep the code simple. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing tests. Closes apache#32164 from sarutak/followup-SPARK-34225. Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com> Signed-off-by: Dongjoon Hyun <dhyun@apple.com> (cherry picked from commit 767ea86) Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
1 parent e2ca7fe commit 9c92a3b

File tree

1 file changed

+2
-14
lines changed

1 file changed

+2
-14
lines changed

core/src/main/scala/org/apache/spark/SparkContext.scala

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1586,15 +1586,7 @@ class SparkContext(config: SparkConf) extends Logging {
15861586
private def addFile(
15871587
path: String, recursive: Boolean, addedOnSubmit: Boolean, isArchive: Boolean = false
15881588
): Unit = {
1589-
val uri = if (!isArchive) {
1590-
if (Utils.isAbsoluteURI(path) && path.contains("%")) {
1591-
new URI(path)
1592-
} else {
1593-
new Path(path).toUri
1594-
}
1595-
} else {
1596-
Utils.resolveURI(path)
1597-
}
1589+
val uri = Utils.resolveURI(path)
15981590
val schemeCorrectedURI = uri.getScheme match {
15991591
case null => new File(path).getCanonicalFile.toURI
16001592
case "local" =>
@@ -1982,11 +1974,7 @@ class SparkContext(config: SparkConf) extends Logging {
19821974
// For local paths with backslashes on Windows, URI throws an exception
19831975
addLocalJarFile(new File(path))
19841976
} else {
1985-
val uri = if (Utils.isAbsoluteURI(path) && path.contains("%")) {
1986-
new URI(path)
1987-
} else {
1988-
new Path(path).toUri
1989-
}
1977+
val uri = Utils.resolveURI(path)
19901978
// SPARK-17650: Make sure this is a valid URL before adding it to the list of dependencies
19911979
Utils.validateURL(uri)
19921980
uri.getScheme match {

0 commit comments

Comments
 (0)