Skip to content

Commit 767ea86

Browse files
sarutakdongjoon-hyun
authored andcommitted
[SPARK-34225][CORE][FOLLOWUP] Replace Hadoop's Path with Utils.resolveURI to make the way to get URI simple
### What changes were proposed in this pull request? This PR proposes to replace Hadoop's `Path` with `Utils.resolveURI` to make the way to get URI simple in `SparkContext`. ### Why are the changes needed? Keep the code simple. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing tests. Closes apache#32164 from sarutak/followup-SPARK-34225. Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com> Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
1 parent 9b2e0d6 commit 767ea86

File tree

1 file changed

+2
-14
lines changed

1 file changed

+2
-14
lines changed

core/src/main/scala/org/apache/spark/SparkContext.scala

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1583,15 +1583,7 @@ class SparkContext(config: SparkConf) extends Logging {
15831583
private def addFile(
15841584
path: String, recursive: Boolean, addedOnSubmit: Boolean, isArchive: Boolean = false
15851585
): Unit = {
1586-
val uri = if (!isArchive) {
1587-
if (Utils.isAbsoluteURI(path) && path.contains("%")) {
1588-
new URI(path)
1589-
} else {
1590-
new Path(path).toUri
1591-
}
1592-
} else {
1593-
Utils.resolveURI(path)
1594-
}
1586+
val uri = Utils.resolveURI(path)
15951587
val schemeCorrectedURI = uri.getScheme match {
15961588
case null => new File(path).getCanonicalFile.toURI
15971589
case "local" =>
@@ -1979,11 +1971,7 @@ class SparkContext(config: SparkConf) extends Logging {
19791971
// For local paths with backslashes on Windows, URI throws an exception
19801972
(addLocalJarFile(new File(path)), "local")
19811973
} else {
1982-
val uri = if (Utils.isAbsoluteURI(path) && path.contains("%")) {
1983-
new URI(path)
1984-
} else {
1985-
new Path(path).toUri
1986-
}
1974+
val uri = Utils.resolveURI(path)
19871975
// SPARK-17650: Make sure this is a valid URL before adding it to the list of dependencies
19881976
Utils.validateURL(uri)
19891977
val uriScheme = uri.getScheme

0 commit comments

Comments
 (0)