Skip to content

Commit 00e730b

Browse files
pzzssrowen
authored andcommitted
[SPARK-6300][Spark Core] sc.addFile(path) does not support the relative path.
when i run cmd like that sc.addFile("../test.txt"), it did not work and throwed an exception: java.lang.IllegalArgumentException: java.net.URISyntaxException: Relative path in absolute URI: file:../test.txt at org.apache.hadoop.fs.Path.initialize(Path.java:206) at org.apache.hadoop.fs.Path.<init>(Path.java:172) ........ ....... Caused by: java.net.URISyntaxException: Relative path in absolute URI: file:../test.txt at java.net.URI.checkPath(URI.java:1804) at java.net.URI.<init>(URI.java:752) at org.apache.hadoop.fs.Path.initialize(Path.java:203) Author: DoingDone9 <799203320@qq.com> Closes #4993 from DoingDone9/relativePath and squashes the following commits: ee375cd [DoingDone9] Update SparkContextSuite.scala d594e16 [DoingDone9] Update SparkContext.scala 0ff3fa8 [DoingDone9] test for add file dced8eb [DoingDone9] Update SparkContext.scala e4a13fe [DoingDone9] getCanonicalPath 161cae3 [DoingDone9] Merge pull request #4 from apache/master c87e8b6 [DoingDone9] Merge pull request #3 from apache/master cb1852d [DoingDone9] Merge pull request #2 from apache/master c3f046f [DoingDone9] Merge pull request #1 from apache/master
1 parent 45f4c66 commit 00e730b

File tree

2 files changed

+38
-15
lines changed

2 files changed

+38
-15
lines changed

core/src/main/scala/org/apache/spark/SparkContext.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1093,7 +1093,7 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
10931093
def addFile(path: String, recursive: Boolean): Unit = {
10941094
val uri = new URI(path)
10951095
val schemeCorrectedPath = uri.getScheme match {
1096-
case null | "local" => "file:" + uri.getPath
1096+
case null | "local" => new File(path).getCanonicalFile.toURI.toString
10971097
case _ => path
10981098
}
10991099

core/src/test/scala/org/apache/spark/SparkContextSuite.scala

Lines changed: 37 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -79,34 +79,57 @@ class SparkContextSuite extends FunSuite with LocalSparkContext {
7979
val byteArray2 = converter.convert(bytesWritable)
8080
assert(byteArray2.length === 0)
8181
}
82-
82+
8383
test("addFile works") {
84-
val file = File.createTempFile("someprefix", "somesuffix")
85-
val absolutePath = file.getAbsolutePath
84+
val file1 = File.createTempFile("someprefix1", "somesuffix1")
85+
val absolutePath1 = file1.getAbsolutePath
86+
87+
val pluto = Utils.createTempDir()
88+
val file2 = File.createTempFile("someprefix2", "somesuffix2", pluto)
89+
val relativePath = file2.getParent + "/../" + file2.getParentFile.getName + "/" + file2.getName
90+
val absolutePath2 = file2.getAbsolutePath
91+
8692
try {
87-
Files.write("somewords", file, UTF_8)
88-
val length = file.length()
93+
Files.write("somewords1", file1, UTF_8)
94+
Files.write("somewords2", file2, UTF_8)
95+
val length1 = file1.length()
96+
val length2 = file2.length()
97+
8998
sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
90-
sc.addFile(file.getAbsolutePath)
99+
sc.addFile(file1.getAbsolutePath)
100+
sc.addFile(relativePath)
91101
sc.parallelize(Array(1), 1).map(x => {
92-
val gotten = new File(SparkFiles.get(file.getName))
93-
if (!gotten.exists()) {
94-
throw new SparkException("file doesn't exist")
102+
val gotten1 = new File(SparkFiles.get(file1.getName))
103+
val gotten2 = new File(SparkFiles.get(file2.getName))
104+
if (!gotten1.exists()) {
105+
throw new SparkException("file doesn't exist : " + absolutePath1)
106+
}
107+
if (!gotten2.exists()) {
108+
throw new SparkException("file doesn't exist : " + absolutePath2)
95109
}
96-
if (length != gotten.length()) {
110+
111+
if (length1 != gotten1.length()) {
112+
throw new SparkException(
113+
s"file has different length $length1 than added file ${gotten1.length()} : " + absolutePath1)
114+
}
115+
if (length2 != gotten2.length()) {
97116
throw new SparkException(
98-
s"file has different length $length than added file ${gotten.length()}")
117+
s"file has different length $length2 than added file ${gotten2.length()} : " + absolutePath2)
99118
}
100-
if (absolutePath == gotten.getAbsolutePath) {
101-
throw new SparkException("file should have been copied")
119+
120+
if (absolutePath1 == gotten1.getAbsolutePath) {
121+
throw new SparkException("file should have been copied :" + absolutePath1)
122+
}
123+
if (absolutePath2 == gotten2.getAbsolutePath) {
124+
throw new SparkException("file should have been copied : " + absolutePath2)
102125
}
103126
x
104127
}).count()
105128
} finally {
106129
sc.stop()
107130
}
108131
}
109-
132+
110133
test("addFile recursive works") {
111134
val pluto = Utils.createTempDir()
112135
val neptune = Utils.createTempDir(pluto.getAbsolutePath)

0 commit comments

Comments
 (0)