Skip to content

Commit ee375cd

Browse files
committed
Update SparkContextSuite.scala
1 parent d594e16 commit ee375cd

File tree

1 file changed

+35
-41
lines changed

1 file changed

+35
-41
lines changed

core/src/test/scala/org/apache/spark/SparkContextSuite.scala

Lines changed: 35 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -79,63 +79,57 @@ class SparkContextSuite extends FunSuite with LocalSparkContext {
7979
val byteArray2 = converter.convert(bytesWritable)
8080
assert(byteArray2.length === 0)
8181
}
82-
83-
test("addFile works") {
84-
val file = File.createTempFile("someprefix", "somesuffix")
85-
val absolutePath = file.getAbsolutePath
86-
try {
87-
Files.write("somewords", file, UTF_8)
88-
val length = file.length()
89-
sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
90-
sc.addFile(file.getAbsolutePath)
91-
sc.parallelize(Array(1), 1).map(x => {
92-
val gotten = new File(SparkFiles.get(file.getName))
93-
if (!gotten.exists()) {
94-
throw new SparkException("file doesn't exist")
95-
}
96-
if (length != gotten.length()) {
97-
throw new SparkException(
98-
s"file has different length $length than added file ${gotten.length()}")
99-
}
100-
if (absolutePath == gotten.getAbsolutePath) {
101-
throw new SparkException("file should have been copied")
102-
}
103-
x
104-
}).count()
105-
} finally {
106-
sc.stop()
107-
}
108-
}
10982

110-
test("addFile works with relative path") {
83+
test("addFile works") {
84+
val file1 = File.createTempFile("someprefix1", "somesuffix1")
85+
val absolutePath1 = file1.getAbsolutePath
86+
11187
val pluto = Utils.createTempDir()
112-
val file = File.createTempFile("someprefix", "somesuffix", pluto)
113-
val relativePath = file.getParent + "/../" + file.getParentFile.getName + "/" + file.getName
114-
val absolutePath = file.getAbsolutePath
88+
val file2 = File.createTempFile("someprefix2", "somesuffix2", pluto)
89+
val relativePath = file2.getParent + "/../" + file2.getParentFile.getName + "/" + file2.getName
90+
val absolutePath2 = file2.getAbsolutePath
91+
11592
try {
116-
Files.write("somewords", file, UTF_8)
117-
val length = file.length()
93+
Files.write("somewords1", file1, UTF_8)
94+
Files.write("somewords2", file2, UTF_8)
95+
val length1 = file1.length()
96+
val length2 = file2.length()
97+
11898
sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
99+
sc.addFile(file1.getAbsolutePath)
119100
sc.addFile(relativePath)
120101
sc.parallelize(Array(1), 1).map(x => {
121-
val gotten = new File(SparkFiles.get(file.getName))
122-
if (!gotten.exists()) {
123-
throw new SparkException("file doesn't exist")
102+
val gotten1 = new File(SparkFiles.get(file1.getName))
103+
val gotten2 = new File(SparkFiles.get(file2.getName))
104+
if (!gotten1.exists()) {
105+
throw new SparkException("file doesn't exist : " + absolutePath1)
106+
}
107+
if (!gotten2.exists()) {
108+
throw new SparkException("file doesn't exist : " + absolutePath2)
124109
}
125-
if (length != gotten.length()) {
110+
111+
if (length1 != gotten1.length()) {
126112
throw new SparkException(
127-
s"file has different length $length than added file ${gotten.length()}")
113+
s"file has different length $length1 than added file ${gotten1.length()} : " + absolutePath1)
128114
}
129-
if (absolutePath == gotten.getAbsolutePath) {
130-
throw new SparkException("file should have been copied")
115+
if (length2 != gotten2.length()) {
116+
throw new SparkException(
117+
s"file has different length $length2 than added file ${gotten2.length()} : " + absolutePath2)
118+
}
119+
120+
if (absolutePath1 == gotten1.getAbsolutePath) {
121+
throw new SparkException("file should have been copied :" + absolutePath1)
122+
}
123+
if (absolutePath2 == gotten2.getAbsolutePath) {
124+
throw new SparkException("file should have been copied : " + absolutePath2)
131125
}
132126
x
133127
}).count()
134128
} finally {
135129
sc.stop()
136130
}
137131
}
138-
132+
139133
test("addFile recursive works") {
140134
val pluto = Utils.createTempDir()
141135
val neptune = Utils.createTempDir(pluto.getAbsolutePath)

0 commit comments

Comments
 (0)