@@ -79,63 +79,57 @@ class SparkContextSuite extends FunSuite with LocalSparkContext {
79
79
val byteArray2 = converter.convert(bytesWritable)
80
80
assert(byteArray2.length === 0 )
81
81
}
82
-
83
- test(" addFile works" ) {
84
- val file = File .createTempFile(" someprefix" , " somesuffix" )
85
- val absolutePath = file.getAbsolutePath
86
- try {
87
- Files .write(" somewords" , file, UTF_8 )
88
- val length = file.length()
89
- sc = new SparkContext (new SparkConf ().setAppName(" test" ).setMaster(" local" ))
90
- sc.addFile(file.getAbsolutePath)
91
- sc.parallelize(Array (1 ), 1 ).map(x => {
92
- val gotten = new File (SparkFiles .get(file.getName))
93
- if (! gotten.exists()) {
94
- throw new SparkException (" file doesn't exist" )
95
- }
96
- if (length != gotten.length()) {
97
- throw new SparkException (
98
- s " file has different length $length than added file ${gotten.length()}" )
99
- }
100
- if (absolutePath == gotten.getAbsolutePath) {
101
- throw new SparkException (" file should have been copied" )
102
- }
103
- x
104
- }).count()
105
- } finally {
106
- sc.stop()
107
- }
108
- }
109
82
110
- test(" addFile works with relative path" ) {
83
+ test(" addFile works" ) {
84
+ val file1 = File .createTempFile(" someprefix1" , " somesuffix1" )
85
+ val absolutePath1 = file1.getAbsolutePath
86
+
111
87
val pluto = Utils .createTempDir()
112
- val file = File .createTempFile(" someprefix" , " somesuffix" , pluto)
113
- val relativePath = file.getParent + " /../" + file.getParentFile.getName + " /" + file.getName
114
- val absolutePath = file.getAbsolutePath
88
+ val file2 = File .createTempFile(" someprefix2" , " somesuffix2" , pluto)
89
+ val relativePath = file2.getParent + " /../" + file2.getParentFile.getName + " /" + file2.getName
90
+ val absolutePath2 = file2.getAbsolutePath
91
+
115
92
try {
116
- Files .write(" somewords" , file, UTF_8 )
117
- val length = file.length()
93
+ Files .write(" somewords1" , file1, UTF_8 )
94
+ Files .write(" somewords2" , file2, UTF_8 )
95
+ val length1 = file1.length()
96
+ val length2 = file2.length()
97
+
118
98
sc = new SparkContext (new SparkConf ().setAppName(" test" ).setMaster(" local" ))
99
+ sc.addFile(file1.getAbsolutePath)
119
100
sc.addFile(relativePath)
120
101
sc.parallelize(Array (1 ), 1 ).map(x => {
121
- val gotten = new File (SparkFiles .get(file.getName))
122
- if (! gotten.exists()) {
123
- throw new SparkException (" file doesn't exist" )
102
+ val gotten1 = new File (SparkFiles .get(file1.getName))
103
+ val gotten2 = new File (SparkFiles .get(file2.getName))
104
+ if (! gotten1.exists()) {
105
+ throw new SparkException (" file doesn't exist : " + absolutePath1)
106
+ }
107
+ if (! gotten2.exists()) {
108
+ throw new SparkException (" file doesn't exist : " + absolutePath2)
124
109
}
125
- if (length != gotten.length()) {
110
+
111
+ if (length1 != gotten1.length()) {
126
112
throw new SparkException (
127
- s " file has different length $length than added file ${gotten .length()}" )
113
+ s " file has different length $length1 than added file ${gotten1 .length()} : " + absolutePath1 )
128
114
}
129
- if (absolutePath == gotten.getAbsolutePath) {
130
- throw new SparkException (" file should have been copied" )
115
+ if (length2 != gotten2.length()) {
116
+ throw new SparkException (
117
+ s " file has different length $length2 than added file ${gotten2.length()} : " + absolutePath2)
118
+ }
119
+
120
+ if (absolutePath1 == gotten1.getAbsolutePath) {
121
+ throw new SparkException (" file should have been copied :" + absolutePath1)
122
+ }
123
+ if (absolutePath2 == gotten2.getAbsolutePath) {
124
+ throw new SparkException (" file should have been copied : " + absolutePath2)
131
125
}
132
126
x
133
127
}).count()
134
128
} finally {
135
129
sc.stop()
136
130
}
137
131
}
138
-
132
+
139
133
test(" addFile recursive works" ) {
140
134
val pluto = Utils .createTempDir()
141
135
val neptune = Utils .createTempDir(pluto.getAbsolutePath)
0 commit comments