|
20 | 20 | import java.io.*;
|
21 | 21 | import java.nio.channels.FileChannel;
|
22 | 22 | import java.nio.ByteBuffer;
|
23 |
| -import java.net.URI; |
24 | 23 | import java.nio.charset.StandardCharsets;
|
25 | 24 | import java.util.ArrayList;
|
26 | 25 | import java.util.Arrays;
|
|
46 | 45 | import com.google.common.collect.Lists;
|
47 | 46 | import com.google.common.base.Throwables;
|
48 | 47 | import com.google.common.io.Files;
|
| 48 | +import org.apache.hadoop.fs.Path; |
49 | 49 | import org.apache.hadoop.io.IntWritable;
|
50 | 50 | import org.apache.hadoop.io.Text;
|
51 | 51 | import org.apache.hadoop.io.compress.DefaultCodec;
|
@@ -1075,18 +1075,23 @@ public void wholeTextFiles() throws Exception {
|
1075 | 1075 | byte[] content2 = "spark is also easy to use.\n".getBytes(StandardCharsets.UTF_8);
|
1076 | 1076 |
|
1077 | 1077 | String tempDirName = tempDir.getAbsolutePath();
|
1078 |
| - Files.write(content1, new File(tempDirName + "/part-00000")); |
1079 |
| - Files.write(content2, new File(tempDirName + "/part-00001")); |
| 1078 | + String path1 = new Path(tempDirName, "part-00000").toUri().getPath(); |
| 1079 | + String path2 = new Path(tempDirName, "part-00001").toUri().getPath(); |
| 1080 | + |
| 1081 | + Files.write(content1, new File(path1)); |
| 1082 | + Files.write(content2, new File(path2)); |
1080 | 1083 |
|
1081 | 1084 | Map<String, String> container = new HashMap<>();
|
1082 |
| - container.put(tempDirName+"/part-00000", new Text(content1).toString()); |
1083 |
| - container.put(tempDirName+"/part-00001", new Text(content2).toString()); |
| 1085 | + container.put(path1, new Text(content1).toString()); |
| 1086 | + container.put(path2, new Text(content2).toString()); |
1084 | 1087 |
|
1085 | 1088 | JavaPairRDD<String, String> readRDD = sc.wholeTextFiles(tempDirName, 3);
|
1086 | 1089 | List<Tuple2<String, String>> result = readRDD.collect();
|
1087 | 1090 |
|
1088 | 1091 | for (Tuple2<String, String> res : result) {
|
1089 |
| - assertEquals(res._2(), container.get(new URI(res._1()).getPath())); |
| 1092 | + // Note that the paths from `wholeTextFiles` are in URI format on Windows, |
| 1093 | + // for example, file:/C:/a/b/c. |
| 1094 | + assertEquals(res._2(), container.get(new Path(res._1()).toUri().getPath())); |
1090 | 1095 | }
|
1091 | 1096 | }
|
1092 | 1097 |
|
|
0 commit comments