Merge pull request apache#595 from mapr/SPARK-638

MapR [SPARK-638] Wrong permissions when creating files under directory

Author: avenherak

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala

@@ -233,6 +233,16 @@ class DynamicPartitionDataWriter(
       committer.newTaskTempFile(taskAttemptContext, partDir, ext)
     }
 
+    // Creation of intermediate folders with mkdirs() required because
+    // intermediate folders created by create() don't inherit permission bits
+    // https://maprdrill.atlassian.net/browse/SPARK-638
+    def createIntermediateFolder(filePath: String, taskAttemptContext: TaskAttemptContext) {
+      val folderPath = new Path(filePath).getParent
+      val fileSystem = folderPath.getFileSystem(taskAttemptContext.getConfiguration)
+      fileSystem.mkdirs(folderPath)
+    }
+    createIntermediateFolder(currentPath, taskAttemptContext)
+
     currentWriter = description.outputWriterFactory.newInstance(
       path = currentPath,
       dataSchema = description.dataColumns.toStructType,