KYLIN-4688 Too many tmp files in HDFS tmp dictionary

yanshuai · Sep 2, 2020 · b519ecd · b519ecd
1 parent e6d9fb0
commit b519ecd
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 9 deletions.
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -27,7 +27,6 @@
 import java.util.Map;
 
 import org.apache.kylin.shaded.com.google.common.collect.Lists;
-import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -69,13 +68,6 @@ public static Configuration getCurrentConfiguration() {
     }
 
     public static Configuration healSickConfig(Configuration conf) {
-        // https://issues.apache.org/jira/browse/KYLIN-953
-        if (StringUtils.isBlank(conf.get("hadoop.tmp.dir"))) {
-            conf.set("hadoop.tmp.dir", "/tmp");
-        }
-        if (StringUtils.isBlank(conf.get("hbase.fs.tmp.dir"))) {
-            conf.set("hbase.fs.tmp.dir", "/tmp");
-        }
         //  https://issues.apache.org/jira/browse/KYLIN-3064
         conf.set("yarn.timeline-service.enabled", "false");
         return conf;

diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileJob.java
@@ -58,6 +58,7 @@ public class CubeHFileJob extends AbstractHadoopJob {
 
     public int run(String[] args) throws Exception {
         Options options = new Options();
+        Path tmpPartitionFilePath = null;
 
         try {
             options.addOption(OPTION_JOB_NAME);
@@ -101,6 +102,7 @@ public int run(String[] args) throws Exception {
 
             // Automatic config !
             HFileOutputFormat3.configureIncrementalLoad(job, htable);
+            tmpPartitionFilePath = new Path(TotalOrderPartitioner.getPartitionFile(job.getConfiguration()));
             HFileOutputFormat3.configureHConnection(job, hbaseConf, getJobTempDir());
 
             reconfigurePartitions(configuration, partitionFilePath);
@@ -119,8 +121,10 @@ public int run(String[] args) throws Exception {
 
             return waitForCompletion(job);
         } finally {
-            if (job != null)
+            if (job != null) {
                 cleanupTempConfFile(job.getConfiguration());
+                this.deletePath(job.getConfiguration(), tmpPartitionFilePath);
+            }
         }
     }