KYLIN-4616 Auto detect the default value of config kylin.source.hive.…

…databasedir
yanshuai · Jul 9, 2020 · 43c3ef8 · 43c3ef8
1 parent 2c70819
commit 43c3ef8
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 6 deletions.
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
@@ -207,5 +207,7 @@ hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog}
 verbose "hive dependency is $hive_dependency"
 export hive_dependency
 export hive_conf_path
+export hiveWarehouseDir=`hive -e 'set hive.metastore.warehouse.dir;' | awk '{split($0,a,"="); print a[2]}'`
+echo "export hiveWarehouseDir=$hiveWarehouseDir"
 echo "export hive_dependency=$hive_dependency
 export hive_conf_path=$hive_conf_path" > ${dir}/cached-hive-dependency.sh
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
@@ -154,6 +154,7 @@ function retrieveStartCommand() {
     -Dkylin.flink.dependency=${flink_dependency} \
     -Dkylin.hadoop.conf.dir=${kylin_hadoop_conf_dir} \
     -Dkylin.server.host-address=${kylin_rest_address} \
+    -Dkylin.source.hive.warehouse-dir=${hiveWarehouseDir} \
     -Dspring.profiles.active=${spring_profile} \
     org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar  org.apache.catalina.startup.Bootstrap start"
 }

diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -29,6 +29,7 @@
 import org.apache.kylin.common.util.ClassUtil;
 import org.apache.kylin.common.util.CliCommandExecutor;
 import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.StringUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -611,8 +612,8 @@ public String[] getMrHiveDictColumns() {
     /**
      * @return the hdfs path for Hive Global dictionary table
      */
-    public String getHiveDatabaseDir() {
-        return this.getOptional("kylin.source.hive.databasedir", "");
+    public String getGlobalDictHiveDatabaseDir() {
+        return this.getOptional("kylin.source.hive.global-dict-database-dir", getHiveDatabaseDir(getMrHiveDictDB()));
     }
 
     public String[] getMrHiveDictColumnsExcludeRefColumns() {
@@ -1057,6 +1058,17 @@ public int getJobOutputMaxSize() {
     // SOURCE.HIVE
     // ============================================================================
 
+    public String getHiveDatabaseDir(String databaseName) {
+        String dbDir = System.getProperty("kylin.source.hive.warehouse-dir");
+        if (!StringUtil.isEmpty(databaseName) && !databaseName.equalsIgnoreCase(DEFAULT)) {
+            if (!dbDir.endsWith("/")) {
+                dbDir += "/";
+            }
+            dbDir += databaseName + ".db";
+        }
+        return dbDir;
+    }
+
     public int getDefaultSource() {
         return Integer.parseInt(getOptional("kylin.source.default", "0"));
     }
@@ -1126,7 +1138,6 @@ public String getHiveDatabaseForIntermediateTable() {
         return CliCommandExecutor.checkHiveProperty(this.getOptional("kylin.source.hive.database-for-flat-table", DEFAULT));
     }
 
-
     public String getFlatTableStorageFormat() {
         return this.getOptional("kylin.source.hive.flat-table-storage-format", "SEQUENCEFILE");
     }

diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -389,7 +389,7 @@ public String getBuildGlobalHiveDicTotalBuildJobInputPath(String jobId) {
 
     public String getBuildGlobalDictionaryMaxDistinctCountPath(String jobId) {
         KylinConfig conf = seg.getConfig();
-        String dbDir = conf.getHiveDatabaseDir();
+        String dbDir = conf.getGlobalDictHiveDatabaseDir();
         IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(seg);
         String tableName = flatDesc.getTableName() + conf.getMrHiveDistinctValueTableSuffix();
         String outPut = dbDir + "/" + tableName + "/dict_column=" + BatchConstants.CFG_GLOBAL_DICT_STATS_PARTITION_VALUE;
@@ -401,7 +401,7 @@ public String getBuildGlobalDictionaryPartReduceStatsPathV2(String jobId) {
     }
 
     public String getBuildGlobalDictionaryTotalOutput(KylinConfig config) {
-        String dbDir = config.getHiveDatabaseDir();
+        String dbDir = config.getGlobalDictHiveDatabaseDir();
         String tableName = EngineFactory.getJoinedFlatTableDesc(seg).getTableName() + config.getMrHiveDictTableSuffix();
         String path = dbDir + "/" + tableName;
         return path;

diff --git a/...ne-mr/src/main/java/org/apache/kylin/engine/mr/steps/BuildGlobalHiveDictPartBuildJob.java b/...ne-mr/src/main/java/org/apache/kylin/engine/mr/steps/BuildGlobalHiveDictPartBuildJob.java
@@ -144,7 +144,7 @@ private void setReduceNum(Job job, KylinConfig config) {
     }
 
     private String getInputPath(KylinConfig config, CubeSegment segment) {
-        String dbDir = config.getHiveDatabaseDir();
+        String dbDir = config.getGlobalDictHiveDatabaseDir();
         String tableName = EngineFactory.getJoinedFlatTableDesc(segment).getTableName() + config.getMrHiveDistinctValueTableSuffix();
         String input = dbDir + "/" + tableName;
         logger.info("part build base input path:" + input);