Skip to content

Commit

Permalink
KYLIN-4121 Cleanup hive view intermediate tables after job be finished
Browse files Browse the repository at this point in the history
  • Loading branch information
RupengWang authored and nichunen committed Aug 6, 2019
1 parent cb7de1d commit 40cab4f
Showing 1 changed file with 7 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,6 @@ protected ExecuteResult doWork(ExecutableContext context) throws ExecuteExceptio
StringBuffer output = new StringBuffer();
try {
output.append(cleanUpIntermediateFlatTable(config));
// don't drop view to avoid concurrent issue
//output.append(cleanUpHiveViewIntermediateTable(config));
} catch (IOException e) {
logger.error("job:" + getId() + " execute finished with exception", e);
return ExecuteResult.createError(e);
Expand All @@ -56,6 +54,7 @@ protected ExecuteResult doWork(ExecutableContext context) throws ExecuteExceptio
return new ExecuteResult(ExecuteResult.State.SUCCEED, output.toString());
}

//clean up both hive intermediate flat table and view table
private String cleanUpIntermediateFlatTable(KylinConfig config) throws IOException {
StringBuffer output = new StringBuffer();
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
Expand Down Expand Up @@ -92,9 +91,14 @@ public void setIntermediateTables(List<String> tableIdentity) {
setParam("oldHiveTables", StringUtil.join(tableIdentity, ","));
}

//get intermediate fact table and lookup table(if exists)
private List<String> getIntermediateTables() {
List<String> intermediateTables = Lists.newArrayList();
String[] tables = StringUtil.splitAndTrim(getParam("oldHiveTables"), ",");
String hiveTables = getParam("oldHiveTables");
if (this.getParams().containsKey("oldHiveViewIntermediateTables")) {
hiveTables += ("," + getParam("oldHiveViewIntermediateTables"));
}
String[] tables = StringUtil.splitAndTrim(hiveTables, ",");
for (String t : tables) {
intermediateTables.add(t);
}
Expand Down

0 comments on commit 40cab4f

Please sign in to comment.