From e9e427a73ef08314f983a2f3dfb37ecb37c752be Mon Sep 17 00:00:00 2001 From: Yichen Zhou Date: Wed, 24 Oct 2018 15:12:13 +0800 Subject: [PATCH] KYLIN-3597 Fix sonar issues --- .../org/apache/kylin/cube/CubeManager.java | 3 +- .../query/enumerator/HiveEnumerator.java | 2 +- .../kylin/source/hive/BeelineHiveClient.java | 50 +++++++++++-------- .../hbase/util/GridTableHBaseBenchmark.java | 2 +- .../apache/kylin/tool/CubeMetaIngester.java | 45 +++++++---------- 5 files changed, 52 insertions(+), 50 deletions(-) diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 7e3be40cdb0..7801185f1db 100755 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -125,6 +125,8 @@ static CubeManager newInstance(KylinConfig config) throws IOException { private SegmentAssist segAssist = new SegmentAssist(); private DictionaryAssist dictAssist = new DictionaryAssist(); + private Random ran = new Random(); + private CubeManager(KylinConfig cfg) throws IOException { logger.info("Initializing CubeManager with config {}", cfg); this.config = cfg; @@ -524,7 +526,6 @@ private String getSnapshotResPath(CubeSegment cubeSegment, String tableName, Sna String namePrefix = config.getHBaseTableNamePrefix(); String namespace = config.getHBaseStorageNameSpace(); String tableName = ""; - Random ran = new Random(); do { StringBuffer sb = new StringBuffer(); if ((namespace.equals("default") || namespace.equals("")) == false) { diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/HiveEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/HiveEnumerator.java index 2b258f03263..f1b674b6323 100644 --- a/query/src/main/java/org/apache/kylin/query/enumerator/HiveEnumerator.java +++ b/query/src/main/java/org/apache/kylin/query/enumerator/HiveEnumerator.java @@ -62,7 +62,7 @@ private ResultSet executeQuery() { String url = olapContext.olapSchema.getStarSchemaUrl(); String user = olapContext.olapSchema.getStarSchemaUser(); String pwd = olapContext.olapSchema.getStarSchemaPassword(); - String sql = olapContext.sql; + String sql = olapContext.sql.toString(); Statement stmt = null; try { conn = DriverManager.getConnection(url, user, pwd); diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java index b1021dc8850..6ad8593d8a9 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java @@ -112,7 +112,8 @@ public long getHiveTableRows(String database, String tableName) throws Exception ResultSet resultSet = null; long count = 0; try { - resultSet = stmt.executeQuery("select count(*) from " + database + "." + tableName); + String query = "select count(*) from "; + resultSet = stmt.executeQuery(query.concat(database + "." + tableName)); if (resultSet.next()) { count = resultSet.getLong(1); } @@ -147,8 +148,10 @@ public HiveTableMeta getHiveTableMeta(String database, String tableName) throws } builder.setAllColumns(allColumns); DBUtils.closeQuietly(columns); - stmt.execute("use " + database); - ResultSet resultSet = stmt.executeQuery("describe formatted " + tableName); + String exe = "use "; + stmt.execute(exe.concat(database)); + String des = "describe formatted "; + ResultSet resultSet = stmt.executeQuery(des.concat(tableName)); extractHiveTableMeta(resultSet, builder); DBUtils.closeQuietly(resultSet); return builder.createHiveTableMeta(); @@ -169,7 +172,11 @@ public static String considerDataTypePrecision(String dataType, String precision private void extractHiveTableMeta(ResultSet resultSet, HiveTableMetaBuilder builder) throws SQLException { while (resultSet.next()) { + parseResultEntry(resultSet, builder); + } + } + private void parseResultEntry(ResultSet resultSet, HiveTableMetaBuilder builder) throws SQLException{ List partitionColumns = Lists.newArrayList(); if ("# Partition Information".equals(resultSet.getString(1).trim())) { resultSet.next(); @@ -204,23 +211,7 @@ private void extractHiveTableMeta(ResultSet resultSet, HiveTableMetaBuilder buil builder.setTableType(resultSet.getString(2).trim()); } if ("Table Parameters:".equals(resultSet.getString(1).trim())) { - while (resultSet.next()) { - if (resultSet.getString(2) == null) { - break; - } - if ("storage_handler".equals(resultSet.getString(2).trim())) { - builder.setIsNative(false);//default is true - } - if ("totalSize".equals(resultSet.getString(2).trim())) { - builder.setFileSize(Long.parseLong(resultSet.getString(3).trim()));//default is false - } - if ("numFiles".equals(resultSet.getString(2).trim())) { - builder.setFileNum(Long.parseLong(resultSet.getString(3).trim())); - } - if ("skip.header.line.count".equals(resultSet.getString(2).trim())) { - builder.setSkipHeaderLineCount(resultSet.getString(3).trim()); - } - } + extractTableParam(resultSet, builder); } if ("InputFormat:".equals(resultSet.getString(1).trim())) { builder.setSdInputFormat(resultSet.getString(2).trim()); @@ -228,6 +219,25 @@ private void extractHiveTableMeta(ResultSet resultSet, HiveTableMetaBuilder buil if ("OutputFormat:".equals(resultSet.getString(1).trim())) { builder.setSdOutputFormat(resultSet.getString(2).trim()); } + } + + private void extractTableParam(ResultSet resultSet, HiveTableMetaBuilder builder) throws SQLException { + while (resultSet.next()) { + if (resultSet.getString(2) == null) { + break; + } + if ("storage_handler".equals(resultSet.getString(2).trim())) { + builder.setIsNative(false);//default is true + } + if ("totalSize".equals(resultSet.getString(2).trim())) { + builder.setFileSize(Long.parseLong(resultSet.getString(3).trim()));//default is false + } + if ("numFiles".equals(resultSet.getString(2).trim())) { + builder.setFileNum(Long.parseLong(resultSet.getString(3).trim())); + } + if ("skip.header.line.count".equals(resultSet.getString(2).trim())) { + builder.setSkipHeaderLineCount(resultSet.getString(3).trim()); + } } } diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java index fb944dce0d0..40c28907357 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java @@ -56,6 +56,7 @@ public class GridTableHBaseBenchmark { private static final double DFT_HIT_RATIO = 0.3; private static final double DFT_INDEX_RATIO = 0.1; private static final int ROUND = 3; + private static final Random rand = new Random(); protected static final Logger logger = LoggerFactory.getLogger(GridTableHBaseBenchmark.class); public static void main(String[] args) throws IOException { @@ -257,7 +258,6 @@ private static void dot(int i, int nRows) { private static byte[] randomBytes() { byte[] bytes = new byte[CELL_SIZE]; - Random rand = new Random(); rand.nextBytes(bytes); return bytes; } diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java index a8e0ab3fb56..4030913796e 100644 --- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java +++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java @@ -20,6 +20,8 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Set; @@ -28,6 +30,7 @@ import org.apache.commons.cli.Options; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceTool; +import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.common.util.AbstractApplication; import org.apache.kylin.common.util.OptionsHelper; import org.apache.kylin.common.util.ZipFileUtils; @@ -112,10 +115,9 @@ protected void execute(OptionsHelper optionsHelper) throws Exception { throw new IllegalArgumentException(OPTION_SRC.getArgName() + " file does does exist"); } - File tempFolder = File.createTempFile("_unzip", "folder"); + Path tempPath = Files.createTempDirectory("_unzip"); + File tempFolder = tempPath.toFile(); tempFolder.deleteOnExit(); - tempFolder.delete(); - tempFolder.mkdir(); ZipFileUtils.decompressZipfileToDirectory(srcPath, tempFolder); if (tempFolder.list().length != 1) { throw new IllegalStateException(Arrays.toString(tempFolder.list())); @@ -182,45 +184,34 @@ private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelMana DataModelManager modelManager = DataModelManager.getInstance(kylinConfig); for (DataModelDesc dataModelDesc : srcModelManager.listDataModels()) { - DataModelDesc existing = modelManager.getDataModelDesc(dataModelDesc.getName()); - if (existing != null) { - if (!forceIngest) { - throw new IllegalStateException("Already exist a model called " + dataModelDesc.getName()); - } else { - logger.warn("Overwriting the old model desc: " + dataModelDesc.getName()); - } - } + checkExesting(modelManager.getDataModelDesc(dataModelDesc.getName()), "model", dataModelDesc.getName()); requiredResources.add(DataModelDesc.concatResourcePath(dataModelDesc.getName())); } CubeDescManager cubeDescManager = CubeDescManager.getInstance(kylinConfig); for (CubeDesc cubeDesc : srcCubeDescManager.listAllDesc()) { - CubeDesc existing = cubeDescManager.getCubeDesc(cubeDesc.getName()); - if (existing != null) { - if (!forceIngest) { - throw new IllegalStateException("Already exist a cube desc called " + cubeDesc.getName()); - } else { - logger.warn("Overwriting the old cube desc: " + cubeDesc.getName()); - } - } + checkExesting(cubeDescManager.getCubeDesc(cubeDesc.getName()), "cube desc", cubeDesc.getName()); requiredResources.add(CubeDesc.concatResourcePath(cubeDesc.getName())); } CubeManager cubeManager = CubeManager.getInstance(kylinConfig); for (CubeInstance cube : srcCubeManager.listAllCubes()) { - CubeInstance existing = cubeManager.getCube(cube.getName()); - if (existing != null) { - if (!forceIngest) { - throw new IllegalStateException("Already exist a cube called " + cube.getName()); - } else { - logger.warn("Overwriting the old cube: " + cube.getName()); - } - } + checkExesting(cubeManager.getCube(cube.getName()), "cube", cube.getName()); requiredResources.add(CubeInstance.concatResourcePath(cube.getName())); } } + private void checkExesting(RootPersistentEntity existing, String type, String name) { + if (existing != null) { + if (!forceIngest) { + throw new IllegalStateException("Already exist a " + type + " called " + name); + } else { + logger.warn("Overwriting the old {0} desc: {1}", type, name); + } + } + } + public static void main(String[] args) { CubeMetaIngester extractor = new CubeMetaIngester(); extractor.execute(args);