Skip to content

Commit

Permalink
KYLIN-3597 Fix sonar issues
Browse files Browse the repository at this point in the history
  • Loading branch information
@ZhouYC627 @shaofengshi
ZhouYC627 authored and shaofengshi committed Oct 25, 2018
1 parent 0f0346b commit e9e427a
Show file tree
Hide file tree
Showing 5 changed files with 52 additions and 50 deletions.
3 changes: 2 additions & 1 deletion core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ static CubeManager newInstance(KylinConfig config) throws IOException {
private SegmentAssist segAssist = new SegmentAssist();
private DictionaryAssist dictAssist = new DictionaryAssist();

private Random ran = new Random();

private CubeManager(KylinConfig cfg) throws IOException {
logger.info("Initializing CubeManager with config {}", cfg);
this.config = cfg;
Expand Down Expand Up @@ -524,7 +526,6 @@ private String getSnapshotResPath(CubeSegment cubeSegment, String tableName, Sna
String namePrefix = config.getHBaseTableNamePrefix();
String namespace = config.getHBaseStorageNameSpace();
String tableName = "";
Random ran = new Random();
do {
StringBuffer sb = new StringBuffer();
if ((namespace.equals("default") || namespace.equals("")) == false) {
Expand Down
2 changes: 1 addition & 1 deletion query/src/main/java/org/apache/kylin/query/enumerator/HiveEnumerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ private ResultSet executeQuery() {
String url = olapContext.olapSchema.getStarSchemaUrl();
String user = olapContext.olapSchema.getStarSchemaUser();
String pwd = olapContext.olapSchema.getStarSchemaPassword();
String sql = olapContext.sql;
String sql = olapContext.sql.toString();
Statement stmt = null;
try {
conn = DriverManager.getConnection(url, user, pwd);
Expand Down
50 changes: 30 additions & 20 deletions source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,8 @@ public long getHiveTableRows(String database, String tableName) throws Exception
ResultSet resultSet = null;
long count = 0;
try {
resultSet = stmt.executeQuery("select count(*) from " + database + "." + tableName);
String query = "select count(*) from ";
resultSet = stmt.executeQuery(query.concat(database + "." + tableName));
if (resultSet.next()) {
count = resultSet.getLong(1);
}
Expand Down Expand Up @@ -147,8 +148,10 @@ public HiveTableMeta getHiveTableMeta(String database, String tableName) throws
}
builder.setAllColumns(allColumns);
DBUtils.closeQuietly(columns);
stmt.execute("use " + database);
ResultSet resultSet = stmt.executeQuery("describe formatted " + tableName);
String exe = "use ";
stmt.execute(exe.concat(database));
String des = "describe formatted ";
ResultSet resultSet = stmt.executeQuery(des.concat(tableName));
extractHiveTableMeta(resultSet, builder);
DBUtils.closeQuietly(resultSet);
return builder.createHiveTableMeta();
Expand All @@ -169,7 +172,11 @@ public static String considerDataTypePrecision(String dataType, String precision

private void extractHiveTableMeta(ResultSet resultSet, HiveTableMetaBuilder builder) throws SQLException {
while (resultSet.next()) {
parseResultEntry(resultSet, builder);
}
}

private void parseResultEntry(ResultSet resultSet, HiveTableMetaBuilder builder) throws SQLException{
List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = Lists.newArrayList();
if ("# Partition Information".equals(resultSet.getString(1).trim())) {
resultSet.next();
Expand Down Expand Up @@ -204,30 +211,33 @@ private void extractHiveTableMeta(ResultSet resultSet, HiveTableMetaBuilder buil
builder.setTableType(resultSet.getString(2).trim());
}
if ("Table Parameters:".equals(resultSet.getString(1).trim())) {
while (resultSet.next()) {
if (resultSet.getString(2) == null) {
break;
}
if ("storage_handler".equals(resultSet.getString(2).trim())) {
builder.setIsNative(false);//default is true
}
if ("totalSize".equals(resultSet.getString(2).trim())) {
builder.setFileSize(Long.parseLong(resultSet.getString(3).trim()));//default is false
}
if ("numFiles".equals(resultSet.getString(2).trim())) {
builder.setFileNum(Long.parseLong(resultSet.getString(3).trim()));
}
if ("skip.header.line.count".equals(resultSet.getString(2).trim())) {
builder.setSkipHeaderLineCount(resultSet.getString(3).trim());
}
}
extractTableParam(resultSet, builder);
}
if ("InputFormat:".equals(resultSet.getString(1).trim())) {
builder.setSdInputFormat(resultSet.getString(2).trim());
}
if ("OutputFormat:".equals(resultSet.getString(1).trim())) {
builder.setSdOutputFormat(resultSet.getString(2).trim());
}
}

private void extractTableParam(ResultSet resultSet, HiveTableMetaBuilder builder) throws SQLException {
while (resultSet.next()) {
if (resultSet.getString(2) == null) {
break;
}
if ("storage_handler".equals(resultSet.getString(2).trim())) {
builder.setIsNative(false);//default is true
}
if ("totalSize".equals(resultSet.getString(2).trim())) {
builder.setFileSize(Long.parseLong(resultSet.getString(3).trim()));//default is false
}
if ("numFiles".equals(resultSet.getString(2).trim())) {
builder.setFileNum(Long.parseLong(resultSet.getString(3).trim()));
}
if ("skip.header.line.count".equals(resultSet.getString(2).trim())) {
builder.setSkipHeaderLineCount(resultSet.getString(3).trim());
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ public class GridTableHBaseBenchmark {
private static final double DFT_HIT_RATIO = 0.3;
private static final double DFT_INDEX_RATIO = 0.1;
private static final int ROUND = 3;
private static final Random rand = new Random();
protected static final Logger logger = LoggerFactory.getLogger(GridTableHBaseBenchmark.class);

public static void main(String[] args) throws IOException {
Expand Down Expand Up @@ -257,7 +258,6 @@ private static void dot(int i, int nRows) {

private static byte[] randomBytes() {
byte[] bytes = new byte[CELL_SIZE];
Random rand = new Random();
rand.nextBytes(bytes);
return bytes;
}
Expand Down
45 changes: 18 additions & 27 deletions tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Set;

Expand All @@ -28,6 +30,7 @@
import org.apache.commons.cli.Options;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceTool;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.ZipFileUtils;
Expand Down Expand Up @@ -112,10 +115,9 @@ protected void execute(OptionsHelper optionsHelper) throws Exception {
throw new IllegalArgumentException(OPTION_SRC.getArgName() + " file does does exist");
}

File tempFolder = File.createTempFile("_unzip", "folder");
Path tempPath = Files.createTempDirectory("_unzip");
File tempFolder = tempPath.toFile();
tempFolder.deleteOnExit();
tempFolder.delete();
tempFolder.mkdir();
ZipFileUtils.decompressZipfileToDirectory(srcPath, tempFolder);
if (tempFolder.list().length != 1) {
throw new IllegalStateException(Arrays.toString(tempFolder.list()));
Expand Down Expand Up @@ -182,45 +184,34 @@ private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelMana

DataModelManager modelManager = DataModelManager.getInstance(kylinConfig);
for (DataModelDesc dataModelDesc : srcModelManager.listDataModels()) {
DataModelDesc existing = modelManager.getDataModelDesc(dataModelDesc.getName());
if (existing != null) {
if (!forceIngest) {
throw new IllegalStateException("Already exist a model called " + dataModelDesc.getName());
} else {
logger.warn("Overwriting the old model desc: " + dataModelDesc.getName());
}
}
checkExesting(modelManager.getDataModelDesc(dataModelDesc.getName()), "model", dataModelDesc.getName());
requiredResources.add(DataModelDesc.concatResourcePath(dataModelDesc.getName()));
}

CubeDescManager cubeDescManager = CubeDescManager.getInstance(kylinConfig);
for (CubeDesc cubeDesc : srcCubeDescManager.listAllDesc()) {
CubeDesc existing = cubeDescManager.getCubeDesc(cubeDesc.getName());
if (existing != null) {
if (!forceIngest) {
throw new IllegalStateException("Already exist a cube desc called " + cubeDesc.getName());
} else {
logger.warn("Overwriting the old cube desc: " + cubeDesc.getName());
}
}
checkExesting(cubeDescManager.getCubeDesc(cubeDesc.getName()), "cube desc", cubeDesc.getName());
requiredResources.add(CubeDesc.concatResourcePath(cubeDesc.getName()));
}

CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
for (CubeInstance cube : srcCubeManager.listAllCubes()) {
CubeInstance existing = cubeManager.getCube(cube.getName());
if (existing != null) {
if (!forceIngest) {
throw new IllegalStateException("Already exist a cube called " + cube.getName());
} else {
logger.warn("Overwriting the old cube: " + cube.getName());
}
}
checkExesting(cubeManager.getCube(cube.getName()), "cube", cube.getName());
requiredResources.add(CubeInstance.concatResourcePath(cube.getName()));
}

}

private void checkExesting(RootPersistentEntity existing, String type, String name) {
if (existing != null) {
if (!forceIngest) {
throw new IllegalStateException("Already exist a " + type + " called " + name);
} else {
logger.warn("Overwriting the old {0} desc: {1}", type, name);
}
}
}

public static void main(String[] args) {
CubeMetaIngester extractor = new CubeMetaIngester();
extractor.execute(args);
Expand Down

0 comments on commit e9e427a

Please sign in to comment.