Skip to content

Commit

Permalink
[Enhancement] make table-stats compatible with column-stats (StarRock…
Browse files Browse the repository at this point in the history
…s#50683)

Signed-off-by: Murphy <mofei@starrocks.com>
  • Loading branch information
murphyatwork authored Sep 26, 2024
1 parent 6083ecd commit 5137be6
Show file tree
Hide file tree
Showing 16 changed files with 439 additions and 55 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ private void updateLakeTabletStat() {
}

private void adjustStatUpdateRows(long tableId, long totalRowCount) {
BasicStatsMeta meta = GlobalStateMgr.getCurrentState().getAnalyzeMgr().getBasicStatsMetaMap().get(tableId);
BasicStatsMeta meta = GlobalStateMgr.getCurrentState().getAnalyzeMgr().getTableBasicStatsMeta(tableId);
if (meta != null) {
meta.setUpdateRows(totalRowCount);
}
Expand Down
5 changes: 4 additions & 1 deletion fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java
Original file line number Diff line number Diff line change
Expand Up @@ -1287,6 +1287,7 @@ private void analyzePlanWithExecStats(ExecPlan execPlan) {
}
}

// TODO: move to DdlExecutor
private void handleAnalyzeStmt() throws IOException {
AnalyzeStmt analyzeStmt = (AnalyzeStmt) parsedStmt;
TableName tableName = analyzeStmt.getTableName();
Expand Down Expand Up @@ -1344,10 +1345,12 @@ private void handleAnalyzeStmt() throws IOException {
} catch (RejectedExecutionException e) {
analyzeStatus.setStatus(StatsConstants.ScheduleStatus.FAILED);
analyzeStatus.setReason("The statistics tasks running concurrently exceed the upper limit");
LOG.warn("analyze statement exceed concurrency limit {}", analyzeStmt.toString(), e);
GlobalStateMgr.getCurrentState().getAnalyzeMgr().addAnalyzeStatus(analyzeStatus);
} catch (ExecutionException | InterruptedException e) {
analyzeStatus.setStatus(StatsConstants.ScheduleStatus.FAILED);
analyzeStatus.setReason("The statistics tasks running failed");
analyzeStatus.setReason("analyze failed due to " + e.getMessage());
LOG.warn("analyze statement failed {}", analyzeStmt.toString(), e);
GlobalStateMgr.getCurrentState().getAnalyzeMgr().addAnalyzeStatus(analyzeStatus);
} finally {
context.getSessionVariable().setQueryTimeoutS(timeout);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,12 @@ public ShowBasicStatsMetaStmt(Predicate predicate, NodePosition pos) {
.addColumn(new Column("UpdateTime", ScalarType.createVarchar(60)))
.addColumn(new Column("Properties", ScalarType.createVarchar(200)))
.addColumn(new Column("Healthy", ScalarType.createVarchar(5)))
.addColumn(new Column("ColumnStats", ScalarType.createVarcharType(128)))
.build();

public static List<String> showBasicStatsMeta(ConnectContext context,
BasicStatsMeta basicStatsMeta) throws MetaNotFoundException {
List<String> row = Lists.newArrayList("", "", "ALL", "", "", "", "");
List<String> row = Lists.newArrayList("", "", "ALL", "", "", "", "", "");
long dbId = basicStatsMeta.getDbId();
long tableId = basicStatsMeta.getTableId();
List<String> columns = basicStatsMeta.getColumns();
Expand Down Expand Up @@ -93,6 +94,7 @@ public static List<String> showBasicStatsMeta(ConnectContext context,
row.set(4, basicStatsMeta.getUpdateTime().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")));
row.set(5, basicStatsMeta.getProperties() == null ? "{}" : basicStatsMeta.getProperties().toString());
row.set(6, (int) (basicStatsMeta.getHealthy() * 100) + "%");
row.set(7, basicStatsMeta.getColumnStatsString());

return row;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ public static long getTableRowCount(Table table, Operator node, OptimizerContext
long rowCount = 0;

BasicStatsMeta basicStatsMeta =
GlobalStateMgr.getCurrentState().getAnalyzeMgr().getBasicStatsMetaMap().get(table.getId());
GlobalStateMgr.getCurrentState().getAnalyzeMgr().getTableBasicStatsMeta(table.getId());
StatsConstants.AnalyzeType analyzeType = basicStatsMeta == null ? null : basicStatsMeta.getType();
LocalDateTime lastWorkTimestamp = GlobalStateMgr.getCurrentState().getTabletStatMgr().getLastWorkTimestamp();
if (StatsConstants.AnalyzeType.FULL == analyzeType) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,10 @@ public void replayRemoveBasicStatsMeta(BasicStatsMeta basicStatsMeta) {
basicStatsMetaMap.remove(basicStatsMeta.getTableId());
}

public BasicStatsMeta getTableBasicStatsMeta(long tableId) {
return basicStatsMetaMap.get(tableId);
}

public Map<Long, BasicStatsMeta> getBasicStatsMetaMap() {
return basicStatsMetaMap;
}
Expand Down Expand Up @@ -816,7 +820,8 @@ public void load(SRMetaBlockReader reader) throws IOException, SRMetaBlockExcept
}

private void updateBasicStatsMeta(long dbId, long tableId, long loadedRows) {
BasicStatsMeta basicStatsMeta = GlobalStateMgr.getCurrentState().getAnalyzeMgr().getBasicStatsMetaMap().get(tableId);
BasicStatsMeta basicStatsMeta =
GlobalStateMgr.getCurrentState().getAnalyzeMgr().getTableBasicStatsMeta(tableId);
if (basicStatsMeta == null) {
// first load without analyze op, we need fill a meta with loaded rows for cardinality estimation
BasicStatsMeta meta = new BasicStatsMeta(dbId, tableId, Lists.newArrayList(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

package com.starrocks.statistic;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
import com.starrocks.catalog.Database;
import com.starrocks.catalog.OlapTable;
Expand All @@ -22,6 +24,7 @@
import com.starrocks.common.io.Writable;
import com.starrocks.persist.gson.GsonUtils;
import com.starrocks.server.GlobalStateMgr;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.collections4.MapUtils;

import java.io.DataInput;
Expand All @@ -32,6 +35,7 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;

public class BasicStatsMeta implements Writable {
@SerializedName("dbId")
Expand All @@ -40,6 +44,10 @@ public class BasicStatsMeta implements Writable {
@SerializedName("tableId")
private long tableId;

// Deprecated by columnStatsMetaMap
// But for backward compatibility, we still need to write into this field, to make sure the behavior is still
// correct after rollback
@Deprecated
@SerializedName("columns")
private List<String> columns;

Expand All @@ -61,6 +69,15 @@ public class BasicStatsMeta implements Writable {
@SerializedName("deltaRows")
private long deltaRows;

// TODO: use ColumnId
@SerializedName("columnStats")
private Map<String, ColumnStatsMeta> columnStatsMetaMap = Maps.newConcurrentMap();

// Used for deserialization
public BasicStatsMeta() {
columnStatsMetaMap = Maps.newConcurrentMap();
}

public BasicStatsMeta(long dbId, long tableId, List<String> columns,
StatsConstants.AnalyzeType type,
LocalDateTime updateTime,
Expand Down Expand Up @@ -102,6 +119,9 @@ public long getTableId() {
}

public List<String> getColumns() {
if (MapUtils.isNotEmpty(columnStatsMetaMap)) {
return Lists.newArrayList(columnStatsMetaMap.keySet());
}
// Just for compatibility, there are no columns in the old code,
// and the columns may be null after deserialization.
if (columns == null) {
Expand Down Expand Up @@ -190,4 +210,43 @@ public boolean isUpdatedAfterLoad(LocalDateTime loadTime) {
return updateTime.isAfter(loadTime);
}
}

public void setProperties(Map<String, String> properties) {
this.properties = properties;
}

public void setUpdateTime(LocalDateTime updateTime) {
this.updateTime = updateTime;
}

public void setAnalyzeType(StatsConstants.AnalyzeType analyzeType) {
this.type = analyzeType;
}

public Map<String, ColumnStatsMeta> getAnalyzedColumns() {
Map<String, ColumnStatsMeta> deduplicate = Maps.newHashMap();
// TODO: just for compatible, we can remove it at next version
for (String column : ListUtils.emptyIfNull(columns)) {
deduplicate.put(column, new ColumnStatsMeta(column, type, updateTime));
}
deduplicate.putAll(columnStatsMetaMap);
return deduplicate;
}

public String getColumnStatsString() {
if (MapUtils.isEmpty(columnStatsMetaMap)) {
return "";
}
return columnStatsMetaMap.values().stream()
.map(ColumnStatsMeta::simpleString).collect(Collectors.joining(","));
}

public void addColumnStatsMeta(ColumnStatsMeta columnStatsMeta) {
this.columnStatsMetaMap.put(columnStatsMeta.getColumnName(), columnStatsMeta);
}

public BasicStatsMeta clone() {
String json = GsonUtils.GSON.toJson(this);
return GsonUtils.GSON.fromJson(json, BasicStatsMeta.class);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
// Copyright 2021-present StarRocks, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.starrocks.statistic;

import com.google.gson.annotations.SerializedName;

import java.time.LocalDateTime;
import java.util.Objects;

/**
* Meta of column-level statistics
*/
public class ColumnStatsMeta {

@SerializedName("columnName")
private String columnName;

@SerializedName("type")
private StatsConstants.AnalyzeType type;

@SerializedName("updateTime")
private LocalDateTime updateTime;

public ColumnStatsMeta(String columnName, StatsConstants.AnalyzeType type, LocalDateTime updateTime) {
this.columnName = columnName;
this.type = type;
this.updateTime = updateTime;
}

public String getColumnName() {
return columnName;
}

public void setColumnName(String columnName) {
this.columnName = columnName;
}

public StatsConstants.AnalyzeType getType() {
return type;
}

public void setType(StatsConstants.AnalyzeType type) {
this.type = type;
}

public LocalDateTime getUpdateTime() {
return updateTime;
}

public void setUpdateTime(LocalDateTime updateTime) {
this.updateTime = updateTime;
}

public String simpleString() {
return String.format("(%s,%s)", columnName, type.toString());
}

@Override
public String toString() {
final StringBuffer sb = new StringBuffer("ColumnStatsMeta{");
sb.append("columnName='").append(columnName).append('\'');
sb.append(", type=").append(type);
sb.append('}');
return sb.toString();
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
ColumnStatsMeta that = (ColumnStatsMeta) o;
return Objects.equals(columnName, that.columnName) && type == that.type;
}

@Override
public int hashCode() {
return Objects.hash(columnName, type);
}
}
Loading

0 comments on commit 5137be6

Please sign in to comment.