Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[opt](merge-on-write) Reduce the version not continuous logs for merge-on-write table #40946

Merged
merged 3 commits into from
Sep 19, 2024
Merged

[opt](merge-on-write) Reduce the version not continuous logs for merge-on-write table #40946

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4b34979
tmp
bobhan1 Sep 18, 2024
38ab203
opt
bobhan1 Sep 18, 2024
4e98148
update
bobhan1 Sep 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
tmp
  • Loading branch information
@bobhan1
bobhan1 committed Sep 19, 2024
commit 4b34979164b2c38adb0d2d6f2133089bd9bd4aad
4 changes: 4 additions & 0 deletions fe/fe-common/src/main/java/org/apache/doris/common/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,10 @@ public class Config extends ConfigBase {
"print log interval for publish transaction failed interval"})
public static long publish_fail_log_interval_second = 5 * 60;

@ConfField(mutable = true, masterOnly = true, description = {"一个 PUBLISH_VERSION 任务失败打印日志次数上限",
"print log interval for publish transaction failed interval"})
public static long publish_version_task_failed_times_log_threshold = 100;

@ConfField(mutable = true, masterOnly = true, description = {"提交事务的最大超时时间，单位是秒。"
+ "该参数仅用于事务型 insert 操作中。",
"Maximal waiting time for all data inserted before one transaction to be committed, in seconds. "
Expand Down
10 changes: 7 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
import org.apache.doris.thrift.TTabletInfo;
import org.apache.doris.thrift.TTaskType;

import com.amazonaws.services.glue.model.TaskType;
import com.google.common.base.Preconditions;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand All @@ -88,11 +89,13 @@ public TMasterResult finishTask(TFinishTaskRequest request) {
// check task status
// retry task by report process
TStatus taskStatus = request.getTaskStatus();
TTaskType taskType = request.getTaskType();
long signature = request.getSignature();
if (LOG.isDebugEnabled()) {
LOG.debug("get task report: {}", request);
}

if (taskStatus.getStatusCode() != TStatusCode.OK) {
if (taskStatus.getStatusCode() != TStatusCode.OK && taskType != TTaskType.PUBLISH_VERSION) {
LOG.warn("finish task reports bad. request: {}", request);
}

Expand All @@ -111,8 +114,6 @@ public TMasterResult finishTask(TFinishTaskRequest request) {
}

long backendId = backend.getId();
TTaskType taskType = request.getTaskType();
long signature = request.getSignature();

AgentTask task = AgentTaskQueue.getTask(backendId, taskType, signature);
if (task == null) {
Expand All @@ -130,6 +131,9 @@ public TMasterResult finishTask(TFinishTaskRequest request) {
} else {
if (taskStatus.getStatusCode() != TStatusCode.OK) {
task.failed();
if (taskType == TTaskType.PUBLISH_VERSION && task.getFailedTimes() < ) {
LOG.warn("finish task reports bad. request: {}", request);
}
String errMsg = "task type: " + taskType + ", status_code: " + taskStatus.getStatusCode().toString()
+ (taskStatus.isSetErrorMsgs() ? (", status_message: " + taskStatus.getErrorMsgs()) : "")
+ ", backendId: " + backend + ", signature: " + signature;
Expand Down