Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Enhancement] Add agent task metrics #29707

Merged
merged 2 commits into from
Aug 25, 2023
Merged

[Enhancement] Add agent task metrics #29707

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
03b5710
add
gengjun-git Aug 22, 2023
4f73ba9
add
gengjun-git Aug 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/metric/MetricRepo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@
import com.starrocks.service.ExecuteEnv;
import com.starrocks.system.Backend;
import com.starrocks.system.SystemInfoService;
import com.starrocks.task.AgentTaskQueue;
import com.starrocks.transaction.TransactionState;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand Down Expand Up @@ -727,6 +728,16 @@ public Long getValue() {
};
queryCoordinatorCount.addLabel(new MetricLabel("type", "query_coordinator_count"));
STARROCKS_METRIC_REGISTER.addMetric(queryCoordinatorCount);

GaugeMetric<Long> agentTaskCount = new GaugeMetric<Long>("memory", MetricUnit.NOUNIT,
"The count of agent task") {
@Override
public Long getValue() {
return (long) AgentTaskQueue.getTaskNum();
}
};
agentTaskCount.addLabel(new MetricLabel("type", "agent_task_count"));
STARROCKS_METRIC_REGISTER.addMetric(agentTaskCount);
}

// to generate the metrics related to tablets of each backends
Expand Down
130 changes: 68 additions & 62 deletions test/sql/test_metrics/R/test_memory_metrics
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
-- name: test_memory_metrics_txn_callback_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"txn_callback_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- name: test_memory_metrics_load_profile_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"load_profile_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_delete_job_info_count
-- name: test_memory_metrics_txn_bytes
CREATE TABLE `t1` (
a int
) ENGINE=OLAP
Expand All @@ -18,23 +18,20 @@ PROPERTIES (
insert into t1 values (1);
-- result:
-- !result
delete from t1 where a = 1;
-- result:
-- !result
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"delete_job_info_count\" | awk '{if ($NF >= 1) print "true"; else print "false";}'
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"txn_bytes\" | awk '{if ($NF >= 832) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_compaction_stats_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"compaction_stats_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- name: test_memory_metrics_agent_task_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"agent_task_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_tablet_count
-- name: test_memory_metrics_tablet_bytes
CREATE TABLE `t1` (
a int
a int
) ENGINE=OLAP
DUPLICATE KEY(`a`)
DISTRIBUTED BY HASH(`a`) BUCKETS 10
Expand All @@ -43,12 +40,12 @@ PROPERTIES (
);
-- result:
-- !result
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"tablet_count\" | awk '{if ($NF >= 10) print "true"; else print "false";}'
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"tablet_bytes\" | awk '{if ($NF >= 1360) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_query_profile_count
-- name: test_memory_metrics_tablet_count
CREATE TABLE `t1` (
a int
) ENGINE=OLAP
Expand All @@ -59,27 +56,26 @@ PROPERTIES (
);
-- result:
-- !result
set enable_profile=true;
-- result:
-- !result
select count(*) from t1;
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"tablet_count\" | awk '{if ($NF >= 10) print "true"; else print "false";}'
-- result:
0
true
-- !result
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"query_profile_count\" | awk '{if ($NF >= 1) print "true"; else print "false";}'
-- name: test_memory_metrics_running_task_run_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"running_task_run_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_pending_task_run_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"pending_task_run_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- name: test_memory_metrics_delete_job_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"delete_job_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_replica_bytes
-- name: test_memory_metrics_query_profile_count
CREATE TABLE `t1` (
a int
a int
) ENGINE=OLAP
DUPLICATE KEY(`a`)
DISTRIBUTED BY HASH(`a`) BUCKETS 10
Expand All @@ -88,24 +84,19 @@ PROPERTIES (
);
-- result:
-- !result
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"replica_bytes\" | awk '{if ($NF >= 1440) print "true"; else print "false";}'
set enable_profile=true;
-- result:
0
true
-- !result
-- name: test_memory_metrics_delete_job_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"delete_job_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
select count(*) from t1;
-- result:
0
true
-- !result
-- name: test_memory_metrics_task_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"task_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"query_profile_count\" | awk '{if ($NF >= 1) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_txn_count
-- name: test_memory_metrics_replica_count
CREATE TABLE `t1` (
a int
) ENGINE=OLAP
Expand All @@ -116,27 +107,40 @@ PROPERTIES (
);
-- result:
-- !result
insert into t1 values (1);
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"replica_count\" | awk '{if ($NF >= 10) print "true"; else print "false";}'
-- result:
0
true
-- !result
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"txn_count\" | awk '{if ($NF >= 1) print "true"; else print "false";}'
-- name: test_memory_metrics_query_detail_count
admin set frontend config ("enable_collect_query_detail_info" = "true");
-- result:
-- !result
[UC]show frontends;
-- result:
172.26.92.139_9312_1692690164465 172.26.92.139 9312 8332 9332 9322 LEADER 1640562713 true true 2292 2023-08-22 17:44:15 true 2023-08-22 17:42:09 add_agent_task_metrics-ba5647e
-- !result
admin set frontend config ("enable_collect_query_detail_info" = "false");
-- result:
-- !result
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"query_detail_count\" | awk '{if ($NF >= 1) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_catalogs_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"catalogs_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- name: test_memory_metrics_history_task_run_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"history_task_run_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_stream_load_task_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"stream_load_task_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- name: test_memory_metrics_compaction_stats_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"compaction_stats_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_txn_bytes
-- name: test_memory_metrics_delete_job_info_count
CREATE TABLE `t1` (
a int
) ENGINE=OLAP
Expand All @@ -150,12 +154,21 @@ PROPERTIES (
insert into t1 values (1);
-- result:
-- !result
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"txn_bytes\" | awk '{if ($NF >= 832) print "true"; else print "false";}'
delete from t1 where a = 1;
-- result:
-- !result
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"delete_job_info_count\" | awk '{if ($NF >= 1) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_tablet_bytes
-- name: test_memory_metrics_pending_task_run_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"pending_task_run_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_replica_bytes
CREATE TABLE `t1` (
a int
) ENGINE=OLAP
Expand All @@ -166,40 +179,36 @@ PROPERTIES (
);
-- result:
-- !result
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"tablet_bytes\" | awk '{if ($NF >= 1360) print "true"; else print "false";}'
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"replica_bytes\" | awk '{if ($NF >= 1440) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_running_task_run_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"running_task_run_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- name: test_memory_metrics_catalogs_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"catalogs_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_load_profile_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"load_profile_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- name: test_memory_metrics_stream_load_task_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"stream_load_task_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_query_detail_count
admin set frontend config ("enable_collect_query_detail_info" = "true");
-- result:
-- !result
[UC]show frontends;
-- result:
172.26.92.139_9312_1690292126093 172.26.92.139 9312 8332 9332 9322 LEADER 2056272365 true true 232533 2023-08-03 19:36:20 true 2023-08-03 16:04:39 add_memory_metrics-40b18af
-- !result
admin set frontend config ("enable_collect_query_detail_info" = "false");
-- name: test_memory_metrics_txn_callback_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"txn_callback_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- result:
0
true
-- !result
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"query_detail_count\" | awk '{if ($NF >= 1) print "true"; else print "false";}'
-- name: test_memory_metrics_task_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"task_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
-- result:
0
true
-- !result
-- name: test_memory_metrics_replica_count
-- name: test_memory_metrics_txn_count
CREATE TABLE `t1` (
a int
) ENGINE=OLAP
Expand All @@ -210,13 +219,10 @@ PROPERTIES (
);
-- result:
-- !result
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"replica_count\" | awk '{if ($NF >= 10) print "true"; else print "false";}'
insert into t1 values (1);
-- result:
0
true
-- !result
-- name: test_memory_metrics_history_task_run_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"history_task_run_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"txn_count\" | awk '{if ($NF >= 1) print "true"; else print "false";}'
-- result:
0
true
Expand All @@ -232,4 +238,4 @@ shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"query_coordinat
-- result:
0
true
-- !result
-- !result
3 changes: 3 additions & 0 deletions test/sql/test_metrics/T/test_memory_metrics
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,3 +134,6 @@ shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"load_profile_co
-- name: test_memory_metrics_query_coordinator_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"query_coordinator_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'

-- name: test_memory_metrics_agent_task_count
shell: curl '${url}/metrics' | grep starrocks_fe_memory | grep \"agent_task_count\" | awk '{if ($NF >= 0) print "true"; else print "false";}'