Skip to content

Commit f36af62

Browse files
SpriCoderSpriCoder
authored
Add database and region tag into tsfile metrics (#11228)
1 parent c0260ea commit f36af62

File tree

12 files changed

+944
-556
lines changed

12 files changed

+944
-556
lines changed

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java‎

Lines changed: 46 additions & 496 deletions
Large diffs are not rendered by default.

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/CompactionFileMetrics.java‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.db.service.metrics.file;
21+
22+
import org.apache.iotdb.commons.service.metric.enums.Metric;
23+
import org.apache.iotdb.commons.service.metric.enums.Tag;
24+
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.AbstractCompactionTask;
25+
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskSummary;
26+
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
27+
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
28+
import org.apache.iotdb.metrics.AbstractMetricService;
29+
import org.apache.iotdb.metrics.MetricConstant;
30+
import org.apache.iotdb.metrics.metricsets.IMetricSet;
31+
import org.apache.iotdb.metrics.utils.MetricLevel;
32+
import org.apache.iotdb.metrics.utils.MetricType;
33+
34+
import java.util.List;
35+
import java.util.concurrent.atomic.AtomicInteger;
36+
import java.util.concurrent.atomic.AtomicLong;
37+
38+
public class CompactionFileMetrics implements IMetricSet {
39+
private static final String INNER_SEQ_TEMP = "inner-seq-temp";
40+
private static final String INNER_UNSEQ_TEMP = "inner-unseq-temp";
41+
private static final String CROSS_TEMP = "cross-temp";
42+
// compaction temporal files
43+
private final AtomicLong innerSeqCompactionTempFileSize = new AtomicLong(0);
44+
private final AtomicLong innerUnseqCompactionTempFileSize = new AtomicLong(0);
45+
private final AtomicLong crossCompactionTempFileSize = new AtomicLong(0);
46+
private final AtomicInteger innerSeqCompactionTempFileNum = new AtomicInteger(0);
47+
private final AtomicInteger innerUnseqCompactionTempFileNum = new AtomicInteger(0);
48+
private final AtomicInteger crossCompactionTempFileNum = new AtomicInteger(0);
49+
50+
private long lastUpdateTime = 0;
51+
52+
@Override
53+
public void bindTo(AbstractMetricService metricService) {
54+
metricService.createAutoGauge(
55+
Metric.FILE_SIZE.toString(),
56+
MetricLevel.CORE,
57+
this,
58+
o -> o.getInnerCompactionTempFileSize(true),
59+
Tag.NAME.toString(),
60+
INNER_SEQ_TEMP);
61+
metricService.createAutoGauge(
62+
Metric.FILE_SIZE.toString(),
63+
MetricLevel.CORE,
64+
this,
65+
o -> o.getInnerCompactionTempFileSize(false),
66+
Tag.NAME.toString(),
67+
INNER_UNSEQ_TEMP);
68+
metricService.createAutoGauge(
69+
Metric.FILE_SIZE.toString(),
70+
MetricLevel.CORE,
71+
this,
72+
CompactionFileMetrics::getCrossCompactionTempFileSize,
73+
Tag.NAME.toString(),
74+
CROSS_TEMP);
75+
metricService.createAutoGauge(
76+
Metric.FILE_COUNT.toString(),
77+
MetricLevel.CORE,
78+
this,
79+
o -> o.getInnerCompactionTempFileNum(true),
80+
Tag.NAME.toString(),
81+
INNER_SEQ_TEMP);
82+
metricService.createAutoGauge(
83+
Metric.FILE_COUNT.toString(),
84+
MetricLevel.CORE,
85+
this,
86+
o -> o.getInnerCompactionTempFileNum(false),
87+
Tag.NAME.toString(),
88+
INNER_UNSEQ_TEMP);
89+
metricService.createAutoGauge(
90+
Metric.FILE_COUNT.toString(),
91+
MetricLevel.CORE,
92+
this,
93+
CompactionFileMetrics::getCrossCompactionTempFileNum,
94+
Tag.NAME.toString(),
95+
CROSS_TEMP);
96+
}
97+
98+
@Override
99+
public void unbindFrom(AbstractMetricService metricService) {
100+
metricService.remove(
101+
MetricType.AUTO_GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), INNER_SEQ_TEMP);
102+
metricService.remove(
103+
MetricType.AUTO_GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), INNER_UNSEQ_TEMP);
104+
metricService.remove(
105+
MetricType.AUTO_GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), CROSS_TEMP);
106+
metricService.remove(
107+
MetricType.AUTO_GAUGE, Metric.FILE_SIZE.toString(), Tag.NAME.toString(), INNER_SEQ_TEMP);
108+
metricService.remove(
109+
MetricType.AUTO_GAUGE, Metric.FILE_SIZE.toString(), Tag.NAME.toString(), INNER_UNSEQ_TEMP);
110+
metricService.remove(
111+
MetricType.AUTO_GAUGE, Metric.FILE_SIZE.toString(), Tag.NAME.toString(), CROSS_TEMP);
112+
}
113+
114+
public long getInnerCompactionTempFileSize(boolean seq) {
115+
updateCompactionTempSize();
116+
return seq ? innerSeqCompactionTempFileSize.get() : innerUnseqCompactionTempFileSize.get();
117+
}
118+
119+
private synchronized void updateCompactionTempSize() {
120+
if (System.currentTimeMillis() - lastUpdateTime <= MetricConstant.UPDATE_INTERVAL) {
121+
return;
122+
}
123+
lastUpdateTime = System.currentTimeMillis();
124+
125+
innerSeqCompactionTempFileSize.set(0);
126+
innerSeqCompactionTempFileNum.set(0);
127+
innerUnseqCompactionTempFileSize.set(0);
128+
innerUnseqCompactionTempFileNum.set(0);
129+
crossCompactionTempFileSize.set(0);
130+
crossCompactionTempFileNum.set(0);
131+
132+
List<AbstractCompactionTask> runningTasks =
133+
CompactionTaskManager.getInstance().getRunningCompactionTaskList();
134+
for (AbstractCompactionTask task : runningTasks) {
135+
CompactionTaskSummary summary = task.getSummary();
136+
if (task instanceof InnerSpaceCompactionTask) {
137+
if (task.isInnerSeqTask()) {
138+
innerSeqCompactionTempFileSize.addAndGet(summary.getTemporalFileSize());
139+
innerSeqCompactionTempFileNum.addAndGet(1);
140+
} else {
141+
innerUnseqCompactionTempFileSize.addAndGet(summary.getTemporalFileSize());
142+
innerUnseqCompactionTempFileNum.addAndGet(1);
143+
}
144+
} else {
145+
crossCompactionTempFileSize.addAndGet(summary.getTemporalFileSize());
146+
crossCompactionTempFileNum.addAndGet(summary.getTemporalFileNum());
147+
}
148+
}
149+
}
150+
151+
public long getCrossCompactionTempFileSize() {
152+
updateCompactionTempSize();
153+
return crossCompactionTempFileSize.get();
154+
}
155+
156+
public long getInnerCompactionTempFileNum(boolean seq) {
157+
updateCompactionTempSize();
158+
return seq ? innerSeqCompactionTempFileNum.get() : innerUnseqCompactionTempFileNum.get();
159+
}
160+
161+
public long getCrossCompactionTempFileNum() {
162+
updateCompactionTempSize();
163+
return crossCompactionTempFileNum.get();
164+
}
165+
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ModsFileMetrics.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.db.service.metrics.file;
21+
22+
import org.apache.iotdb.commons.service.metric.enums.Metric;
23+
import org.apache.iotdb.commons.service.metric.enums.Tag;
24+
import org.apache.iotdb.metrics.AbstractMetricService;
25+
import org.apache.iotdb.metrics.metricsets.IMetricSet;
26+
import org.apache.iotdb.metrics.utils.MetricLevel;
27+
import org.apache.iotdb.metrics.utils.MetricType;
28+
29+
import java.util.concurrent.atomic.AtomicInteger;
30+
import java.util.concurrent.atomic.AtomicLong;
31+
32+
public class ModsFileMetrics implements IMetricSet {
33+
private final AtomicInteger modFileNum = new AtomicInteger(0);
34+
private final AtomicLong modFileSize = new AtomicLong(0);
35+
36+
@Override
37+
public void bindTo(AbstractMetricService metricService) {
38+
metricService.createAutoGauge(
39+
Metric.FILE_SIZE.toString(),
40+
MetricLevel.CORE,
41+
this,
42+
ModsFileMetrics::getModFileSize,
43+
Tag.NAME.toString(),
44+
"mods");
45+
metricService.createAutoGauge(
46+
Metric.FILE_COUNT.toString(),
47+
MetricLevel.CORE,
48+
this,
49+
ModsFileMetrics::getModFileNum,
50+
Tag.NAME.toString(),
51+
"mods");
52+
}
53+
54+
@Override
55+
public void unbindFrom(AbstractMetricService metricService) {
56+
metricService.remove(
57+
MetricType.AUTO_GAUGE, Metric.FILE_SIZE.toString(), Tag.NAME.toString(), "mods");
58+
metricService.remove(
59+
MetricType.AUTO_GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), "mods");
60+
}
61+
62+
private int getModFileNum() {
63+
return modFileNum.get();
64+
}
65+
66+
private long getModFileSize() {
67+
return modFileSize.get();
68+
}
69+
70+
public void increaseModFileNum(int num) {
71+
modFileNum.addAndGet(num);
72+
}
73+
74+
public void decreaseModFileNum(int num) {
75+
modFileNum.addAndGet(-num);
76+
}
77+
78+
public void increaseModFileSize(long size) {
79+
modFileSize.addAndGet(size);
80+
}
81+
82+
public void decreaseModFileSize(long size) {
83+
modFileSize.addAndGet(-size);
84+
}
85+
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/SystemRelatedFileMetrics.java‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.db.service.metrics.file;
21+
22+
import org.apache.iotdb.commons.service.metric.enums.Metric;
23+
import org.apache.iotdb.commons.service.metric.enums.Tag;
24+
import org.apache.iotdb.metrics.AbstractMetricService;
25+
import org.apache.iotdb.metrics.config.MetricConfig;
26+
import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
27+
import org.apache.iotdb.metrics.metricsets.IMetricSet;
28+
import org.apache.iotdb.metrics.utils.MetricLevel;
29+
import org.apache.iotdb.metrics.utils.MetricType;
30+
import org.apache.iotdb.metrics.utils.SystemType;
31+
32+
import org.slf4j.Logger;
33+
import org.slf4j.LoggerFactory;
34+
35+
import java.io.BufferedReader;
36+
import java.io.File;
37+
import java.io.IOException;
38+
import java.io.InputStreamReader;
39+
40+
public class SystemRelatedFileMetrics implements IMetricSet {
41+
private static final Logger LOGGER = LoggerFactory.getLogger(SystemRelatedFileMetrics.class);
42+
private static final MetricConfig CONFIG = MetricConfigDescriptor.getInstance().getMetricConfig();
43+
private final Runtime runtime = Runtime.getRuntime();
44+
private String[] getOpenFileNumberCommand;
45+
46+
@SuppressWarnings("squid:S1075")
47+
private String fileHandlerCntPathInLinux = "/proc/%s/fd";
48+
49+
public SystemRelatedFileMetrics() {
50+
fileHandlerCntPathInLinux = String.format(fileHandlerCntPathInLinux, CONFIG.getPid());
51+
}
52+
53+
@Override
54+
public void bindTo(AbstractMetricService metricService) {
55+
if ((CONFIG.getSystemType() == SystemType.LINUX || CONFIG.getSystemType() == SystemType.MAC)
56+
&& !CONFIG.getPid().isEmpty()) {
57+
this.getOpenFileNumberCommand =
58+
new String[] {"/bin/sh", "-c", String.format("lsof -p %s | wc -l", CONFIG.getPid())};
59+
metricService.createAutoGauge(
60+
Metric.FILE_COUNT.toString(),
61+
MetricLevel.IMPORTANT,
62+
this,
63+
SystemRelatedFileMetrics::getOpenFileHandlersNumber,
64+
Tag.NAME.toString(),
65+
"open_file_handlers");
66+
}
67+
}
68+
69+
private long getOpenFileHandlersNumber() {
70+
long fdCount = 0;
71+
try {
72+
if (CONFIG.getSystemType() == SystemType.LINUX) {
73+
// count the fd in the system directory instead of
74+
// calling runtime.exec() which could be much slower
75+
File fdDir = new File(fileHandlerCntPathInLinux);
76+
if (fdDir.exists()) {
77+
File[] fds = fdDir.listFiles();
78+
fdCount = fds == null ? 0 : fds.length;
79+
}
80+
} else if ((CONFIG.getSystemType() == SystemType.MAC) && !CONFIG.getPid().isEmpty()) {
81+
Process process = runtime.exec(getOpenFileNumberCommand);
82+
StringBuilder result = new StringBuilder();
83+
try (BufferedReader input =
84+
new BufferedReader(new InputStreamReader(process.getInputStream()))) {
85+
String line;
86+
while ((line = input.readLine()) != null) {
87+
result.append(line);
88+
}
89+
}
90+
fdCount = Long.parseLong(result.toString().trim());
91+
}
92+
} catch (IOException e) {
93+
LOGGER.warn("Failed to get open file number, because ", e);
94+
}
95+
return fdCount;
96+
}
97+
98+
@Override
99+
public void unbindFrom(AbstractMetricService metricService) {
100+
if ((CONFIG.getSystemType() == SystemType.LINUX || CONFIG.getSystemType() == SystemType.MAC)
101+
&& !CONFIG.getPid().isEmpty()) {
102+
metricService.remove(
103+
MetricType.AUTO_GAUGE,
104+
Metric.FILE_COUNT.toString(),
105+
Tag.NAME.toString(),
106+
"open_file_handlers");
107+
}
108+
}
109+
}

0 commit comments

Comments
 (0)