Skip to content

add metric for largest segment size #8445

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Jackie-Jiang merged 4 commits into from
Apr 1, 2022
Merged

add metric for largest segment size #8445

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6c41a24
add metric for largest segment size
jadami-stripe Mar 31, 2022
73b6dca
add to prometheous; rename metric; don't use stream api; add unit tests
jadami-stripe Apr 1, 2022
725c60f
use regular long
jadami-stripe Apr 1, 2022
0e8b052
make DEFAULT_SIZE_WHEN_MISSING_OR_ERROR final
jadami-stripe Apr 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,11 @@ rules:
cache: true
labels:
table: "$1"
- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\", name=\"pinot.controller.largestSegmentSizeOnServer.(\\w+)\"><>(\\w+)"
name: "pinot_controller_largestSegmentSizeOnServer_$2"
cache: true
labels:
table: "$1"
- pattern: "\"org.apache.pinot.common.metrics\"<type=\"ControllerMetrics\", name=\"pinot.controller.tableTotalSizeOnServer.(\\w+)_(\\w+)\"><>(\\w+)"
name: "pinot_controller_tableTotalSizeOnServer_$3"
labels:
Expand Down Expand Up @@ -403,4 +408,4 @@ rules:
name: "pinot_minion_$2_$3"
cache: true
labels:
id: "$1"
id: "$1"
2 changes: 2 additions & 0 deletions pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ public enum ControllerGauge implements AbstractMetrics.Gauge {
@Deprecated // Instead use TABLE_TOTAL_SIZE_ON_SERVER
OFFLINE_TABLE_ESTIMATED_SIZE("OfflineTableEstimatedSize", false),

LARGEST_SEGMENT_SIZE_ON_SERVER("LargestSegmentSizeOnServer", false),

// Total size of table across replicas on servers
TABLE_TOTAL_SIZE_ON_SERVER("TableTotalSizeOnServer", false),

Expand Down
37 changes: 31 additions & 6 deletions pinot-controller/src/main/java/org/apache/pinot/controller/util/TableSizeReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
*/
public class TableSizeReader {
private static final Logger LOGGER = LoggerFactory.getLogger(TableSizeReader.class);
public static final long DEFAULT_SIZE_WHEN_MISSING_OR_ERROR = -1L;

private final Executor _executor;
private final HttpConnectionManager _connectionManager;
Expand Down Expand Up @@ -99,6 +100,18 @@ public TableSizeDetails getTableSizeDetails(@Nonnull String tableName, @Nonnegat
_controllerMetrics.setValueOfTableGauge(realtimeTableName, ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER,
tableSizeDetails._realtimeSegments._estimatedSizeInBytes / _helixResourceManager.getNumReplicas(
realtimeTableConfig));

long largestSegmentSizeOnServer = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
for (SegmentSizeDetails segmentSizeDetail : tableSizeDetails._realtimeSegments._segments.values()) {
for (SegmentSizeInfo segmentSizeInfo : segmentSizeDetail._serverInfo.values()) {
largestSegmentSizeOnServer = Math.max(largestSegmentSizeOnServer, segmentSizeInfo.getDiskSizeInBytes());
}
}
if (largestSegmentSizeOnServer != DEFAULT_SIZE_WHEN_MISSING_OR_ERROR) {
_controllerMetrics.setValueOfTableGauge(realtimeTableName,
ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER,
largestSegmentSizeOnServer);
}
}
if (offlineTableConfig != null) {
String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(tableName);
Expand All @@ -111,6 +124,18 @@ public TableSizeDetails getTableSizeDetails(@Nonnull String tableName, @Nonnegat
_controllerMetrics.setValueOfTableGauge(offlineTableName, ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER,
tableSizeDetails._offlineSegments._estimatedSizeInBytes / _helixResourceManager.getNumReplicas(
offlineTableConfig));

long largestSegmentSizeOnServer = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
for (SegmentSizeDetails segmentSizeDetail : tableSizeDetails._offlineSegments._segments.values()) {
for (SegmentSizeInfo segmentSizeInfo : segmentSizeDetail._serverInfo.values()) {
largestSegmentSizeOnServer = Math.max(largestSegmentSizeOnServer, segmentSizeInfo.getDiskSizeInBytes());
}
}
if (largestSegmentSizeOnServer != DEFAULT_SIZE_WHEN_MISSING_OR_ERROR) {
_controllerMetrics.setValueOfTableGauge(offlineTableName,
ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER,
largestSegmentSizeOnServer);
}
}

return tableSizeDetails;
Expand Down Expand Up @@ -205,7 +230,7 @@ public TableSubTypeSizeDetails getTableSubtypeSize(String tableNameWithType, int
for (String segment : segments) {
SegmentSizeDetails segmentSizeDetails =
segmentToSizeDetailsMap.computeIfAbsent(segment, k -> new SegmentSizeDetails());
segmentSizeDetails._serverInfo.put(server, new SegmentSizeInfo(segment, -1L));
segmentSizeDetails._serverInfo.put(server, new SegmentSizeInfo(segment, DEFAULT_SIZE_WHEN_MISSING_OR_ERROR));
}
}
}
Expand All @@ -226,7 +251,7 @@ public TableSubTypeSizeDetails getTableSubtypeSize(String tableNameWithType, int
long segmentLevelMax = -1L;
int errors = 0;
for (SegmentSizeInfo sizeInfo : sizeDetails._serverInfo.values()) {
if (sizeInfo.getDiskSizeInBytes() != -1) {
if (sizeInfo.getDiskSizeInBytes() != DEFAULT_SIZE_WHEN_MISSING_OR_ERROR) {
sizeDetails._reportedSizeInBytes += sizeInfo.getDiskSizeInBytes();
segmentLevelMax = Math.max(segmentLevelMax, sizeInfo.getDiskSizeInBytes());
} else {
Expand All @@ -242,8 +267,8 @@ public TableSubTypeSizeDetails getTableSubtypeSize(String tableNameWithType, int
} else {
// Segment is missing from all servers
missingSegments.add(segment);
sizeDetails._reportedSizeInBytes = -1L;
sizeDetails._estimatedSizeInBytes = -1L;
sizeDetails._reportedSizeInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
sizeDetails._estimatedSizeInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
subTypeSizeDetails._missingSegments++;
}
}
Expand All @@ -257,8 +282,8 @@ public TableSubTypeSizeDetails getTableSubtypeSize(String tableNameWithType, int
missingPercent);
if (subTypeSizeDetails._missingSegments == numSegments) {
LOGGER.warn("Failed to get size report for all {} segments of table: {}", numSegments, tableNameWithType);
subTypeSizeDetails._reportedSizeInBytes = -1;
subTypeSizeDetails._estimatedSizeInBytes = -1;
subTypeSizeDetails._reportedSizeInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
subTypeSizeDetails._estimatedSizeInBytes = DEFAULT_SIZE_WHEN_MISSING_OR_ERROR;
} else {
LOGGER.warn("Missing size report for {} out of {} segments for table {}", subTypeSizeDetails._missingSegments,
numSegments, tableNameWithType);
Expand Down
39 changes: 24 additions & 15 deletions pinot-controller/src/test/java/org/apache/pinot/controller/api/TableSizeReaderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,15 +90,15 @@ public void setUp()

when(mockPropertyStore.get(ArgumentMatchers.anyString(), ArgumentMatchers.eq(null),
ArgumentMatchers.eq(AccessOption.PERSISTENT))).thenAnswer((Answer) invocationOnMock -> {
String path = (String) invocationOnMock.getArguments()[0];
if (path.contains("realtime_REALTIME")) {
return TableConfigUtils.toZNRecord(tableConfig);
}
if (path.contains("offline_OFFLINE")) {
return TableConfigUtils.toZNRecord(tableConfig);
}
return null;
});
String path = (String) invocationOnMock.getArguments()[0];
Copy link
Contributor Author

@jadami10 jadami10 Apr 1, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not changes here. the reformat changed the whitespace

if (path.contains("realtime_REALTIME")) {
return TableConfigUtils.toZNRecord(tableConfig);
}
if (path.contains("offline_OFFLINE")) {
return TableConfigUtils.toZNRecord(tableConfig);
}
return null;
});

when(_helix.getPropertyStore()).thenReturn(mockPropertyStore);
when(_helix.getNumReplicas(ArgumentMatchers.eq(tableConfig))).thenReturn(NUM_REPLICAS);
Expand Down Expand Up @@ -271,10 +271,10 @@ private void validateTableSubTypeSize(String[] servers, TableSizeReader.TableSub
for (Map.Entry<String, List<String>> segmentEntry : segmentServers.entrySet()) {
final String segmentName = segmentEntry.getKey();
final TableSizeReader.SegmentSizeDetails segmentDetails = tableSize._segments.get(segmentName);
if (segmentDetails._reportedSizeInBytes != -1) {
if (segmentDetails._reportedSizeInBytes != TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR) {
reportedSize += segmentDetails._reportedSizeInBytes;
}
if (segmentDetails._estimatedSizeInBytes != -1) {
if (segmentDetails._estimatedSizeInBytes != TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR) {
estimatedSize += segmentDetails._estimatedSizeInBytes;
}

Expand All @@ -293,8 +293,8 @@ private void validateTableSubTypeSize(String[] servers, TableSizeReader.TableSub
Assert.assertEquals(segmentDetails._reportedSizeInBytes, numResponses * expectedSegmentSize);
Assert.assertEquals(segmentDetails._estimatedSizeInBytes, expectedServers.size() * expectedSegmentSize);
} else {
Assert.assertEquals(segmentDetails._reportedSizeInBytes, -1);
Assert.assertEquals(segmentDetails._estimatedSizeInBytes, -1);
Assert.assertEquals(segmentDetails._reportedSizeInBytes, TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR);
Assert.assertEquals(segmentDetails._estimatedSizeInBytes, TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR);
}
}
Assert.assertEquals(tableSize._reportedSizeInBytes, reportedSize);
Expand Down Expand Up @@ -326,6 +326,8 @@ public void testGetTableSubTypeSizeAllSuccess()
ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER), offlineSizes._estimatedSizeInBytes / NUM_REPLICAS);
Assert.assertEquals(_controllerMetrics.getValueOfTableGauge(tableNameWithType,
ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER), offlineSizes._estimatedSizeInBytes);
Assert.assertEquals(_controllerMetrics.getValueOfTableGauge(tableNameWithType,
ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER), 160);
}

@Test
Expand All @@ -338,15 +340,18 @@ public void testGetTableSubTypeSizeAllErrors()
Assert.assertNotNull(offlineSizes);
Assert.assertEquals(offlineSizes._missingSegments, 3);
Assert.assertEquals(offlineSizes._segments.size(), 3);
Assert.assertEquals(offlineSizes._reportedSizeInBytes, -1);
Assert.assertEquals(tableSizeDetails._estimatedSizeInBytes, -1);
Assert.assertEquals(offlineSizes._reportedSizeInBytes, TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR);
Assert.assertEquals(tableSizeDetails._estimatedSizeInBytes, TableSizeReader.DEFAULT_SIZE_WHEN_MISSING_OR_ERROR);
String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType(table);
Assert.assertEquals(_controllerMetrics
.getValueOfTableGauge(tableNameWithType, ControllerGauge.TABLE_STORAGE_EST_MISSING_SEGMENT_PERCENT), 100);
Assert.assertEquals(_controllerMetrics.getValueOfTableGauge(tableNameWithType,
ControllerGauge.TABLE_SIZE_PER_REPLICA_ON_SERVER), offlineSizes._estimatedSizeInBytes / NUM_REPLICAS);
Assert.assertEquals(_controllerMetrics.getValueOfTableGauge(tableNameWithType,
ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER), offlineSizes._estimatedSizeInBytes);
// 0 means not found for the gauge
Assert.assertEquals(_controllerMetrics.getValueOfTableGauge(tableNameWithType,
ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER), 0);
}

@Test
Expand All @@ -370,6 +375,8 @@ public void testGetTableSubTypeSizesWithErrors()
Assert.assertEquals(
_controllerMetrics.getValueOfTableGauge(tableNameWithType, ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER),
offlineSizes._estimatedSizeInBytes);
Assert.assertEquals(_controllerMetrics.getValueOfTableGauge(tableNameWithType,
ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER), 160);
}

@Test
Expand All @@ -389,5 +396,7 @@ public void getTableSizeDetailsRealtimeOnly()
realtimeSegments._estimatedSizeInBytes / NUM_REPLICAS);
Assert.assertEquals(_controllerMetrics.getValueOfTableGauge(tableNameWithType,
ControllerGauge.TABLE_TOTAL_SIZE_ON_SERVER), realtimeSegments._estimatedSizeInBytes);
Assert.assertEquals(_controllerMetrics.getValueOfTableGauge(tableNameWithType,
ControllerGauge.LARGEST_SEGMENT_SIZE_ON_SERVER), 120);
}
}