Skip to content

Commit

Permalink
Add deleted doc count in _cat/shards (opensearch-project#11678)
Browse files Browse the repository at this point in the history 
Signed-off-by: Shreyansh Ray <rayshrey@amazon.com>
  • Loading branch information
@rayshrey
rayshrey committed Mar 18, 2024
1 parent 8d6b001 commit 75103c2
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 3 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Interpret byte array as primitive using VarHandles ([#11362](https://github.com/opensearch-project/OpenSearch/pull/11362))
- Automatically add scheme to discovery.ec2.endpoint ([#11512](https://github.com/opensearch-project/OpenSearch/pull/11512))
- Restore support for Java 8 for RestClient ([#11562](https://github.com/opensearch-project/OpenSearch/pull/11562))
- Add deleted doc count in _cat/shards ([#11678](https://github.com/opensearch-project/OpenSearch/pull/11678))

### Deprecated

Expand Down
94 changes: 92 additions & 2 deletions rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,103 @@
"Help":
- skip:
version: " - 2.3.99"
version: " - 2.11.99"
reason: deleted docs added in 2.12.0
features: node_selector
- do:
cat.shards:
help: true
node_selector:
version: "2.12.0 - "

- match:
$body: |
/^ index .+ \n
shard .+ \n
prirep .+ \n
state .+ \n
docs .+ \n
store .+ \n
ip .+ \n
id .+ \n
node .+ \n
sync_id .+ \n
unassigned.reason .+ \n
unassigned.at .+ \n
unassigned.for .+ \n
unassigned.details .+ \n
recoverysource.type .+ \n
completion.size .+ \n
fielddata.memory_size .+ \n
fielddata.evictions .+ \n
query_cache.memory_size .+ \n
query_cache.evictions .+ \n
flush.total .+ \n
flush.total_time .+ \n
get.current .+ \n
get.time .+ \n
get.total .+ \n
get.exists_time .+ \n
get.exists_total .+ \n
get.missing_time .+ \n
get.missing_total .+ \n
indexing.delete_current .+ \n
indexing.delete_time .+ \n
indexing.delete_total .+ \n
indexing.index_current .+ \n
indexing.index_time .+ \n
indexing.index_total .+ \n
indexing.index_failed .+ \n
merges.current .+ \n
merges.current_docs .+ \n
merges.current_size .+ \n
merges.total .+ \n
merges.total_docs .+ \n
merges.total_size .+ \n
merges.total_time .+ \n
refresh.total .+ \n
refresh.time .+ \n
refresh.external_total .+ \n
refresh.external_time .+ \n
refresh.listeners .+ \n
search.fetch_current .+ \n
search.fetch_time .+ \n
search.fetch_total .+ \n
search.open_contexts .+ \n
search.query_current .+ \n
search.query_time .+ \n
search.query_total .+ \n
search.scroll_current .+ \n
search.scroll_time .+ \n
search.scroll_total .+ \n
search.point_in_time_current .+ \n
search.point_in_time_time .+ \n
search.point_in_time_total .+ \n
segments.count .+ \n
segments.memory .+ \n
segments.index_writer_memory .+ \n
segments.version_map_memory .+ \n
segments.fixed_bitset_memory .+ \n
seq_no.max .+ \n
seq_no.local_checkpoint .+ \n
seq_no.global_checkpoint .+ \n
warmer.current .+ \n
warmer.total .+ \n
warmer.total_time .+ \n
path.data .+ \n
path.state .+ \n
docs.deleted .+ \n
$/
---
"Help from 2.4.0 to 2.11.0":
- skip:
version: " - 2.3.99 , 2.12.0 - "
reason: point in time stats were added in 2.4.0
features: node_selector
- do:
cat.shards:
help: true
node_selector:
version: "2.4.0 - "
version: "2.4.0 - 2.11.99"

- match:
$body: |
Expand Down
2 changes: 2 additions & 0 deletions server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,7 @@ protected Table getTableWithHeader(final RestRequest request) {

table.addCell("path.data", "alias:pd,dataPath;default:false;text-align:right;desc:shard data path");
table.addCell("path.state", "alias:ps,statsPath;default:false;text-align:right;desc:shard state path");
table.addCell("docs.deleted", "alias:dd,docsDeleted;default:false;text-align:right;desc:number of deleted docs in shard");

table.endHeaders();
return table;
Expand Down Expand Up @@ -448,6 +449,7 @@ Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsRe

table.addCell(getOrNull(shardStats, ShardStats::getDataPath, s -> s));
table.addCell(getOrNull(shardStats, ShardStats::getStatePath, s -> s));
table.addCell(getOrNull(commonStats, CommonStats::getDocs, DocsStats::getDeleted));

table.endRow();
}
Expand Down
10 changes: 9 additions & 1 deletion server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.opensearch.cluster.routing.ShardRoutingState;
import org.opensearch.cluster.routing.TestShardRouting;
import org.opensearch.common.Table;
import org.opensearch.index.shard.DocsStats;
import org.opensearch.index.shard.ShardPath;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.test.rest.FakeRestRequest;
Expand All @@ -65,6 +66,8 @@ public class RestShardsActionTests extends OpenSearchTestCase {

public void testBuildTable() {
final int numShards = randomIntBetween(1, 5);
long numDocs = randomLongBetween(0, 10000);
long numDeletedDocs = randomLongBetween(0, 100);
DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT);

List<ShardRouting> shardRoutings = new ArrayList<>(numShards);
Expand All @@ -76,10 +79,12 @@ public void testBuildTable() {
Path path = createTempDir().resolve("indices")
.resolve(shardRouting.shardId().getIndex().getUUID())
.resolve(String.valueOf(shardRouting.shardId().id()));
CommonStats commonStats = new CommonStats();
commonStats.docs = new DocsStats(numDocs, numDeletedDocs, 0);
ShardStats shardStats = new ShardStats(
shardRouting,
new ShardPath(false, path, path, shardRouting.shardId()),
null,
commonStats,
null,
null,
null
Expand Down Expand Up @@ -120,6 +125,7 @@ public void testBuildTable() {
assertThat(headers.get(6).value, equalTo("ip"));
assertThat(headers.get(7).value, equalTo("id"));
assertThat(headers.get(8).value, equalTo("node"));
assertThat(headers.get(74).value, equalTo("docs.deleted"));

final List<List<Table.Cell>> rows = table.getRows();
assertThat(rows.size(), equalTo(numShards));
Expand All @@ -132,10 +138,12 @@ public void testBuildTable() {
assertThat(row.get(1).value, equalTo(shardRouting.getId()));
assertThat(row.get(2).value, equalTo(shardRouting.primary() ? "p" : "r"));
assertThat(row.get(3).value, equalTo(shardRouting.state()));
assertThat(row.get(4).value, equalTo(shardStats.getStats().getDocs().getCount()));
assertThat(row.get(6).value, equalTo(localNode.getHostAddress()));
assertThat(row.get(7).value, equalTo(localNode.getId()));
assertThat(row.get(72).value, equalTo(shardStats.getDataPath()));
assertThat(row.get(73).value, equalTo(shardStats.getStatePath()));
assertThat(row.get(74).value, equalTo(shardStats.getStats().getDocs().getDeleted()));
}
}
}

0 comments on commit 75103c2

Please sign in to comment.