Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Enhancement](show-backends-disks) Add show backends disks #24229

Merged
merged 15 commits into from
Oct 12, 2023
Merged

[Enhancement](show-backends-disks) Add show backends disks #24229

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
6ba16bb
Add statement to query disk information corresponding to data directo…
yongjinhou Sep 12, 2023
534e928
resolve conflict
yongjinhou Sep 15, 2023
cb543b9
fix build be failed
yongjinhou Sep 15, 2023
402ec52
format
yongjinhou Sep 15, 2023
46f66fe
add disk info of deploy dir
yongjinhou Sep 20, 2023
46b51e1
remove comment
yongjinhou Sep 20, 2023
27545e7
set to true before checking health
yongjinhou Sep 20, 2023
dcf03ba
format
yongjinhou Sep 20, 2023
d75d011
fix fe ut
yongjinhou Sep 21, 2023
e0e44fd
fix
yongjinhou Oct 8, 2023
c34ef85
fix
yongjinhou Oct 8, 2023
55c9dbc
format
yongjinhou Oct 9, 2023
e17d768
modify disk type to enum
yongjinhou Oct 11, 2023
8b6003a
remove include
yongjinhou Oct 11, 2023
402de28
format
yongjinhou Oct 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
modify disk type to enum
  • Loading branch information
@yongjinhou @adonis0147
yongjinhou authored and adonis0147 committed Oct 12, 2023
commit e17d768a116c8aaee2f47c664ff6396b5f96ff79
12 changes: 6 additions & 6 deletions be/src/agent/task_worker_pool.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
#include <fmt/format.h>
#include <gen_cpp/AgentService_types.h>
#include <gen_cpp/HeartbeatService_types.h>
#include <gen_cpp/MasterService_types.h>
// #include <gen_cpp/MasterService_types.h>
#include <gen_cpp/Status_types.h>
#include <gen_cpp/Types_types.h>
#include <unistd.h>
Expand Down Expand Up @@ -678,12 +678,12 @@ void TaskWorkerPool::_report_disk_state_worker_thread_callback() {
disk.__set_disk_available_capacity(root_path_info.available);
disk.__set_trash_used_capacity(root_path_info.trash_used_capacity);
disk.__set_used(root_path_info.is_used);
disk.__set_dir_type("STORAGE");
disk.__set_dir_type(TDiskType::STORAGE);
request.disks[root_path_info.path] = disk;
}

_set_disk_infos(request, DiskType::LOG);
_set_disk_infos(request, DiskType::DEPLOY);
_set_disk_infos(request, TDiskType::LOG);
_set_disk_infos(request, TDiskType::DEPLOY);

request.__set_num_cores(CpuInfo::num_cores());
request.__set_pipeline_executor_size(config::pipeline_executor_size > 0
Expand Down Expand Up @@ -1102,7 +1102,7 @@ void TaskWorkerPool::_handle_report(const TReportRequest& request, ReportType ty
}
}

void TaskWorkerPool::_set_disk_infos(TReportRequest& request, DiskType type) {
void TaskWorkerPool::_set_disk_infos(TReportRequest& request, TDiskType::type type) {
SpecialDirInfo dir_info;
StorageEngine::instance()->get_special_dir_info(&dir_info, type);

Expand All @@ -1112,7 +1112,7 @@ void TaskWorkerPool::_set_disk_infos(TReportRequest& request, DiskType type) {
special_disk.__set_disk_total_capacity(dir_info.capacity);
special_disk.__set_disk_available_capacity(dir_info.available);
special_disk.__set_used(dir_info.is_used);
special_disk.__set_dir_type(TYPE_STRING(type));
special_disk.__set_dir_type(type);
request.disks[dir_info.path] = special_disk;
}

Expand Down
16 changes: 2 additions & 14 deletions be/src/agent/task_worker_pool.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

#include <butil/macros.h>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

warning: 'butil/macros.h' file not found [clang-diagnostic-error]

#include <butil/macros.h>
         ^

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

不要在这乱指挥了

#include <gen_cpp/AgentService_types.h>
#include <gen_cpp/MasterService_types.h>
#include <gen_cpp/Types_types.h>
#include <stdint.h>

Expand Down Expand Up @@ -85,8 +86,6 @@ class TaskWorkerPool {

enum ReportType { TASK, DISK, TABLET };

enum DiskType { LOG, DEPLOY };

enum class ThreadModel {
SINGLE_THREAD, // Only 1 thread allowed in the pool
MULTI_THREADS // 1 or more threads allowed in the pool
Expand Down Expand Up @@ -164,17 +163,6 @@ class TaskWorkerPool {
}
}

const std::string TYPE_STRING(DiskType type) {
switch (type) {
case LOG:
return "LOG";
case DEPLOY:
return "DEPLOY";
default:
return "Unknown";
}
}

TaskWorkerPool(const TaskWorkerType task_worker_type, ExecEnv* env,
const TMasterInfo& master_info, ThreadModel thread_model);
virtual ~TaskWorkerPool();
Expand Down Expand Up @@ -224,7 +212,7 @@ class TaskWorkerPool {
TFinishTaskRequest* finish_task_request);

void _handle_report(const TReportRequest& request, ReportType type);
void _set_disk_infos(TReportRequest& request, DiskType type);
void _set_disk_infos(TReportRequest& request, TDiskType::type type);

Status _get_tablet_info(const TTabletId tablet_id, const TSchemaHash schema_hash,
int64_t signature, TTabletInfo* tablet_info);
Expand Down
6 changes: 3 additions & 3 deletions be/src/olap/storage_engine.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,14 +363,14 @@ Status StorageEngine::get_all_data_dir_info(std::vector<DataDirInfo>* data_dir_i
}

void StorageEngine::get_special_dir_info(SpecialDirInfo* special_dir_infos,
TaskWorkerPool::DiskType type) {
TDiskType::type type) {
switch (type) {
case TaskWorkerPool::DiskType::LOG:
case TDiskType::LOG:
_log_dir->health_check();
static_cast<void>(_log_dir->update_capacity());
_log_dir->get_dir_info(special_dir_infos);
break;
case TaskWorkerPool::DiskType::DEPLOY:
case TDiskType::DEPLOY:
_deploy_dir->health_check();
static_cast<void>(_deploy_dir->update_capacity());
_deploy_dir->get_dir_info(special_dir_infos);
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/storage_engine.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ class StorageEngine {

// get all info of root_path
Status get_all_data_dir_info(std::vector<DataDirInfo>* data_dir_infos, bool need_update);
void get_special_dir_info(SpecialDirInfo* dir_infos, TaskWorkerPool::DiskType type);
void get_special_dir_info(SpecialDirInfo* dir_infos, TDiskType::type type);

int64_t get_file_or_directory_size(const std::string& file_path);

Expand Down
27 changes: 14 additions & 13 deletions fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ public class Backend implements Writable {

// Represent a meaningless IP
public static final String DUMMY_IP = "0.0.0.0";
public static final String DATA_DIR_TYPE = "STORAGE";

@SerializedName("id")
private long id;
Expand Down Expand Up @@ -376,7 +377,7 @@ public int getHeartbeatFailureCounter() {
public ImmutableMap<String, DiskInfo> getDisks() {
Map<String, DiskInfo> disks = Maps.newHashMap();
for (Map.Entry<String, DiskInfo> entry : disksRef.entrySet()) {
if (entry.getValue().getDirType().equals("STORAGE")) {
if (entry.getValue().getDirType().equals(DATA_DIR_TYPE)) {
disks.put(entry.getKey(), entry.getValue());
}
}
Expand All @@ -390,7 +391,7 @@ public ImmutableMap<String, DiskInfo> getAllDisks() {
public boolean hasPathHash() {
Map<String, DiskInfo> disks = Maps.newHashMap();
for (Map.Entry<String, DiskInfo> entry : disksRef.entrySet()) {
if (entry.getValue().getDirType().equals("STORAGE")) {
if (entry.getValue().getDirType().equals(DATA_DIR_TYPE)) {
disks.put(entry.getKey(), entry.getValue());
}
}
Expand All @@ -400,7 +401,7 @@ public boolean hasPathHash() {
public boolean hasSpecifiedStorageMedium(TStorageMedium storageMedium) {
Map<String, DiskInfo> disks = Maps.newHashMap();
for (Map.Entry<String, DiskInfo> entry : disksRef.entrySet()) {
if (entry.getValue().getDirType().equals("STORAGE")) {
if (entry.getValue().getDirType().equals(DATA_DIR_TYPE)) {
disks.put(entry.getKey(), entry.getValue());
}
}
Expand All @@ -411,7 +412,7 @@ public long getTotalCapacityB() {
ImmutableMap<String, DiskInfo> disks = disksRef;
long totalCapacityB = 0L;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals("STORAGE")) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
totalCapacityB += diskInfo.getTotalCapacityB();
}
}
Expand All @@ -423,7 +424,7 @@ public long getAvailableCapacityB() {
ImmutableMap<String, DiskInfo> disks = disksRef;
long availableCapacityB = 1L;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals("STORAGE")) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
availableCapacityB += diskInfo.getAvailableCapacityB();
}
}
Expand All @@ -434,7 +435,7 @@ public long getDataUsedCapacityB() {
ImmutableMap<String, DiskInfo> disks = disksRef;
long dataUsedCapacityB = 0L;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals("STORAGE")) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
dataUsedCapacityB += diskInfo.getDataUsedCapacityB();
}
}
Expand All @@ -445,7 +446,7 @@ public long getTrashUsedCapacityB() {
ImmutableMap<String, DiskInfo> disks = disksRef;
long trashUsedCapacityB = 0L;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals("STORAGE")) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
trashUsedCapacityB += diskInfo.getTrashUsedCapacityB();
}
}
Expand All @@ -456,7 +457,7 @@ public long getRemoteUsedCapacityB() {
ImmutableMap<String, DiskInfo> disks = disksRef;
long totalRemoteUsedCapacityB = 0L;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals("STORAGE")) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
totalRemoteUsedCapacityB += diskInfo.getRemoteUsedCapacity();
}
}
Expand All @@ -467,7 +468,7 @@ public double getMaxDiskUsedPct() {
ImmutableMap<String, DiskInfo> disks = disksRef;
double maxPct = 0.0;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals("STORAGE")) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
double percent = diskInfo.getUsedPct();
if (percent > maxPct) {
maxPct = percent;
Expand All @@ -485,7 +486,7 @@ public boolean diskExceedLimitByStorageMedium(TStorageMedium storageMedium) {
boolean exceedLimit = true;
for (DiskInfo diskInfo : diskInfos.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getStorageMedium()
== storageMedium && !diskInfo.exceedLimit(true) && diskInfo.getDirType().equals("STORAGE")) {
== storageMedium && !diskInfo.exceedLimit(true) && diskInfo.getDirType().equals(DATA_DIR_TYPE)) {
exceedLimit = false;
break;
}
Expand All @@ -500,7 +501,7 @@ public boolean diskExceedLimit() {
ImmutableMap<String, DiskInfo> diskInfos = disksRef;
boolean exceedLimit = true;
for (DiskInfo diskInfo : diskInfos.values()) {
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals("STORAGE")
if (diskInfo.getState() == DiskState.ONLINE && diskInfo.getDirType().equals(DATA_DIR_TYPE)
&& !diskInfo.exceedLimit(true)) {
exceedLimit = false;
break;
Expand All @@ -515,7 +516,7 @@ public void updateDisks(Map<String, TDisk> backendDisks) {
if (!initPathInfo) {
boolean allPathHashUpdated = true;
for (DiskInfo diskInfo : disks.values()) {
if (diskInfo.getDirType().equals("STORAGE") && diskInfo.getPathHash() == 0) {
if (diskInfo.getDirType().equals(DATA_DIR_TYPE) && diskInfo.getPathHash() == 0) {
allPathHashUpdated = false;
break;
}
Expand All @@ -542,7 +543,7 @@ public void updateDisks(Map<String, TDisk> backendDisks) {
long trashUsedCapacityB = tDisk.getTrashUsedCapacity();
long diskAvailableCapacityB = tDisk.getDiskAvailableCapacity();
boolean isUsed = tDisk.isUsed();
String dirType = tDisk.getDirType();
String dirType = tDisk.getDirType().toString();
DiskInfo diskInfo = disks.get(rootPath);
if (diskInfo == null) {
diskInfo = new DiskInfo(rootPath);
Expand Down
7 changes: 4 additions & 3 deletions fe/fe-core/src/test/java/org/apache/doris/catalog/BackendTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.doris.resource.Tag;
import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TDisk;
import org.apache.doris.thrift.TDiskType;
import org.apache.doris.thrift.TStorageMedium;

import com.google.common.collect.ImmutableMap;
Expand Down Expand Up @@ -92,11 +93,11 @@ public void diskInfoTest() {
Map<String, TDisk> diskInfos = new HashMap<String, TDisk>();

TDisk disk1 = new TDisk("/data1/", 1000, 800, true);
disk1.setDirType("STORAGE");
disk1.setDirType(TDiskType.STORAGE);
TDisk disk2 = new TDisk("/data2/", 2000, 700, true);
disk2.setDirType("STORAGE");
disk2.setDirType(TDiskType.STORAGE);
TDisk disk3 = new TDisk("/data3/", 3000, 600, false);
disk3.setDirType("STORAGE");
disk3.setDirType(TDiskType.STORAGE);

diskInfos.put(disk1.getRootPath(), disk1);
diskInfos.put(disk2.getRootPath(), disk2);
Expand Down
5 changes: 3 additions & 2 deletions fe/fe-core/src/test/java/org/apache/doris/clone/DecommissionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TDisk;
import org.apache.doris.thrift.TDiskType;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.utframe.UtFrameUtils;

Expand Down Expand Up @@ -91,7 +92,7 @@ public static void beforeClass() throws Exception {
tDisk1.setDiskAvailableCapacity(tDisk1.disk_total_capacity - tDisk1.data_used_capacity);
tDisk1.setPathHash(random.nextLong());
tDisk1.setStorageMedium(TStorageMedium.HDD);
tDisk1.setDirType("STORAGE");
tDisk1.setDirType(TDiskType.STORAGE);
backendDisks.put(tDisk1.getRootPath(), tDisk1);

TDisk tDisk2 = new TDisk();
Expand All @@ -102,7 +103,7 @@ public static void beforeClass() throws Exception {
tDisk2.setDiskAvailableCapacity(tDisk2.disk_total_capacity - tDisk2.data_used_capacity);
tDisk2.setPathHash(random.nextLong());
tDisk2.setStorageMedium(TStorageMedium.HDD);
tDisk2.setDirType("STORAGE");
tDisk2.setDirType(TDiskType.STORAGE);
backendDisks.put(tDisk2.getRootPath(), tDisk2);

be.updateDisks(backendDisks);
Expand Down
5 changes: 3 additions & 2 deletions fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TDisk;
import org.apache.doris.thrift.TDiskType;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.utframe.UtFrameUtils;

Expand Down Expand Up @@ -134,7 +135,7 @@ public static void beforeClass() throws Exception {
tDisk1.setDiskAvailableCapacity(tDisk1.disk_total_capacity - tDisk1.data_used_capacity);
tDisk1.setPathHash(random.nextLong());
tDisk1.setStorageMedium(TStorageMedium.HDD);
tDisk1.setDirType("STORAGE");
tDisk1.setDirType(TDiskType.STORAGE);
backendDisks.put(tDisk1.getRootPath(), tDisk1);

TDisk tDisk2 = new TDisk();
Expand All @@ -145,7 +146,7 @@ public static void beforeClass() throws Exception {
tDisk2.setDiskAvailableCapacity(tDisk2.disk_total_capacity - tDisk2.data_used_capacity);
tDisk2.setPathHash(random.nextLong());
tDisk2.setStorageMedium(TStorageMedium.SSD);
tDisk2.setDirType("STORAGE");
tDisk2.setDirType(TDiskType.STORAGE);
backendDisks.put(tDisk2.getRootPath(), tDisk2);

be.updateDisks(backendDisks);
Expand Down
5 changes: 3 additions & 2 deletions fe/fe-core/src/test/java/org/apache/doris/clone/TabletReplicaTooSlowTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.doris.system.Diagnoser;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TDisk;
import org.apache.doris.thrift.TDiskType;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.utframe.UtFrameUtils;

Expand Down Expand Up @@ -103,7 +104,7 @@ public static void beforeClass() throws Exception {
tDisk1.setDiskAvailableCapacity(tDisk1.disk_total_capacity - tDisk1.data_used_capacity);
tDisk1.setPathHash(random.nextLong());
tDisk1.setStorageMedium(TStorageMedium.HDD);
tDisk1.setDirType("STORAGE");
tDisk1.setDirType(TDiskType.STORAGE);
backendDisks.put(tDisk1.getRootPath(), tDisk1);

TDisk tDisk2 = new TDisk();
Expand All @@ -114,7 +115,7 @@ public static void beforeClass() throws Exception {
tDisk2.setDiskAvailableCapacity(tDisk2.disk_total_capacity - tDisk2.data_used_capacity);
tDisk2.setPathHash(random.nextLong());
tDisk2.setStorageMedium(TStorageMedium.SSD);
tDisk2.setDirType("STORAGE");
tDisk2.setDirType(TDiskType.STORAGE);
backendDisks.put(tDisk2.getRootPath(), tDisk2);

be.updateDisks(backendDisks);
Expand Down
3 changes: 2 additions & 1 deletion fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TDisk;
import org.apache.doris.thrift.TDiskType;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.utframe.UtFrameUtils;

Expand Down Expand Up @@ -95,7 +96,7 @@ private static void setDiskInfos(int diskNum, long diskCapacity, Backend be) {
disk.setDiskAvailableCapacity(disk.disk_total_capacity - disk.data_used_capacity);
disk.setPathHash(random.nextLong());
disk.setStorageMedium(TStorageMedium.HDD);
disk.setDirType("STORAGE");
disk.setDirType(TDiskType.STORAGE);
backendDisks.put(disk.getRootPath(), disk);
}
be.updateDisks(backendDisks);
Expand Down
5 changes: 3 additions & 2 deletions fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.apache.doris.resource.Tag;
import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TDisk;
import org.apache.doris.thrift.TDiskType;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.utframe.UtFrameUtils;

Expand Down Expand Up @@ -118,7 +119,7 @@ public static void beforeClass() throws Exception {
tDisk1.setDiskAvailableCapacity(tDisk1.disk_total_capacity - tDisk1.data_used_capacity);
tDisk1.setPathHash(random.nextLong());
tDisk1.setStorageMedium(TStorageMedium.HDD);
tDisk1.setDirType("STORAGE");
tDisk1.setDirType(TDiskType.STORAGE);
backendDisks.put(tDisk1.getRootPath(), tDisk1);

TDisk tDisk2 = new TDisk();
Expand All @@ -129,7 +130,7 @@ public static void beforeClass() throws Exception {
tDisk2.setDiskAvailableCapacity(tDisk2.disk_total_capacity - tDisk2.data_used_capacity);
tDisk2.setPathHash(random.nextLong());
tDisk2.setStorageMedium(TStorageMedium.SSD);
tDisk2.setDirType("STORAGE");
tDisk2.setDirType(TDiskType.STORAGE);
backendDisks.put(tDisk2.getRootPath(), tDisk2);

be.updateDisks(backendDisks);
Expand Down
Loading