Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add more table in meta dump #3870

Merged
merged 6 commits into from
Mar 8, 2022
Merged

add more table in meta dump #3870

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
050b65c
add more table in meta dump
pengweisong Feb 9, 2022
45cc04b
add some prefix func
pengweisong Mar 2, 2022
ec196fd
Merge branch 'master' into meta-dump-upgrade
Sophie-Xie Mar 4, 2022
e1ac5dd
Merge branch 'master' into meta-dump-upgrade
Sophie-Xie Mar 4, 2022
a8b2fcf
add meta version
pengweisong Mar 7, 2022
e046a10
Merge branch 'master' into meta-dump-upgrade
Sophie-Xie Mar 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion src/common/utils/MetaKeyUtils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,10 @@ std::string MetaKeyUtils::schemaEdgesPrefix(GraphSpaceID spaceId) {
return key;
}

const std::string& MetaKeyUtils::schemaEdgesPrefix() {
return kEdgesTable;
}

std::string MetaKeyUtils::schemaEdgeKey(GraphSpaceID spaceId,
EdgeType edgeType,
SchemaVer version) {
Expand Down Expand Up @@ -590,6 +594,10 @@ std::string MetaKeyUtils::schemaTagPrefix(GraphSpaceID spaceId, TagID tagId) {
return key;
}

const std::string& MetaKeyUtils::schemaTagsPrefix() {
return kTagsTable;
}

std::string MetaKeyUtils::schemaTagsPrefix(GraphSpaceID spaceId) {
std::string key;
key.reserve(kTagsTable.size() + sizeof(GraphSpaceID));
Expand Down Expand Up @@ -625,6 +633,10 @@ std::string MetaKeyUtils::indexVal(const nebula::meta::cpp2::IndexItem& item) {
return value;
}

const std::string& MetaKeyUtils::indexPrefix() {
return kIndexesTable;
}

std::string MetaKeyUtils::indexPrefix(GraphSpaceID spaceId) {
std::string key;
key.reserve(kIndexesTable.size() + sizeof(GraphSpaceID));
Expand Down Expand Up @@ -1237,7 +1249,10 @@ GraphSpaceID MetaKeyUtils::parseLocalIdSpace(folly::StringPiece rawData) {
}

/**
* diskPartsKey = kDiskPartsTable + len(serialized(hostAddr)) + serialized(hostAddr) + path
* diskPartsKey = kDiskPartsTable +
* len(serialized(hostAddr)) + serialized(hostAddr) +
* space id +
* disk path
*/

HostAddr MetaKeyUtils::parseDiskPartsHost(const folly::StringPiece& rawData) {
Expand Down
6 changes: 6 additions & 0 deletions src/common/utils/MetaKeyUtils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,8 @@ class MetaKeyUtils final {

static std::string schemaEdgesPrefix(GraphSpaceID spaceId);

static const std::string& schemaEdgesPrefix();

static std::string schemaEdgeKey(GraphSpaceID spaceId, EdgeType edgeType, SchemaVer version);

static EdgeType parseEdgeType(folly::StringPiece key);
Expand All @@ -198,6 +200,8 @@ class MetaKeyUtils final {

static std::string schemaTagsPrefix(GraphSpaceID spaceId);

static const std::string& schemaTagsPrefix();

static meta::cpp2::Schema parseSchema(folly::StringPiece rawData);

static std::string indexKey(GraphSpaceID spaceId, IndexID indexID);
Expand All @@ -206,6 +210,8 @@ class MetaKeyUtils final {

static std::string indexPrefix(GraphSpaceID spaceId);

static const std::string& indexPrefix();

static IndexID parseIndexesKeyIndexID(folly::StringPiece key);

static meta::cpp2::IndexItem parseIndex(const folly::StringPiece& rawData);
Expand Down
137 changes: 121 additions & 16 deletions src/tools/meta-dump/MetaDumpTool.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,49 @@ class MetaDumper {
if (!iter) {
return Status::Error("Init iterator failed");
}

std::string prefix;
{
LOG(INFO) << "Space info";
prefix = "__spaces__";
LOG(INFO) << "------------------------------------------\n\n";
LOG(INFO) << "Meta version:";
enum class MetaVersion {
UNKNOWN = 0,
V1 = 1,
V2 = 2,
V3 = 3,
};

prefix = "__meta_version__";
iter->Seek(rocksdb::Slice(prefix));
bool found = false;
while (iter->Valid() && iter->key().starts_with(prefix)) {
auto version = *reinterpret_cast<const MetaVersion*>(iter->value().data());
found = true;
LOG(INFO) << "Meta version=" << static_cast<int>(version);
break;
}

if (!found) {
prefix = MetaKeyUtils::hostPrefix();
iter->Seek(rocksdb::Slice(prefix));
while (iter->Valid() && iter->key().starts_with(prefix)) {
found = true;
auto v1KeySize = prefix.size() + sizeof(int64_t);
auto version = (iter->key().size() == v1KeySize) ? MetaVersion::V1 : MetaVersion::V3;
LOG(INFO) << "Meta version=" << static_cast<int>(version);
iter->Next();
break;
}

if (!found) {
LOG(INFO) << "Meta version= Unkown";
}
}
}
{
LOG(INFO) << "------------------------------------------\n\n";
LOG(INFO) << "Space info:";
prefix = MetaKeyUtils::spacePrefix();
iter->Seek(rocksdb::Slice(prefix));
while (iter->Valid() && iter->key().starts_with(prefix)) {
auto key = folly::StringPiece(iter->key().data(), iter->key().size());
Expand All @@ -52,8 +91,9 @@ class MetaDumper {
}
}
{
LOG(INFO) << "Partition info";
prefix = "__parts__";
LOG(INFO) << "------------------------------------------\n\n";
LOG(INFO) << "Partition info::";
prefix = MetaKeyUtils::partPrefix();
iter->Seek(rocksdb::Slice(prefix));
while (iter->Valid() && iter->key().starts_with(prefix)) {
auto key = folly::StringPiece(iter->key().data(), iter->key().size());
Expand All @@ -71,8 +111,21 @@ class MetaDumper {
}
}
{
LOG(INFO) << "Host info";
prefix = "__hosts__";
LOG(INFO) << "------------------------------------------\n\n";
LOG(INFO) << "Registered machine info:";
prefix = MetaKeyUtils::machinePrefix();
iter->Seek(rocksdb::Slice(prefix));
while (iter->Valid() && iter->key().starts_with(prefix)) {
auto key = folly::StringPiece(iter->key().data(), iter->key().size());
auto machine = MetaKeyUtils::parseMachineKey(key);
LOG(INFO) << folly::sformat("registered machine: {}", machine.toString());
iter->Next();
}
}
{
LOG(INFO) << "------------------------------------------\n\n";
LOG(INFO) << "Host info:";
prefix = MetaKeyUtils::hostPrefix();
iter->Seek(rocksdb::Slice(prefix));
while (iter->Valid() && iter->key().starts_with(prefix)) {
auto key = folly::StringPiece(iter->key().data(), iter->key().size());
Expand All @@ -87,8 +140,56 @@ class MetaDumper {
}
}
{
LOG(INFO) << "Tag info";
prefix = "__tags__";
LOG(INFO) << "------------------------------------------\n\n";
LOG(INFO) << "Host directories info:";
prefix = MetaKeyUtils::hostDirPrefix();
iter->Seek(rocksdb::Slice(prefix));
while (iter->Valid() && iter->key().starts_with(prefix)) {
auto key = folly::StringPiece(iter->key().data(), iter->key().size());
auto val = folly::StringPiece(iter->value().data(), iter->value().size());
auto addr = MetaKeyUtils::parseHostDirKey(key);
auto dir = MetaKeyUtils::parseHostDir(val);

std::string dataDirs = "";
for (auto d : dir.get_data()) {
dataDirs += d + ", ";
}
LOG(INFO) << folly::sformat("host addr: {}, data dirs: {}, root dir: {}",
addr.toString(),
dataDirs,
dir.get_root());
iter->Next();
}
}
{
LOG(INFO) << "------------------------------------------\n\n";
LOG(INFO) << "Disk partitions info:";
prefix = MetaKeyUtils::diskPartsPrefix();
iter->Seek(rocksdb::Slice(prefix));
while (iter->Valid() && iter->key().starts_with(prefix)) {
auto key = folly::StringPiece(iter->key().data(), iter->key().size());
auto val = folly::StringPiece(iter->value().data(), iter->value().size());
auto addr = MetaKeyUtils::parseDiskPartsHost(key);
auto spaceId = MetaKeyUtils::parseDiskPartsSpace(key);
auto diskPath = MetaKeyUtils::parseDiskPartsPath(key);
auto parts = MetaKeyUtils::parseDiskPartsVal(val);

std::string partsStr = "";
for (auto p : parts.get_part_list()) {
partsStr += (std::to_string(p) + ", ");
}
LOG(INFO) << folly::sformat("host addr: {}, data dir: {}, space id: {}, parts: {}",
addr.toString(),
diskPath,
spaceId,
partsStr);
iter->Next();
}
}
{
LOG(INFO) << "------------------------------------------\n\n";
LOG(INFO) << "Tag info:";
prefix = MetaKeyUtils::schemaTagsPrefix();
iter->Seek(rocksdb::Slice(prefix));
while (iter->Valid() && iter->key().starts_with(prefix)) {
auto key = folly::StringPiece(iter->key().data(), iter->key().size());
Expand All @@ -100,8 +201,9 @@ class MetaDumper {
}
}
{
LOG(INFO) << "Edge info";
prefix = "__edges__";
LOG(INFO) << "------------------------------------------\n\n";
LOG(INFO) << "Edge info:";
prefix = MetaKeyUtils::schemaEdgesPrefix();
iter->Seek(rocksdb::Slice(prefix));
while (iter->Valid() && iter->key().starts_with(prefix)) {
auto key = folly::StringPiece(iter->key().data(), iter->key().size());
Expand All @@ -113,8 +215,9 @@ class MetaDumper {
}
}
{
LOG(INFO) << "Index info";
prefix = "__indexes__";
LOG(INFO) << "------------------------------------------\n\n";
LOG(INFO) << "Index info:";
prefix = MetaKeyUtils::indexPrefix();
iter->Seek(rocksdb::Slice(prefix));
while (iter->Valid() && iter->key().starts_with(prefix)) {
auto key = folly::StringPiece(iter->key().data(), iter->key().size());
Expand All @@ -125,8 +228,9 @@ class MetaDumper {
}
}
{
LOG(INFO) << "Leader info";
prefix = "__leader_terms__";
LOG(INFO) << "------------------------------------------\n\n";
LOG(INFO) << "Leader info:";
prefix = MetaKeyUtils::leaderPrefix();
HostAddr host;
TermID term;
nebula::cpp2::ErrorCode code;
Expand All @@ -152,8 +256,9 @@ class MetaDumper {
}
}
{
LOG(INFO) << "Zone info";
prefix = "__zones__";
LOG(INFO) << "------------------------------------------\n\n";
LOG(INFO) << "Zone info:";
prefix = MetaKeyUtils::zonePrefix();
iter->Seek(rocksdb::Slice(prefix));
while (iter->Valid() && iter->key().starts_with(prefix)) {
auto key = folly::StringPiece(iter->key().data(), iter->key().size());
Expand Down