Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix dedup for GetDstBySrc #4582

Merged
merged 3 commits into from
Aug 25, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
replace std::unordered_set with robin_hood in find path executor
  • Loading branch information
jievince committed Aug 25, 2022
commit d659754c9b519d76c2fa0351680168cf8c13cc06
6 changes: 3 additions & 3 deletions src/graph/executor/algo/BFSShortestPathExecutor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ folly::Future<Status> BFSShortestPathExecutor::execute() {
allRightEdges_.emplace_back();
auto& currentEdges = allRightEdges_.back();
auto rIter = ectx_->getResult(pathNode_->rightVidVar()).iter();
std::unordered_set<Value> rightVids;
HashSet rightVids;
for (; rIter->valid(); rIter->next()) {
auto& vid = rIter->getColumn(0);
if (rightVids.emplace(vid).second) {
Expand Down Expand Up @@ -61,7 +61,7 @@ Status BFSShortestPathExecutor::buildPath(bool reverse) {
auto iterSize = iter->size();
visitedVids.reserve(visitedVids.size() + iterSize);

std::unordered_set<Value> uniqueDst;
HashSet uniqueDst;
uniqueDst.reserve(iterSize);
DataSet nextStepVids;
nextStepVids.colNames = {nebula::kVid};
Expand Down Expand Up @@ -108,7 +108,7 @@ Status BFSShortestPathExecutor::buildPath(bool reverse) {
folly::Future<Status> BFSShortestPathExecutor::conjunctPath() {
const auto& leftEdges = allLeftEdges_.back();
const auto& preRightEdges = allRightEdges_[step_ - 1];
std::unordered_set<Value> meetVids;
HashSet meetVids;
bool oddStep = true;
for (const auto& edge : leftEdges) {
if (preRightEdges.find(edge.first) != preRightEdges.end()) {
Expand Down
7 changes: 5 additions & 2 deletions src/graph/executor/algo/BFSShortestPathExecutor.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

#ifndef GRAPH_EXECUTOR_ALGO_BFSSHORTESTPATHEXECUTOR_H_
#define GRAPH_EXECUTOR_ALGO_BFSSHORTESTPATHEXECUTOR_H_
#include <robin_hood.h>

#include "graph/executor/Executor.h"

// BFSShortestPath has two inputs. GetNeighbors(From) & GetNeighbors(To)
Expand Down Expand Up @@ -42,6 +44,7 @@ namespace graph {
class BFSShortestPath;
class BFSShortestPathExecutor final : public Executor {
public:
using HashSet = robin_hood::unordered_flat_set<Value, std::hash<Value>>;
BFSShortestPathExecutor(const PlanNode* node, QueryContext* qctx)
: Executor("BFSShortestPath", node, qctx) {}

Expand All @@ -61,8 +64,8 @@ class BFSShortestPathExecutor final : public Executor {
private:
const BFSShortestPath* pathNode_{nullptr};
size_t step_{1};
std::unordered_set<Value> leftVisitedVids_;
std::unordered_set<Value> rightVisitedVids_;
HashSet leftVisitedVids_;
HashSet rightVisitedVids_;
std::vector<std::unordered_multimap<Value, Edge>> allLeftEdges_;
std::vector<std::unordered_multimap<Value, Edge>> allRightEdges_;
DataSet currentDs_;
Expand Down
10 changes: 4 additions & 6 deletions src/graph/executor/algo/BatchShortestPath.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ using nebula::storage::StorageClient;
DECLARE_uint32(num_path_thread);
namespace nebula {
namespace graph {
folly::Future<Status> BatchShortestPath::execute(const std::unordered_set<Value>& startVids,
const std::unordered_set<Value>& endVids,
folly::Future<Status> BatchShortestPath::execute(const HashSet& startVids,
const HashSet& endVids,
DataSet* result) {
size_t rowSize = init(startVids, endVids);
std::vector<folly::Future<Status>> futures;
Expand All @@ -36,8 +36,7 @@ folly::Future<Status> BatchShortestPath::execute(const std::unordered_set<Value>
});
}

size_t BatchShortestPath::init(const std::unordered_set<Value>& startVids,
const std::unordered_set<Value>& endVids) {
size_t BatchShortestPath::init(const HashSet& startVids, const HashSet& endVids) {
size_t rowSize = splitTask(startVids, endVids);

leftVids_.reserve(rowSize);
Expand Down Expand Up @@ -473,8 +472,7 @@ void BatchShortestPath::setNextStepVid(const PathMap& paths, size_t rowNum, bool
}
}

size_t BatchShortestPath::splitTask(const std::unordered_set<Value>& startVids,
const std::unordered_set<Value>& endVids) {
size_t BatchShortestPath::splitTask(const HashSet& startVids, const HashSet& endVids) {
size_t threadNum = FLAGS_num_path_thread;
size_t startVidsSize = startVids.size();
size_t endVidsSize = endVids.size();
Expand Down
9 changes: 4 additions & 5 deletions src/graph/executor/algo/BatchShortestPath.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,17 @@ class BatchShortestPath final : public ShortestPathBase {
std::unordered_map<std::string, std::string>* stats)
: ShortestPathBase(node, qctx, stats) {}

folly::Future<Status> execute(const std::unordered_set<Value>& startVids,
const std::unordered_set<Value>& endVids,
folly::Future<Status> execute(const HashSet& startVids,
const HashSet& endVids,
DataSet* result) override;

using CustomPath = Row;
using PathMap = std::unordered_map<DstVid, std::unordered_map<StartVid, std::vector<CustomPath>>>;

private:
size_t splitTask(const std::unordered_set<Value>& startVids,
const std::unordered_set<Value>& endVids);
size_t splitTask(const HashSet& startVids, const HashSet& endVids);

size_t init(const std::unordered_set<Value>& startVids, const std::unordered_set<Value>& endVids);
size_t init(const HashSet& startVids, const HashSet& endVids);

folly::Future<Status> getNeighbors(size_t rowNum, size_t stepNum, bool reverse);

Expand Down
6 changes: 5 additions & 1 deletion src/graph/executor/algo/ProduceAllPathsExecutor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
// This source code is licensed under Apache 2.0 License.
#include "graph/executor/algo/ProduceAllPathsExecutor.h"

#include <robin_hood.h>

#include "graph/planner/plan/Algo.h"

DECLARE_int32(num_operator_threads);
namespace nebula {
namespace graph {
Expand All @@ -14,7 +17,8 @@ folly::Future<Status> ProduceAllPathsExecutor::execute() {

if (step_ == 1) {
auto rIter = ectx_->getResult(pathNode_->rightVidVar()).iter();
std::unordered_set<Value> rightVids;
using HashSet = robin_hood::unordered_flat_set<Value, std::hash<Value>>;
HashSet rightVids;
for (; rIter->valid(); rIter->next()) {
auto& vid = rIter->getColumn(0);
if (rightVids.emplace(vid).second) {
Expand Down
7 changes: 5 additions & 2 deletions src/graph/executor/algo/ShortestPathBase.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
#ifndef GRAPH_EXECUTOR_ALGO_SHORTESTPATHBASE_H_
#define GRAPH_EXECUTOR_ALGO_SHORTESTPATHBASE_H_

#include <robin_hood.h>

#include "graph/planner/plan/Algo.h"

using nebula::storage::StorageRpcResponse;
Expand All @@ -14,6 +16,7 @@ namespace nebula {
namespace graph {
class ShortestPathBase {
public:
using HashSet = robin_hood::unordered_flat_set<Value, std::hash<Value>>;
ShortestPathBase(const ShortestPath* node,
QueryContext* qctx,
std::unordered_map<std::string, std::string>* stats)
Expand All @@ -24,8 +27,8 @@ class ShortestPathBase {

virtual ~ShortestPathBase() {}

virtual folly::Future<Status> execute(const std::unordered_set<Value>& startVids,
const std::unordered_set<Value>& endVids,
virtual folly::Future<Status> execute(const HashSet& startVids,
const HashSet& endVids,
DataSet* result) = 0;

using DstVid = Value;
Expand Down
7 changes: 3 additions & 4 deletions src/graph/executor/algo/ShortestPathExecutor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ folly::Future<Status> ShortestPathExecutor::execute() {
DataSet result;
result.colNames = pathNode_->colNames();

std::unordered_set<Value> startVids;
std::unordered_set<Value> endVids;
HashSet startVids;
HashSet endVids;
size_t rowSize = checkInput(startVids, endVids);
std::unique_ptr<ShortestPathBase> pathPtr = nullptr;
if (rowSize <= FLAGS_num_path_thread) {
Expand All @@ -36,8 +36,7 @@ folly::Future<Status> ShortestPathExecutor::execute() {
return finish(ResultBuilder().value(Value(std::move(result))).build());
}

size_t ShortestPathExecutor::checkInput(std::unordered_set<Value>& startVids,
std::unordered_set<Value>& endVids) {
size_t ShortestPathExecutor::checkInput(HashSet& startVids, HashSet& endVids) {
auto iter = ectx_->getResult(pathNode_->inputVar()).iter();
const auto& vidType = *(qctx()->rctx()->session()->space().spaceDesc.vid_type_ref());
for (; iter->valid(); iter->next()) {
Expand Down
6 changes: 5 additions & 1 deletion src/graph/executor/algo/ShortestPathExecutor.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,24 @@
#ifndef GRAPH_EXECUTOR_ALGO_SHORTESTPATHEXECUTOR_H_
#define GRAPH_EXECUTOR_ALGO_SHORTESTPATHEXECUTOR_H_

#include <robin_hood.h>

#include "graph/executor/Executor.h"
#include "graph/planner/plan/Algo.h"

namespace nebula {
namespace graph {
class ShortestPathExecutor final : public Executor {
public:
using HashSet = robin_hood::unordered_flat_set<Value, std::hash<Value>>;
ShortestPathExecutor(const PlanNode* node, QueryContext* qctx)
: Executor("ShortestPath", node, qctx) {
pathNode_ = asNode<ShortestPath>(node);
}

folly::Future<Status> execute() override;

size_t checkInput(std::unordered_set<Value>& startVids, std::unordered_set<Value>& endVids);
size_t checkInput(HashSet& startVids, HashSet& endVids);

private:
const ShortestPath* pathNode_{nullptr};
Expand Down
10 changes: 4 additions & 6 deletions src/graph/executor/algo/SingleShortestPath.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ using nebula::storage::StorageClient;

namespace nebula {
namespace graph {
folly::Future<Status> SingleShortestPath::execute(const std::unordered_set<Value>& startVids,
const std::unordered_set<Value>& endVids,
folly::Future<Status> SingleShortestPath::execute(const HashSet& startVids,
const HashSet& endVids,
DataSet* result) {
size_t rowSize = startVids.size() * endVids.size();
init(startVids, endVids, rowSize);
Expand All @@ -35,9 +35,7 @@ folly::Future<Status> SingleShortestPath::execute(const std::unordered_set<Value
});
}

void SingleShortestPath::init(const std::unordered_set<Value>& startVids,
const std::unordered_set<Value>& endVids,
size_t rowSize) {
void SingleShortestPath::init(const HashSet& startVids, const HashSet& endVids, size_t rowSize) {
leftVids_.reserve(rowSize);
rightVids_.reserve(rowSize);

Expand Down Expand Up @@ -139,7 +137,7 @@ Status SingleShortestPath::doBuildPath(size_t rowNum, GetNeighborsIter* iter, bo
allSteps.emplace_back();
auto& currentStep = allSteps.back();

std::unordered_set<Value> uniqueDst;
HashSet uniqueDst;
uniqueDst.reserve(iterSize);
std::vector<Row> nextStepVids;
nextStepVids.reserve(iterSize);
Expand Down
13 changes: 6 additions & 7 deletions src/graph/executor/algo/SingleShortestPath.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,20 @@ namespace nebula {
namespace graph {
class SingleShortestPath final : public ShortestPathBase {
public:
using HashSet = robin_hood::unordered_flat_set<Value, std::hash<Value>>;
SingleShortestPath(const ShortestPath* node,
QueryContext* qctx,
std::unordered_map<std::string, std::string>* stats)
: ShortestPathBase(node, qctx, stats) {}

folly::Future<Status> execute(const std::unordered_set<Value>& startVids,
const std::unordered_set<Value>& endVids,
folly::Future<Status> execute(const HashSet& startVids,
const HashSet& endVids,
DataSet* result) override;

using HalfPath = std::vector<std::unordered_map<DstVid, std::vector<CustomStep>>>;

private:
void init(const std::unordered_set<Value>& startVids,
const std::unordered_set<Value>& endVids,
size_t rowSize);
void init(const HashSet& startVids, const HashSet& endVids, size_t rowSize);

folly::Future<Status> shortestPath(size_t rowNum, size_t stepNum);

Expand All @@ -48,8 +47,8 @@ class SingleShortestPath final : public ShortestPathBase {
std::vector<Row> createLeftPath(size_t rowNum, const Value& meetVid);

private:
std::vector<std::unordered_set<Value>> leftVisitedVids_;
std::vector<std::unordered_set<Value>> rightVisitedVids_;
std::vector<HashSet> leftVisitedVids_;
std::vector<HashSet> rightVisitedVids_;
std::vector<HalfPath> allLeftPaths_;
std::vector<HalfPath> allRightPaths_;
};
Expand Down
4 changes: 3 additions & 1 deletion src/graph/executor/query/MinusExecutor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

#include "graph/executor/query/MinusExecutor.h"

#include <robin_hood.h>

#include "graph/planner/plan/Query.h"

namespace nebula {
Expand All @@ -17,7 +19,7 @@ folly::Future<Status> MinusExecutor::execute() {
auto left = getLeftInputData();
auto right = getRightInputData();

std::unordered_set<const Row*> hashSet;
robin_hood::unordered_flat_set<const Row*, std::hash<const Row*>> hashSet;
hashSet.reserve(right.iterRef()->size());
for (; right.iterRef()->valid(); right.iterRef()->next()) {
hashSet.insert(right.iterRef()->row());
Expand Down