Skip to content

unify calling cudaSetDevice #30470

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 19, 2021
Merged

unify calling cudaSetDevice #30470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1e06ec1
unify calling cudaSetDevice
zhiqiu Jan 12, 2021
265db22
fix compile
zhiqiu Jan 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion paddle/fluid/framework/details/nccl_op_handle.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ class NCCLOpHandleBase : public OpHandleBase {
continue;
}

PADDLE_ENFORCE_CUDA_SUCCESS(cudaSetDevice(dev_id));
platform::SetDeviceId(dev_id);
PADDLE_ENFORCE_CUDA_SUCCESS(cudaEventCreateWithFlags(
&inter_events_[dev_id], cudaEventDisableTiming));
PADDLE_ENFORCE_CUDA_SUCCESS(cudaEventCreateWithFlags(
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/framework/details/op_handle_base.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ void OpHandleBase::InitCUDA() {
#ifdef PADDLE_WITH_CUDA
for (auto &p : dev_ctxes_) {
int dev_id = BOOST_GET_CONST(platform::CUDAPlace, p.first).device;
PADDLE_ENFORCE_CUDA_SUCCESS(cudaSetDevice(dev_id));
platform::SetDeviceId(dev_id);
PADDLE_ENFORCE_CUDA_SUCCESS(
cudaEventCreateWithFlags(&events_[dev_id], cudaEventDisableTiming));
}
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/framework/fleet/nccl_wrapper.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ void NCCLWrapper::SetRankInfo(const int local_rank, const int global_rank,
nccl_info_.local_rank_ = local_rank;
nccl_info_.my_global_rank_ = global_rank;
nccl_info_.global_ranks_ = ranks;
PADDLE_ENFORCE_CUDA_SUCCESS(cudaSetDevice(local_rank));
platform::SetDeviceId(local_rank);
PADDLE_ENFORCE_CUDA_SUCCESS(cudaStreamCreate(&(nccl_info_.stream_)));
#endif
return;
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/inference/tensorrt/engine.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ void TensorRTEngine::freshDeviceId() {
platform::errors::OutOfRange(
"Device id %d exceeds the current device count: %d.",
device_id_, count));
cudaSetDevice(device_id_);
platform::SetDeviceId(device_id_);
}

} // namespace tensorrt
Expand Down
4 changes: 2 additions & 2 deletions paddle/fluid/memory/malloc_test.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ void MultiStreamCompute(float **data, float **second_data,

TEST(Malloc, CUDADeviceContextMultiStream) {
auto place = platform::CUDAPlace(0);
EXPECT_TRUE(cudaSuccess == cudaSetDevice(0));
platform::SetDeviceId(0);

AllocationPtr main_stream_alloc_ptr = Alloc(place, N * sizeof(float));
EXPECT_GE(main_stream_alloc_ptr->size(), N * sizeof(float));
Expand Down Expand Up @@ -94,7 +94,7 @@ TEST(Malloc, CUDADeviceContextMultiStream) {

TEST(Malloc, CUDADeviceContextMultiThreadMultiStream) {
auto place = platform::CUDAPlace(0);
EXPECT_TRUE(cudaSuccess == cudaSetDevice(0));
platform::SetDeviceId(0);

AllocationPtr main_stream_alloc_ptr = Alloc(place, N * sizeof(float));
EXPECT_GE(main_stream_alloc_ptr->size(), N * sizeof(float));
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/platform/collective_helper.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ NCCLComm* NCCLCommContext::CreateNCCLComm(ncclUniqueId* nccl_id, int nranks,
"Expected dev_id >= 0. But received dev_id is %d.", dev_id));

ncclComm_t comm = nullptr;
PADDLE_ENFORCE_CUDA_SUCCESS(cudaSetDevice(dev_id));
SetDeviceId(dev_id);
PADDLE_ENFORCE_CUDA_SUCCESS(
platform::dynload::ncclCommInitRank(&comm, nranks, *nccl_id, rank));

Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/platform/gpu_info.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ void SetDeviceId(int id) {
"Device id must be less than GPU count, "
"but received id is: %d. GPU count is: %d.",
id, GetCUDADeviceCount()));
PADDLE_ENFORCE_CUDA_SUCCESS(cudaSetDevice(id));
PADDLE_RETRY_CUDA_SUCCESS(cudaSetDevice(id));
}

void GpuMemoryUsage(size_t *available, size_t *total) {
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/platform/nccl_helper.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ struct NCCLContextMap {
}
VLOG(1) << "init nccl rank:" << rank << ", nranks:" << nranks
<< ", gpu_id:" << gpu_id << ", dev_id:" << order_[i];
PADDLE_RETRY_CUDA_SUCCESS(cudaSetDevice(gpu_id));
SetDeviceId(gpu_id);
PADDLE_RETRY_CUDA_SUCCESS(platform::dynload::ncclCommInitRank(
comms.get() + i, nranks, *nccl_id, rank));
}
Expand Down