fix device id and add get/set_cuda_rng_state api

Author: yaoxuefeng6

paddle/fluid/framework/generator.cc

@@ -43,9 +43,11 @@ const std::shared_ptr<Generator>& GetDefaultCUDAGenerator(int64_t device_id) {
     cuda_device_flags.resize(num_cuda_devices);
     default_cuda_generators.resize(num_cuda_devices);
   });
-  platform::Place place;
-  if (device_id == -1)
+  if (device_id == -1) {
+    platform::Place place;
     device_id = BOOST_GET_CONST(platform::CUDAPlace, place).GetDeviceId();
+  }
+
   std::call_once(cuda_device_flags[device_id], [device_id]() {
     default_cuda_generators[device_id] =
         std::make_shared<Generator>(GetRandomSeed(), device_id);

paddle/fluid/operators/dropout_op.cu

@@ -186,7 +186,9 @@ class GPUDropoutKernel : public framework::OpKernel<T> {
             context.Attr<bool>("fix_seed") ? context.Attr<int>("seed") : rnd();
       }
 
-      auto gen_cuda = framework::GetDefaultCUDAGenerator(-1);
+      int device_id = BOOST_GET_CONST(platform::CUDAPlace, context.GetPlace())
+                          .GetDeviceId();
+      auto gen_cuda = framework::GetDefaultCUDAGenerator(device_id);
       if (gen_cuda->GetIsInitPy() && (!context.Attr<bool>("fix_seed"))) {
         auto seed_offset = gen_cuda->IncrementOffset(1);
         RandomGeneratorWithGenerator<T, uint8_t><<<grid, threads, 0, stream>>>(

paddle/fluid/operators/gaussian_random_op.cu

@@ -64,7 +64,10 @@ class GPUGaussianRandomKernel : public framework::OpKernel<T> {
     T* data = tensor->mutable_data<T>(context.GetPlace());
 
     int64_t size = tensor->numel();
-    auto gen_cuda = framework::GetDefaultCUDAGenerator(-1);
+
+    int device_id =
+        BOOST_GET_CONST(platform::CUDAPlace, context.GetPlace()).GetDeviceId();
+    auto gen_cuda = framework::GetDefaultCUDAGenerator(device_id);
 
     if (gen_cuda->GetIsInitPy() && seed_flag) {
       auto seed_offset = gen_cuda->IncrementOffset(1);
@@ -102,7 +105,10 @@ class GPUGaussianRandomBatchSizeLikeKernel : public framework::OpKernel<T> {
     T std = static_cast<T>(context.Attr<float>("std"));
     thrust::counting_iterator<unsigned int> index_sequence_begin(0);
     int64_t size = tensor->numel();
-    auto gen_cuda = framework::GetDefaultCUDAGenerator(-1);
+
+    int device_id =
+        BOOST_GET_CONST(platform::CUDAPlace, context.GetPlace()).GetDeviceId();
+    auto gen_cuda = framework::GetDefaultCUDAGenerator(device_id);
 
     if (gen_cuda->GetIsInitPy() && seed_flag) {
       auto seed_offset = gen_cuda->IncrementOffset(1);

paddle/fluid/operators/truncated_gaussian_random_op.cu

@@ -97,7 +97,10 @@ class GPUTruncatedGaussianRandomKernel : public framework::OpKernel<T> {
     thrust::counting_iterator<unsigned int> index_sequence_begin(0);
     int64_t size = tensor->numel();
 
-    auto gen_cuda = framework::GetDefaultCUDAGenerator(-1);
+    int device_id =
+        BOOST_GET_CONST(platform::CUDAPlace, context.GetPlace()).GetDeviceId();
+    auto gen_cuda = framework::GetDefaultCUDAGenerator(device_id);
+
     if (gen_cuda->GetIsInitPy() && seed_flag) {
       auto seed_offset = gen_cuda->IncrementOffset(1);
       int offset_step = 100;

paddle/fluid/operators/uniform_random_op.cu

@@ -138,7 +138,8 @@ class GPUUniformRandomKernel : public framework::OpKernel<T> {
     T diag_val = static_cast<T>(context.Attr<float>("diag_val"));
     thrust::counting_iterator<unsigned int> index_sequence_begin(0);
     int64_t size = tensor->numel();
-    int64_t device_id = -1;
+    int device_id =
+        BOOST_GET_CONST(platform::CUDAPlace, context.GetPlace()).GetDeviceId();
     auto gen_cuda = framework::GetDefaultCUDAGenerator(device_id);
     if (gen_cuda->GetIsInitPy() && seed_flag) {
       auto seed_offset = gen_cuda->IncrementOffset(1);

python/paddle/__init__.py

@@ -217,8 +217,8 @@
 from .tensor.search import nonzero  #DEFINE_ALIAS
 from .tensor.search import sort  #DEFINE_ALIAS
 from .framework.random import manual_seed  #DEFINE_ALIAS
-from .framework.random import get_cuda_state  #DEFINE_ALIAS
-from .framework.random import set_cuda_state  #DEFINE_ALIAS
+from .framework.random import get_cuda_rng_state  #DEFINE_ALIAS
+from .framework.random import set_cuda_rng_state  #DEFINE_ALIAS
 from .framework import Variable  #DEFINE_ALIAS
 from .framework import ParamAttr  #DEFINE_ALIAS
 from .framework import create_global_var  #DEFINE_ALIAS

python/paddle/fluid/tests/unittests/test_cuda_random_seed.py

@@ -36,9 +36,8 @@ def test_gen_dropout_dygraph(self):
         fluid.enable_dygraph()
 
         gen.manual_seed(111111111)
-        st = paddle.get_cuda_state()
-        #x_data = np.arange(1, 101).reshape(2, 50).astype("float32")
-        #x = paddle.to_variable(x_data)
+        st = paddle.get_cuda_rng_state()
+
         x = fluid.layers.uniform_random(
             [2, 10], dtype="float32", min=0.0, max=1.0)
         x_again = fluid.layers.uniform_random(
@@ -49,12 +48,9 @@ def test_gen_dropout_dygraph(self):
         print("x_again: {}".format(x_again.numpy()))
         x = x + x_again + x_third
         y = fluid.layers.dropout(x, 0.5)
-        #gen.manual_seed(111111111)
-        # gen.set_state(st)
-        paddle.set_cuda_state(st)
-        #gen = paddle.manual_seed(123431)
-        #x1 = np.arange(1,101).reshape(2,50).astype("float32")
-        #x1 = paddle.to_variable(x_data)
+
+        paddle.set_cuda_rng_state(st)
+
         x1 = fluid.layers.uniform_random(
             [2, 10], dtype="float32", min=0.0, max=1.0)
         x1_again = fluid.layers.uniform_random(
@@ -65,8 +61,7 @@ def test_gen_dropout_dygraph(self):
         y1 = fluid.layers.dropout(x1, 0.5)
         y_np = y.numpy()
         y1_np = y1.numpy()
-        #print(y_np)
-        #print(y1_np)
+
         if core.is_compiled_with_cuda():
             print(">>>>>>> dropout dygraph >>>>>>>")
             self.assertTrue(np.allclose(y_np, y1_np))
@@ -77,20 +72,16 @@ def test_generator_gaussian_random_dygraph(self):
 
         paddle.manual_seed(12312321111)
         x = fluid.layers.gaussian_random([120], dtype="float32")
-        st1 = paddle.get_cuda_state()
+        st1 = paddle.get_cuda_rng_state()
         x1 = fluid.layers.gaussian_random([120], dtype="float32")
-        paddle.set_cuda_state(st1)
+        paddle.set_cuda_rng_state(st1)
         x2 = fluid.layers.gaussian_random([120], dtype="float32")
         paddle.manual_seed(12312321111)
         x3 = fluid.layers.gaussian_random([120], dtype="float32")
         x_np = x.numpy()
         x1_np = x1.numpy()
         x2_np = x2.numpy()
         x3_np = x3.numpy()
-        #print("x: {}".format(x_np))
-        #print("x1: {}".format(x1_np))
-        #print("x2: {}".format(x2_np))
-        #print("x3: {}".format(x3_np))
 
         if core.is_compiled_with_cuda():
             print(">>>>>>> gaussian random dygraph >>>>>>>")
@@ -124,7 +115,7 @@ def test_gen_TruncatedNormal_initializer(self):
         fluid.disable_dygraph()
 
         gen = paddle.manual_seed(123123143)
-        cur_state = paddle.get_cuda_state()
+        cur_state = paddle.get_cuda_rng_state()
 
         startup_program = fluid.Program()
         train_program = fluid.Program()

python/paddle/framework/random.py

@@ -16,7 +16,7 @@
 import paddle.fluid as fluid
 from paddle.fluid import core
 
-__all__ = ['manual_seed', 'get_cuda_state', 'set_cuda_state']
+__all__ = ['manual_seed', 'get_cuda_rng_state', 'set_cuda_rng_state']
 
 
 def manual_seed(seed):
@@ -51,7 +51,7 @@ def manual_seed(seed):
     return core.default_cpu_generator().manual_seed(seed)
 
 
-def get_cuda_state():
+def get_cuda_rng_state():
     """
 
     Get random state of cuda generators.
@@ -66,8 +66,7 @@ def get_cuda_state():
         .. code-block:: python
 
             import paddle
-            gen = paddle.manual_seed(102)
-            sts = paddle.get_cuda_state()
+            sts = paddle.get_cuda_rng_state()
 
     """
     state_list = []
@@ -78,13 +77,13 @@ def get_cuda_state():
     return state_list
 
 
-def set_cuda_state(state_list):
+def set_cuda_rng_state(state_list):
     """
 
     Sets generator state for all cuda generators
 
     Args:
-        state_list(list): The cuda states to set back to cuda generators. state_list is obtained from get_cuda_state().
+        state_list(list): The cuda states to set back to cuda generators. state_list is obtained from get_cuda_rng_state().
 
     Returns:
         None
@@ -93,9 +92,8 @@ def set_cuda_state(state_list):
         .. code-block:: python
 
             import paddle
-            gen = paddle.manual_seed(102)
-            sts = paddle.get_cuda_state()
-            paddle.set_cuda_state(sts)
+            sts = paddle.get_cuda_rng_state()
+            paddle.set_cuda_rng_state(sts)
 
     """
     if core.is_compiled_with_cuda():