mv nightly dist tests (#45475)

python/paddle/fluid/tests/unittests/CMakeLists.txt

@@ -147,17 +147,6 @@ if(WIN32)
   list(REMOVE_ITEM TEST_OPS test_ops_nms)
   list(REMOVE_ITEM TEST_OPS test_trt_convert_preln_residual_bias)
 endif()
-
-list(REMOVE_ITEM TEST_OPS test_fleet_checkpoint)
-list(REMOVE_ITEM TEST_OPS test_auto_checkpoint)
-list(REMOVE_ITEM TEST_OPS test_auto_checkpoint1)
-list(REMOVE_ITEM TEST_OPS test_auto_checkpoint2)
-list(REMOVE_ITEM TEST_OPS test_auto_checkpoint3)
-list(REMOVE_ITEM TEST_OPS test_auto_checkpoint_multiple)
-list(REMOVE_ITEM TEST_OPS test_auto_checkpoint_dist_basic)
-list(REMOVE_ITEM TEST_OPS test_hdfs1)
-list(REMOVE_ITEM TEST_OPS test_hdfs2)
-list(REMOVE_ITEM TEST_OPS test_hdfs3)
 list(REMOVE_ITEM TEST_OPS test_checkpoint_saver)
 
 if(APPLE OR WIN32)
@@ -687,11 +676,6 @@ if(WITH_DISTRIBUTE)
 
     endif()
   endif()
-  if(NOT WITH_DGC)
-    # if not with dgc, must close all dgc tests
-    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_mnist_dgc_nccl")
-    list(REMOVE_ITEM DIST_TEST_OPS "test_dist_se_resnext_dgc")
-  endif()
 
   if(NOT APPLE)
     if(WITH_GPU OR WITH_ROCM)
@@ -859,87 +843,6 @@ if(NOT WIN32)
   set_tests_properties(test_parallel_executor_fetch_feed PROPERTIES TIMEOUT 450)
 endif()
 
-if(WITH_DISTRIBUTE
-   AND NOT APPLE
-   AND NOT WIN32)
-  py_test_modules(test_fleet_checkpoint MODULES test_fleet_checkpoint)
-  set_tests_properties(test_fleet_checkpoint PROPERTIES TIMEOUT 200)
-  set_tests_properties(test_fleet_checkpoint
-                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE:NIGHTLY")
-  bash_test_modules(
-    test_auto_checkpoint
-    START_BASH
-    dist_test.sh
-    TIMEOUT
-    200
-    LABELS
-    "RUN_TYPE=EXCLUSIVE:NIGHTLY")
-  bash_test_modules(
-    test_auto_checkpoint1
-    START_BASH
-    dist_test.sh
-    TIMEOUT
-    200
-    LABELS
-    "RUN_TYPE=EXCLUSIVE:NIGHTLY")
-  bash_test_modules(
-    test_auto_checkpoint2
-    START_BASH
-    dist_test.sh
-    TIMEOUT
-    200
-    LABELS
-    "RUN_TYPE=EXCLUSIVE:NIGHTLY")
-  bash_test_modules(
-    test_auto_checkpoint3
-    START_BASH
-    dist_test.sh
-    TIMEOUT
-    200
-    LABELS
-    "RUN_TYPE=EXCLUSIVE:NIGHTLY")
-  bash_test_modules(
-    test_auto_checkpoint_multiple
-    START_BASH
-    dist_test.sh
-    TIMEOUT
-    200
-    LABELS
-    "RUN_TYPE=EXCLUSIVE:NIGHTLY")
-  bash_test_modules(
-    test_auto_checkpoint_dist_basic
-    START_BASH
-    dist_test.sh
-    TIMEOUT
-    200
-    LABELS
-    "RUN_TYPE=EXCLUSIVE:NIGHTLY")
-  bash_test_modules(
-    test_hdfs1
-    START_BASH
-    dist_test.sh
-    TIMEOUT
-    200
-    LABELS
-    "RUN_TYPE=EXCLUSIVE:NIGHTLY")
-  bash_test_modules(
-    test_hdfs2
-    START_BASH
-    dist_test.sh
-    TIMEOUT
-    200
-    LABELS
-    "RUN_TYPE=EXCLUSIVE:NIGHTLY")
-  bash_test_modules(
-    test_hdfs3
-    START_BASH
-    dist_test.sh
-    TIMEOUT
-    200
-    LABELS
-    "RUN_TYPE=EXCLUSIVE:NIGHTLY")
-endif()
-
 add_subdirectory(sequence)
 add_subdirectory(dygraph_to_static)
 add_subdirectory(rnn)

python/paddle/fluid/tests/unittests/collective/README.md

@@ -4,15 +4,15 @@
 ### step 2. Edit the `testslist.csv` file
     Add an item like test_c_identity in testslist.csv
     and specify the properties for the new unit test
-    the properties are the following:  
+    the properties are the following:
 * `name`: the test's name
 * `os`: The supported operator system, ignoring case. If the test run in multiple operator systems, use ";" to split systems, for example, `apple;linux` means the test runs on both Apple and Linux. The supported values are `linux`,`win32` and `apple`. If the value is empty, this means the test runs on all opertaor systems.
 * `arch`: the device's architecture. similar to `os`, multiple valuse ars splited by ";" and ignoring case. The supported architectures are `gpu`, `xpu`, `ASCEND`, `ASCEND_CL` and `rocm`.
-* `timeout`: timeout of a unittest, whose unit is second.
+* `timeout`: timeout of a unittest, whose unit is second. Blank means defalut.
 * `run_type`: run_type of a unittest. Supported values are `NIGHTLY`, `EXCLUSIVE`, `CINN`, `DIST`, `GPUPS`, `INFER`, `EXCLUSIVE:NIGHTLY`, `DIST:NIGHTLY`，which are case-insensitive. 
-* `launcher`: the test launcher.Supported values are test_runner.py, dist_test.sh and custom scripts' name.
-* `num_port`: the number os port used in a distributed unit test
-* `run_serial`: whether in serial mode. the value can be 1 or 0.Default (empty) is 0.
+* `launcher`: the test launcher.Supported values are test_runner.py, dist_test.sh and custom scripts' name. Blank means test_runner.py.
+* `num_port`: the number of port used in a distributed unit test. Blank means automatically distributed port. 
+* `run_serial`: whether in serial mode. the value can be 1 or 0.Default (empty) is 0. Blank means defalut.
 * `ENVS`: required environments. multiple envirenmonts are splited by ";".
 * `conditions`: extra required conditions for some tests. The value is a list of boolean expression in cmake programmer, splited with ";". For example, the value can be `WITH_DGC;NOT WITH_NCCL` or `WITH_NCCL;${NCCL_VERSION} VERSION_GREATER_EQUAL 2212`,The relationship between these expressions is a conjunction.
 

python/paddle/fluid/tests/unittests/collective/fleet/CMakeLists.txt

@@ -94,7 +94,7 @@ if((WITH_ROCM) AND LOCAL_ALL_PLAT)
     LABELS
     "RUN_TYPE=DIST"
     ENVS
-    "PADDLE_DIST_UT_PORT=21206;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
+    "PADDLE_DIST_UT_PORT=21204;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
   )
   set_tests_properties(test_parallel_dygraph_transformer PROPERTIES RUN_SERIAL
                                                                     1)
@@ -574,7 +574,7 @@ if((WITH_GPU
     OR WITH_ASCEND
     OR WITH_ASCEND_CL
    )
-   AND (WIN32 OR LINUX))
+   AND (LINUX OR WIN32))
   py_test_modules(
     test_fleet_hybrid_meta_optimizer MODULES test_fleet_hybrid_meta_optimizer
     ENVS
@@ -619,7 +619,7 @@ if((WITH_ROCM) AND LOCAL_ALL_PLAT)
     LABELS
     "RUN_TYPE=DIST"
     ENVS
-    "PADDLE_DIST_UT_PORT=21258;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
+    "PADDLE_DIST_UT_PORT=21256;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
   )
   set_tests_properties(test_parallel_dygraph_sparse_embedding
                        PROPERTIES TIMEOUT "200" RUN_SERIAL 1)
@@ -660,7 +660,7 @@ if((WITH_ROCM) AND LOCAL_ALL_PLAT)
     LABELS
     "RUN_TYPE=DIST"
     ENVS
-    "PADDLE_DIST_UT_PORT=21262;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
+    "PADDLE_DIST_UT_PORT=21260;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
   )
   set_tests_properties(test_parallel_dygraph_sparse_embedding_over_height
                        PROPERTIES TIMEOUT "350" RUN_SERIAL 1)
@@ -863,3 +863,117 @@ if(WITH_NCCL OR WITH_RCCL)
     endif()
   endif()
 endif()
+if(LOCAL_ALL_ARCH AND (LINUX))
+  bash_test_modules(
+    test_auto_checkpoint
+    START_BASH
+    ../../dist_test.sh
+    LABELS
+    "RUN_TYPE=EXCLUSIVE:NIGHTLY"
+    ENVS
+    "PADDLE_DIST_UT_PORT=21380;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
+  )
+  set_tests_properties(test_auto_checkpoint PROPERTIES TIMEOUT "200")
+endif()
+if(LOCAL_ALL_ARCH AND (LINUX))
+  bash_test_modules(
+    test_auto_checkpoint1
+    START_BASH
+    ../../dist_test.sh
+    LABELS
+    "RUN_TYPE=EXCLUSIVE:NIGHTLY"
+    ENVS
+    "PADDLE_DIST_UT_PORT=21382;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
+  )
+  set_tests_properties(test_auto_checkpoint1 PROPERTIES TIMEOUT "200")
+endif()
+if(LOCAL_ALL_ARCH AND (LINUX))
+  bash_test_modules(
+    test_auto_checkpoint2
+    START_BASH
+    ../../dist_test.sh
+    LABELS
+    "RUN_TYPE=EXCLUSIVE:NIGHTLY"
+    ENVS
+    "PADDLE_DIST_UT_PORT=21384;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
+  )
+  set_tests_properties(test_auto_checkpoint2 PROPERTIES TIMEOUT "200")
+endif()
+if(LOCAL_ALL_ARCH AND (LINUX))
+  bash_test_modules(
+    test_auto_checkpoint3
+    START_BASH
+    ../../dist_test.sh
+    LABELS
+    "RUN_TYPE=EXCLUSIVE:NIGHTLY"
+    ENVS
+    "PADDLE_DIST_UT_PORT=21386;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
+  )
+  set_tests_properties(test_auto_checkpoint3 PROPERTIES TIMEOUT "200")
+endif()
+if(LOCAL_ALL_ARCH AND (LINUX))
+  bash_test_modules(
+    test_auto_checkpoint_multiple
+    START_BASH
+    ../../dist_test.sh
+    LABELS
+    "RUN_TYPE=EXCLUSIVE:NIGHTLY"
+    ENVS
+    "PADDLE_DIST_UT_PORT=21388;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
+  )
+  set_tests_properties(test_auto_checkpoint_multiple PROPERTIES TIMEOUT "200")
+endif()
+if(LOCAL_ALL_ARCH AND (LINUX))
+  bash_test_modules(
+    test_auto_checkpoint_dist_basic
+    START_BASH
+    ../../dist_test.sh
+    LABELS
+    "RUN_TYPE=EXCLUSIVE:NIGHTLY"
+    ENVS
+    "PADDLE_DIST_UT_PORT=21390;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
+  )
+  set_tests_properties(test_auto_checkpoint_dist_basic PROPERTIES TIMEOUT "200")
+endif()
+if(LOCAL_ALL_ARCH AND (LINUX))
+  bash_test_modules(
+    test_hdfs1
+    START_BASH
+    ../../dist_test.sh
+    LABELS
+    "RUN_TYPE=EXCLUSIVE:NIGHTLY"
+    ENVS
+    "PADDLE_DIST_UT_PORT=21392;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
+  )
+  set_tests_properties(test_hdfs1 PROPERTIES TIMEOUT "200")
+endif()
+if(LOCAL_ALL_ARCH AND (LINUX))
+  bash_test_modules(
+    test_hdfs2
+    START_BASH
+    ../../dist_test.sh
+    LABELS
+    "RUN_TYPE=EXCLUSIVE:NIGHTLY"
+    ENVS
+    "PADDLE_DIST_UT_PORT=21394;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
+  )
+  set_tests_properties(test_hdfs2 PROPERTIES TIMEOUT "200")
+endif()
+if(LOCAL_ALL_ARCH AND (LINUX))
+  bash_test_modules(
+    test_hdfs3
+    START_BASH
+    ../../dist_test.sh
+    LABELS
+    "RUN_TYPE=EXCLUSIVE:NIGHTLY"
+    ENVS
+    "PADDLE_DIST_UT_PORT=21396;http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python"
+  )
+  set_tests_properties(test_hdfs3 PROPERTIES TIMEOUT "200")
+endif()
+if((WITH_GPU OR WITH_ROCM) AND (LINUX))
+  py_test_modules(
+    test_fleet_checkpoint MODULES test_fleet_checkpoint ENVS
+    "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python")
+  set_tests_properties(test_fleet_checkpoint PROPERTIES TIMEOUT "200")
+endif()

python/paddle/fluid/tests/unittests/collective/fleet/testslist.csv

@@ -1,6 +1,6 @@
 name,os,arch,timeout,run_type,launcher,num_port,run_serial,envs,conditions
 test_fleet_sharding_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,350,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_static_mp_layers,linux;win32,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_static_mp_layers,LINUX;WIN32,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_dgc_op,,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC
 test_dgc_optimizer,,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC
 test_parallel_margin_cross_entropy,,GPU,120,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL
@@ -32,21 +32,21 @@ test_recv_save_op,,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH
 test_communicator_sync,,,,DIST,test_runner.py,2,1,FLAGS_communicator_send_queue_size=1;FLAGS_communicator_max_merge_var_num=1;http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_pipeline_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_gradient_merge_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_amp_init,linux;win32,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_amp_init,LINUX;WIN32,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_dygraph_sharding_optimizer_stage2,,,120,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_meta_optimizer_base,linux;win32,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_meta_optimizer_base,LINUX;WIN32,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_raw_program_meta_optimizer,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_sharding_parallel,,,120,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_tensor_parallel,,,200,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_dygraph_group_sharded_api_for_eager,,,120,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_distributed_strategy,linux;win32,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_distributed_strategy,LINUX;WIN32,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_dgc_meta_optimizer,,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC
 test_parallel_dygraph_unused_variables,,,350,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_lamb_meta_optimizer,linux,GPU;XPU;ASCEND;ASCEND_CL,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_lamb_meta_optimizer,LINUX,GPU;XPU;ASCEND;ASCEND_CL,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_dgc_momentum_op,,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_DGC
 test_parallel_dygraph_no_sync_gradient_check,,,60,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_fleet_pipeline_meta_optimizer_with_recompute,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_hybrid_meta_optimizer,WIN32;LINUX,GPU;XPU;ASCEND;ASCEND_CL,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_hybrid_meta_optimizer,LINUX;WIN32,GPU;XPU;ASCEND;ASCEND_CL,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_qat,,,120,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_sparse_embedding,,GPU,200,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL;${NCCL_VERSION} VERSION_GREATER_EQUAL 2212
 test_parallel_dygraph_sparse_embedding,,ROCM,200,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
@@ -55,11 +55,11 @@ test_parallel_dygraph_sparse_embedding_over_height,,GPU,150,DIST,../../dist_test
 test_parallel_dygraph_sparse_embedding_over_height,,ROCM,350,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_distributed_strategy,LINUX;APPLE,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_auto_parallel_parallelizer,,,120,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_recompute_meta_optimizer,linux;win32,GPU;XPU;ASCEND;ASCEND_CL,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_recompute_meta_optimizer,LINUX;WIN32,GPU;XPU;ASCEND;ASCEND_CL,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_dygraph_group_sharded_api,,,120,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
-test_fleet_private_function,linux;win32,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_private_function,LINUX;WIN32,,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_new_group,,GPU;XPU;ASCEND;ASCEND_CL,,DIST,test_new_group.sh,2,1,http_proxy=;https_proxy=,
-test_c_comm_init_op,linux,GPU;XPU;ASCEND;ASCEND_CL,120,DIST,test_c_comm_init_op.sh,2,1,http_proxy=;https_proxy=,
+test_c_comm_init_op,LINUX,GPU;XPU;ASCEND;ASCEND_CL,120,DIST,test_c_comm_init_op.sh,2,1,http_proxy=;https_proxy=,
 test_ir_pass_pipeline,,,120,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_mnist,,GPU;ROCM,200,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_parallel_dygraph_se_resnext,,GPU;ROCM,200,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
@@ -71,3 +71,13 @@ test_dygraph_recompute,,GPU;ROCM,,DIST,test_runner.py,2,1,http_proxy=;https_prox
 test_dygraph_recompute_for_eager,,GPU;ROCM,,DIST,test_runner.py,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,
 test_dist_mnist_dgc_nccl,,,,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL OR WITH_RCCL;WITH_DGC
 test_dist_se_resnext_dgc,,,,DIST,../../dist_test.sh,2,1,http_proxy=;https_proxy=;PYTHONPATH=../..,WITH_NCCL OR WITH_RCCL;WITH_DGC
+test_auto_checkpoint,LINUX,,200,EXCLUSIVE:NIGHTLY,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_auto_checkpoint1,LINUX,,200,EXCLUSIVE:NIGHTLY,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_auto_checkpoint2,LINUX,,200,EXCLUSIVE:NIGHTLY,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_auto_checkpoint3,LINUX,,200,EXCLUSIVE:NIGHTLY,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_auto_checkpoint_multiple,LINUX,,200,EXCLUSIVE:NIGHTLY,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_auto_checkpoint_dist_basic,LINUX,,200,EXCLUSIVE:NIGHTLY,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_hdfs1,LINUX,,200,EXCLUSIVE:NIGHTLY,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_hdfs2,LINUX,,200,EXCLUSIVE:NIGHTLY,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_hdfs3,LINUX,,200,EXCLUSIVE:NIGHTLY,../../dist_test.sh,2,,http_proxy=;https_proxy=;PYTHONPATH=../..,
+test_fleet_checkpoint,LINUX,GPU;ROCM,200,EXCLUSIVE:NIGHTLY,test_runner.py,,,http_proxy=;https_proxy=;PYTHONPATH=../..,