Rm oneflow export and experimental

.github/workflows/simple.yml

@@ -27,7 +27,7 @@ jobs:
     if: github.ref == 'refs/heads/master' || (github.event.pull_request.draft == false && contains(github.event.pull_request.requested_reviewers.*.login, 'oneflow-ci-bot') && contains(github.event.pull_request.labels.*.name, 'need-simple-ci'))
     runs-on: ${{ matrix.os }}
     needs: [cancel_previous]
-    timeout-minutes: 120
+    timeout-minutes: 240
     strategy:
       fail-fast: true
       max-parallel: 3

.github/workflows/test.yml

@@ -53,11 +53,13 @@ jobs:
         id: license_check
         run: |
           python3 ci/check/run_license_format.py -i oneflow -c
+          python3 ci/check/run_license_format.py -i python -c
       - name: Add license
         id: license_fmt
         if: ${{ failure() }}
         run: |
           python3 ci/check/run_license_format.py -i oneflow --fix
+          python3 ci/check/run_license_format.py -i python --fix
       - name: Check C++/CUDA format
         id: cpp_check
         run: |
@@ -521,7 +523,7 @@ jobs:
         run: |
           docker run \
             ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
-            -e ONEFLOW_TEST_DIR=$PWD/oneflow/python/test/modules \
+            -e ONEFLOW_TEST_DIR=$PWD/python/oneflow/test/modules \
             ${{ env.image_tag }} \
             bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test_multi_client/generic_test.sh"
       - name: Dataloader API test
@@ -530,7 +532,7 @@ jobs:
         run: |
           docker run \
             ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
-            -e ONEFLOW_TEST_DIR=$PWD/oneflow/python/test/dataloader \
+            -e ONEFLOW_TEST_DIR=$PWD/python/oneflow/test/dataloader \
             ${{ env.image_tag }} \
             bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/generic_test.sh"
       - name: Tensor API test
@@ -539,23 +541,23 @@ jobs:
         run: |
           docker run \
             ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
-            -e ONEFLOW_TEST_DIR=$PWD/oneflow/python/test/tensor \
+            -e ONEFLOW_TEST_DIR=$PWD/python/oneflow/test/tensor \
             ${{ env.image_tag }} \
             bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/generic_test.sh"
       - name: Graph API test
         if: matrix.test_suite == 'cuda_new_interface'
         run: |
           docker run \
             ${{ env.extra_docker_args }} ${{ env.pip_cache_docker_args }} \
-            -e ONEFLOW_TEST_DIR=$PWD/oneflow/python/test/graph \
+            -e ONEFLOW_TEST_DIR=$PWD/python/oneflow/test/graph \
             ${{ env.image_tag }} \
             bash -c "python3 -m pip config set global.index-url ${{ env.pip_index_mirror }} && bash ci/test/try_install.sh && bash ci/test/generic_test.sh"
       - name: Checkout Oneflow-Inc/models
         if: matrix.test_suite == 'cuda_new_interface'
         uses: actions/checkout@v2
         with:
           repository: Oneflow-Inc/models
-          ref: 830a6b91f10c0a04a68843370cea6319a21ed9c2
+          ref: c822b17f9ed598185cf1fa70f570871d12db7442
           path: oneflow-models
       - name: Speed test
         id: speed

.gitignore

@@ -31,4 +31,4 @@ compile_commands.json
 /distributed-tmp
 /serving-tmp
 test_tmp_dir
-oneflow/python/version.py
+unittest-log-*

CMakeLists.txt

@@ -105,8 +105,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(THIRD_PARTY_DIR "${PROJECT_BINARY_DIR}/third_party_install"
   CACHE PATH "Where to install third party headers and libs")
 
-set(THIRD_PARTY_SUBMODULE_DIR "${PROJECT_SOURCE_DIR}/build/third_party"
-  CACHE PATH "Where the third party submodules are")
+set(ONEFLOW_PYTHON_DIR "${PROJECT_SOURCE_DIR}/python" CACHE PATH "oneflow python src dir")
 
 if(WIN32)
   set(CMAKE_BUILD_TYPE Debug)

README.md

@@ -3,7 +3,7 @@
 **OneFlow is a performance-centered and open-source deep learning framework.**
 
 [![Simple CI](https://github.com/Oneflow-Inc/oneflow/actions/workflows/simple.yml/badge.svg)](https://github.com/Oneflow-Inc/oneflow/actions/workflows/simple.yml)
-
+[![Documentation Status](https://readthedocs.org/projects/oneflow/badge/?version=master)](https://oneflow.readthedocs.io/en/master/?badge=master)
 ## Latest News
 - Version 0.4.0 is out!
   - New Pytorch flavored APIs (`import oneflow.experimental as flow`)

ci/check/run_license_format.py

@@ -85,6 +85,7 @@ def glob_files(path):
     parser.add_argument(
         "-v", "--verbose", default=False, action="store_true", required=False
     )
+    parser.add_argument("--silent", default=False, action="store_true", required=False)
     parser.add_argument(
         "-c", "--check", default=False, action="store_true", required=False
     )
@@ -109,4 +110,5 @@ def glob_files(path):
                     if args.verbose:
                         print("license already added:", p)
                 else:
-                    print("license just added:", p)
+                    if args.silent == False:
+                        print("license just added:", p)

ci/test/1node_benchmark_test.sh

@@ -1,7 +1,7 @@
 set -xe
 
 rm -rf /benchmarks
-cp -r oneflow/compatible_single_client_python/benchmarks /benchmarks
+cp -r python/oneflow/compatible/single_client/benchmarks /benchmarks
 cd /benchmarks
 
 python3 cnn_benchmark/of_cnn_benchmarks.py \
@@ -42,7 +42,7 @@ python3 cnn_benchmark/of_cnn_benchmarks.py \
     --iter_num=5 \
     --learning_rate=0.01 \
     --optimizer="sgd" \
-    --loss_print_every_n_iter=1 
+    --loss_print_every_n_iter=1
 
 python3 bert_benchmark/run_pretraining.py \
     --gpu_num_per_node=1 \

ci/test/1node_benchmark_test_fp16.sh

@@ -1,7 +1,7 @@
 set -ex
 
 rm -rf /benchmarks
-cp -r oneflow/compatible_single_client_python/benchmarks /benchmarks
+cp -r python/oneflow/compatible/single_client/benchmarks /benchmarks
 cd /benchmarks
 
 python3 cnn_benchmark/of_cnn_benchmarks.py \

ci/test/1node_custom_op_test.sh

@@ -3,11 +3,11 @@
 set -xe
 
 src_dir=${ONEFLOW_SRC_DIR:-"$PWD"}
-test_tmp_dir=${ONEFLOW_TEST_TMP_DIR:-"/test_tmp_dir"}
+test_tmp_dir=${ONEFLOW_TEST_TMP_DIR:-"./test_tmp_dir"}
 
 rm -rf $test_tmp_dir
 mkdir -p $test_tmp_dir
-cp -r $src_dir/oneflow/compatible_single_client_python/test/custom_ops $test_tmp_dir
+cp -r $src_dir/python/oneflow/compatible/single_client/test/custom_ops $test_tmp_dir
 cd $test_tmp_dir
 
 export ONEFLOW_TEST_DEVICE_NUM=1

ci/test/1node_model_eager_test.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -xe
 
-cp -r oneflow/python/test /test_dir
+cp -r python/oneflow/test /test_dir
 cd /test_dir
 
 python3 models/eager_1node_test.py

ci/test/1node_model_serve_test.sh

@@ -6,7 +6,7 @@ test_tmp_dir=${ONEFLOW_TEST_TMP_DIR:-"/test_tmp_dir"}
 
 rm -rf $test_tmp_dir
 mkdir -p $test_tmp_dir
-cp -r $src_dir/oneflow/compatible_single_client_python/test $test_tmp_dir
+cp -r $src_dir/python/oneflow/compatible/single_client/test $test_tmp_dir
 cd $test_tmp_dir
 
 export ONEFLOW_TEST_DEVICE_NUM=1

ci/test/1node_model_test.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -xe
 
-cp -r oneflow/compatible_single_client_python/test /test_dir
+cp -r python/oneflow/compatible/single_client/test /test_dir
 cd /test_dir
 
 python3 models/1node_test.py

ci/test/1node_op_test.sh

@@ -10,7 +10,7 @@ test_tmp_dir=${ONEFLOW_TEST_TMP_DIR:-"./test_tmp_dir"}
 
 rm -rf $test_tmp_dir
 mkdir -p $test_tmp_dir
-cp -r $src_dir/oneflow/compatible_single_client_python/test $test_tmp_dir
+cp -r $src_dir/python/oneflow/compatible/single_client/test $test_tmp_dir
 cd $test_tmp_dir
 
 python3 -m oneflow --doctor

ci/test/2node_op_test.sh

@@ -10,7 +10,7 @@ test_tmp_dir=${ONEFLOW_TEST_TMP_DIR:-"/test_tmp_dir"}
 rm -rf $test_tmp_dir
 mkdir -p $test_tmp_dir
 chmod -R o+w $test_tmp_dir
-cp -r $src_dir/oneflow/compatible_single_client_python/test $test_tmp_dir
+cp -r $src_dir/python/oneflow/compatible/single_client/test $test_tmp_dir
 cd $test_tmp_dir
 
 ONEFLOW_TEST_DEVICE_NUM=1 python3 test/ops/test_assign.py --failfast --verbose

ci/test/doctest.sh

@@ -11,7 +11,7 @@ python3 -c 'import oneflow; f=open("oneflow_path.txt", "w"); f.write(oneflow.__p
 gpu_num=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)
 python3 $src_dir/ci/test/parallel_run.py \
     --gpu_num=${gpu_num} \
-    --dir=$(cat oneflow_path.txt)/python \
+    --dir=$(cat oneflow_path.txt) \
     --timeout=1 \
     --verbose \
     --chunk=1 \

ci/test/dry_run_test.sh

@@ -9,7 +9,7 @@ test_tmp_dir=${ONEFLOW_TEST_TMP_DIR:-"./test_tmp_dir"}
 
 rm -rf $test_tmp_dir
 mkdir -p $test_tmp_dir
-cp -r $src_dir/oneflow/compatible_single_client_python/benchmarks $test_tmp_dir
+cp -r $src_dir/python/oneflow/compatible/single_client/benchmarks $test_tmp_dir
 cd $test_tmp_dir/benchmarks
 
 export ONEFLOW_DRY_RUN=1

ci/test/generic_test.sh

@@ -5,9 +5,9 @@ export TF_CPP_MIN_LOG_LEVEL=3
 export PYTHONUNBUFFERED=1
 
 src_dir=${ONEFLOW_SRC_DIR:-"$PWD"}
-test_dir=${ONEFLOW_TEST_DIR:-"$PWD/oneflow/python/test/ops"}
+test_dir=${ONEFLOW_TEST_DIR:-"$PWD/python/oneflow/test/ops"}
 test_tmp_dir=${ONEFLOW_TEST_TMP_DIR:-"./test_tmp_dir"}
-export ONEFLOW_TEST_UTILS_DIR=$src_dir/oneflow/python/test_utils
+export ONEFLOW_TEST_UTILS_DIR=$src_dir/python/oneflow/test_utils
 
 rm -rf $test_tmp_dir
 mkdir -p $test_tmp_dir

ci/test/parallel_run.py

@@ -13,13 +13,14 @@
 def gen_cmds(cmd=None, dir=None, doctest=False):
     if doctest:
         paths = glob.glob(os.path.join(dir, "**/*.py"), recursive=True)
-        print(paths)
+        paths = [p for p in paths if "compatible" not in p and "single_client" not in p]
         with_doctest = []
         for p in paths:
             with open(p) as f:
                 content = f.read()
-                if "doctest" in content and "__" not in p:
+                if "doctest" in content:
                     with_doctest.append("{} {} -v".format(cmd, p))
+        print(with_doctest)
         return with_doctest
     else:
         paths = glob.glob(os.path.join(dir, "test_*.py"), recursive=False)

ci/test/test_xla.sh

@@ -5,7 +5,7 @@ test_tmp_dir=${ONEFLOW_TEST_TMP_DIR:-"/test_tmp_dir"}
 
 rm -rf $test_tmp_dir
 mkdir -p $test_tmp_dir
-cp -r $src_dir/oneflow/compatible_single_client_python/test/xrt $test_tmp_dir
+cp -r $src_dir/python/oneflow/compatible/single_client/test/xrt $test_tmp_dir
 cd $test_tmp_dir
 python3 -c "import oneflow.compatible.single_client as flow; assert flow.sysconfig.with_xla()"
-for f in $src_dir/oneflow/compatible_single_client_python/test/xrt/*.py; do python3 "$f"; done
+for f in $src_dir/python/oneflow/compatible/single_client/test/xrt/*.py; do python3 "$f"; done

ci/test_multi_client/generic_test.sh

@@ -4,9 +4,9 @@ set -xe
 export PYTHONUNBUFFERED=1
 
 src_dir=${ONEFLOW_SRC_DIR:-"$PWD"}
-test_dir=${ONEFLOW_TEST_DIR:-"$PWD/oneflow/python/test/modules"}
+test_dir=${ONEFLOW_TEST_DIR:-"$PWD/python/oneflow/test/modules"}
 test_tmp_dir=${ONEFLOW_TEST_TMP_DIR:-"./test_tmp_dir"}
-export ONEFLOW_TEST_UTILS_DIR=$src_dir/oneflow/python/test_utils
+export ONEFLOW_TEST_UTILS_DIR=$src_dir/python/oneflow/test_utils
 
 
 rm -rf $test_tmp_dir

cmake/oneflow.cmake

@@ -166,6 +166,7 @@ endforeach()
 # clang format
 add_custom_target(of_format
   COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ci/check/run_license_format.py -i ${CMAKE_CURRENT_SOURCE_DIR}/oneflow --fix
+  COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ci/check/run_license_format.py -i ${ONEFLOW_PYTHON_DIR} --fix
   COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ci/check/run_clang_format.py --source_dir ${CMAKE_CURRENT_SOURCE_DIR}/oneflow --fix --quiet
   COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ci/check/run_py_format.py --source_dir ${CMAKE_CURRENT_SOURCE_DIR} --fix
   )
@@ -276,7 +277,7 @@ pybind11_add_module(oneflow_internal ${PYBIND11_SRCS} ${of_pybind_obj_cc} ${PYBI
 set_property(TARGET oneflow_internal PROPERTY CXX_VISIBILITY_PRESET "default")
 add_dependencies(oneflow_internal of_cfgobj generate_py_cfg)
 set_target_properties(oneflow_internal PROPERTIES PREFIX "_")
-set_target_properties(oneflow_internal PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/python_scripts/oneflow")
+set_target_properties(oneflow_internal PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${ONEFLOW_PYTHON_DIR}/oneflow")
 target_link_libraries(oneflow_internal PRIVATE ${of_libs} ${oneflow_third_party_libs} of_pyext_obj ${oneflow_exe_third_party_libs})
 target_include_directories(oneflow_internal PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS})
 
@@ -288,55 +289,16 @@ if (WITH_XLA)
   list(APPEND gen_pip_args --xla)
 endif()
 
-set(of_pyscript_dir "${PROJECT_BINARY_DIR}/python_scripts")
 add_custom_target(of_pyscript_copy ALL
-    COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/clean_generated_api.py --root_path=${of_pyscript_dir}
-    COMMAND "${CMAKE_COMMAND}" -E copy
-        "${PROJECT_SOURCE_DIR}/oneflow/init.py" "${of_pyscript_dir}/oneflow/__init__.py"
-    COMMAND "${CMAKE_COMMAND}" -E copy
-        "${PROJECT_SOURCE_DIR}/oneflow/__main__.py" "${of_pyscript_dir}/oneflow/__main__.py"
-    COMMAND rm -rf ${of_pyscript_dir}/oneflow/python
-    COMMAND rm -rf ${of_pyscript_dir}/oneflow/compatible
-    COMMAND ${CMAKE_COMMAND} -E create_symlink "${PROJECT_SOURCE_DIR}/oneflow/python" "${of_pyscript_dir}/oneflow/python"
-    COMMAND "${CMAKE_COMMAND}" -E copy
-        "${PROJECT_SOURCE_DIR}/oneflow/single_client_init.py" "${of_pyscript_dir}/oneflow/compatible/single_client/__init__.py"
-    COMMAND ${CMAKE_COMMAND} -E create_symlink "${PROJECT_SOURCE_DIR}/oneflow/compatible_single_client_python" "${of_pyscript_dir}/oneflow/compatible/single_client/python"
-    COMMAND ${CMAKE_COMMAND} -E touch "${of_pyscript_dir}/oneflow/compatible/__init__.py"
-    COMMAND "${CMAKE_COMMAND}" -E copy
-        "${PROJECT_SOURCE_DIR}/oneflow/single_client_main.py" "${of_pyscript_dir}/oneflow/compatible/single_client/__main__.py"
-    COMMAND ${CMAKE_COMMAND} -E make_directory "${of_pyscript_dir}/oneflow/distributed"
-    COMMAND ${CMAKE_COMMAND} -E create_symlink "${PROJECT_SOURCE_DIR}/oneflow/python/distributed/launch.py" "${of_pyscript_dir}/oneflow/distributed/launch.py"
-    COMMAND ${CMAKE_COMMAND} -E copy_directory "${of_proto_python_dir}/oneflow/core" "${of_pyscript_dir}/oneflow/core"
-    COMMAND ${CMAKE_COMMAND} -E touch "${of_pyscript_dir}/oneflow/core/__init__.py"
-    COMMAND ${CMAKE_COMMAND} -E copy_directory "${of_proto_python_dir}/oneflow/core" "${of_pyscript_dir}/oneflow/compatible/single_client/core"
-    COMMAND ${CMAKE_COMMAND} -E touch "${of_pyscript_dir}/oneflow/compatible/single_client/core/__init__.py"
-    COMMAND ${CMAKE_COMMAND} -E make_directory "${of_pyscript_dir}/oneflow/F"
-    COMMAND ${CMAKE_COMMAND} -E touch "${of_pyscript_dir}/oneflow/F/__init__.py"
-    COMMAND ${CMAKE_COMMAND} -E make_directory "${of_pyscript_dir}/oneflow/compatible/single_client/F"
-    COMMAND ${CMAKE_COMMAND} -E touch "${of_pyscript_dir}/oneflow/compatible/single_client/F/__init__.py"
-    COMMAND ${CMAKE_COMMAND} -E make_directory "${of_pyscript_dir}/oneflow/experimental/F"
-    COMMAND ${CMAKE_COMMAND} -E touch "${of_pyscript_dir}/oneflow/experimental/F/__init__.py"
-    COMMAND ${CMAKE_COMMAND} -E make_directory "${of_pyscript_dir}/oneflow/compatible/single_client/experimental/F"
-    COMMAND ${CMAKE_COMMAND} -E touch "${of_pyscript_dir}/oneflow/compatible/single_client/experimental/F/__init__.py"
-    COMMAND ${CMAKE_COMMAND} -E make_directory "${of_pyscript_dir}/oneflow/python_gen"
-    COMMAND ${CMAKE_COMMAND} -E touch "${of_pyscript_dir}/oneflow/python_gen/__init__.py"
-    COMMAND ${CMAKE_COMMAND} -E make_directory "${of_pyscript_dir}/oneflow/compatible/single_client/python_gen"
-    COMMAND ${CMAKE_COMMAND} -E touch "${of_pyscript_dir}/oneflow/compatible/single_client/python_gen/__init__.py"
-    COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/generate_pip_version.py ${gen_pip_args} --src=${PROJECT_SOURCE_DIR}
-    COMMAND ${Python_EXECUTABLE} "${PROJECT_SOURCE_DIR}/tools/generate_oneflow_symbols_export_file.py"
-        "${PROJECT_SOURCE_DIR}/oneflow/python" "${of_pyscript_dir}/oneflow/python_gen/__export_symbols__.py" "python"
-    COMMAND ${Python_EXECUTABLE} "${PROJECT_SOURCE_DIR}/tools/generate_oneflow_symbols_export_file.py"
-        "${of_pyscript_dir}/oneflow/compatible" "${of_pyscript_dir}/oneflow/compatible/single_client/python_gen/__export_symbols__.py" "compatible")
+    COMMAND ${CMAKE_COMMAND} -E touch "${of_proto_python_dir}/oneflow/core/__init__.py"
+    COMMAND ${CMAKE_COMMAND} -E create_symlink "${of_proto_python_dir}/oneflow/core" "${ONEFLOW_PYTHON_DIR}/oneflow/core"
+    COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/generate_pip_version.py ${gen_pip_args} --src=${PROJECT_SOURCE_DIR} --out=${ONEFLOW_PYTHON_DIR}/oneflow/version.py
+)
 
 # source this file to add oneflow in PYTHONPATH
-file(WRITE "${PROJECT_BINARY_DIR}/source.sh" "export PYTHONPATH=${of_pyscript_dir}:$PYTHONPATH")
+file(WRITE "${PROJECT_BINARY_DIR}/source.sh" "export PYTHONPATH=${ONEFLOW_PYTHON_DIR}:$PYTHONPATH")
 
 add_dependencies(of_pyscript_copy of_protoobj)
-add_custom_target(generate_api ALL
-  COMMAND rm -rf ${of_pyscript_dir}/oneflow/generated
-  COMMAND export PYTHONPATH=${of_pyscript_dir}:$ENV{PYTHONPATH} && ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/generate_oneflow_api.py --root_path=${of_pyscript_dir}/oneflow)
-add_dependencies(generate_api of_pyscript_copy)
-add_dependencies(generate_api oneflow_internal)
 
 file(RELATIVE_PATH PROJECT_BINARY_DIR_RELATIVE ${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR})
 
@@ -373,10 +335,10 @@ if(BUILD_TESTING)
 endif()
 
 # build include
-set(ONEFLOW_INCLUDE_DIR "${PROJECT_BINARY_DIR}/python_scripts/oneflow/include")
+set(ONEFLOW_INCLUDE_DIR "${ONEFLOW_PYTHON_DIR}/oneflow/include")
 add_custom_target(of_include_copy
   COMMAND ${CMAKE_COMMAND} -E remove_directory "${ONEFLOW_INCLUDE_DIR}" && ${CMAKE_COMMAND} -E make_directory "${ONEFLOW_INCLUDE_DIR}")
-add_dependencies(of_include_copy generate_api)
+add_dependencies(of_include_copy oneflow_internal)
 foreach(of_include_src_dir ${ONEFLOW_INCLUDE_SRC_DIRS})
   set(oneflow_all_include_file)
   file(GLOB_RECURSE oneflow_all_include_file "${of_include_src_dir}/*.*")