[python] Submit to PyPI (#635)

* add make command to the python package.

* Update README.rst

* Update README.rst

* Update README.rst

* fix tests.

* fix unix build

* update readme

* fix setup.py

* update travis

* Update .travis.yml

* Update test.py

* some fixes.

* check the 64-bit python

* fix build.

* refine MANIFEST.in

* update Manifest.in

* add more build options.

* Add fatal in cmake

* fix a endif.

* fix bugs.

* fix pep8

* add test for the pip package build

* add test pip install in travis.

* fix version with pre-compile dll

* fix readme.rst

* update readme

.travis.yml

@@ -36,24 +36,30 @@ install:
 
 script:
 - cd $TRAVIS_BUILD_DIR
-- mkdir build && cd build && cmake -DUSE_MPI=ON ..&& make
-- cd $TRAVIS_BUILD_DIR/python-package && python setup.py install
+- mkdir build && cd build && cmake -DUSE_MPI=ON -DBUILD_EXE=OFF -DBUILD_LIB=ON ..&& make
+- cd $TRAVIS_BUILD_DIR/python-package && python setup.py install --precompile
 - cd $TRAVIS_BUILD_DIR && pytest tests/c_api_test/test.py
 - cd $TRAVIS_BUILD_DIR && pytest tests/python_package_test
 - cd $TRAVIS_BUILD_DIR && pep8 --ignore=E501 --exclude=./compute,./docs .
 - cd $TRAVIS_BUILD_DIR
-- rm -rf build && mkdir build && cd build && cmake -DUSE_GPU=ON -DBOOST_ROOT="$HOME/miniconda/" -DOpenCL_INCLUDE_DIR=$AMDAPPSDK/include/ ..
+- rm -rf build && mkdir build && cd build && cmake -DBUILD_EXE=OFF -DBUILD_LIB=ON -DUSE_GPU=ON -DBOOST_ROOT="$HOME/miniconda/" -DOpenCL_INCLUDE_DIR=$AMDAPPSDK/include/ ..
 - sed -i 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' ../include/LightGBM/config.h
 - make
 - sed -i 's/std::string device_type = "gpu";/std::string device_type = "cpu";/' ../include/LightGBM/config.h
-- cd $TRAVIS_BUILD_DIR/python-package && python setup.py install
+- cd $TRAVIS_BUILD_DIR/python-package && python setup.py install --precompile
 - cd $TRAVIS_BUILD_DIR && pytest tests/c_api_test/test.py
 - cd $TRAVIS_BUILD_DIR && pytest tests/python_package_test
 - cd $TRAVIS_BUILD_DIR
-- rm -rf build && mkdir build && cd build && cmake .. && make
+- rm -rf build && rm -f lib_lightgbm.so
 - cd $TRAVIS_BUILD_DIR/python-package && python setup.py install
-- cd $TRAVIS_BUILD_DIR && pytest tests/c_api_test/test.py
 - cd $TRAVIS_BUILD_DIR && pytest tests/python_package_test
+- cd $TRAVIS_BUILD_DIR
+- LGB_VER=$(head -n 1 VERSION.txt)
+- cd $TRAVIS_BUILD_DIR/python-package && python setup.py sdist
+- cd $TRAVIS_BUILD_DIR/python-package/dist && pip install lightgbm-$LGB_VER.tar.gz -v
+- cd $TRAVIS_BUILD_DIR && pytest tests/python_package_test
+- cd $TRAVIS_BUILD_DIR
+- rm -rf build && mkdir build && cd build && cmake .. && make
 - cd $TRAVIS_BUILD_DIR/tests/cpp_test && ../../lightgbm config=train.conf && ../../lightgbm config=predict.conf output_result=origin.pred
 - cd $TRAVIS_BUILD_DIR/build && make
 - cd $TRAVIS_BUILD_DIR/tests/cpp_test && ../../lightgbm config=predict.conf output_result=ifelse.pred && python test.py

CMakeLists.txt

@@ -10,6 +10,12 @@ PROJECT(lightgbm)
 OPTION(USE_MPI "MPI based parallel learning" OFF)
 OPTION(USE_OPENMP "Enable OpenMP" ON)
 OPTION(USE_GPU "Enable GPU-acclerated training (EXPERIMENTAL)" OFF)
+OPTION(BUILD_EXE "Build execute program" ON)
+OPTION(BUILD_LIB "Build dynamic library" OFF)
+
+if(NOT BUILD_EXE AND NOT BUILD_LIB)
+    message( FATAL_ERROR "You cannot disable BUILD_EXE and BUILD_LIB at the same time." )
+endif()
 
 if(APPLE)
     OPTION(APPLE_OUTPUT_DYLIB "Output dylib shared library" OFF)
@@ -107,33 +113,51 @@ file(GLOB SOURCES
     src/treelearner/*.cpp
 )
 
-add_executable(lightgbm src/main.cpp ${SOURCES})
-add_library(_lightgbm SHARED src/c_api.cpp src/lightgbm_R.cpp ${SOURCES})
+if(BUILD_EXE)
+    add_executable(lightgbm src/main.cpp ${SOURCES})
+endif()
+if(BUILD_LIB)
+    add_library(_lightgbm SHARED src/c_api.cpp src/lightgbm_R.cpp ${SOURCES})
+endif()
 
-if(MSVC)
+if(MSVC AND BUILD_LIB)
     set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm")
-endif(MSVC)
+endif()
 
 if(USE_MPI)
-  TARGET_LINK_LIBRARIES(lightgbm ${MPI_CXX_LIBRARIES})
-  TARGET_LINK_LIBRARIES(_lightgbm ${MPI_CXX_LIBRARIES})
+    if(BUILD_EXE)
+        TARGET_LINK_LIBRARIES(lightgbm ${MPI_CXX_LIBRARIES})
+    endif()
+    if(BUILD_LIB)
+        TARGET_LINK_LIBRARIES(_lightgbm ${MPI_CXX_LIBRARIES})
+    endif()
 endif(USE_MPI)
 
 if(USE_GPU)
-  TARGET_LINK_LIBRARIES(lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES})
-  TARGET_LINK_LIBRARIES(_lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES})
+    if(BUILD_EXE)
+        TARGET_LINK_LIBRARIES(lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES})
+    endif()
+    if(BUILD_LIB)
+        TARGET_LINK_LIBRARIES(_lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES})
+    endif()
 endif(USE_GPU)
 
 if(WIN32 AND (MINGW OR CYGWIN))
-    TARGET_LINK_LIBRARIES(lightgbm Ws2_32)
-    TARGET_LINK_LIBRARIES(_lightgbm Ws2_32)
-    TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI)
-    TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI)
+    if(BUILD_EXE)
+        TARGET_LINK_LIBRARIES(lightgbm Ws2_32)
+        TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI)
+    endif()
+    if(BUILD_LIB)
+        TARGET_LINK_LIBRARIES(_lightgbm Ws2_32)
+        TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI)
+    endif()
 endif()
 
-install(TARGETS lightgbm _lightgbm
-        RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin
-        LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib
-        ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib)
+if(BUILD_EXE AND BUILD_LIB)
+    install(TARGETS lightgbm _lightgbm
+            RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin
+            LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib
+            ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib)
+    install(DIRECTORY ${LightGBM_HEADER_DIR}/LightGBM DESTINATION ${CMAKE_INSTALL_PREFIX}/include)
+endif()
 
-install(DIRECTORY ${LightGBM_HEADER_DIR}/LightGBM DESTINATION ${CMAKE_INSTALL_PREFIX}/include)

R-package/src/install.libs.R

@@ -34,7 +34,7 @@ if (!use_precompile) {
   setwd(build_dir)
 
   # Prepare installation steps
-  cmake_cmd <- "cmake"
+  cmake_cmd <- "cmake -DBUILD_EXE=OFF -DBUILD_LIB=ON "
   build_cmd <- "make -j4"
   lib_folder <- file.path(R_PACKAGE_SOURCE, "src", fsep = "/")
 
@@ -51,7 +51,7 @@ if (!use_precompile) {
   }
 
   if (use_gpu) {
-    cmake_cmd <- paste0(cmake_cmd, " -DUSE_GPU=1 ")
+    cmake_cmd <- paste0(cmake_cmd, " -DUSE_GPU=ON ")
   }
 
   # Install

README.md

@@ -3,6 +3,7 @@ LightGBM, Light Gradient Boosting Machine
 [![Build Status](https://travis-ci.org/Microsoft/LightGBM.svg?branch=master)](https://travis-ci.org/Microsoft/LightGBM)
 [![Windows Build status](https://ci.appveyor.com/api/projects/status/1ys5ot401m0fep6l/branch/master?svg=true)](https://ci.appveyor.com/project/guolinke/lightgbm/branch/master)
 [![Documentation Status](https://readthedocs.org/projects/lightgbm/badge/?version=latest)](http://lightgbm.readthedocs.io/)
+[![PyPI version](https://badge.fury.io/py/lightgbm.svg)](https://badge.fury.io/py/lightgbm)
 
 LightGBM is a gradient boosting framework that uses tree based learning algorithms. It is designed to be distributed and efficient with the following advantages:
 
@@ -19,6 +20,8 @@ For more details, please refer to [Features](https://github.com/Microsoft/LightG
 News
 ----
 
+06/20/2017: Python-package is on PyPI now.
+
 06/09/2017: [LightGBM Slack team](https://lightgbm.slack.com) is available.
 
 05/03/2017: LightGBM v2 stable release.

VERSION.txt

@@ -0,0 +1 @@
+2.0.2

appveyor.yml

@@ -13,7 +13,7 @@ init:
 
 build_script:
   - mkdir build && cd build
-  - cmake -DCMAKE_GENERATOR_PLATFORM=x64 .. && cmake --build . --target ALL_BUILD --config Release
+  - cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DBUILD_EXE=ON -DBUILD_LIB=ON .. && cmake --build . --target ALL_BUILD --config Release
   - cd ..
 
 test_script:
@@ -23,10 +23,16 @@ test_script:
   - conda info -a
   - conda install --yes numpy scipy scikit-learn pandas matplotlib
   - pip install pep8 pytest
-  - cd python-package && python setup.py install && cd ..
+  - cd python-package && python setup.py install --precompile && cd ..
   - pytest tests/c_api_test/test.py
   - pytest tests/python_package_test
   - pep8 --ignore=E501 --exclude=./compute,./docs .
+  - "set /p LGB_VER=< VERSION.txt"
+  - cd python-package && python setup.py sdist --formats gztar
+  - cd dist 
+  - "pip install lightgbm-%LGB_VER%.tar.gz -v"
+  - cd ../..
+  - pytest tests/python_package_test
 
 nuget:
   project_feed: true

python-package/MANIFEST.in

@@ -0,0 +1,8 @@
+prune build
+include *.rst *.txt
+recursive-include lightgbm *.py *.txt *.so
+recursive-include lightgbm/Release *.dll
+recursive-include lightgbm/include *
+recursive-include lightgbm/src *
+global-exclude *.pyo
+global-exclude *.pyc

python-package/README.rst

@@ -1,31 +1,60 @@
 LightGBM Python Package
 =======================
 
+|PyPI version|
+
+
 Installation
 ------------
 
-1. Following `Installation Guide <https://github.com/Microsoft/LightGBM/wiki/Installation-Guide>`__ to build first.
-   For the windows user, please change the build config to ``DLL``.
-2. Install with ``cd python-package; python setup.py install`` 
+Preparation
+'''''''''''
+
+You need to install `cmake <https://cmake.org/>`_ and `setuptools <https://pypi.python.org/pypi/setuptools>`_ first. 
+
+For Windows users, Visual Studio (or `MS Build <https://www.visualstudio.com/downloads/#build-tools-for-visual-studio-2017>`_) is needed. You also can use MinGW instead if installing from GitHub.
+
+For Mac OS X users, gcc with OpenMP support must be installed first. Refer to `wiki <https://github.com/Microsoft/LightGBM/wiki/Installation-Guide#osx>`_ for installing gcc with OpenMP support.
+
+Note: 32-bit python is not supported. Please install 64-bit version.
+
+Install from pip
+''''''''''''''''
+
+``pip install lightgbm``
+
+For the MinGW build in Windows and GPU support, please install the latest version from GitHub.
 
-Note: Make sure you have `setuptools <https://pypi.python.org/pypi/setuptools>`__
+Install from GitHub
+'''''''''''''''''''
 
+.. code:: sh
+
+    git clone --recursive https://github.com/Microsoft/LightGBM
+    cd LightGBM/python-package
+    python setup.py install
+
+You may need to use ``sudo`` (or administrator rights in Windows) to perform ``python setup.py install``.
+
+Use ``python setup.py install --mingw`` to use MinGW in Windows.
+
+Use ``python setup.py install --gpu`` to enable GPU support. You will need to install Boost and OpenCL first: details for installation can be found in `gpu-support <https://github.com/Microsoft/LightGBM/wiki/Installation-Guide#with-gpu-support>`_.
 
 Examples
 --------
 
-Refer to the walk through examples in `python-guide folder <https://github.com/Microsoft/LightGBM/tree/master/examples/python-guide>`__
+Refer to the walk through examples in `python-guide folder <https://github.com/Microsoft/LightGBM/tree/master/examples/python-guide>`_
 
 
 Troubleshooting
---------
+---------------
 
-Refer to `FAQ <https://github.com/Microsoft/LightGBM/tree/master/docs/FAQ.md>`__ 
+Refer to `FAQ <https://github.com/Microsoft/LightGBM/tree/master/docs/FAQ.md>`_ 
 
 Developments
---------
+------------
 
-The code style of python package follows `pep8 <https://www.python.org/dev/peps/pep-0008/>`__. If you would like to make a contribution and not familiar with pep-8, please check the pep8 style guide first. Otherwise, you won't pass the check. You should be careful about:
+The code style of python package follows `pep8 <https://www.python.org/dev/peps/pep-0008/>`_. If you would like to make a contribution and not familiar with pep-8, please check the pep8 style guide first. Otherwise, you won't pass the check. You should be careful about:
 
 - E1 Indentation (check pep8 link above)
 - E202 whitespace before and after brackets
@@ -36,3 +65,7 @@ The code style of python package follows `pep8 <https://www.python.org/dev/peps/
 - E302 expected 2 blank lines in front of and at the end of a function or a class
 
 You can ignore E501 (line too long).
+
+
+.. |PyPI version| image:: https://badge.fury.io/py/lightgbm.svg
+    :target: https://badge.fury.io/py/lightgbm

python-package/lightgbm/libpath.py

@@ -16,11 +16,9 @@ def find_lib_path():
         return []
 
     curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
-    dll_path = [curr_path, os.path.join(curr_path, '../../lib/'),
-                os.path.join(curr_path, '../../'),
-                os.path.join(curr_path, './lib/'),
-                os.path.join(sys.prefix, 'lightgbm')]
+    dll_path = [curr_path, os.path.join(curr_path, '../../'), os.path.join(curr_path, '../../lib/')]
     if os.name == 'nt':
+        dll_path.append(os.path.join(curr_path, './Release/'))
         dll_path.append(os.path.join(curr_path, '../../Release/'))
         dll_path.append(os.path.join(curr_path, '../../windows/x64/DLL/'))
         dll_path = [os.path.join(p, 'lib_lightgbm.dll') for p in dll_path]