Create multiple requirements to avoid install unused packages (nubank#93

)

* Create multiple requirements to avoid install unused packages

* Add requirements*.txt to MANIFEST.in

* Move swifter import to the method that uses it

* Update tqdm version

.circleci/config.yml

@@ -22,7 +22,7 @@
     name: Install package
     command: |
       . scripts/helpers.sh
-      install_package [test_deps]
+      install_package [devel]
 
   install-pandoc: &install-pandoc
     name: Install pandoc

MANIFEST.in

@@ -1,5 +1,4 @@
 include README.md
-include requirements.txt
-include requirements_test.txt
+include requirements*.txt
 include conda.txt
 recursive-include src/fklearn/resources *

requirements.txt

@@ -1,19 +1,7 @@
-boto3>=1.9.158,<2
 cloudpickle>=0.8.0,<0.9.0
 joblib>=0.13.2,<0.14.0
-lightgbm>=2.2.2,<2.3.0
-matplotlib>=3.0.2,<3.1.0
 numpy>=1.16.4,<1.17.0
 pandas>=0.24.1,<0.25
-pyarrow>=0.13.0,<0.14.0
-s3fs>=0.2.0,<0.3.0
-schema>=0.6.2,<0.7.0
 scikit-learn>=0.21.2,<0.22.0
-shap>=0.28.5,<0.29.0
 statsmodels>=0.9.0,<1
 toolz>=0.9.0,<1
-tqdm>=4.32.1,<4.33.0
-xgboost>=0.81,<0.90
-swifter>=0.284,<0.300
-catboost>=0.14.2,<0.15.0
-scipy>=1.2.1,<1.3.0

requirements_catboost.txt

@@ -0,0 +1 @@
+catboost>=0.14.2,<0.15.0

requirements_demos.txt

@@ -0,0 +1,3 @@
+matplotlib>=3.0.2,<3.1.0
+tqdm>=4.32.1,<4.36.0
+scipy>=1.2.1,<1.3.0

requirements_lgbm.txt

@@ -0,0 +1 @@
+lightgbm>=2.2.2,<2.3.0

requirements_tools.txt

@@ -0,0 +1,2 @@
+shap>=0.28.5,<0.29.0
+swifter>=0.284,<0.300

requirements_xgboost.txt

@@ -0,0 +1 @@
+xgboost>=0.81,<0.90

setup.py

@@ -12,6 +12,19 @@ def requirements_from_pip(filename='requirements.txt'):
     with open(filename, 'r') as pip:
         return [l.strip() for l in pip if not l.startswith('#') and l.strip()]
 
+core_deps = requirements_from_pip()
+demos_deps = requirements_from_pip("requirements_demos.txt")
+test_deps = requirements_from_pip("requirements_test.txt")
+
+tools_deps = requirements_from_pip("requirements_tools.txt")
+
+lgbm_deps = requirements_from_pip("requirements_lgbm.txt")
+xgboost_deps = requirements_from_pip("requirements_xgboost.txt")
+catboost_deps = requirements_from_pip("requirements_catboost.txt")
+
+all_models_deps = lgbm_deps + xgboost_deps + catboost_deps
+all_deps = all_models_deps + tools_deps
+devel_deps = test_deps + all_deps
 
 setup(name=MODULE_NAME,
       description="Functional machine learning",
@@ -21,8 +34,16 @@ def requirements_from_pip(filename='requirements.txt'):
       packages=find_packages('src'),
       version=(open(join('src', MODULE_NAME, 'resources', 'VERSION'))
                .read().strip()),
-      install_requires=requirements_from_pip(),
-      extras_require={"test_deps": requirements_from_pip('requirements_test.txt')},
+      install_requires=core_deps,
+      extras_require={"test_deps": test_deps,
+                      "lgbm": lgbm_deps,
+                      "xgboost": xgboost_deps,
+                      "catboost": catboost_deps,
+                      "tools": tools_deps,
+                      "devel": devel_deps,
+                      "all_models": all_models_deps,
+                      "devel": devel_deps,
+                      "all": all_deps},
       include_package_data=True,
       zip_safe=False,
       classifiers=['Programming Language :: Python :: 3.6'])

src/fklearn/training/transformation.py

@@ -3,7 +3,6 @@
 import numpy as np
 import pandas as pd
 from numpy import nan
-import swifter  # NOQA
 from sklearn.preprocessing import StandardScaler
 from statsmodels.distributions import empirical_distribution as ed
 from toolz import curry, merge, compose, mapcat
@@ -786,6 +785,7 @@ def custom_transformer(df: pd.DataFrame,
         and returns another DataFrame.
 
     """
+    import swifter  # NOQA
 
     def p(df: pd.DataFrame) -> pd.DataFrame:
         if is_vectorized: