[KED-1388] Print Python environment (#519)

.circleci/config.yml

@@ -26,9 +26,6 @@ commands:
   setup_requirements:
     description: Install PIP dependencies
     steps:
-      - run:
-          name: Echo python version for debugging
-          command: 'echo "Python version: $(python --version 2>&1)"'
       - run:
           name: Install pip setuptools
           command: make install-pip-setuptools
@@ -41,6 +38,18 @@ commands:
       - run:
           name: Install test requirements
           command: pip install -r test_requirements.txt -U
+      - run:
+          # Since recently Spark installation for some reason does not have enough permissions to execute
+          # /home/circleci/miniconda/envs/kedro_builder/lib/python3.X/site-packages/pyspark/bin/spark-class.
+          # So fixing it manually here.
+          name: Fix Spark permissions
+          command: sudo chmod -R u+x /home/circleci/miniconda/envs/kedro_builder/lib/
+      - run:
+          name: Print Python environment
+          command: make print-python-env
+      - run:
+          name: Pip freeze
+          command: pip freeze
 
   setup_pre_commit:
     description: Install pre-commit hooks
@@ -58,12 +67,6 @@ commands:
       - checkout
       - setup_conda
       - setup_requirements
-      - run:
-          # Since recently Spark installation for some reason does not have permissions to execute
-          # /home/circleci/miniconda/envs/kedro_builder/lib/python3.X/site-packages/pyspark/bin/spark-class.
-          # So fixing it manually here.
-          name: Fix Spark permissions
-          command: sudo chmod -R u+x /home/circleci/miniconda/envs/kedro_builder/lib/
       - run:
           name: Run unit tests
           command: make test
@@ -92,7 +95,7 @@ commands:
           name: Run e2e tests
           command: make e2e-tests
 
-  build_docs:  # NOTE: requires python3.6+
+  build_docs:
     description: Build docs
     steps:
       - checkout

Makefile

@@ -10,7 +10,7 @@ clean:
 	pre-commit clean || true
 
 install-pip-setuptools:
-	python -m pip install -U "pip>=18.0, <19.0" "setuptools>=38.0, <39.0" wheel
+	python -m pip install -U "pip>=18.0, <21.0" "setuptools>=38.0, <47.0" wheel
 
 legal:
 	python tools/license_and_headers.py
@@ -53,3 +53,6 @@ install-pre-commit: install-test-requirements
 uninstall-pre-commit:
 	pre-commit uninstall
 	pre-commit uninstall --hook-type pre-push
+
+print-python-env:
+	@./tools/print_env.sh

features/environment.py

@@ -92,8 +92,8 @@ def call(cmd):
             "pip",
             "install",
             "-U",
-            "pip>=18.0, <19.0",
-            "setuptools>=38.0, <39.0",
+            "pip>=18.0, <21.0",
+            "setuptools>=38.0, <47.0",
             "wheel",
         ]
     )

tools/print_env.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 QuantumBlack Visual Analytics Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
+# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS
+# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo
+# (either separately or in combination, "QuantumBlack Trademarks") are
+# trademarks of QuantumBlack. The License does not grant you any right or
+# license to the QuantumBlack Trademarks. You may not use the QuantumBlack
+# Trademarks or any confusingly similar mark as a trademark for your product,
+#     or use the QuantumBlack Trademarks in any other manner that might cause
+# confusion in the marketplace, including but not limited to in advertising,
+# on websites, or on software.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+print_sep="=============================="
+
+eval_command() {
+    local title="$1"
+    local command="$2"
+
+    echo "$print_sep $title $print_sep"
+    eval "$command"
+    echo
+}
+
+eval_command CONDA "conda info 2>/dev/null || echo \"Conda not found\""
+eval_command PYTHON "which python && python -V"
+eval_command PIP "python -m pip -V"
+eval_command PYLINT "python -m pylint --version"
+eval_command PYTEST "python -m pytest --version"
+eval_command BLACK "python -m black --version"
+eval_command BEHAVE "python -m behave --version"
+eval_command MYPY "python -m mypy --version"
+eval_command FLAKE8 "python -m flake8 --version"
+eval_command ISORT "python -m isort --version"
+eval_command PRE-COMMIT "python -m pre_commit --version"
+eval_command SPARK "python -c \\
+  \"import pyspark; print(f'PySpark: {pyspark.__version__}')\" 2>/dev/null && \\
+  spark-submit --version || echo \"Spark not found\""
+eval_command SPHINX "python -m sphinx --version 2>/dev/null || echo \"Sphinx not found\""
+eval_command KEDRO "python -m kedro info 2>/dev/null || echo \"Kedro not found\""