Check the Partition state in setup (#30)

Author: payoto

setup.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Copyright (c) 2022 Graphcore Ltd. All rights reserved.
 # Script to be sourced on launch of the Gradient Notebook
-
+EXIT_CODE=0
 echo "Graphcore setup - Starting notebook setup"
 DETECTED_NUMBER_OF_IPUS=$(python .gradient/available_ipus.py)
 if [[ "$1" == "test" ]]; then
@@ -19,7 +19,31 @@ if [[ "${DETECTED_NUMBER_OF_IPUS}" == "0" ]]; then
     echo " https://docs.paperspace.com/contact-support/ "
     echo " referencing the Notebook ID: ${PAPERSPACE_METRIC_WORKLOAD_ID:-unknown}"
     echo "=============================================================================="
-    exit -1
+    exit 255
+fi
+# Check the state of the partition
+GC_INFO_OUTPUT=$(timeout 5 gc-info -l 2>&1)
+if [[ "$(echo ${GC_INFO_OUTPUT} | grep 'Partition.* \[active\]')" ]]
+then
+    echo "Graphcore setup - Partition check - passed"
+elif [[ "$(echo ${GC_INFO_OUTPUT} | grep 'partition is not ACTIVE')" ]]
+then
+    echo "=============================================================================="
+    echo "                         IPU ERROR  DETECTED"
+    echo "=============================================================================="
+    echo " IPU Partition is not active. This error indicates a problem with the "
+    echo "hardware you are running on. Please contact Paperspace Support at "
+    echo " https://docs.paperspace.com/contact-support/ "
+    echo " referencing the Notebook ID: ${PAPERSPACE_METRIC_WORKLOAD_ID:-unknown}"
+    echo "=============================================================================="
+    gc-info -l
+    exit 254
+else
+    echo "[WARNING] IPU Partition in an unrecognised state - Notebook will start normally but"
+    echo "[WARNING] you may encounter hardware related errors. Get in touch with Paperspace and/or"
+    echo "[WARNING] Graphcore support if you encounter unexpected behaviours or errors."
+    echo "[WARNING] Output was: ${GC_INFO_OUTPUT}"
+    EXIT_CODE=253
 fi
 
 export NUM_AVAILABLE_IPU=${IPU_ARG}
@@ -53,3 +77,5 @@ echo "Graphcore setup - Starting Jupyter kernel"
 jupyter lab --allow-root --ip=0.0.0.0 --no-browser --ServerApp.trust_xheaders=True \
             --ServerApp.disable_check_xsrf=False --ServerApp.allow_remote_access=True \
             --ServerApp.allow_origin='*' --ServerApp.allow_credentials=True
+
+exit $EXIT_CODE