Merge pull request #11 from f4lco/feature/basic-tests

Add basic tests for notebook (test) execution (#7)

.travis.yml

@@ -9,3 +9,4 @@ install:
 
 script:
  - make lint
+ - make test

Makefile

@@ -1,3 +1,6 @@
 
 lint:
-	pylint treon
+	pylint treon tests
+
+test:
+	pytest tests

README.md

@@ -110,8 +110,13 @@ $ pip list --local | grep treon # should point to the source tree
 treon                0.1.2                /workspace/treon
 ```
 
-Please refer to the `Makefile` for supplementary development tasks, such as linting treon's source code.
-For instance, to run the linter before committing, invoke `make lint`. 
+Please refer to the `Makefile` for supplementary development tasks.
+In particular, the following targets may be relevant when validating changes before committing:
+
+```
+$ make lint # check treon's source for code style errors
+$ make test # run all tests
+```
 
 ## Motivation
 Our aim at [ReviewNB](https://www.reviewnb.com/) is to make notebooks a first class entity in the production workflow. We've built a code review system for Notebooks. The next step is to [build a CI pipeline](https://github.com/ReviewNB/support/issues/19) & treon is the core tool in that effort. It is licensed librerally (MIT) & I foresee it being used as an independent tool as well. You can use it locally and/or integrate with CI system of your choice.

requirements-dev.txt

@@ -1 +1,2 @@
 pylint
+pytest

tests/resources/basic.ipynb

@@ -0,0 +1,91 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Basic Notebook\n",
+    "\n",
+    "This notebook showcases normal execution within treon, without errors."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "hello world\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('hello world')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert 1 + 2 == 3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import unittest\n",
+    "\n",
+    "class SpamTest(unittest.TestCase):\n",
+    "    \n",
+    "    def test_eggs(self):\n",
+    "        assert 2 + 2 == 4"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def function_with_doctests():\n",
+    "    \"\"\"\n",
+    "    >>> 1 + 2\n",
+    "    3\n",
+    "    >>> 1 / 0\n",
+    "    Traceback (most recent call last):\n",
+    "    ZeroDivisionError: division by zero\n",
+    "    \"\"\"\n",
+    "    pass"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

tests/resources/doctest_failed.ipynb

@@ -0,0 +1,39 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def function_failed_doctest():\n",
+    "    \"\"\"\n",
+    "    >>> 1 + 1\n",
+    "    42\n",
+    "    \"\"\"\n",
+    "    pass"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

tests/resources/runtime_error.ipynb

@@ -0,0 +1,53 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This notebook will not execute successfully."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ZeroDivisionError",
+     "evalue": "division by zero",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mZeroDivisionError\u001b[0m                         Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-1-bc757c3fda29>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;36m1\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mZeroDivisionError\u001b[0m: division by zero"
+     ]
+    }
+   ],
+   "source": [
+    "1 / 0"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

tests/resources/unittest_failed.ipynb

@@ -0,0 +1,39 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import unittest\n",
+    "\n",
+    "class FailingTest(unittest.TestCase):\n",
+    "    \n",
+    "    def test_fail(self):\n",
+    "        self.assertEqual(-1, 1)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

tests/test_execution.py

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+import os
+import pytest
+
+from nbconvert.preprocessors import CellExecutionError
+from treon.test_execution import execute_notebook
+
+
+def test_successful_execution():
+    successful, _ = _run('resources/basic.ipynb')
+
+    assert successful
+
+
+def test_failed_execution():
+    with pytest.raises(CellExecutionError) as exc_info:
+        _run('resources/runtime_error.ipynb')
+
+    assert 'ZeroDivisionError' in exc_info.value.traceback
+
+
+def test_failed_unittest():
+    successful, output = _run('resources/unittest_failed.ipynb')
+
+    assert not successful
+    assert 'AssertionError' in output
+
+
+def test_failed_doctest():
+    successful, output = _run('resources/doctest_failed.ipynb')
+
+    assert not successful
+    assert 'Test Failed' in output
+
+
+def _run(notebook):
+    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), notebook)
+    return execute_notebook(path)