Merge pull request #54 from TeoZosa/add-mutation-testing-component

✨ Add Mutation Testing Component

.cruft.json

@@ -1,6 +1,6 @@
 {
     "template": "https://github.com/TeoZosa/cookiecutter-cruft-poetry-tox-pre-commit-ci-cd",
-    "commit": "c8abdc017d11343aa1a40cae3ca1d34084276cd7",
+    "commit": "be6a3ef0e83cf23489f664c68006971d362decfe",
     "context": {
         "cookiecutter": {
             "project_name": "Structlog-Sentry-Logger",

.gitattributes

@@ -0,0 +1 @@
+.mutmut-cache filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci.yml

@@ -109,8 +109,6 @@ jobs:
     steps:
       - name: Check out the repository
         uses: actions/checkout@v2.3.3
-        with:
-          fetch-depth: 2
 
       - name: Set up Python
         uses: actions/setup-python@v2.2.1
@@ -144,6 +142,40 @@ jobs:
           file: ./.tox/coverage.xml
           fail_ci_if_error: true
 
+  test-mutations:
+    name: Mutation testing
+    runs-on: "ubuntu-latest"
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2.3.3
+        with:
+          lfs: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v2.2.1
+        with:
+          python-version: "3.9"
+
+      - name: Install Poetry
+        run: |
+          pip install --constraint=.github/workflows/constraints.txt poetry
+          poetry --version
+
+      - name: Configure Poetry
+        run: |
+          poetry config cache-dir "${GITHUB_WORKSPACE}/.cache/pypoetry"
+          poetry config virtualenvs.in-project true
+          poetry config --list
+
+      - name: Install dependencies
+        run: |
+          make provision_environment
+
+      - name: Run mutation testing
+        run: |
+          make test-mutations
+
   # Install Verification ----------------------
 
   verify-user-install:

.github/workflows/release.yml

@@ -18,8 +18,6 @@ jobs:
     steps:
       - name: Check out the repository
         uses: actions/checkout@v2.3.3
-        with:
-          fetch-depth: 2
 
       - name: Set up Python
         uses: actions/setup-python@v2.2.1

CONTRIBUTING_GUIDE.rst

@@ -82,6 +82,20 @@ Run the full test suite:
 
    $ make test
 
+Run `mutation tests`_ to validate test suite robustness (Optional):
+-------------------------------------------------------------------
+
+.. highlight:: bash
+.. code-block:: bash
+
+   $ make test-mutations
+
+.. note::
+   Test time scales with the complexity of the codebase. Results are cached
+   in ``.mutmut-cache``, so once you get past the initial `cold start problem`_,
+   subsequent mutation test runs will be much faster; new mutations will only
+   be applied to modified code paths.
+
 Lint the code:
 --------------
 
@@ -120,6 +134,8 @@ Build the documentation:
 
 .. _pytest: https://pytest.readthedocs.io/
 .. _tox: https://tox.readthedocs.io/
+.. _`mutation tests`: https://opensource.com/article/20/7/mutmut-python
+.. _`cold start problem`: https://en.wikipedia.org/wiki/Cold_start_(recommender_systems)
 
 
 How to submit changes

Makefile

@@ -89,6 +89,11 @@ test-%: clean update-dependencies generate-requirements
 	poetry run tox -e $*,coverage
 	$(MAKE) clean-requirements
 
+.PHONY: test-mutations
+## Test against mutated code to validate test suite robustness
+test-mutations:
+	$(MAKE) tox-mutmut
+
 .PHONY: lint
 ## Run full static analysis suite for local development
 lint:

README.md

@@ -243,18 +243,32 @@ Testing
 
 We use [`tox`](https://tox.readthedocs.io/en/latest/) for our test automation framework
 and [`pytest`](https://pytest.readthedocs.io/) for our testing framework.
+
 To invoke the tests, run:
 
 ```shell script
 make test
 ```
 
+Run [mutation tests](https://opensource.com/article/20/7/mutmut-python) to validate test suite robustness (Optional):
+
+```shell script
+make test-mutations
+```
+
+> 📝 **Note**  
+>  Test time scales with the complexity of the codebase. Results are cached
+>  in `.mutmut-cache`, so once you get past the initial [cold start problem](https://en.wikipedia.org/wiki/Cold_start_(recommender_systems)),
+>  subsequent mutation test runs will be much faster; new mutations will only
+>  be applied to modified code paths.
+
 Code Quality
 ------------
 
 We are using [`pre-commit`](https://pre-commit.com/) for our code quality
-static analysis automation and management framework. To invoke the analyses and
-auto-formatting over all version-controlled files, run:
+static analysis automation and management framework.
+
+To invoke the analyses and auto-formatting over all version-controlled files, run:
 
 ```shell script
 make lint

pyproject.toml

@@ -55,6 +55,7 @@ pytest-mock = "^3.5.1"
 pytest-sugar = "^0.9.4"
 pytest-xdist = "^2.2.0"
 xdoctest = "^0.15.0"
+mutmut = "^2.1.0"
 tox = "^3.21.2"
 tox-gh-actions = "^2.3.0"
 # Code formatter

tox.ini

@@ -33,6 +33,63 @@ commands = pytest \
            {posargs:tests}
 parallel_show_output = True
 
+[testenv:mutmut]
+setenv =
+    {[testenv]setenv}
+    JUNIT_XML_REPORT_OUTPUT_ROOT={toxworkdir}/junit.{envname}
+    JUNIT_XML_REPORT_OUTPUT_PATH={env:JUNIT_XML_REPORT_OUTPUT_ROOT}.xml
+    JUNIT_HTML_REPORT_OUTPUT_PATH={env:JUNIT_XML_REPORT_OUTPUT_ROOT}.html
+# To read emoji in the mutmut source:
+    LANG=en_US.UTF-8
+deps =
+    {[testenv]deps}
+    nodeenv
+passenv = {[testenv]passenv}
+allowlist_externals = bash
+commands =
+# Run mutmut in a shell to remove escape metacharacters in "{[testenv]commands}"
+# Note: mutmut 1 exit code is a fatal error.
+#   Other non-zero exit codes correspond to mutant statuses but indicate that
+#   mutmut itself was successful and therefore are ignored for this test environment.
+#   see: https://github.com/boxed/mutmut/blob/8bd1331429234ba572471d2ab6854b38e880f25d/mutmut/__init__.py#L1258
+    bash -c '\
+        mutmut run \
+            --paths-to-mutate=structlog_sentry_logger/ \
+            --runner="{[testenv]commands}" \
+            --tests-dir=tests/ || \
+        MUTMUT_EXIT_CODE="$\{?\}"; \
+        if [[ "$\{MUTMUT_EXIT_CODE\}" -eq 1 ]]; \
+            then \
+                exit "$\{MUTMUT_EXIT_CODE\}"; \
+        fi'
+    mutmut results
+
+# Export JUnit XML results to user-friendly static HTML page
+    bash -c '\
+        mutmut junitxml \
+        > {env:JUNIT_XML_REPORT_OUTPUT_PATH}'
+
+    nodeenv --python-virtualenv --jobs=8
+    npm install -g --no-package-lock --no-save xunit-viewer
+    # Inside of the virtualenv, '-g' installs into the virtualenv,
+    # as opposed to installing into the local `node_modules`.
+
+# Run in bash to avoid infinite hang due to xunit-viewer async execution
+    bash -c '\
+        xunit-viewer \
+            --results {env:JUNIT_XML_REPORT_OUTPUT_PATH} \
+            --title="Mutation Tests Report" \
+            --output {env:JUNIT_HTML_REPORT_OUTPUT_PATH} &> /dev/null & \
+        XUNIT_VIEWER_PID="$\{!\}"; \
+        sleep 5; \
+        ps -p "$\{XUNIT_VIEWER_PID\}" > /dev/null; \
+        XUNIT_VIEWER_EXIT_CODE="$\{?\}"; \
+        exit "$\{XUNIT_VIEWER_EXIT_CODE\}";'
+    bash -c '\
+        echo -e "Test report available at \033[4;36m \
+            {env:JUNIT_HTML_REPORT_OUTPUT_PATH} \
+            \033[0m"'
+
 [testenv:coverage]
 description = [Run locally after tests]: Combine coverage data and create reports;
               generates a diff coverage against `{env:DIFF_AGAINST:origin/master}`