Add a check for tests which are always skipped

This takes the form of a new script which crawls pytest XML reports
and collates them into a single aggregate. It checks for tests which
are skipped or missing in all of the reports.

The aggregator can be run along with a suite of tox environments via
`make collatd-test-report`, and a new CI job runs this in a build.

Author: sirosen

.github/workflows/build.yaml

@@ -78,6 +78,33 @@ jobs:
           python -m tox run-parallel -m ci
           python -m tox run -e cov
 
+  collate-tests:
+    runs-on: ubuntu-latest
+    name: "Collate results to check for skipped tests"
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: get date for caching
+        run: /bin/date -u "+%U" > cachedate.txt
+        shell: bash
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: |
+            3.7
+            3.10
+            3.11
+          cache: "pip"
+          cache-dependency-path: |
+            .github/workflows/build.yaml
+            setup.cfg
+            tox.ini
+            cachedate.txt
+
+      - run: python -m pip install tox
+
+      - run: make collated-test-report
+
   self-check:
     name: "Self-Check"
     runs-on: ubuntu-latest

Makefile

@@ -17,6 +17,16 @@ release:
 	-git push $(shell git rev-parse --abbrev-ref @{push} | cut -d '/' -f1) refs/tags/$(PKG_VERSION)
 	tox run -e publish-release
 
+.PHONY: collated-test-report
+collated-test-report:
+	tox p -e py37-mindeps,py311,py310-notoml,py310-tomli-format,py311-json5  -- '--junitxml={envdir}/pytest.xml'
+	python ./scripts/aggregate-pytest-reports.py \
+		.tox/py37-mindeps/pytest.xml \
+		.tox/py311/pytest.xml \
+		.tox/py310-notoml/pytest.xml \
+		.tox/py310-tomli-format/pytest.xml \
+		.tox/py311-json5/pytest.xml
+
 .PHONY: clean
 clean:
 	rm -rf dist build *.egg-info .tox .coverage.*

scripts/aggregate-pytest-reports.py

@@ -0,0 +1,43 @@
+import argparse
+import sys
+from collections import defaultdict
+from xml.etree import ElementTree  # nosec
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("FILES", nargs="+")
+    args = parser.parse_args()
+
+    tests_by_name = defaultdict(dict)
+    for filename in args.FILES:
+        tree = ElementTree.parse(filename)
+        root = tree.getroot()
+
+        for testcase in root.findall("./testsuite/testcase"):
+            classname = testcase.get("classname")
+            name = testcase.get("name")
+            nodename = f"{classname.replace('.', '/')}.py::{name}"
+
+            skip_node = testcase.find("skipped")
+            if skip_node is not None:
+                if "skipped" not in tests_by_name[nodename]:
+                    tests_by_name[nodename]["skipped"] = True
+            else:
+                tests_by_name[nodename]["skipped"] = False
+
+    fail = False
+    for nodename, attributes in tests_by_name.items():
+        if attributes.get("skipped") is True:
+            print(f"ALWAYS SKIPPED: {nodename}")
+            fail = True
+
+    if fail:
+        print("fail")
+        sys.exit(1)
+    print("ok")
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()