Add tox testing, with code-quality tests (#39)

* tox testing, add QC

* try fix tests

* use isolated builds in tox

* remove test autogen

* try codecov within tox

* split code qual

* codecov token

* change env syntax

* remove codecov altogehter

* newline

* review updates

* whitespace

* add pre-commit

* add pre-commit to tox

* update readme

.github/workflows/test.yml

@@ -17,24 +17,21 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest pytest-cov
-          pip install -e .[autogen]
-      - name: Test with pytest
-        run: pytest -v --cov-report=xml --cov=ome_autogen
-      - name: Coverage
-        uses: codecov/codecov-action@v1
+          pip install tox tox-gh-actions
+      - name: Test with tox
+        run: tox
 
-  mypy:
+  code-quality:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
           python-version: 3.8
-      - name: Run mypy
+      - name: Install dependencies
         run: |
-          pip install mypy
-          pip install -e .[autogen]
-          python src/ome_autogen.py
-          mypy
+          python -m pip install --upgrade pip
+          pip install tox
+      - name: Code Quality
+        run: tox -e pre-commit

.gitignore

@@ -109,4 +109,4 @@ venv.bak/
 _test_data
 src/ome_types/model/
 src/ome_types/_version.py
-docs/source/_autosummary
+docs/source/_autosummary

.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0 # Use the ref you want to point at
+    hooks:
+      - id: trailing-whitespace
+        exclude: ^testing|^docs|.xsd
+      - id: end-of-file-fixer
+        exclude: ^testing|^docs
+  - repo: https://github.com/timothycrosley/isort
+    rev: 5.3.2
+    hooks:
+      - id: isort
+  - repo: https://github.com/python/black
+    rev: 20.8b1
+    hooks:
+      - id: black
+        exclude: ^docs
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.3
+    hooks:
+      - id: flake8
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: "v0.782"
+    hooks:
+      - id: mypy
+        exclude: ^testing|^docs

MANIFEST.in

@@ -1,3 +1,3 @@
 include LICENSE README.md setup.py pyproject.toml
 recursive-include src *.xsd
-recursive-exclude testing *
+recursive-exclude testing *

README.md

@@ -100,14 +100,38 @@ from ome_types import from_xml
 metadata = from_xml(xml)
 ```
 
-where `xml` in that example can be a path to a file, a URI of a
-resource, an opened file-like object, an Element instance, an ElementTree
-instance, or a literal string containing the XML data.
+where `xml` in that example can be a path to a file, a URI of a resource, an
+opened file-like object, an Element instance, an ElementTree instance, or a
+literal string containing the XML data.
 
-all attributes and variable names follow the OME data model, but `camelCaseNames`
-have been replaced with pythonic `snake_case_names`
+all attributes and variable names follow the OME data model, but
+`camelCaseNames` have been replaced with pythonic `snake_case_names`
 
 ## Work in progress!
 
 This is a work in progress and will absolutely need refining.  Feel free to
-submit an issue or a PR if you try it out and have requests.
+submit an issue or a PR if you try it out and have requests.
+
+## Contributing
+
+We use `pre-commit` to run various code-quality checks (black, mypy, flake8)
+during continuous integration.  If you'd like to make sure that your code will
+pass these checks before you commit your code, you can install `pre-commit`
+after cloning this repository:
+
+```shell
+pip install pre-commit
+pre-commit install
+```
+
+or, you can install and run `tox` which will run tests and code-quality checks
+in an isolated environment.
+
+## Testing
+
+To run tests quickly, just install and run `pytest`.  Note, however, that this
+requires that the `ome_types.model` module has already been built with python
+`src/ome_autogen.py`.
+
+Alternatively, you can install and run `tox` which will run tests and
+code-quality checks in an isolated environment.

docs/source/conf.py

@@ -16,7 +16,6 @@
 sys.path.insert(0, os.path.abspath("../../src"))
 from ome_types import __version__  # noqa: E402
 
-
 # -- Project information -----------------------------------------------------
 
 project = "ome-types"

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4", "black", "isort<5.0", "xmlschema", "numpydoc", "pydantic"]
+requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4", "black", "isort<5.0", "xmlschema", "autoflake", "numpydoc", "pydantic"]
 build-backend = "setuptools.build_meta"
 
 [tool.check-manifest]
@@ -9,3 +9,26 @@ ignore = [
     "coverage.yml",
     "src/ome_types/_version.py",  # added by setuptools_scm during build
 ]
+
+
+[tool.isort]
+profile = "black"
+line_length = 88
+skip_glob = ["*examples/*", "*vendored*"]
+
+[tool.black]
+target-version = ['py37', 'py38']
+exclude = '''
+/(
+    \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+  | docs
+)/
+'''

setup.cfg

@@ -24,7 +24,6 @@ classifiers =
 package_dir=
     =src
 packages = find:
-py_modules = ome_autogen
 include_package_data = true
 python_requires = >=3.7
 install_requires =
@@ -36,8 +35,49 @@ install_requires =
 * = *.xsd
 
 [options.extras_require]
-autogen = isort<5.0; black; numpydoc
-
+autogen = isort<5.0; black; autoflake; numpydoc
 
 [options.packages.find]
-where=src
+where=src
+
+[flake8]
+ignore = W503, C901, E501
+max-line-length = 88
+
+[mypy]
+follow_imports = silent
+strict_optional = True
+warn_redundant_casts = True
+disallow_any_generics = True
+check_untyped_defs = True
+no_implicit_reexport = True
+ignore_missing_imports = True
+files = src
+disallow_untyped_defs = True
+
+[tox:tox]
+envlist = py{37,38},pre-commit
+toxworkdir=/tmp/.tox
+isolated_build = True
+
+[gh-actions]
+python =
+    3.7: py37
+    3.8: py38
+
+[testenv]
+passenv = CI GITHUB_ACTIONS CODECOV_TOKEN
+deps = pytest
+commands = pytest -v --basetemp={envtmpdir} {posargs}
+
+[testenv:mypy]
+deps = mypy
+commands = mypy
+
+[testenv:lint]
+deps = flake8
+commands = flake8
+
+[testenv:pre-commit]
+deps = pre-commit
+commands = pre-commit run --all-files

setup.py

@@ -1,7 +1,8 @@
 from distutils.command.build_py import build_py as _build_py
-from setuptools import setup
 from runpy import run_path
 
+from setuptools import setup
+
 
 class build_py(_build_py):
     run_path("src/ome_autogen.py", run_name="__main__")

src/ome_autogen.py

@@ -4,36 +4,36 @@
 import os
 import re
 import shutil
+from dataclasses import dataclass
 from itertools import chain
-from textwrap import dedent, indent, wrap
 from pathlib import Path
+from textwrap import dedent, indent, wrap
 from typing import (
+    Any,
+    Dict,
     Generator,
     Iterable,
-    List,
-    Dict,
-    Set,
-    Union,
     Iterator,
-    Tuple,
+    List,
     Optional,
+    Tuple,
+    Union,
 )
-from dataclasses import dataclass
 
 import black
 import isort
+from autoflake import fix_code
 from numpydoc.docscrape import NumpyDocString, Parameter
 from xmlschema import XMLSchema, qnames
 from xmlschema.validators import (
     XsdAnyAttribute,
     XsdAnyElement,
     XsdAttribute,
+    XsdComponent,
     XsdElement,
     XsdType,
-    XsdComponent,
 )
 
-
 # FIXME: Work out a better way to implement these override hacks.
 
 
@@ -283,6 +283,13 @@ def ref(self) -> Any:
 }
 
 
+def autoflake(text: str, **kwargs: Any) -> str:
+
+    kwargs.setdefault("remove_all_unused_imports", True)
+    kwargs.setdefault("remove_unused_variables", True)
+    return fix_code(text, **kwargs)
+
+
 def black_format(text: str, line_length: int = 79) -> str:
     return black.format_str(text, mode=black.FileMode(line_length=line_length))
 
@@ -881,7 +888,7 @@ def lines(self) -> str:
         return "\n".join(lines)
 
     def format(self) -> str:
-        return black_format(sort_imports(self.lines() + "\n"))
+        return black_format(sort_imports(autoflake(self.lines() + "\n")))
 
     def write(self, filename: str) -> None:
         os.makedirs(os.path.dirname(filename), exist_ok=True)
@@ -941,7 +948,7 @@ def convert_schema(url: str = _url, target_dir: str = _target) -> None:
         {k: Member.plurals_registry[k] for k in sorted(Member.plurals_registry)}
     )
     text = black_format(text)
-    with open(os.path.join(target_dir, f"__init__.py"), "w") as f:
+    with open(os.path.join(target_dir, "__init__.py"), "w") as f:
         f.write(text)
 
 

src/ome_types/__init__.py

@@ -4,28 +4,26 @@
     __version__ = "unknown"
 
 import os
-from .schema import to_dict, validate
-from typing import Union, Type
 from pathlib import Path
-
-__all__ = ["to_dict", "validate", "from_xml"]
-
+from typing import Union
 
 try:
     from .model import OME
 except ImportError:
-    print("OME dataclasses not found ... running autogeneration")
-    import runpy
+    raise ImportError(
+        "Could not import 'ome_types.model.OME'.\nIf you are in a dev environment, "
+        "you may need to run 'python -m src.ome_autogen'"
+    ) from None
+from .schema import to_dict, validate
 
-    runpy.run_module("ome_autogen", run_name="__main__")
-    from .model import OME
+__all__ = ["to_dict", "validate", "from_xml"]
 
 
-def from_xml(xml: Union[Path, str], OME=OME) -> OME:  # type: ignore
+def from_xml(xml: Union[Path, str]) -> OME:  # type: ignore
     xml = os.fspath(xml)
     d = to_dict(xml)
     for key in list(d.keys()):
         if key.startswith(("xml", "xsi")):
             d.pop(key)
 
-    return OME(**d)
+    return OME(**d)  # type: ignore

src/ome_types/dataclasses.py

@@ -108,8 +108,9 @@ def quantity(self: Any) -> Optional[pint.Quantity]:
         value = getattr(self, field)
         if value is None:
             return None
-        unit = getattr(self, field + "_unit").value.replace(' ', '_')
+        unit = getattr(self, field + "_unit").value.replace(" ", "_")
         return ureg.Quantity(value, unit)
+
     return property(quantity)
 
 

src/ome_types/util.py

@@ -1,9 +1,8 @@
 import dataclasses
-import weakref
 from typing import Any, Dict, List
 
-from .model.simple_types import LSID
 from .model.reference import Reference
+from .model.simple_types import LSID
 
 
 def collect_references(value: Any) -> List[Reference]: