Merge pull request #929 from eslavich/eslavich-fix-failing-tests

Fix tests impacted by numpy 1.20 deprecations

CHANGES.rst

@@ -50,6 +50,9 @@
 
 - Fix bug when decompressing arrays with numpy 1.20.  [#901, #909]
 
+- Add pytest plugin options to skip and xfail individual tests
+  and xfail the unsupported ndarray-1.0.0 example. [#929]
+
 2.7.1 (2020-08-18)
 ------------------
 

asdf/tags/core/tests/test_ndarray.py

@@ -368,7 +368,7 @@ def test_string(tmpdir):
 
 def test_string_table(tmpdir):
     tree = {
-        'table': np.array([(b'foo', 'სამეცნიერო', 42, 53.0)])
+        'table': np.array([(b'foo', 'სამეცნიერო', '42', '53.0')])
         }
 
     helpers.assert_roundtrip_tree(tree, tmpdir)

pytest_asdf/plugin.py

@@ -2,6 +2,7 @@
 import os
 from importlib.util import find_spec
 from pkg_resources import parse_version
+import pathlib
 
 import yaml
 import pytest
@@ -16,6 +17,12 @@ def pytest_addoption(parser):
         "asdf_schema_root", "Root path indicating where schemas are stored")
     parser.addini(
         "asdf_schema_skip_names", "Base names of files to skip in schema tests")
+    parser.addini(
+        "asdf_schema_skip_tests",
+        "List of tests to skip, one per line, in format <schema path suffix>::<test name>")
+    parser.addini(
+        "asdf_schema_xfail_tests",
+        "List of tests to xfail, one per line, in format <schema path suffix>::<test name>")
     parser.addini(
         "asdf_schema_skip_examples",
         "Base names of schemas whose examples should not be tested")
@@ -50,7 +57,7 @@ def pytest_addoption(parser):
 class AsdfSchemaFile(pytest.File):
     @classmethod
     def from_parent(cls, parent, *, fspath, skip_examples=False, validate_default=True,
-        ignore_unrecognized_tag=False, ignore_version_mismatch=False, **kwargs):
+        ignore_unrecognized_tag=False, ignore_version_mismatch=False, skip_tests=[], xfail_tests=[], **kwargs):
         if hasattr(super(), "from_parent"):
             result = super().from_parent(parent, fspath=fspath, **kwargs)
         else:
@@ -60,19 +67,36 @@ def from_parent(cls, parent, *, fspath, skip_examples=False, validate_default=Tr
         result.validate_default = validate_default
         result.ignore_unrecognized_tag = ignore_unrecognized_tag
         result.ignore_version_mismatch = ignore_version_mismatch
+        result.skip_tests = skip_tests
+        result.xfail_tests = xfail_tests
+
         return result
 
+    def _set_markers(self, item):
+        if item.name in self.skip_tests or "*" in self.skip_tests:
+            item.add_marker(pytest.mark.skip)
+        if item.name in self.xfail_tests or "*" in self.xfail_tests:
+            item.add_marker(pytest.mark.xfail)
+
     def collect(self):
-        yield AsdfSchemaItem.from_parent(self, self.fspath, validate_default=self.validate_default)
+        item = AsdfSchemaItem.from_parent(self, self.fspath, validate_default=self.validate_default, name="test_schema")
+        self._set_markers(item)
+        yield item
+
         if not self.skip_examples:
-            for example in self.find_examples_in_schema():
-                yield AsdfSchemaExampleItem.from_parent(
+            for index, example in enumerate(self.find_examples_in_schema()):
+                name = f"test_example_{index}"
+                item = AsdfSchemaExampleItem.from_parent(
                     self,
                     self.fspath,
                     example,
+                    index,
                     ignore_unrecognized_tag=self.ignore_unrecognized_tag,
                     ignore_version_mismatch=self.ignore_version_mismatch,
+                    name=name,
                 )
+                self._set_markers(item)
+                yield item
 
     def find_examples_in_schema(self):
         """Returns generator for all examples in schema at given path"""
@@ -93,7 +117,7 @@ class AsdfSchemaItem(pytest.Item):
     @classmethod
     def from_parent(cls, parent, schema_path, validate_default=True, **kwargs):
         if hasattr(super(), "from_parent"):
-            result = super().from_parent(parent, name=str(schema_path), **kwargs)
+            result = super().from_parent(parent, **kwargs)
         else:
             result = AsdfSchemaItem(str(schema_path), parent, **kwargs)
 
@@ -111,6 +135,9 @@ def runtest(self):
             resolve_references=True)
         schema.check_schema(schema_tree, validate_default=self.validate_default)
 
+    def reportinfo(self):
+        return self.fspath, 0, ""
+
 
 ASTROPY_4_0_TAGS = {
     'tag:stsci.edu:asdf/transform/rotate_sequence_3d',
@@ -149,13 +176,12 @@ def parse_schema_filename(filename):
 
 class AsdfSchemaExampleItem(pytest.Item):
     @classmethod
-    def from_parent(cls, parent, schema_path, example,
+    def from_parent(cls, parent, schema_path, example, example_index,
         ignore_unrecognized_tag=False, ignore_version_mismatch=False, **kwargs):
-        test_name = "{}-example".format(schema_path)
         if hasattr(super(), "from_parent"):
-            result = super().from_parent(parent, name=test_name, **kwargs)
+            result = super().from_parent(parent, **kwargs)
         else:
-            result = AsdfSchemaExampleItem(test_name, parent, **kwargs)
+            result = AsdfSchemaExampleItem(parent, **kwargs)
         result.filename = str(schema_path)
         result.example = example
         result.ignore_unrecognized_tag = ignore_unrecognized_tag
@@ -224,6 +250,31 @@ def runtest(self):
             buff = io.BytesIO()
             ff.write_to(buff)
 
+    def reportinfo(self):
+        return self.fspath, 0, ""
+
+
+def _parse_test_list(content):
+    result = {}
+
+    for line in content.split("\n"):
+        line = line.strip()
+        if len(line) > 0:
+            parts = line.split("::", 1)
+            path_suffix = pathlib.Path(parts[0]).as_posix()
+
+            if len(parts) == 1:
+                name = "*"
+            else:
+                name = parts[-1]
+
+            if path_suffix not in result:
+                result[path_suffix] = []
+
+            result[path_suffix].append(name)
+
+    return result
+
 
 def pytest_collect_file(path, parent):
     if not (parent.config.getini('asdf_schema_tests_enabled') or
@@ -240,6 +291,9 @@ def pytest_collect_file(path, parent):
     ignore_unrecognized_tag = parent.config.getini('asdf_schema_ignore_unrecognized_tag')
     ignore_version_mismatch = parent.config.getini('asdf_schema_ignore_version_mismatch')
 
+    skip_tests = _parse_test_list(parent.config.getini('asdf_schema_skip_tests'))
+    xfail_tests = _parse_test_list(parent.config.getini('asdf_schema_xfail_tests'))
+
     schema_roots = [os.path.join(str(parent.config.rootdir), os.path.normpath(root))
                         for root in schema_roots]
 
@@ -248,13 +302,25 @@ def pytest_collect_file(path, parent):
 
     for root in schema_roots:
         if str(path).startswith(root) and path.purebasename not in skip_names:
+            posix_path = pathlib.Path(path).as_posix()
+            schema_skip_tests = []
+            for suffix, names in skip_tests.items():
+                if posix_path.endswith(suffix):
+                    schema_skip_tests.extend(names)
+            schema_xfail_tests = []
+            for suffix, names in xfail_tests.items():
+                if posix_path.endswith(suffix):
+                    schema_xfail_tests.extend(names)
+
             return AsdfSchemaFile.from_parent(
                 parent,
                 fspath=path,
                 skip_examples=(path.purebasename in skip_examples),
                 validate_default=validate_default,
                 ignore_unrecognized_tag=ignore_unrecognized_tag,
                 ignore_version_mismatch=ignore_version_mismatch,
+                skip_tests=schema_skip_tests,
+                xfail_tests=schema_xfail_tests,
             )
 
     return None

setup.cfg

@@ -83,22 +83,27 @@ open_files_ignore = test.fits asdf.fits
 # which pytest trips over during collection:
 filterwarnings =
     ignore::asdf.exceptions.AsdfDeprecationWarning:asdf.asdftypes
+    ignore:numpy.ndarray size changed:astropy.utils.exceptions.AstropyWarning
+    ignore:numpy.ndarray size changed:RuntimeWarning
 # Configuration for pytest-doctestplus
 text_file_format = rst
 # Account for both the astropy test runner case and the native pytest case
 asdf_schema_root = asdf-standard/schemas asdf/schemas
-asdf_schema_skip_names =
-    asdf-schema-1.0.0
-    draft-01
-    celestial_frame-1.0.0
-    celestial_frame-1.1.0
-    frame-1.1.0
-    spectral_frame-1.1.0
-    step-1.1.0
-    step-1.2.0
-    wcs-1.1.0
-    wcs-1.2.0
-asdf_schema_skip_examples = domain-1.0.0 frame-1.0.0
+asdf_schema_skip_tests =
+    stsci.edu/asdf/asdf-schema-1.0.0.yaml
+    stsci.edu/asdf/transform/domain-1.0.0.yaml
+    stsci.edu/asdf/wcs/celestial_frame-1.0.0.yaml
+    stsci.edu/asdf/wcs/celestial_frame-1.1.0.yaml
+    stsci.edu/asdf/wcs/frame-1.0.0.yaml
+    stsci.edu/asdf/wcs/frame-1.1.0.yaml
+    stsci.edu/asdf/wcs/spectral_frame-1.1.0.yaml
+    stsci.edu/asdf/wcs/step-1.1.0.yaml
+    stsci.edu/asdf/wcs/step-1.2.0.yaml
+    stsci.edu/asdf/wcs/wcs-1.1.0.yaml
+    stsci.edu/asdf/wcs/wcs-1.2.0.yaml
+    stsci.edu/yaml-schema/draft-01.yaml
+asdf_schema_xfail_tests =
+    stsci.edu/asdf/core/ndarray-1.0.0.yaml::test_example_2
 # Enable the schema tests by default
 asdf_schema_tests_enabled = true
 asdf_schema_ignore_unrecognized_tag = true

tox.ini

@@ -39,7 +39,10 @@ pip_pre= true
 [testenv:warnings]
 basepython= python3.8
 commands=
-    pytest --remote-data -W error -W ignore::asdf.exceptions.AsdfDeprecationWarning:asdf.asdftypes
+    pytest --remote-data -W error \
+      -W ignore::asdf.exceptions.AsdfDeprecationWarning:asdf.asdftypes \
+      -W 'ignore:numpy.ndarray size changed:astropy.utils.exceptions.AstropyWarning' \
+      -W 'ignore:numpy.ndarray size changed:RuntimeWarning'
 
 [testenv:packaged]
 basepython= python3.8