Move dqflags from romancal to roman_datamodels and make them en…

…ums. (#293)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

CHANGES.rst

@@ -8,6 +8,7 @@
 
 - Allow assignment to or creation of node attributes using dot notation of object instances
   with validation. [#284]
+- Move ``dqflags`` from ``romancal`` to ``roman_datamodels``. [#293]
 
 - Bugfix for ``model.meta.filename`` not matching the filename of the file on disk. [#295]
 

src/roman_datamodels/dqflags.py

@@ -0,0 +1,76 @@
+""" Roman Data Quality Flags
+
+The definitions are documented in the Roman RTD:
+
+[NOTE: Documentation not yet implemented. Fix this URL when completed.]
+
+https://roman-cal-pipeline.readthedocs.io/en/latest/roman/references_general/references_general.html#data-quality-flags
+
+
+Implementation
+-------------
+
+The flags are implemented as "bit flags": Each flag is assigned a bit position
+in a byte, or multi-byte word, of memory. If that bit is set, the flag assigned
+to that bit is interpreted as being set or active.
+
+The data structure that stores bit flags is just the standard Python `int`,
+which provides 32 bits. Bits of an integer are most easily referred to using
+the formula `2**bit_number` where `bit_number` is the 0-index bit of interest.
+"""
+
+from enum import IntEnum, unique
+
+
+# fmt: off
+@unique
+class pixel(IntEnum):
+    """Pixel-specific data quality flags"""
+
+    GOOD             = 0      # No bits set, all is good
+    DO_NOT_USE       = 2**0   # Bad pixel. Do not use.
+    SATURATED        = 2**1   # Pixel saturated during exposure
+    JUMP_DET         = 2**2   # Jump detected during exposure
+    DROPOUT          = 2**3   # Data lost in transmission
+    GW_AFFECTED_DATA = 2**4   # Data affected by the GW read window
+    PERSISTENCE      = 2**5   # High persistence (was RESERVED_2)
+    AD_FLOOR         = 2**6   # Below A/D floor (0 DN, was RESERVED_3)
+    OUTLIER          = 2**7   # Flagged by outlier detection (was RESERVED_4)
+    UNRELIABLE_ERROR = 2**8   # Uncertainty exceeds quoted error
+    NON_SCIENCE      = 2**9   # Pixel not on science portion of detector
+    DEAD             = 2**10  # Dead pixel
+    HOT              = 2**11  # Hot pixel
+    WARM             = 2**12  # Warm pixel
+    LOW_QE           = 2**13  # Low quantum efficiency
+    TELEGRAPH        = 2**15  # Telegraph pixel
+    NONLINEAR        = 2**16  # Pixel highly nonlinear
+    BAD_REF_PIXEL    = 2**17  # Reference pixel cannot be used
+    NO_FLAT_FIELD    = 2**18  # Flat field cannot be measured
+    NO_GAIN_VALUE    = 2**19  # Gain cannot be measured
+    NO_LIN_CORR      = 2**20  # Linearity correction not available
+    NO_SAT_CHECK     = 2**21  # Saturation check not available
+    UNRELIABLE_BIAS  = 2**22  # Bias variance large
+    UNRELIABLE_DARK  = 2**23  # Dark variance large
+    UNRELIABLE_SLOPE = 2**24  # Slope variance large (i.e., noisy pixel)
+    UNRELIABLE_FLAT  = 2**25  # Flat variance large
+    RESERVED_5       = 2**26  #
+    RESERVED_6       = 2**27  #
+    UNRELIABLE_RESET = 2**28  # Sensitive to reset anomaly
+    RESERVED_7       = 2**29  #
+    OTHER_BAD_PIXEL  = 2**30  # A catch-all flag
+    REFERENCE_PIXEL  = 2**31  # Pixel is a reference pixel
+
+
+@unique
+class group(IntEnum):
+    """Group-specific data quality flags
+        Once groups are combined, these flags are equivalent to the pixel-specific flags.
+    """
+    GOOD       = pixel.GOOD
+    DO_NOT_USE = pixel.DO_NOT_USE
+    SATURATED  = pixel.SATURATED
+    JUMP_DET   = pixel.JUMP_DET
+    DROPOUT    = pixel.DROPOUT
+    AD_FLOOR   = pixel.AD_FLOOR
+
+# fmt: on

tests/test_dqflags.py

@@ -0,0 +1,104 @@
+from math import log10
+
+import pytest
+
+from roman_datamodels import datamodels as rdm
+from roman_datamodels import dqflags
+from roman_datamodels.maker_utils import mk_datamodel
+
+
+def _is_power_of_two(x):
+    return (log10(x) / log10(2)) % 1 == 0
+
+
+def test_pixel_uniqueness():
+    """
+    Test that there are no duplicate names in dqflags.pixel
+
+        Note: The @unique decorator should ensure that no flag names have the
+            same value as another in the enum raising an error at first import
+            of this module. However, this test is just a sanity check on this.
+    """
+
+    assert len(dqflags.pixel) == len(dqflags.pixel.__members__)
+
+
+@pytest.mark.parametrize("flag", dqflags.pixel)
+def test_pixel_flags(flag):
+    """Test that each pixel flag follows the defined rules"""
+    # Test that the pixel flags are dqflags.pixel instances
+    assert isinstance(flag, dqflags.pixel)
+
+    # Test that the pixel flags are ints
+    assert isinstance(flag, int)
+
+    # Test that the pixel flags are dict accessible
+    assert dqflags.pixel[flag.name] is flag
+
+    # Test that the pixel flag is a power of 2
+    if flag.name == "GOOD":
+        # GOOD is the only non-power-of-two flag (it is 0)
+        assert flag.value == 0
+    else:
+        assert _is_power_of_two(flag.value)
+
+
+@pytest.mark.parametrize("flag", dqflags.pixel)
+def test_write_pixel_flags(tmp_path, flag):
+    filename = tmp_path / "test_dq.asdf"
+
+    ramp = mk_datamodel(rdm.RampModel, shape=(2, 8, 8))
+
+    # Set all pixels to the flag value
+    ramp.pixeldq[...] = flag
+
+    # Check that we can write the model to disk (i.e. the flag validates)
+    ramp.save(filename)
+
+    # Check that we can read the model back in and the flag is preserved
+    with rdm.open(filename) as dm:
+        assert (dm.pixeldq == flag).all()
+
+
+def test_group_uniqueness():
+    """
+    Test that there are no duplicate names in dqflags.group
+
+        Note: The @unique decorator should ensure that no flag names have the
+            same value as another in the enum raising an error at first import
+            of this module. However, this test is just a sanity check on this.
+    """
+    assert len(dqflags.group) == len(dqflags.group.__members__)
+
+
+@pytest.mark.parametrize("flag", dqflags.group)
+def test_group_flags(flag):
+    """Test that each group flag follows the defined rules"""
+    # Test that the group flags are dqflags.group instances
+    assert isinstance(flag, dqflags.group)
+
+    # Test that the group flags are ints
+    assert isinstance(flag, int)
+
+    # Test that the group flags are dict accessible
+    assert dqflags.group[flag.name] is flag
+
+    # Test that each group flag matches a pixel flag of the same name
+    assert dqflags.pixel[flag.name] == flag
+
+
+@pytest.mark.parametrize("flag", dqflags.group)
+def test_write_group_flags(tmp_path, flag):
+    filename = tmp_path / "test_dq.asdf"
+
+    ramp = mk_datamodel(rdm.RampModel, shape=(2, 8, 8))
+
+    # Set all pixels to the flag value
+    ramp.groupdq[...] = flag
+
+    # Check that we can write the model to disk (i.e. the flag validates)
+    ramp.save(filename)
+
+    # Check that we can read the model back in and the flag is preserved
+    with rdm.open(filename) as dm:
+        assert (dm.groupdq == flag).all()