Merge branch 'spacetelescope:main' into JP-3721

CHANGES.rst

@@ -1,3 +1,53 @@
+2.1.0 (2024-09-17)
+==================
+
+Bug Fixes
+---------
+
+- Fix invalid ``abvegaoffset`` and ``coords`` schemas. (`#327
+  <https://github.com/spacetelescope/stdatamodels/issues/327>`_)
+
+
+Documentation
+-------------
+
+- use ``towncrier`` to handle change log entries (`#326
+  <https://github.com/spacetelescope/stdatamodels/issues/326>`_)
+
+
+New Features
+------------
+
+- Add ``mt_v2`` and ``mt_v3`` keywords to ``moving_target`` schema (`#263
+  <https://github.com/spacetelescope/stdatamodels/issues/263>`_)
+- Added ``MEDIUMDEEP2`` and ``MEDIUMDEEP8`` to allowed readout patterns in JWST
+  core schema, ``READPATT``, and ``PREADPATT``. (`#315
+  <https://github.com/spacetelescope/stdatamodels/issues/315>`_)
+- add grating keyword to dark and superbias schemas (`#317
+  <https://github.com/spacetelescope/stdatamodels/issues/317>`_)
+- Update JWST datamodel ``irs2`` datatype to provide ``numpy>=2.0``
+  compatibility. (`#319
+  <https://github.com/spacetelescope/stdatamodels/issues/319>`_)
+- Add datamodel and schema for ``PastasossModel``, a new reference file type
+  for JWST NIRISS SOSS exposures. (`#320
+  <https://github.com/spacetelescope/stdatamodels/issues/320>`_)
+- Add keyword to JWST core schema to track status of new step
+  ``clean_flicker_noise``. (`#328
+  <https://github.com/spacetelescope/stdatamodels/issues/328>`_)
+
+
+Deprecations and Removals
+-------------------------
+
+- replace usages of ``copy_arrays`` with ``memmap`` (`#306
+  <https://github.com/spacetelescope/stdatamodels/issues/306>`_)
+- remove uses of now unused ``ignore_version_mismatch`` (`#313
+  <https://github.com/spacetelescope/stdatamodels/issues/313>`_)
+- Remove deprecated ``R_DRIZPAR`` keyword from core schema as well as
+  ``jwst.datamodels.DrizParsModel``. (`#316
+  <https://github.com/spacetelescope/stdatamodels/issues/316>`_)
+
+
 2.0.0 (2024-06-24)
 ===================
 

pyproject.toml

@@ -45,6 +45,7 @@ jwst_pipeline = "stdatamodels.jwst.transforms.integration:get_resource_mappings"
 [project.optional-dependencies]
 test = [
     "psutil",
+    "pyyaml",
     "pytest>=4.6.0",
     "pytest-doctestplus",
     "crds>=11.17.1",

src/stdatamodels/jwst/_tests/test_schemas.py

@@ -0,0 +1,81 @@
+from pathlib import Path
+
+import asdf
+import pytest
+import yaml
+
+from stdatamodels.fits_support import _get_validators
+from stdatamodels.schema import walk_schema
+
+
+# relative paths to schema directories
+SCHEMA_RELATIVE_PATHS = [
+    "../datamodels/schemas",
+    "../transforms/resources/schemas/stsci.edu/jwst_pipeline",
+]
+
+
+def _get_schema_ids():
+    root_path = Path(__file__).parent
+    schema_ids = []
+    for schema_relative_path in SCHEMA_RELATIVE_PATHS:
+        path = root_path / schema_relative_path
+        for schema_path in path.glob("*.yaml"):
+            with open(schema_path, "r") as f:
+                schema_ids.append(yaml.load(f, yaml.SafeLoader)["id"])
+    return schema_ids
+
+
+SCHEMA_IDS = _get_schema_ids()
+
+
+@pytest.fixture(scope="module")
+def known_validators():
+    # what validators do we understand?
+    # since we aren't going to use these we can feed it
+    # anything in place of a valid hdulist
+    class Foo:
+        pass
+
+    validators = _get_validators(Foo())[0]
+    known_validators = set(validators.keys())
+    return known_validators
+
+
+@pytest.fixture(scope="module")
+def valid_keywords(known_validators):
+    return known_validators | {
+        "id",
+        "$schema",
+        "title",
+        "description",
+        "default",
+        "examples",
+        "blend_table",
+        "blend_rule",
+        "fits_hdu",
+        "definitions",
+        "allow_extra_columns",
+    }
+
+
+def test_found_schemas():
+    """
+    Make sure we found some schemas
+    """
+    assert SCHEMA_IDS
+
+
+@pytest.mark.parametrize("schema_id", SCHEMA_IDS)
+def test_schema_contains_only_known_keywords(schema_id, valid_keywords):
+    # load the schema from asdf instead of the file to
+    # verify that asdf knows about the schema
+    schema = asdf.schema.load_schema(schema_id)
+
+
+    def callback(schema, path, combiner, ctx, recurse):
+        extra = schema.keys() - ctx["valid_keywords"]
+        assert not extra, f"{extra} found at {path} in {schema_id}"
+
+    ctx = {"valid_keywords": valid_keywords}
+    walk_schema(schema, callback, ctx)

src/stdatamodels/jwst/datamodels/__init__.py

@@ -65,6 +65,7 @@
 from .saturation import SaturationModel
 from .segmap import SegmentationMapModel
 from .slit import SlitModel, SlitDataModel
+from .pastasossmodel import PastasossModel
 from .sossextractmodel import SossExtractModel
 from .sosswavegrid import SossWaveGridModel
 from .spec import SpecModel
@@ -132,6 +133,7 @@
     'NrcImgPhotomModel', 'NrcWfssPhotomModel',
     'NisImgPhotomModel', 'NisSossPhotomModel', 'NisWfssPhotomModel',
     'NrsFsPhotomModel', 'NrsMosPhotomModel',
+    'PastasossModel',
     'PsfMaskModel',
     'QuadModel', 'RampModel',
     'RampFitOutputModel', 'ReadnoiseModel',

src/stdatamodels/jwst/datamodels/pastasossmodel.py

@@ -0,0 +1,18 @@
+from .reference import ReferenceFileModel
+
+
+__all__ = ['PastasossModel']
+
+
+class PastasossModel(ReferenceFileModel):
+    """
+    A data model to hold NIRISS SOSS wavelength grids.
+    This 1-D array of wavelengths can be saved from a processing run
+    and applied to future input products.
+
+    Parameters
+    __________
+    wavegrid : numpy float32 array
+        1-D array of the wavelengths corresponding to the ATOCA fit.
+    """
+    schema_url = "http://stsci.edu/schemas/jwst_datamodel/pastasossmodel.schema"

src/stdatamodels/jwst/datamodels/schemas/abvegaoffset.schema.yaml

@@ -10,18 +10,4 @@ allOf:
     abvega_offset:
       title: Offsets to convert AB to Vega magnitudes
       type: object
-      properties:
-        abvega_offset:
-          title: AB_mag - Vega_mag
-          type: number
-        anyOf:
-          detector:
-            title: Detector name
-            type: string
-          filter:
-            title: Filter wheel element name
-            type: string
-          pupil:
-            title: Pupil wheel element name
-            type: string
 ...

src/stdatamodels/jwst/datamodels/schemas/core.schema.yaml

@@ -2037,6 +2037,15 @@ properties:
                 type: string
                 fits_keyword: R_OTE
                 blend_table: True
+          pastasoss:
+            title: NIRISS SOSS trace and wavecal reference file information
+            type: object
+            properties:
+              name:
+                title: SOSS trace and wavecal reference file name
+                type: string
+                fits_keyword: R_PSTSOS
+                blend_table: True
           pathloss:
             title: Pathloss reference file information
             type: object
@@ -2325,6 +2334,11 @@ properties:
             type: string
             fits_keyword: S_CHGMIG
             blend_table: True
+          clean_flicker_noise:
+            title: Clean flicker noise
+            type: string
+            fits_keyword: S_CLNFNS
+            blend_table: True
           combine_1d:
             title: 1-D Spectral Combination
             type: string

src/stdatamodels/jwst/datamodels/schemas/irs2.schema.yaml

@@ -34,8 +34,8 @@ allOf:
       fits_hdu: DQ
       datatype:
       - name: OUTPUT
-        datatype: uint8
+        datatype: uint16
       - name: ODD_EVEN
-        datatype: uint8
+        datatype: uint16
       - name: MASK
         datatype: uint32

src/stdatamodels/jwst/datamodels/schemas/pastasossmodel.schema.yaml

@@ -0,0 +1,86 @@
+%YAML 1.1
+---
+$schema: "http://stsci.edu/schemas/fits-schema/fits-schema"
+id: "http://stsci.edu/schemas/jwst_datamodel/pastasossmodel.schema"
+allOf:
+- $ref: referencefile.schema
+- $ref: keyword_exptype.schema
+- $ref: keyword_filter.schema
+- type: object
+  properties:
+    meta:
+      type: object
+      properties:
+        pwcpos_cmd:
+          title: Commanded position
+          type: number
+    traces:
+      type: array
+      title: An array of trace models by order
+      items:
+        type: object
+        properties:
+          spectral_order:
+            title: Spectral order
+            type: integer
+            minimum: 1
+          pivot_x:
+            title: Pixel rotation point
+            type: number
+          pivot_y:
+            title: Pixel rotation point
+            type: number
+          trace:
+            title: The trace position
+            ndim: 2
+            datatype: float64
+          oversampling:
+            title: Pixel oversampling
+            type: integer
+            default: 1
+          padding:
+            title: Native pixel-size padding around the image
+            type: integer
+            default: 0
+    wavecal_models:
+      type: array
+      title: Polynomial regression model fitting wavelength to x, offset
+      items:
+        type: object
+        properties:
+          spectral_order:
+            title: Spectral order
+            type: integer
+            minimum: 1
+          polynomial_degree:
+            title: Polynomial degree of model
+            type: integer
+          coefficients:
+            title: List of polynomial coefficients
+            ndim: 1
+            datatype: float64
+          coefficient_labels:
+            title: Coefficient order specification
+            type: string
+          scale_extents:
+            title: Range of values for normalization
+            ndim: 2
+            datatype: float64
+    throughputs:
+      type: array
+      title: An array of GR700XD throughputs by order
+      items:
+        type: object
+        properties:
+          spectral_order:
+            title: Spectral order
+            type: integer
+            minimum: 1
+          throughput:
+            title: The throughput array
+            ndim: 1
+            datatype: float64
+          wavelength:
+            title: The wavelength array
+            ndim: 1
+            datatype: float64

src/stdatamodels/jwst/datamodels/tests/test_schema_against_crds.py

@@ -150,6 +150,7 @@ def flatten(xs):
     'msaoper': None,
     'nrm': dm.NRMModel,
     'ote': dm.OTEModel,
+    'pastasoss': dm.PastasossModel,
     'persat': dm.PersistenceSatModel,
     'psfmask': dm.PsfMaskModel,
     'readnoise': dm.ReadnoiseModel,

src/stdatamodels/jwst/transforms/resources/schemas/stsci.edu/jwst_pipeline/coords-0.7.0.yaml

@@ -29,7 +29,7 @@ examples:
 
 allOf:
   - $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.1.0"
-  - object:
+  - type: object
     properties:
       model_type:
         description: |

src/stdatamodels/jwst/transforms/resources/schemas/stsci.edu/jwst_pipeline/coords-1.0.0.yaml

@@ -29,7 +29,7 @@ examples:
 
 allOf:
   - $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.2.0"
-  - object:
+  - type: object
     properties:
       model_type:
         description: |

src/stdatamodels/jwst/transforms/resources/schemas/stsci.edu/jwst_pipeline/coords-1.1.0.yaml

@@ -28,7 +28,7 @@ examples:
 
 allOf:
   - $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.3.0"
-  - object:
+  - type: object
     properties:
       model_type:
         description: |