A few updates for ATMS end-to-end testing (#768)

In [ previous PR](#757) for ATMS, the end-to-end testing was done without data thinning. And the excessive QC filtering in clear-sky areas, compared to GSI, was found. (See [plots](#757 (comment))) This PR includes two fixes for ATMS in the end-to-end testing and one new feature: 1. fix the excess QC filtering in clear-sky areas. ---> This has a paring [UFO PR #3122](JCSDA-internal/ufo#3122) 2. add data thinning 3. add the diagnostic flags (QC) --- **This can [reproduce QC flags](#768 (comment)) from GSI in UFO** **To test updates in this PR, please check out the UFO branch: [feature/satrad](https://github.com/JCSDA-internal/ufo/tree/feature/satrad) from JCSDA-internal in gdas-validation. This branch consolidates all proposed code changes to UFO for gdas validation: UFO PR #JCSDA-internal/ufo#3122 UFO PR #JCSDA-internal/ufo#3121 UFO PR #JCSDA-internal/ufo#3094 --------- Co-authored-by: Cory Martin <cory.r.martin@noaa.gov>
NOAA-EMC · Nov 27, 2023 · e7fc7cb · e7fc7cb
1 parent 94abd9a
commit e7fc7cb
Show file tree

Hide file tree

Showing 6 changed files with 242 additions and 147 deletions.
diff --git a/parm/atm/obs/config/atms_n20.yaml b/parm/atm/obs/config/atms_n20.yaml
@@ -18,6 +18,7 @@ obs operator:
   Absorbers: [H2O,O3]
   Clouds: [Water, Ice]
   Cloud_Fraction: 1.0
+  Cloud_Seeding: true 
   obs options:
     Sensor_ID: &Sensor_ID atms_n20
     EndianType: little_endian
@@ -56,7 +57,6 @@ obs bias:
 
 obs pre filters:
 # Step 0-A: Create Diagnostic Flags
-# Diagnostic flag for CLW retrieval
 - filter: Create Diagnostic Flags
   filter variables:
   - name: brightnessTemperature
@@ -83,6 +83,9 @@ obs pre filters:
   - name: InterChannelConsistency
     initial value: false
     force reinitialization: false 
+  - name: UseflagCheck 
+    initial value: false
+    force reinitialization: false 
 
 obs post filters:
 # Step 0-B: Calculate derived variables
@@ -224,7 +227,7 @@ obs post filters:
           channels: *atms_n20_channels
         coefs: [1, -1]
 
-# Step 0-C: Assign initial all-sky observation error
+# Step 0-C: Assign Initial All-Sky Observation Error
 - filter: Perform Action
   filter variables:
   - name: brightnessTemperature
@@ -235,7 +238,7 @@ obs post filters:
       name: InitialObsError@DerivedMetaData
       channels: *atms_n20_channels
 
-# Step 1: Remove observations from the edge of the scan
+# Step 1: Remove Observations from the Edge of the Scan
 - filter: Domain Check
   filter variables:
   - name: brightnessTemperature
@@ -250,15 +253,16 @@ obs post filters:
     - name: reject
 
 # Step 2: Data Thinning
-#- filter: Gaussian Thinning
-#  horizontal_mesh: 145 
-#  use_reduced_horizontal_grid: true
-##  round_horizontal_bin_count_to_nearest: true
-##  partition_longitude_bins_using_mesh: true
-#  actions:
-#    - name: set
-#      flag: Thinning 
-#    - name: reject
+- filter: Gaussian Thinning
+  horizontal_mesh: 145 
+  use_reduced_horizontal_grid: true
+  distance_norm: geodesic
+#  round_horizontal_bin_count_to_nearest: true
+#  partition_longitude_bins_using_mesh: true
+  actions:
+    - name: set
+      flag: Thinning 
+    - name: reject
 
 # Step 3A: CLW Retrieval Check (observation_based)
 - filter: Bounds Check
@@ -286,7 +290,7 @@ obs post filters:
       flag: CLWRetrievalCheck
     - name: reject
 
-# Step 4: Window channel sanity check
+# Step 4: Window Channel Sanity Check
 - filter: Bounds Check
   filter variables:
   - name: brightnessTemperature
@@ -338,7 +342,7 @@ obs post filters:
       ignore: rejected observations
     - name: reject
 
-# Step 6: Observation error inflation based on topography check
+# Step 6: Observation Error Inflation based on Topography Check
 - filter: Variable Assignment
   assignments:
   - name: ObsErrorFactorTopo@DerivedMetaData
@@ -361,7 +365,7 @@ obs post filters:
       name: ObsErrorFactorTopo@DerivedMetaData
       channels: *atms_n20_channels
 
-# Step 7: Obs error inflation based on TOA transmittancec check
+# Step 7: Obs Error Inflation based on TOA Transmittancec Check
 - filter: Variable Assignment
   assignments:
   - name: ObsErrorFactorTransmitTop@DerivedMetaData
@@ -383,7 +387,7 @@ obs post filters:
       name: ObsErrorFactorTransmitTop@DerivedMetaData
       channels: *atms_n20_channels
 
-# Step 8: Observation error inflation based on surface jacobian check
+# Step 8: Observation Error Inflation based on Surface Jacobian Check
 - filter: Variable Assignment
   assignments:
   - name: ObsErrorFactorSurfJacobian@DerivedMetaData
@@ -409,7 +413,7 @@ obs post filters:
       name: ObsErrorFactorSurfJacobian@DerivedMetaData
       channels: *atms_n20_channels
 
-# Step 9: Situation dependent check
+# Step 9: Situation Dependent Check
 - filter: Variable Assignment
   assignments:
   - name: ObsErrorFactorSituDepend@DerivedMetaData
@@ -504,7 +508,7 @@ obs post filters:
       ignore: rejected observations
     - name: reject
 
-# Step 11: Inter-channel check
+# Step 11: Inter-Channel Check
 - filter: Bounds Check
   filter variables:
   - name: brightnessTemperature
@@ -518,7 +522,7 @@ obs post filters:
       sensor: *Sensor_ID
       use_flag: [ 1,  1,  1,  1,  1,
                   1,  1,  1,  1,  1,
-                  1,  1,  1,  1,  1,
+                  1,  1,  1,  1,  4,
                   1,  1,  1,  1,  1,
                   1,  1]
   maxvalue: 1.0e-12
@@ -528,3 +532,24 @@ obs post filters:
       ignore: rejected observations
     - name: reject
 
+# Step 12: Useflag Check
+- filter: Bounds Check
+  filter variables:
+  - name: brightnessTemperature
+    channels: *atms_n20_channels
+  test variables:
+  - name: ObsFunction/ChannelUseflagCheckRad
+    channels: *atms_n20_channels
+    options:
+      channels: *atms_n20_channels
+      use_flag: [ 1,  1,  1,  1,  1,
+                  1,  1,  1,  1,  1,
+                  1,  1,  1,  1,  4,
+                  1,  1,  1,  1,  1,
+                  1,  1]
+  minvalue: 1.0e-12
+  actions:
+    - name: set
+      flag: UseflagCheck 
+      ignore: rejected observations
+    - name: reject
diff --git a/parm/atm/obs/config/atms_npp.yaml b/parm/atm/obs/config/atms_npp.yaml
@@ -18,6 +18,7 @@ obs operator:
   Absorbers: [H2O,O3]
   Clouds: [Water, Ice]
   Cloud_Fraction: 1.0
+  Cloud_Seeding: true 
   obs options:
     Sensor_ID: &Sensor_ID atms_npp
     EndianType: little_endian
@@ -56,7 +57,6 @@ obs bias:
 
 obs pre filters:
 # Step 0-A: Create Diagnostic Flags
-# Diagnostic flag for CLW retrieval
 - filter: Create Diagnostic Flags
   filter variables:
   - name: brightnessTemperature
@@ -83,6 +83,9 @@ obs pre filters:
   - name: InterChannelConsistency
     initial value: false
     force reinitialization: false 
+  - name: UseflagCheck 
+    initial value: false
+    force reinitialization: false 
 
 obs post filters:
 # Step 0-B: Calculate derived variables
@@ -224,7 +227,7 @@ obs post filters:
           channels: *atms_npp_channels
         coefs: [1, -1]
 
-# Step 0-C: Assign initial all-sky observation error
+# Step 0-C: Assign Initial All-Sky Observation Error
 - filter: Perform Action
   filter variables:
   - name: brightnessTemperature
@@ -235,7 +238,7 @@ obs post filters:
       name: InitialObsError@DerivedMetaData
       channels: *atms_npp_channels
 
-# Step 1: Remove observations from the edge of the scan
+# Step 1: Remove Observations from the Edge of the Scan
 - filter: Domain Check
   filter variables:
   - name: brightnessTemperature
@@ -250,15 +253,16 @@ obs post filters:
     - name: reject
 
 # Step 2: Data Thinning
-#- filter: Gaussian Thinning
-#  horizontal_mesh: 145 
-#  use_reduced_horizontal_grid: true
-##  round_horizontal_bin_count_to_nearest: true
-##  partition_longitude_bins_using_mesh: true
-#  actions:
-#    - name: set
-#      flag: Thinning 
-#    - name: reject
+- filter: Gaussian Thinning
+  horizontal_mesh: 145 
+  use_reduced_horizontal_grid: true
+  distance_norm: geodesic
+#  round_horizontal_bin_count_to_nearest: true
+#  partition_longitude_bins_using_mesh: true
+  actions:
+    - name: set
+      flag: Thinning 
+    - name: reject
 
 # Step 3A: CLW Retrieval Check (observation_based)
 - filter: Bounds Check
@@ -286,7 +290,7 @@ obs post filters:
       flag: CLWRetrievalCheck
     - name: reject
 
-# Step 4: Window channel sanity check
+# Step 4: Window Channel Sanity Check
 - filter: Bounds Check
   filter variables:
   - name: brightnessTemperature
@@ -338,7 +342,7 @@ obs post filters:
       ignore: rejected observations
     - name: reject
 
-# Step 6: Observation error inflation based on topography check
+# Step 6: Observation Error Inflation based on Topography Check
 - filter: Variable Assignment
   assignments:
   - name: ObsErrorFactorTopo@DerivedMetaData
@@ -361,7 +365,7 @@ obs post filters:
       name: ObsErrorFactorTopo@DerivedMetaData
       channels: *atms_npp_channels
 
-# Step 7: Obs error inflation based on TOA transmittancec check
+# Step 7: Obs Error Inflation based on TOA Transmittancec Check
 - filter: Variable Assignment
   assignments:
   - name: ObsErrorFactorTransmitTop@DerivedMetaData
@@ -383,7 +387,7 @@ obs post filters:
       name: ObsErrorFactorTransmitTop@DerivedMetaData
       channels: *atms_npp_channels
 
-# Step 8: Observation error inflation based on surface jacobian check
+# Step 8: Observation Error Inflation based on Surface Jacobian Check
 - filter: Variable Assignment
   assignments:
   - name: ObsErrorFactorSurfJacobian@DerivedMetaData
@@ -409,7 +413,7 @@ obs post filters:
       name: ObsErrorFactorSurfJacobian@DerivedMetaData
       channels: *atms_npp_channels
 
-# Step 9: Situation dependent check
+# Step 9: Situation Dependent Check
 - filter: Variable Assignment
   assignments:
   - name: ObsErrorFactorSituDepend@DerivedMetaData
@@ -504,7 +508,7 @@ obs post filters:
       ignore: rejected observations
     - name: reject
 
-# Step 11: Inter-channel check
+# Step 11: Inter-Channel Check
 - filter: Bounds Check
   filter variables:
   - name: brightnessTemperature
@@ -518,7 +522,7 @@ obs post filters:
       sensor: *Sensor_ID
       use_flag: [ 1,  1,  1,  1,  1,
                   1,  1,  1,  1,  1,
-                  1,  1,  1,  1,  1,
+                  1,  1,  1,  1,  4,
                   1,  1,  1,  1,  1,
                   1,  1]
   maxvalue: 1.0e-12
@@ -528,3 +532,24 @@ obs post filters:
       ignore: rejected observations
     - name: reject
 
+# Step 12: Useflag Check
+- filter: Bounds Check
+  filter variables:
+  - name: brightnessTemperature
+    channels: *atms_npp_channels
+  test variables:
+  - name: ObsFunction/ChannelUseflagCheckRad
+    channels: *atms_npp_channels
+    options:
+      channels: *atms_npp_channels
+      use_flag: [ 1,  1,  1,  1,  1,
+                  1,  1,  1,  1,  1,
+                  1,  1,  1,  1,  4,
+                  1,  1,  1,  1,  1,
+                  1,  1]
+  minvalue: 1.0e-12
+  actions:
+    - name: set
+      flag: UseflagCheck 
+      ignore: rejected observations
+    - name: reject