From c3ce3251e4f017f4c4ef6e221df3372324ba389e Mon Sep 17 00:00:00 2001 From: Xiaoyan Zhang <45010998+xyzemc@users.noreply.github.com> Date: Fri, 13 Dec 2024 13:11:33 -0500 Subject: [PATCH] Merged the atms_n20 yaml file into RDASApp. (#236) This PR is going to merge the ATMS NOAA_20 JEDI yaml file into RDASApp validated yaml file templates. As the test in issue #203, atms_n20 data has been tested in MPAS_JEDI including all QC filters and bias correction. Not only the ioda data converted from regular feed bufr data has been tested, but also the ioda data converted from combined regular feed, DBNet, and ESRs bufr data has been tested too. In addition, this atms_n20 yaml file has included the online domain check filters as described in and issue #224, and removed the 'circle domain check' filter --- .../templates/obtype_config/atms_n20.yaml | 582 ++++++++++++++++++ 1 file changed, 582 insertions(+) create mode 100644 rrfs-test/validated_yamls/templates/obtype_config/atms_n20.yaml diff --git a/rrfs-test/validated_yamls/templates/obtype_config/atms_n20.yaml b/rrfs-test/validated_yamls/templates/obtype_config/atms_n20.yaml new file mode 100644 index 00000000..5a39f5af --- /dev/null +++ b/rrfs-test/validated_yamls/templates/obtype_config/atms_n20.yaml @@ -0,0 +1,582 @@ + - obs space: + name: atms_n20 + distribution: + name: "@DISTRIBUTION@" + obsdatain: + engine: + type: H5File + obsfile: data/obs/ioda_atms_n20.nc + obsdataout: + engine: + type: H5File + obsfile: jdiag_atms_n20.nc4 + simulated variables: [brightnessTemperature] + observed variables: [brightnessTemperature] + channels: &atms_n20_channels 1-22 + obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + Cloud_Seeding: true + SurfaceWindGeoVars: uv + obs options: + Sensor_ID: atms_n20 + EndianType: little_endian + CoefficientPath: data/crtm/ + IRVISlandCoeff: IGBP + linear obs operator: + Absorbers: [H2O, O3] + obs localizations: + - localization method: Horizontal Gaspari-Cohn + lengthscale: 300e3 # orig + obs bias: + input file: data/obs/atms_n20.satbias.nc4 + output file: out_atms_n20.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: &atms_n20_tlapse data/obs/atms_n20.tlapse.txt + - name: lapseRate + tlapse: *atms_n20_tlapse + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: data/obs/atms_n20.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: out_atms_n20.satbias_cov.nc4 + + obs pre filters: + # Step 0-A: Create Diagnostic Flags + - filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + + obs post filters: + # Step 0-B: Calculate derived variables + # Calculate CLW retrieved from observation + - filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + + # Calculate CLW retrieved from observation + - filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + + # Calculate symmetric retrieved CLW + - filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + + - filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + + # Calculate scattering index from observation + - filter: Variable Assignment + assignments: + - name: SIRetFromObs@DerivedMetaData + type: float + function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 16 + scatret_types: [ObsValue] + + # Calculate CLW obs/bkg match index + - filter: Variable Assignment + assignments: + - name: CLWMatchIndex@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: CLWMatchIndexMW@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + + # Calculate symmetric observation error + - filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorModelRamp@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + xvar: + name: CLWRetSymmetric@DerivedMetaData + x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, + 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, + 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, + 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, + 3.000, 0.800, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 19.000, 30.000, 25.000, 16.500, 12.000, + 9.000, 6.500] + + # Calculate Innovation@DerivedMetaData + - filter: Variable Assignment + assignments: + - name: Innovation@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *atms_n20_channels + options: + variables: + - name: brightnessTemperature@ObsValue + channels: *atms_n20_channels + - name: brightnessTemperature@HofX + channels: *atms_n20_channels + coefs: [1, -1] + + - filter: Bounds Check + filter variables: + #- name: brightnessTemperature + - name: HofX/brightnessTemperature + channels: *atms_n20_channels + minvalue: 50.00001 + maxvalue: 449.99999 + action: + name: reject + + # Regional Domain Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + test variables: + - name: GeoVaLs/observable_domain_mask + flag all filter variables if any test variable is out of bounds: true + minvalue: 0.0 + maxvalue: 0.5 + + # Step 0-C: Assign Initial All-Sky Observation Error + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + + # Step 1: Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-22 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 7-90 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + + # Step 2: data Thinning + - filter: Gaussian Thinning + #horizontal_mesh: 145 + horizontal_mesh: 60 + use_reduced_horizontal_grid: true + distance_norm: geodesic + # round_horizontal_bin_count_to_nearest: true + # partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + + # Step 3A: CLW Retrieval Check (observation_based) + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromObs@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 3B: CLW Retrieval Check (background_based) + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromBkg@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 4: Window Channel Sanity Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16, 17-22 + test variables: + - name: Innovation@DerivedMetaData + channels: 1, 2, 5-7, 16 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + + # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_n20_channels + type: float + function: + name: HydrometeorCheckATMS@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + test variables: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *atms_n20_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + + # Step 6: Observation Error Inflation based on Topography Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *atms_n20_channels + options: + sensor: atms_n20 + channels: *atms_n20_channels + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_n20_channels + + # Step 7: Obs Error Inflation based on TOA Transmittancec Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_n20_channels + + # Step 8: Observation Error Inflation based on Surface Jacobian Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *atms_n20_channels + options: + sensor: atms_n20 + channels: *atms_n20_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *atms_n20_channels + + # Step 9: Situation Dependent Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *atms_n20_channels + options: + sensor: atms_n20 + channels: *atms_n20_channels + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + scatobs_function: + name: SIRetFromObs@DerivedMetaData + clwmatchidx_function: + name: CLWMatchIndex@DerivedMetaData + channels: *atms_n20_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, + 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, + 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *atms_n20_channels + + # Step 10: Gross check + # Remove data if abs(Obs-HofX) > absolute threhold + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorLat@DerivedMetaData + type: float + function: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + + - filter: Variable Assignment + assignments: + - name: ObsErrorBound@DerivedMetaData + channels: *atms_n20_channels + type: float + function: + name: ObsErrorBoundMW@ObsFunction + channels: *atms_n20_channels + options: + sensor: atms_n20 + channels: *atms_n20_channels + obserr_bound_latitude: + name: ObsErrorFactorLat@DerivedMetaData + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + obserr_bound_topo: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *atms_n20_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *atms_n20_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 2.0, 4.5, + 4.5, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0] + + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + function absolute threshold: + - name: ObsErrorBound@DerivedMetaData + channels: *atms_n20_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + + # Step 11: Inter-Channel Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + use passive_bc: true + sensor: atms_n20 + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, -1, -1, + 1, 1, 1, 1, 1, + 1, 1] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + + # Step 12: Useflag Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *atms_n20_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *atms_n20_channels + options: + channels: *atms_n20_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, -1, -1, + 1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject