diff --git a/README.rst b/README.rst index d1d06e7c8..e709ed69d 100644 --- a/README.rst +++ b/README.rst @@ -24,5 +24,5 @@ pyaerocom .. |Coverage| image:: https://codecov.io/gh/metno/pyaerocom/branch/main-dev/graph/badge.svg?token=A0AdX8YciZ :target: https://codecov.io/gh/metno/pyaerocom -.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.13927979.svg - :target: https://doi.org/10.5281/zenodo.13927979 +.. |DOI| image:: https://zenodo.org/badge/DOI/10.5281/zenodo.13927979.svg + :target: https://doi.org/10.5281/zenodo.13927979 diff --git a/docs/_static/aeroval/sample_pm_ratios.py b/docs/_static/aeroval/sample_pm_ratios.py new file mode 100644 index 000000000..5bcad8eca --- /dev/null +++ b/docs/_static/aeroval/sample_pm_ratios.py @@ -0,0 +1,52 @@ +# Example aeroval configuration for pm ratios +# using the EMEP reader (which has a built in pm ratio calculation) + +import os +from getpass import getuser + +from pyaerocom.aeroval import EvalSetup, ExperimentProcessor +from pyaerocom.aeroval.config.pmratios.base_config import get_CFG + +reportyear = year = 2019 +CFG = get_CFG( + reportyear=reportyear, + year=year, + # model directory + model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/", +) +user = getuser() + +CFG.update( + dict( + json_basedir=os.path.abspath( + f"/home/{user}/data/aeroval-local-web/data" + ), # always adjust to your environment + coldata_basedir=os.path.abspath(f"/home/{user}/data/aeroval-local-web/coldata"), + # always adjust to your environment + clear_existing_json=True, + add_model_maps=True, + # if True, the analysis will stop whenever an error occurs (else, errors that + # occurred will be written into the logfiles) + raise_exceptions=False, + modelmaps_opts=dict(maps_freq="monthly", maps_res_deg=5), + # Regional filter for analysis + periods=[f"{year}"], + proj_id="RATPM", + exp_id=f"ratpm testing {year}", + exp_name=f"Evaluation of EMEP runs for {year}", + exp_descr=( + f"Evaluation of EMEP runs for {year} for CAMEO. The EMEP model, is compared against observations from EEA and EBAS." + ), + exp_pi="jang@met.no", + public=True, + # directory where colocated data files are supposed to be stored + weighted_stats=True, + ) +) + +stp = EvalSetup(**CFG) + +ana = ExperimentProcessor(stp) +ana.update_interface() + +res = ana.run() diff --git a/docs/aeroval-examples.rst b/docs/aeroval-examples.rst index 713b3e82f..2838933fa 100644 --- a/docs/aeroval-examples.rst +++ b/docs/aeroval-examples.rst @@ -8,7 +8,7 @@ detailed explanations of the setup parameters. A configuration could be run as t The code blocks below are the Python configuruation files *cfg_examples_example1.py* and *sample_gridded_io_aux.py*. -Example 1 +Example 1: NorESM, CAMS reanalysis against AERONET --------- NorESM2 and CAMS reanalysis vs AERONET and merged satellite AOD dataset. @@ -21,3 +21,8 @@ Example IO aux file for model reading .. literalinclude:: _static/aeroval/sample_gridded_io_aux.py +Example for pm ratios compared to EMEP model data +--------------------- + +.. literalinclude:: _static/aeroval/sample_pm_ratios.py + diff --git a/pyaerocom/_lowlevel_helpers.py b/pyaerocom/_lowlevel_helpers.py index 18fb48deb..ace6258ee 100644 --- a/pyaerocom/_lowlevel_helpers.py +++ b/pyaerocom/_lowlevel_helpers.py @@ -126,13 +126,6 @@ def validate(self, val): return val -class DictType(Validator): - def validate(self, val): - if not isinstance(val, dict): - raise ValueError(f"need dict, got {val}") - return val - - class FlexList(Validator): """list that can be instantated via input str, tuple or list or None""" @@ -170,17 +163,6 @@ def validate(self, val): return val -class DictStrKeysListVals(Validator): - def validate(self, val: dict): - if not isinstance(val, dict): - raise ValueError(f"need dict, got {val}") - if any(not isinstance(x, str) for x in val): - raise ValueError(f"all keys need to be str type in {val}") - if any(not isinstance(x, list) for x in val.values()): - raise ValueError(f"all values need to be list type in {val}") - return val - - class Loc(abc.ABC): """Abstract descriptor representing a path location @@ -194,7 +176,12 @@ class Loc(abc.ABC): """ def __init__( - self, default=None, assert_exists=False, auto_create=False, tooltip=None, logger=None + self, + default=None, + assert_exists=False, + auto_create=False, + tooltip=None, + logger=None, ): self.assert_exists = assert_exists self.auto_create = auto_create diff --git a/pyaerocom/aeroval/collections.py b/pyaerocom/aeroval/collections.py index 96d442b28..dc67e3fc0 100644 --- a/pyaerocom/aeroval/collections.py +++ b/pyaerocom/aeroval/collections.py @@ -214,7 +214,7 @@ def get_entry(self, key) -> ModelEntry: """ try: entry = self[key] - entry["model_name"] = key + entry.model_name = key return entry except (KeyError, AttributeError): raise EntryNotAvailable(f"no such entry {key}") diff --git a/pyaerocom/aeroval/config/pmratios/__init__.py b/pyaerocom/aeroval/config/pmratios/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyaerocom/aeroval/config/pmratios/base_config.py b/pyaerocom/aeroval/config/pmratios/base_config.py new file mode 100644 index 000000000..f90eed2ab --- /dev/null +++ b/pyaerocom/aeroval/config/pmratios/base_config.py @@ -0,0 +1,240 @@ +""" +Global config for ratio pm10 vs pm25 +""" + +import copy +import logging +import os + +logger = logging.getLogger(__name__) + +# Constraints +DEFAULT_RESAMPLE_CONSTRAINTS = dict( + yearly=dict(monthly=9), + monthly=dict( + daily=21, + weekly=3, + ), + daily=dict(hourly=18), +) + +DEFAULT_RESAMPLE_CONSTRAINTS_DAILY = dict( + daily=dict(hourly=18), +) + +# ODCSFUN_EEANRT = "EEAAQeRep.NRT;concpm10/EEAAQeRep.NRT;concpm25" +ODCSFUN_EEAV2 = "EEAAQeRep.v2;concpm10/EEAAQeRep.v2;concpm25" +ODCSFUN_EBAS = "EBASMC;concpm10/EBASMC;concpm25" + + +def get_CFG(reportyear, year, model_dir) -> dict: + """create aeroval configuration dict to run the variable + ratpm10pm25 (ratio pm10 vspm25) + + :returns: a dict of a model configuration usable for EvalSetup + """ + # get current path for reference to local gridded_io_aux.py + + CFG = dict( + json_basedir=os.path.abspath("/home/jang/data/aeroval-local-web/data"), + coldata_basedir=os.path.abspath("/home/jang/data/aeroval-local-web/coldata"), + # io_aux_file=os.path.abspath("/home/jang/data/aeroval-local-web/gridded_io_aux.py"), not needed for ReadMscwCtm + # io_aux_file=os.path.join(base_conf_path, "gridded_io_aux.py"), + # var_scale_colmap_file=os.path.abspath( + # "/home/jang/data/aeroval-local-web/pyaerocom-config/config_files/CAMEO/user_var_scale_colmap.ini" + # ), + # if True, existing colocated data files will be deleted and contours will be overwritten + reanalyse_existing=True, + only_json=False, + add_model_maps=False, + only_model_maps=False, + modelmaps_opts=dict(maps_freq="monthly", maps_res_deg=5), + clear_existing_json=False, + # if True, the analysis will stop whenever an error occurs (else, errors that + # occurred will be written into the logfiles) + raise_exceptions=False, + # Regional filter for analysis + filter_name="ALL-wMOUNTAINS", + # colocation frequency (no statistics in higher resolution can be computed) + ts_type="daily", + map_zoom="Europe", + freqs=[ + "yearly", + "monthly", + "daily", + ], + periods=[f"{year}"], + main_freq="monthly", + zeros_to_nan=False, + use_diurnal=True, + min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, + colocate_time=True, + obs_remove_outliers=False, + model_remove_outliers=False, + harmonise_units=True, + regions_how="country", + annual_stats_constrained=True, + proj_id="emep", + exp_id=f"{reportyear}-reporting", + exp_name=f"Evaluation of EMEP runs for {reportyear} EMEP reporting", + exp_descr=( + f"Evaluation of EMEP runs for {reportyear} EMEP reporting. The EMEP model, simulated for {year}, is compared against observations from EEA and EBAS." + ), + exp_pi="emep.mscw@met.no", + public=True, + # directory where colocated data files are supposed to be stored + weighted_stats=True, + var_order_menu=[ + # Gases + "ratpm10pm25", + "ratpm25pm10", + "concNno", + "concNno2", + "concNtno3", + "concNhno3", + "concNtnh", + "concNnh3", + "concnh4", + "concSso2", + "concso4t", + "concso4c", + "vmro3", + "vmro3max", + "vmro3mda8", + "vmrox", + "vmrco", + # PMs + "concpm10", + "concpm25", + "concno3pm10", + "concno3pm25", + "concnh4pm25", + "concso4pm25", + "concCecpm10", + "concCecpm25", + "concCocpm10", # SURF_ugC_PM_OMCOARSE missing in model-output + "concCocpm25", + "concsspm10", + "concsspm25", + # Depositions + "wetrdn", + "wetoxs", + "wetoxn", + "prmm", + ], + ) + + CFG["model_cfg"] = { + "EMEPcameo": dict( + model_id="EMEP,", + model_data_dir=model_dir, + gridded_reader_id={"model": "ReadMscwCtm"}, + # model_read_aux={}, + model_ts_type_read="daily", + ), + } + + """ + Filters + """ + + BASE_FILTER = { + "latitude": [30, 82], + "longitude": [-30, 90], + } + + EBAS_FILTER = { + **BASE_FILTER, + "data_level": [None, 2], + "set_flags_nan": True, + } + + OBS_GROUNDBASED = { + ################## + # EBAS + ################## + "EBAS-d-10": dict( + obs_id="EBASratd10", + web_interface_name="EBAS-d", + obs_vars=["ratpm10pm25"], + obs_vert_type="Surface", + colocate_time=True, + min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, + ts_type="daily", + obs_filters=EBAS_FILTER, + obs_type="ungridded", + obs_merge_how={ + "ratpm10pm25": "eval", + }, + obs_aux_requires={ + "ratpm10pm25": { + "EBASMC": [ + "concpm10", + "concpm25", + ], + } + }, + obs_aux_funs={ + "ratpm10pm25": + # variables used in computation method need to be based on AeroCom + # units, since the colocated StationData objects (from which the + # new UngriddedData is computed, will perform AeroCom unit check + # and conversion) + "(EBASMC;concpm10/EBASMC;concpm25)" + }, + obs_aux_units={"ratpm10pm25": "1"}, + ), + "EBAS-d-25": dict( + obs_id="EBASratd25", + web_interface_name="EBAS-d", + obs_vars=["ratpm25pm10"], + obs_vert_type="Surface", + colocate_time=True, + min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, + ts_type="daily", + obs_filters=EBAS_FILTER, + obs_type="ungridded", + obs_merge_how={ + "ratpm25pm10": "eval", + }, + obs_aux_requires={ + "ratpm25pm10": { + "EBASMC": [ + "concpm10", + "concpm25", + ], + } + }, + obs_aux_funs={ + "ratpm25pm10": + # variables used in computation method need to be based on AeroCom + # units, since the colocated StationData objects (from which the + # new UngriddedData is computed, will perform AeroCom unit check + # and conversion) + "(EBASMC;concpm25/EBASMC;concpm10)" + }, + obs_aux_units={"ratpm25pm10": "1"}, + ), + "EBAS-d-tc": dict( + obs_id="EBASMC", + web_interface_name="EBAS-d", + obs_vars=[ + "concpm10", + "concpm25", + ], + obs_vert_type="Surface", + colocate_time=True, + min_num_obs=DEFAULT_RESAMPLE_CONSTRAINTS, + ts_type="daily", + obs_filters=EBAS_FILTER, + ), + } + + # Setup for supported satellite evaluations + OBS_SAT = {} + + OBS_CFG = {**OBS_GROUNDBASED, **OBS_SAT} + + CFG["obs_cfg"] = OBS_CFG + + return copy.deepcopy(CFG) diff --git a/pyaerocom/aeroval/experiment_output.py b/pyaerocom/aeroval/experiment_output.py index 1dd43f9a6..9e6ce980e 100644 --- a/pyaerocom/aeroval/experiment_output.py +++ b/pyaerocom/aeroval/experiment_output.py @@ -784,9 +784,8 @@ def _create_menu_dict(self) -> dict: if mod_name not in new[var]["obs"][obs_name][vert_code]: new[var]["obs"][obs_name][vert_code][mod_name] = {} - model_id = mcfg["model_id"] new[var]["obs"][obs_name][vert_code][mod_name] = { - "model_id": model_id, + "model_id": mcfg.model_id, "model_var": mod_var, "obs_var": obs_var, } diff --git a/pyaerocom/aeroval/modelentry.py b/pyaerocom/aeroval/modelentry.py index b04f02ab0..50f9d77ae 100644 --- a/pyaerocom/aeroval/modelentry.py +++ b/pyaerocom/aeroval/modelentry.py @@ -1,12 +1,12 @@ -import inspect +# import inspect from copy import deepcopy -from pyaerocom._lowlevel_helpers import BrowseDict, DictStrKeysListVals, DictType, StrType +from pydantic import BaseModel, ConfigDict from pyaerocom.aeroval.aux_io_helpers import check_aux_info -class ModelEntry(BrowseDict): - """Modeln configuration for evaluation (dictionary) +class ModelEntry(BaseModel): + """Model configuration for evaluation (BaseModel) Note ----model_read_aux @@ -42,23 +42,24 @@ class ModelEntry(BrowseDict): and returns var) """ - model_id = StrType() - model_use_vars = DictType() - model_add_vars = DictStrKeysListVals() - model_read_aux = DictType() - model_rename_vars = DictType() - - def __init__(self, model_id, **kwargs): - self.model_id = model_id - self.model_ts_type_read = "" - self.model_use_vars = {} - self.model_add_vars = {} - self.model_rename_vars = {} - self.model_read_aux = {} - - self.kwargs = kwargs - - self.update(**kwargs) + ## Pydantic ConfigDict + model_config = ConfigDict( + validate_assignment=True, + protected_namespaces=(), + ) + + model_id: str + model_ts_type_read: str | dict | None = "" # TODO: see if can make None + model_name: str | None = None + model_use_vars: dict = {} + model_add_vars: dict[str, tuple[str, ...]] = {} + model_read_aux: dict = {} + model_rename_vars: dict = {} + flex_ts_type: bool = True + model_data_dir: str | None = None + # attributes previously given as kwargs used in CAMS2_83 + gridded_reader_id: dict[str, str] = {"model": "ReadGridded", "obs": "ReadGridded"} + model_kwargs: dict = {} @property def aux_funs_required(self): @@ -68,27 +69,16 @@ def aux_funs_required(self): return True if bool(self.model_read_aux) else False def json_repr(self) -> dict: - sup_rep = super().json_repr() - - # a little hacky, but makes the cams2-82 configs work - try: - for key in sup_rep["model_read_aux"]: - sup_rep["model_read_aux"][key]["fun"] = inspect.getsource( - deepcopy(sup_rep["model_read_aux"][key]["fun"]) - ) - except TypeError: - pass - - return sup_rep + return self.model_dump() - def get_vars_to_process(self, obs_vars: list) -> tuple: + def get_vars_to_process(self, obs_vars: tuple) -> tuple: """ Get lists of obs / mod variables to be processed Parameters ---------- - obs_vars : list - list of observation variables + obs_vars : tuple + tuple of observation variables Returns ------- @@ -109,12 +99,6 @@ def get_vars_to_process(self, obs_vars: list) -> tuple: modout.append(obsvar) for ovar, mvars in self.model_add_vars.items(): - if not isinstance(mvars, list): - raise AttributeError( - f"values of model_add_vars need to be lists, even if " - f"only single variables are to be added: " - f"{self.model_add_vars}" - ) for mvar in mvars: obsout.append(ovar) modout.append(mvar) @@ -134,7 +118,7 @@ def _get_aux_funcs_setup(self, funs): def prep_dict_analysis(self, funs=None) -> dict: if funs is None: funs = {} - output = deepcopy(self.to_dict()) + output = deepcopy(self.model_dump()) if self.aux_funs_required: output["model_read_aux"].update(self._get_aux_funcs_setup(funs)) return output diff --git a/pyaerocom/aeroval/modelmaps_engine.py b/pyaerocom/aeroval/modelmaps_engine.py index aea53c4e2..229c0b87d 100644 --- a/pyaerocom/aeroval/modelmaps_engine.py +++ b/pyaerocom/aeroval/modelmaps_engine.py @@ -112,7 +112,6 @@ def _run_model(self, model_name: str, var_list): make_overlay = OVERLAY in self.cfg.modelmaps_opts.plot_types.get( model_name, False ) - if self.cfg.modelmaps_opts.plot_types == {CONTOUR} or make_contour: _files = self._process_contour_map_var( model_name, var, self.reanalyse_existing @@ -401,6 +400,7 @@ def _read_model_data(self, model_name: str, var: str) -> GriddedData: kwargs.update(**self.cfg.colocation_opts.model_kwargs) if var in self.cfg.colocation_opts.model_read_opts: kwargs.update(self.cfg.colocation_opts.model_read_opts[var]) + kwargs.update(self.cfg.get_model_entry(model_name).get("model_kwargs", {})) if model_reader is not None and model_reader in MODELREADERS_USE_MAP_FREQ: ts_types = reader.ts_types diff --git a/pyaerocom/aeroval/utils.py b/pyaerocom/aeroval/utils.py index e6e50a027..8f680fe05 100644 --- a/pyaerocom/aeroval/utils.py +++ b/pyaerocom/aeroval/utils.py @@ -148,7 +148,7 @@ def compute_model_average_and_diversity( for mname in models: logger.info(f"Adding {mname} ({var_name})") - mid = cfg.cfg.model_cfg.get_entry(mname)["model_id"] + mid = cfg.cfg.model_cfg.get_entry(mname).model_id if mid == data_id or mname == data_id: continue diff --git a/pyaerocom/colocation/colocator.py b/pyaerocom/colocation/colocator.py index 61612065f..21fa3bf6e 100644 --- a/pyaerocom/colocation/colocator.py +++ b/pyaerocom/colocation/colocator.py @@ -22,7 +22,11 @@ colocate_gridded_ungridded, correct_model_stp_coldata, ) -from pyaerocom.exceptions import ColocationError, ColocationSetupError, DataCoverageError +from pyaerocom.exceptions import ( + ColocationError, + ColocationSetupError, + DataCoverageError, +) from pyaerocom.helpers import ( get_lowest_resolution, start_stop, @@ -373,7 +377,7 @@ def run(self, var_list: list = None): calc_mda8 = True data_out = defaultdict(lambda: dict()) - # ToDo: see if the following could be solved via custom context manager + # TODO: see if the following could be solved via custom context manager try: vars_to_process = self.prepare_run(var_list) except Exception as ex: @@ -401,7 +405,11 @@ def run(self, var_list: list = None): logger.debug(e) else: self._save_coldata(mda8) - logger.info("Successfully calculated mda8 for [%s, %s].", obs_var, mod_var) + logger.info( + "Successfully calculated mda8 for [%s, %s].", + obs_var, + mod_var, + ) data_out[f"{mod_var}mda8"][f"{obs_var}mda8"] = mda8 self._processing_status.append([mod_var, obs_var, 1]) diff --git a/pyaerocom/io/mscw_ctm/additional_variables.py b/pyaerocom/io/mscw_ctm/additional_variables.py index 883550722..abdf51e0c 100644 --- a/pyaerocom/io/mscw_ctm/additional_variables.py +++ b/pyaerocom/io/mscw_ctm/additional_variables.py @@ -1,9 +1,12 @@ +import logging + import xarray as xr from geonum.atmosphere import T0_STD, p0 - from pyaerocom.aux_var_helpers import concx_to_vmrx from pyaerocom.molmasses import get_molmass +logger = logging.getLogger(__name__) + def add_dataarrays(arr0: xr.DataArray, *arrs: xr.DataArray) -> xr.DataArray: """ @@ -382,3 +385,61 @@ def calc_concpolyol(concspores): concpolyol = concspores.copy(deep=True) * factor concpolyol.attrs["units"] = "ug m-3" return concpolyol + + +def calc_ratpm10pm25(concpm10: xr.DataArray, concpm25: xr.DataArray) -> xr.DataArray: + """ + Calculate ratio of pm10 and pm25 + + Parameters + ---------- + concpm10 : xr.DataArray + mass concentration pm10 + concpm25 : xr.DataArray + mass concentration of pm25 + + Returns + ------- + xr.DataArray + ratio of concpm10 / concpm25 in units of 1 + + """ + try: + if concpm10.attrs["units"] != concpm25.attrs["units"]: + logger.warning( + f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!" + ) + except KeyError: + pass + ratpm10pm25 = concpm10 / concpm25 + ratpm10pm25.attrs["units"] = "1" + return ratpm10pm25 + + +def calc_ratpm25pm10(concpm25: xr.DataArray, concpm10: xr.DataArray) -> xr.DataArray: + """ + Calculate ratio of pm10 and pm25 + + Parameters + ---------- + concpm10 : xr.DataArray + mass concentration pm10 + concpm25 : xr.DataArray + mass concentration of pm25 + + Returns + ------- + xr.DataArray + ratio of concpm25 / concpm10 in units of 1 + + """ + try: + if concpm10.attrs["units"] != concpm25.attrs["units"]: + logger.warning( + f"concpm10 unit {concpm10.attrs['units']} not equal to concpm25 unit {concpm25.attrs['units']}!" + ) + except KeyError: + pass + ratpm25pm10 = concpm25 / concpm10 + ratpm25pm10.attrs["units"] = "1" + return ratpm25pm10 diff --git a/pyaerocom/io/mscw_ctm/reader.py b/pyaerocom/io/mscw_ctm/reader.py index 6fb3c016e..c717fde3c 100755 --- a/pyaerocom/io/mscw_ctm/reader.py +++ b/pyaerocom/io/mscw_ctm/reader.py @@ -6,7 +6,6 @@ import numpy as np import xarray as xr - from pyaerocom import const from pyaerocom.exceptions import VarNotAvailableError from pyaerocom.griddeddata import GriddedData @@ -36,6 +35,8 @@ identity, subtract_dataarrays, update_EC_units, + calc_ratpm10pm25, + calc_ratpm25pm10, ) from .model_variables import emep_variables @@ -101,6 +102,8 @@ class ReadMscwCtm(GriddedReader): "concNno2": ["concno2"], "concSso2": ["concso2"], "vmro3": ["conco3"], + "ratpm10pm25": ["concpm10", "concpm25"], + "ratpm25pm10": ["concpm25", "concpm10"], # For Pollen # "concpolyol": ["concspores"], } @@ -145,6 +148,8 @@ class ReadMscwCtm(GriddedReader): "concNno2": calc_concNno2, "concSso2": calc_concSso2, "vmro3": calc_vmro3, + "ratpm10pm25": calc_ratpm10pm25, + "ratpm25pm10": calc_ratpm25pm10, # "concpolyol": calc_concpolyol, } @@ -702,10 +707,6 @@ def read_var(self, var_name, ts_type=None, **kwargs): proj_info=proj_info, ) - #!obsolete - # if var.is_deposition: - # implicit_to_explicit_rates(gridded, ts_type) - # At this point a GriddedData object with name gridded should exist gridded.metadata["data_id"] = self._data_id diff --git a/pyaerocom/io/pyaro/read_pyaro.py b/pyaerocom/io/pyaro/read_pyaro.py index 8c9fdc20e..95ef4a09f 100644 --- a/pyaerocom/io/pyaro/read_pyaro.py +++ b/pyaerocom/io/pyaro/read_pyaro.py @@ -148,7 +148,7 @@ def _convert_to_ungriddeddata(self, pyaro_data: dict[str, Data]) -> UngriddedDat var_size = {var: len(pyaro_data[var]) for var in pyaro_data} vars = list(pyaro_data.keys()) total_size = sum(list(var_size.values())) - units = {var: {"units": pyaro_data[var]._units} for var in pyaro_data} + units = {var: {"units": pyaro_data[var].units} for var in pyaro_data} # Object necessary for ungriddeddata var_idx = {var: i for i, var in enumerate(vars)} diff --git a/pyaerocom/ungriddeddata.py b/pyaerocom/ungriddeddata.py index 7c32ad8c8..e547a064f 100644 --- a/pyaerocom/ungriddeddata.py +++ b/pyaerocom/ungriddeddata.py @@ -970,7 +970,11 @@ def to_station_data( continue if freq is not None: stat.resample_time( - var, freq, how=resample_how, min_num_obs=min_num_obs, inplace=True + var, + freq, + how=resample_how, + min_num_obs=min_num_obs, + inplace=True, ) elif insert_nans: stat.insert_nans_timeseries(var) @@ -1249,7 +1253,13 @@ def to_station_data_all( - longitude: list of longitude coordinates """ - out_data = {"stats": [], "station_name": [], "latitude": [], "failed": [], "longitude": []} + out_data = { + "stats": [], + "station_name": [], + "latitude": [], + "failed": [], + "longitude": [], + } _iter = self._generate_station_index(by_station_name, ignore_index) for idx in _iter: @@ -1575,7 +1585,13 @@ def set_flags_nan(self, inplace=False): # TODO: check, confirm and remove Beta version note in docstring def remove_outliers( - self, var_name, inplace=False, low=None, high=None, unit_ref=None, move_to_trash=True + self, + var_name, + inplace=False, + low=None, + high=None, + unit_ref=None, + move_to_trash=True, ): """Method that can be used to remove outliers from data @@ -1900,7 +1916,10 @@ def apply_filters(self, var_outlier_ranges=None, **filter_attributes): var_outlier_ranges = {} for var in data.contains_vars: - lower, upper = None, None # uses pyaerocom default specified in variables.ini + lower, upper = ( + None, + None, + ) # uses pyaerocom default specified in variables.ini if var in var_outlier_ranges: lower, upper = var_outlier_ranges[var] data = data.remove_outliers( @@ -2907,7 +2926,13 @@ def plot_station_coordinates( kwargs["label"] = info_str ax = plot_coordinates( - lons, lats, color=color, marker=marker, markersize=markersize, legend=legend, **kwargs + lons, + lats, + color=color, + marker=marker, + markersize=markersize, + legend=legend, + **kwargs, ) if "title" in kwargs: diff --git a/tests/io/mscw_ctm/test_additional_variables.py b/tests/io/mscw_ctm/test_additional_variables.py index 99dccba70..bccbb057f 100644 --- a/tests/io/mscw_ctm/test_additional_variables.py +++ b/tests/io/mscw_ctm/test_additional_variables.py @@ -6,6 +6,8 @@ calc_concNno3pm25, calc_concNtnh, calc_conNtno3, + calc_ratpm10pm25, + calc_ratpm25pm10, update_EC_units, ) from tests.fixtures.mscw_ctm import create_fake_MSCWCtm_data @@ -80,6 +82,22 @@ def test_calc_concNtnh(): assert (concNtnh == concNtnh_from_func).all() +def test_calc_concpm10pm25(): + concpm10 = create_fake_MSCWCtm_data() + concpm25 = create_fake_MSCWCtm_data() + + ratpm10pm25_from_func = calc_ratpm10pm25(concpm10, concpm25) + assert ratpm10pm25_from_func.attrs["units"] == "1" + + +def test_calc_concpm25pm10(): + concpm10 = create_fake_MSCWCtm_data() + concpm25 = create_fake_MSCWCtm_data() + + ratpm25pm10_from_func = calc_ratpm25pm10(concpm25, concpm10) + assert ratpm25pm10_from_func.attrs["units"] == "1" + + def test_calc_concNnh3(): concnh3 = create_fake_MSCWCtm_data() diff --git a/tests/io/mscw_ctm/test_aeroval_configs.py b/tests/io/mscw_ctm/test_aeroval_configs.py new file mode 100644 index 000000000..cff52fcbb --- /dev/null +++ b/tests/io/mscw_ctm/test_aeroval_configs.py @@ -0,0 +1,13 @@ +from pyaerocom.aeroval.config.pmratios.base_config import get_CFG + + +def test_ratpmconfig(): + """short test if the example configuration for pm ratios is still in the code""" + + reportyear = year = 2019 + CFG = get_CFG( + reportyear=reportyear, + year=year, + model_dir="/lustre/storeB/project/fou/kl/CAMEO/u8_cams0201/", + ) + assert not CFG["raise_exceptions"]