Merge pull request #973 from metno/earlinet

Fix bugs in 3d collocation and profile processing

pyaerocom/aeroval/coldatatojson_engine.py

@@ -3,6 +3,7 @@
 from time import time
 
 from cf_units import Unit
+from numpy.typing import ArrayLike
 
 from pyaerocom import ColocatedData, TsType
 from pyaerocom._lowlevel_helpers import write_json
@@ -180,7 +181,7 @@ def process_coldata(self, coldata: ColocatedData):
         if annual_stats_constrained:
             data = _apply_annual_constraint(data)
 
-        if coldata.data.attrs.get("just_for_viz", True):  # make the regular json output
+        if not coldata.data.attrs.get("just_for_viz", False):  # make the regular json output
             if not diurnal_only:
                 logger.info("Processing statistics timeseries for all regions")
 
@@ -227,6 +228,7 @@ def process_coldata(self, coldata: ColocatedData):
                 station_names=coldata.data.station_name.values,
                 periods=periods,
                 seasons=seasons,
+                obs_name=obs_name,
                 var_name_web=var_name_web,
                 out_dirs=out_dirs,
             )
@@ -267,17 +269,16 @@ def _process_profile_data_for_vizualization(
         self,
         data: dict[str, ColocatedData] = None,
         use_country: bool = False,
-        region_names=None,
-        station_names=None,
+        region_names: dict[str:str] = None,
+        station_names: ArrayLike = None,
         periods: list[str] = None,
         seasons: list[str] = None,
         obs_name: str = None,
         var_name_web: str = None,
         out_dirs: dict = None,
     ):
-        assert (
-            region_names != None and station_names != None
-        ), f"Both region_id and station_name can not both be None"
+        if region_names == None and station_names == None:
+            raise ValueError("Both region_id and station_name can not both be None")
 
         # Loop through regions
         for regid in region_names:

pyaerocom/colocation_3d.py

@@ -166,10 +166,12 @@ def _colocate_vertical_profile_gridded(
             obs_stat_this_layer = obs_stat.copy()
 
             try:
-                obs_stat_this_layer[var_ref] = obs_stat_this_layer.select_altitude(
-                    var_name=var_ref, altitudes=list(vertical_layer.values())
-                ).mean(
-                    "altitude", skipna=True  # very important to skip nans here
+                obs_stat_this_layer[var_ref] = (
+                    obs_stat_this_layer.select_altitude(
+                        var_name=var_ref, altitudes=list(vertical_layer.values())
+                    )
+                    .mean("altitude", skipna=True)  # very important to skip nans here
+                    .to_series()  # make pandas series
                 )
             except ValueError:
                 logger.warning(

pyaerocom/data/variables.ini

@@ -351,8 +351,8 @@ map_cbar_levels = [0, 4, 8, 12, 16, 20, 40, 60, 80, 100, 200, 300, 400]
 description = Aerosol light backscattering coefficient at 550 nm
 wavelength_nm = 550
 unit = km-1 sr-1
-minimum = -1000
-maximum = 1000
+minimum = -1
+maximum = 1
 
 [bsc550dryaer]
 description = Dry aerosol light backscattering coefficient at 550 nm (RH<40)
@@ -365,24 +365,24 @@ maximum = 1000
 description = Aerosol light backscattering coefficient at 532 nm
 wavelength_nm = 532
 unit = km-1 sr-1
-minimum = -1000
-maximum = 1000
+minimum = -1
+maximum = 1
 
 [bsc355aer]
 var_name = bsc355aer
 description = Aerosol light backscattering coefficient at 355 nm
 wavelength_nm = 355
 unit = km-1 sr-1
-minimum = -1000
-maximum = 1000
+minimum = -1
+maximum = 1
 
 [bsc1064aer]
 var_name = bsc1064aer
 description = Aerosol light backscattering coefficient at 1064 nm
 wavelength_nm = 1064
 unit = km-1 sr-1
-minimum = -1000
-maximum = 1000
+minimum = -1
+maximum = 1
 
 [ac550aer]
 description = Aerosol light absorption coefficient at 550 nm

pyaerocom/plugins/ipcforests/metadata.py

@@ -266,7 +266,6 @@ def get_timeseries(self, species: str, quality_limit: float = 0.5) -> pd.Series:
         """
         flags = np.array(self.flags[species])
         quality = np.sum(flags[np.where(flags == 0)]) / len(flags)
-        breakpoint()
         if quality >= quality_limit:
             return pd.Series(self.data[species], index=self.dtime[species])
         else: