Skip to content

Commit

Permalink
Merge pull request #73 from vincelhx/main
Browse files Browse the repository at this point in the history
start streaks + change for sigma0<=0
  • Loading branch information
vincelhx authored Sep 6, 2024
2 parents 81cb510 + c12a2de commit 97265ba
Show file tree
Hide file tree
Showing 2 changed files with 171 additions and 37 deletions.
129 changes: 92 additions & 37 deletions grdwindinversion/inversion.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import re
import string
import os
from grdwindinversion.streaks import get_streaks
from grdwindinversion.load_config import getConf
# optional debug messages
import logging
Expand Down Expand Up @@ -223,7 +224,7 @@ def inverse(dual_pol, inc, sigma0, sigma0_dual, ancillary_wind, dsig_cr, model_v
sigma0 to be inverted for dualpol
ancillary_wind=: xarray.DataArray (numpy.complex28)
ancillary wind
| (for example ecmwf winds), in **GMF convention** (-np.conj included),
| (for example ecmwf winds), in **GMF convention** (-np.conj included),
dsig_cr=: float or xarray.DataArray
parameters used for
Expand Down Expand Up @@ -280,7 +281,7 @@ def inverse(dual_pol, inc, sigma0, sigma0_dual, ancillary_wind, dsig_cr, model_v
return wind_co, None, None


def makeL2asOwi(xr_dataset, dual_pol, copol, crosspol):
def makeL2asOwi(xr_dataset, dual_pol, copol, crosspol, add_streaks):
"""
Rename xr_dataset variables and attributes to match naming convention.
Expand Down Expand Up @@ -318,6 +319,7 @@ def makeL2asOwi(xr_dataset, dual_pol, copol, crosspol):
'winddir_co': 'owiWindDirection_co',
'ancillary_wind_speed': 'owiAncillaryWindSpeed',
'ancillary_wind_direction': 'owiAncillaryWindDirection',
'sigma0_detrend': 'owiNrcs_detrend'
})

if "offboresight" in xr_dataset:
Expand Down Expand Up @@ -346,6 +348,9 @@ def makeL2asOwi(xr_dataset, dual_pol, copol, crosspol):
xr_dataset.owiNrcs.attrs['long_name'] = 'Normalized Radar Cross Section'
xr_dataset.owiNrcs.attrs['definition'] = 'owiNrcs_no_noise_correction - owiNesz'

xr_dataset['owiMask_Nrcs'] = xr_dataset['sigma0_mask'].sel(pol=copol)
xr_dataset.owiMask_Nrcs.attrs = xr_dataset.sigma0_mask.attrs

# NESZ & DSIG
xr_dataset = xr_dataset.assign(
owiNesz=(['line', 'sample'], xr_dataset.nesz.sel(pol=copol).values))
Expand Down Expand Up @@ -391,14 +396,20 @@ def makeL2asOwi(xr_dataset, dual_pol, copol, crosspol):
'winddir_dual': 'owiWindDirection',
'windspeed_cross': 'owiWindSpeed_cross',
'windspeed_dual': 'owiWindSpeed',
'sigma0_detrend_cross': 'owiNrcs_detrend_cross'
})
# nrcs cross
xr_dataset['owiNrcs_cross'] = xr_dataset['sigma0_ocean'].sel(
pol=crosspol)

xr_dataset.owiNrcs_cross.attrs['units'] = 'm^2 / m^2'
xr_dataset.owiNrcs_cross.attrs['long_name'] = 'Normalized Radar Cross Section'
xr_dataset.owiNrcs_cross.attrs['definition'] = 'owiNrcs_cross_no_noise_correction - owiNesz_cross'

xr_dataset['owiMask_Nrcs_cross'] = xr_dataset['sigma0_mask'].sel(
pol=crosspol)
xr_dataset.owiMask_Nrcs_cross.attrs = xr_dataset.sigma0_mask.attrs

# nesz cross
xr_dataset = xr_dataset.assign(owiNesz_cross=(
['line', 'sample'], xr_dataset.nesz.sel(pol=crosspol).values)) # no flattening
Expand All @@ -425,6 +436,11 @@ def makeL2asOwi(xr_dataset, dual_pol, copol, crosspol):

xr_dataset.owiNrcs_cross.attrs['definition'] = 'owiNrcs_cross_no_noise_correction_recalibrated - owiNesz_cross'

if add_streaks:
xr_dataset = xr_dataset.rename({
'streaks_direction': 'owiStreaksDirection',
})

#  other variables

xr_dataset['owiWindQuality'] = xr.full_like(xr_dataset.owiNrcs, 0)
Expand Down Expand Up @@ -483,7 +499,7 @@ def makeL2asOwi(xr_dataset, dual_pol, copol, crosspol):
return xr_dataset, encoding


def preprocess(filename, outdir, config_path, overwrite=False, resolution='1000m'):
def preprocess(filename, outdir, config_path, overwrite=False, add_streaks=False, resolution='1000m'):
"""
Main function to generate L2 product.
Expand All @@ -503,7 +519,7 @@ def preprocess(filename, outdir, config_path, overwrite=False, resolution='1000m
Returns
-------
xarray.Dataset
final dataset
final dataset
"""

sensor, sensor_longname, fct_meta, fct_dataset = getSensorMetaDataset(
Expand Down Expand Up @@ -531,13 +547,13 @@ def preprocess(filename, outdir, config_path, overwrite=False, resolution='1000m
meta, subdir=not no_subdir_cfg)

if os.path.exists(out_file) and overwrite is False:
raise FileExistsError("out_file %s exists already")
raise FileExistsError("outfile %s already exists" % out_file)

ancillary_name = config["ancillary"]
map_model = getAncillary(meta, ancillary_name)
if map_model is None:
raise Exception(
f'the weather model is not set `map_model` is None -> you probably don"t have access to f{ancillary_name} archive')
f"the weather model is not set `map_model` is None -> you probably don't have access to {ancillary_name} archive")

try:
if ((recalibration) & ("SENTINEL" in sensor_longname)):
Expand Down Expand Up @@ -591,6 +607,25 @@ def preprocess(filename, outdir, config_path, overwrite=False, resolution='1000m
copol_gmf = 'HH'
crosspol_gmf = 'VH'

model_vv = config["GMF_"+copol_gmf+"_NAME"]
model_vh = config["GMF_"+crosspol_gmf+"_NAME"]

# need to load gmfs before inversion
gmfs_impl = [x for x in [model_vv, model_vh] if "gmf_" in x]
windspeed.gmfs.GmfModel.activate_gmfs_impl(gmfs_impl)
sarwings_luts = [x for x in [model_vv, model_vh]
if x.startswith("sarwing_lut_")]

if len(sarwings_luts) > 0:
windspeed.register_sarwing_luts(getConf()["sarwing_luts_path"])

nc_luts = [x for x in [model_vv, model_vh] if x.startswith("nc_lut")]

if len(nc_luts) > 0:
windspeed.register_nc_luts(getConf()["nc_luts_path"])

if (model_vv == "gmf_cmod7"):
windspeed.register_cmod7(getConf()["lut_cmod7_path"])
#  Step 2 - clean and prepare dataset

# variables to not keep in the L2
Expand Down Expand Up @@ -680,21 +715,32 @@ def preprocess(filename, outdir, config_path, overwrite=False, resolution='1000m
# nrcs processing
xr_dataset['sigma0_ocean'] = xr.where(xr_dataset['mask'], np.nan,
xr_dataset['sigma0'].compute()).transpose(*xr_dataset['sigma0'].dims)
xr_dataset['sigma0_ocean'] = xr.where(
xr_dataset['sigma0_ocean'] <= 0, np.nan, xr_dataset['sigma0_ocean'])

xr_dataset['sigma0_ocean'].attrs = xr_dataset['sigma0'].attrs
#  we forced it to nan
xr_dataset['sigma0_ocean'].attrs['comment'] = "clipped, no values <=0"
#  we forced it to 1e-15
xr_dataset['sigma0_ocean'].attrs['comment'] = "clipped, no values <=0 ; 1e-15 instread"

# rajout d'un mask pour les valeurs <=0:
xr_dataset['sigma0_mask'] = xr.where(
xr_dataset['sigma0_ocean'] <= 0, 1, 0).transpose(*xr_dataset['sigma0'].dims)
xr_dataset.sigma0_mask.attrs['valid_range'] = np.array([0, 1])
xr_dataset.sigma0_mask.attrs['flag_values'] = np.array([0, 1])
xr_dataset.sigma0_mask.attrs['flag_meanings'] = 'valid no_valid'
xr_dataset['sigma0_ocean'] = xr.where(
xr_dataset['sigma0_ocean'] <= 0, 1e-15, xr_dataset['sigma0_ocean'])

xr_dataset['sigma0_ocean_raw'] = xr.where(xr_dataset['mask'], np.nan,
xr_dataset['sigma0_raw'].compute()).transpose(*xr_dataset['sigma0_raw'].dims)
xr_dataset['sigma0_ocean_raw'] = xr.where(
xr_dataset['sigma0_ocean_raw'] <= 0, np.nan, xr_dataset['sigma0_ocean_raw'])

xr_dataset['sigma0_ocean_raw'].attrs = xr_dataset['sigma0_raw'].attrs

xr_dataset['sigma0_detrend'] = xsarsea.sigma0_detrend(
xr_dataset.sigma0.sel(pol=copol), xr_dataset.incidence, model=model_vv)

# processing
if dual_pol:

xr_dataset['sigma0_detrend_cross'] = xsarsea.sigma0_detrend(
xr_dataset.sigma0.sel(pol=crosspol), xr_dataset.incidence, model=model_vh)
if config["apply_flattening"]:
xr_dataset = xr_dataset.assign(nesz_cross_final=(
['line', 'sample'], windspeed.nesz_flattening(xr_dataset.nesz.sel(pol=crosspol), xr_dataset.incidence)))
Expand All @@ -720,9 +766,6 @@ def preprocess(filename, outdir, config_path, overwrite=False, resolution='1000m
sigma0_ocean_cross = None
dsig_cross = 0.1 # default value set in xsarsea

model_vv = config["GMF_"+copol_gmf+"_NAME"]
model_vh = config["GMF_"+crosspol_gmf+"_NAME"]

if ((recalibration) & ("SENTINEL" in sensor_longname)):
xr_dataset.attrs["path_aux_pp1_new"] = os.path.basename(os.path.dirname(
os.path.dirname(xsar_dataset.datatree['recalibration'].attrs['path_aux_pp1_new'])))
Expand All @@ -734,10 +777,39 @@ def preprocess(filename, outdir, config_path, overwrite=False, resolution='1000m
xr_dataset.attrs["path_aux_cal_old"] = os.path.basename(os.path.dirname(
os.path.dirname(xsar_dataset.datatree['recalibration'].attrs['path_aux_cal_old'])))

if add_streaks:
xsar_dataset_100 = fct_dataset(
meta, resolution='100m')
xr_dataset_100 = xsar_dataset_100.datatree['measurement'].to_dataset()
xr_dataset_100 = xr_dataset_100.rename(map_model)

# adding sigma0 detrend
xr_dataset_100['sigma0_detrend'] = xsarsea.sigma0_detrend(
xr_dataset_100.sigma0.sel(pol=copol), xr_dataset_100.incidence, model=model_vv)

xr_dataset_100['sigma0_detrend_cross'] = xsarsea.sigma0_detrend(
xr_dataset_100.sigma0.sel(pol=crosspol), xr_dataset_100.incidence, model=model_vh)

sigma0_detrend_combined = xr.concat(
[xr_dataset_100['sigma0_detrend'],
xr_dataset_100['sigma0_detrend_cross']],
dim='pol'
)
sigma0_detrend_combined['pol'] = [copol, crosspol]

xr_dataset_100['sigma0_detrend'] = sigma0_detrend_combined
xr_dataset_100.land_mask.values = binary_dilation(xr_dataset_100['land_mask'].values.astype('uint8'),
structure=np.ones((3, 3), np.uint8), iterations=3)
xr_dataset_100['sigma0_detrend'] = xr.where(
xr_dataset_100['land_mask'], np.nan, xr_dataset_100['sigma0'].compute()).transpose(*xr_dataset_100['sigma0'].dims)

xr_dataset['streaks_direction'] = get_streaks(
xr_dataset, xr_dataset_100)

return xr_dataset, dual_pol, copol, crosspol, copol_gmf, crosspol_gmf, model_vv, model_vh, sigma0_ocean_cross, dsig_cross, sensor_longname, out_file, config


def makeL2(filename, outdir, config_path, overwrite=False, generateCSV=True, resolution='1000m'):
def makeL2(filename, outdir, config_path, overwrite=False, generateCSV=True, add_streaks=False, resolution='1000m'):
"""
Main function to generate L2 product.
Expand Down Expand Up @@ -765,7 +837,7 @@ def makeL2(filename, outdir, config_path, overwrite=False, generateCSV=True, res
"""

xr_dataset, dual_pol, copol, crosspol, copol_gmf, crosspol_gmf, model_vv, model_vh, sigma0_ocean_cross, dsig_cross, sensor_longname, out_file, config = preprocess(
filename, outdir, config_path, overwrite, resolution)
filename, outdir, config_path, overwrite, add_streaks, resolution)

kwargs = {
"inc_step_lr": config.pop("inc_step_lr", None),
Expand All @@ -777,23 +849,6 @@ def makeL2(filename, outdir, config_path, overwrite=False, generateCSV=True, res
"resolution": config.pop("resolution", None),
}

# need to load gmfs before

gmfs_impl = [x for x in [model_vv, model_vh] if "gmf_" in x]
windspeed.gmfs.GmfModel.activate_gmfs_impl(gmfs_impl)
sarwings_luts = [x for x in [model_vv, model_vh]
if x.startswith("sarwing_lut_")]
if len(sarwings_luts) > 0:
windspeed.register_sarwing_luts(getConf()["sarwing_luts_path"])

nc_luts = [x for x in [model_vv, model_vh] if x.startswith("nc_lut")]

if len(nc_luts) > 0:
windspeed.register_nc_luts(getConf()["nc_luts_path"])

if (model_vv == "gmf_cmod7"):
windspeed.register_cmod7(getConf()["lut_cmod7_path"])

wind_co, wind_dual, windspeed_cr = inverse(dual_pol,
inc=xr_dataset['incidence'],
sigma0=xr_dataset['sigma0_ocean'].sel(
Expand Down Expand Up @@ -856,7 +911,7 @@ def makeL2(filename, outdir, config_path, overwrite=False, generateCSV=True, res
xr_dataset["winddir_cross"].attrs["model"] = "No model used ; content is a copy of dualpol wind direction"

xr_dataset, encoding = makeL2asOwi(
xr_dataset, dual_pol, copol, crosspol)
xr_dataset, dual_pol, copol, crosspol, add_streaks=add_streaks)

#  add attributes
firstMeasurementTime = None
Expand Down Expand Up @@ -909,7 +964,7 @@ def makeL2(filename, outdir, config_path, overwrite=False, generateCSV=True, res
}

for recalib_attrs in ["path_aux_pp1_new", 'path_aux_pp1_old', "path_aux_cal_new", "path_aux_cal_old"]:
if recalib_attrs in xr_dataset:
if recalib_attrs in xr_dataset.attrs:
attrs[recalib_attrs] = xr_dataset.attrs[recalib_attrs]

# new one to match convention
Expand Down
79 changes: 79 additions & 0 deletions grdwindinversion/streaks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import xarray as xr
import xsarsea.gradients
import xarray as xr
from scipy.ndimage import binary_dilation
import numpy as np


def get_streaks(xr_dataset, xr_dataset_100):
"""
Get the streaks from the wind field.
Parameters
----------
xr_dataset : xarray.Dataset
dataset at user resolution.
xr_dataset_100 : xarray.Dataset
dataset at 100m resolution.
Returns
-------
xarray.Dataset
Extract wind direction from Koch Method using xsarsea tools.
"""

# return empy dataArray, waiting for solution
return xr.DataArray(data=np.nan * np.ones([len(xr_dataset.coords[dim]) for dim in ['line','sample']]),
dims=['line','sample'],
coords=[xr_dataset.coords[dim] for dim in ['line','sample']])
#

"""
gradients = xsarsea.gradients.Gradients(xr_dataset_100['sigma0_detrend'], windows_sizes=[
1600, 3200], downscales_factors=[1, 2], window_step=1)
# get gradients histograms as an xarray dataset
hist = gradients.histogram
# get orthogonals gradients
hist['angles'] = hist['angles'] + np.pi/2
# mean
hist_mean = hist.mean(['downscale_factor', 'window_size', 'pol'])
# smooth
hist_mean_smooth = hist_mean.copy()
hist_mean_smooth['weight'] = xsarsea.gradients.circ_smooth(
hist_mean['weight'])
# smooth only
# hist_smooth = hist.copy()
# hist_smooth['weight'] = xsarsea.gradients.circ_smooth(hist_smooth['weight'])
# select histogram peak
iangle = hist_mean_smooth['weight'].fillna(0).argmax(dim='angles')
streaks_dir = hist_mean_smooth.angles.isel(angles=iangle)
streaks_weight = hist_mean_smooth['weight'].isel(angles=iangle)
streaks = xr.merge(
[dict(angle=streaks_dir, weight=streaks_weight)]).drop('angles')
# streaks are [0, pi]. Remove ambiguity with anciallary wind
ancillary_wind = xr_dataset_100['ancillary_wind'].sel(line=streaks.line,
sample=streaks.sample,
method='nearest').compute()
streaks_c = streaks['weight'] * np.exp(1j * streaks['angle'])
diff_angle = xr.apply_ufunc(np.angle, ancillary_wind / streaks_c)
streaks_c = xr.where(np.abs(diff_angle) > np.pi/2, -streaks_c, streaks_c)
streaks['weight'] = np.abs(streaks_c)
streaks['angle'] = xr.apply_ufunc(np.angle, streaks_c)
streaks_dir = xr.apply_ufunc(
np.angle, streaks_c.interp(line=xr_dataset.line, sample=xr_dataset.sample))
streaks_dir = xr.where(
xr_dataset['land_mask'], np.nan, streaks_dir)
streaks_dir.attrs['comment'] = 'angle in radians, anticlockwise, 0=line'
streaks_dir.attrs['description'] = 'wind direction estimated from local gradient, and direction ambiguity removed with ancillary wind'
return streaks_dir
"""

0 comments on commit 97265ba

Please sign in to comment.