diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a650501..363ded8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,11 +12,11 @@ jobs: test: runs-on: ubuntu-latest timeout-minutes: 360 # 6 hours limit for the job - + steps: - # Checkout the code - - name: Checkout code - uses: actions/checkout@v4 + # Checkout the code + - name: Checkout code + uses: actions/checkout@v4 - uses: mamba-org/setup-micromamba@v1 with: @@ -42,57 +42,56 @@ jobs: micromamba activate grdwind_env conda install --use-local grdwindinversion -y -c tcevaer -c conda-forge - # Cache the test data if previously downloaded (up to 10 GB limit for the cache) - # WARNING : modify the key if the data is modified !! - - name: Cache test data - uses: actions/cache@v4 - id: cache - with: - path: ./test_data - key: test-data-v3 - restore-keys: test-data-v3 + # Cache the test data if previously downloaded (up to 10 GB limit for the cache) + # WARNING : modify the key if the data is modified !! + - name: Cache test data + uses: actions/cache@v4 + id: cache + with: + path: ./test_data + key: test-data-v3 + restore-keys: test-data-v3 - # Download test data if not already cached - - name: Download test data - if: steps.cache.outputs.cache-hit != 'true' # Only download if cache miss - run: | - mkdir -p ./test_data/ - wget https://cloud.ifremer.fr/index.php/s/ExLQ2TnYAqozPWE/download -O /tmp/ecmwf.zip - unzip /tmp/ecmwf.zip -d ./test_data/ - wget https://cloud.ifremer.fr/index.php/s/kRgdOOPsjoZieZR/download -O /tmp/l1.zip - unzip /tmp/l1.zip -d ./test_data/ - timeout-minutes: 200 # Adjust depending on the size of your data + # Download test data if not already cached + - name: Download test data + if: steps.cache.outputs.cache-hit != 'true' # Only download if cache miss + run: | + mkdir -p ./test_data/ + wget https://cloud.ifremer.fr/index.php/s/ExLQ2TnYAqozPWE/download -O /tmp/ecmwf.zip + unzip /tmp/ecmwf.zip -d ./test_data/ + wget https://cloud.ifremer.fr/index.php/s/kRgdOOPsjoZieZR/download -O /tmp/l1.zip + unzip /tmp/l1.zip -d ./test_data/ + timeout-minutes: 200 # Adjust depending on the size of your data - # Set up xsar configuration - - name: Setup xsar configuration - run: | - mkdir -p ~/.xsar - echo "data_dir: /tmp" > ~/.xsar/config.yaml - echo "auxiliary_dir: ./test_data/auxiliary" >> ~/.xsar/config.yaml - echo "path_dataframe_aux: ./test_data/auxiliary/active_aux.csv" >> ~/.xsar/config.yaml + # Set up xsar configuration + - name: Setup xsar configuration + run: | + mkdir -p ~/.xsar + echo "data_dir: /tmp" > ~/.xsar/config.yaml + echo "auxiliary_dir: ./test_data/auxiliary" >> ~/.xsar/config.yaml + echo "path_dataframe_aux: ./test_data/auxiliary/active_aux.csv" >> ~/.xsar/config.yaml - # Set up grdwindinversion configuration - - name: Setup grdwindinversion configuration - run: | - mkdir -p ~/.grdwindinversion - echo "'ecmwf_0100_1h': ./test_data/ECMWF/forecast/hourly/0100deg/netcdf_light/%Y/%j/ECMWF_FORECAST_0100_%Y%m%d%H%M_10U_10V.nc" > ~/.grdwindinversion/data_config.yaml - echo "'ecmwf_0125_1h': ./test_data/ECMWF/0.125deg/1h/forecasts/%Y/%j/ecmwf_%Y%m%d%H%M.nc" >> ~/.grdwindinversion/data_config.yaml - #echo "'sarwing_luts_path': './test_data/GMFS/v1.6/'" >> ~/.grdwindinversion/data_config.yaml - #echo "'nc_luts_path': ./test_data/GMFS/nc_luts" >> ~/.grdwindinversion/data_config.yaml - #echo "'lut_cmod7_path': './test_data/GMFS/v1.6/GMF_cmod7_official/cmod7_and_python_script'" >> ~/.grdwindinversion/data_config.yaml - #echo "'lut_ms1ahw_path': './test_data/GMFS/v1.6/GMF_cmodms1ahw'" >> ~/.grdwindinversion/data_config.yaml + # Set up grdwindinversion configuration + - name: Setup grdwindinversion configuration + run: | + mkdir -p ~/.grdwindinversion + echo "'ecmwf_0100_1h': ./test_data/ECMWF/forecast/hourly/0100deg/netcdf_light/%Y/%j/ECMWF_FORECAST_0100_%Y%m%d%H%M_10U_10V.nc" > ~/.grdwindinversion/data_config.yaml + echo "'ecmwf_0125_1h': ./test_data/ECMWF/0.125deg/1h/forecasts/%Y/%j/ecmwf_%Y%m%d%H%M.nc" >> ~/.grdwindinversion/data_config.yaml + #echo "'nc_luts_path': ./test_data/GMFS/nc_luts" >> ~/.grdwindinversion/data_config.yaml + #echo "'lut_cmod7_path': './test_data/GMFS/v1.6/GMF_cmod7_official/cmod7_and_python_script'" >> ~/.grdwindinversion/data_config.yaml + #echo "'lut_ms1ahw_path': './test_data/GMFS/v1.6/GMF_cmodms1ahw'" >> ~/.grdwindinversion/data_config.yaml - # Run the tests - - name: Run tests - run: | - eval "$(micromamba shell hook --shell bash)" - micromamba activate grdwind_env - pytest + # Run the tests + - name: Run tests + run: | + eval "$(micromamba shell hook --shell bash)" + micromamba activate grdwind_env + pytest - # Optionally, upload test artifacts (NetCDF files or logs) if needed - #- name: Upload test artifacts - # if: failure() # Only upload on failure - # uses: actions/upload-artifact@v2 - # with: - # name: test-output - # path: ./test_output/ + # Optionally, upload test artifacts (NetCDF files or logs) if needed + #- name: Upload test artifacts + # if: failure() # Only upload on failure + # uses: actions/upload-artifact@v2 + # with: + # name: test-output + # path: ./test_output/ diff --git a/grdwindinversion/.travis.yml b/grdwindinversion/.travis.yml deleted file mode 100644 index 371a871..0000000 --- a/grdwindinversion/.travis.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Config file for automatic testing at travis-ci.com - -language: python -python: - - 3.8 - - 3.7 - - 3.6 - -# Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: pip install -U tox-travis - -# Command to run tests, e.g. python setup.py test -script: tox - -# Assuming you have installed the travis-ci CLI tool, after you -# create the Github repo and add it to Travis, run the -# following command to finish PyPI deployment setup: -# $ travis encrypt --add deploy.password -deploy: - provider: pypi - distributions: sdist bdist_wheel - user: agrouaze - password: - secure: PLEASE_REPLACE_ME - on: - tags: true - repo: agrouaze/grdwindinversion - python: 3.8 diff --git a/grdwindinversion/inversion.py b/grdwindinversion/inversion.py index d6feb74..3239239 100644 --- a/grdwindinversion/inversion.py +++ b/grdwindinversion/inversion.py @@ -98,10 +98,10 @@ def getOutputName2(input_file, outdir, sensor, meta, subdir=True): regex = re.compile( "([A-Z0-9]+)_OK([0-9]+)_PK([0-9]+)_(.*?)_(.*?)_(.*?)_(.*?)_(.*?)_(.*?)_(.*?)") template = string.Template( - "${MISSIONID}_OK${DATA1}_PK${DATA2}_${DATA3}_${BEAM_MODE}_${DATE}_${TIME}_${POLARIZATION1}_${POLARIZATION2}_${PRODUCT}") + "${MISSIONID}_OK${DATA1}_PK${DATA2}_${DATA3}_${BEAM}_${DATE}_${TIME}_${POLARIZATION1}_${POLARIZATION2}_${PRODUCT}") match = regex.match(basename_match) MISSIONID, DATA1, DATA2, DATA3, BEAM_MODE, DATE, TIME, POLARIZATION1, POLARIZATION2, LAST = match.groups() - new_format = f"{MISSIONID.lower()}-{BEAM_MODE.lower()}-owi-xx-{meta_start_date.lower()}-{meta_stop_date.lower()}-_____-_____.nc" + new_format = f"{MISSIONID.lower()}-{BEAM.lower()}-owi-xx-{meta_start_date.lower()}-{meta_stop_date.lower()}-_____-_____.nc" else: raise ValueError( "sensor must be S1A|S1B|RS2|RCM, got sensor %s" % sensor) @@ -243,7 +243,7 @@ def inverse(dual_pol, inc, sigma0, sigma0_dual, ancillary_wind, dsig_cr, model_c See Also -------- xsarsea documentation - https://cyclobs.ifremer.fr/static/sarwing_datarmor/xsarsea/examples/windspeed_inversion.html + https://cerweb.ifremer.fr/datarmor/doc_sphinx/xsarsea/ """ logging.debug("inversion") @@ -282,7 +282,7 @@ def inverse(dual_pol, inc, sigma0, sigma0_dual, ancillary_wind, dsig_cr, model_c return wind_co, None, None -def makeL2asOwi(xr_dataset, dual_pol, copol, crosspol, add_streaks): +def makeL2asOwi(xr_dataset, dual_pol, copol, crosspol, add_streaks, apply_flattening): """ Rename xr_dataset variables and attributes to match naming convention. @@ -392,13 +392,18 @@ def makeL2asOwi(xr_dataset, dual_pol, copol, crosspol, add_streaks): xr_dataset = xr_dataset.rename({ 'dsig_cross': 'owiDsig_cross', - 'nesz_cross_final': 'owiNesz_cross_final', 'winddir_cross': 'owiWindDirection_cross', 'winddir_dual': 'owiWindDirection', 'windspeed_cross': 'owiWindSpeed_cross', 'windspeed_dual': 'owiWindSpeed', 'sigma0_detrend_cross': 'owiNrcs_detrend_cross' }) + + if apply_flattening: + xr_dataset = xr_dataset.rename({ + 'nesz_cross_flattened': 'owiNesz_cross_flattened', + }) + # nrcs cross xr_dataset['owiNrcs_cross'] = xr_dataset['sigma0_ocean'].sel( pol=crosspol) @@ -635,15 +640,7 @@ def preprocess(filename, outdir, config_path, overwrite=False, add_streaks=False config["l2_params"]["model_cross"] = model_cross config["sensor_longname"] = sensor_longname - # need to load gmfs before inversion - gmfs_impl = [x for x in [model_co, model_cross] if "gmf_" in x] - windspeed.gmfs.GmfModel.activate_gmfs_impl(gmfs_impl) - sarwings_luts = [x for x in [model_co, model_cross] - if x.startswith("sarwing_lut_")] - - if len(sarwings_luts) > 0: - windspeed.register_sarwing_luts(getConf()["sarwing_luts_path"]) - + # need to load LUTs before inversion nc_luts = [x for x in [model_co, model_cross] if x.startswith("nc_lut")] if len(nc_luts) > 0: @@ -767,22 +764,17 @@ def preprocess(filename, outdir, config_path, overwrite=False, add_streaks=False xr_dataset['sigma0_detrend_cross'] = xsarsea.sigma0_detrend( xr_dataset.sigma0.sel(pol=crosspol), xr_dataset.incidence, model=model_cross) if config["apply_flattening"]: - xr_dataset = xr_dataset.assign(nesz_cross_final=( + xr_dataset = xr_dataset.assign(nesz_cross_flattened=( ['line', 'sample'], windspeed.nesz_flattening(xr_dataset.nesz.sel(pol=crosspol), xr_dataset.incidence))) - xr_dataset['nesz_cross_final'].attrs[ + xr_dataset['nesz_cross_flattened'].attrs[ "comment"] = 'nesz has been flattened using windspeed.nesz_flattening' - + # dsig + xr_dataset["dsig_cross"] = windspeed.get_dsig(config["dsig_"+crosspol_gmf+"_NAME"], xr_dataset.incidence, + xr_dataset['sigma0_ocean'].sel(pol=crosspol), xr_dataset.nesz_cross_flattened) else: - xr_dataset = xr_dataset.assign( - nesz_cross_final=(['line', 'sample'], xr_dataset.nesz.sel(pol=crosspol).values)) - xr_dataset['nesz_cross_final'].attrs["comment"] = 'nesz has not been flattened' - - xr_dataset.nesz_cross_final.attrs['units'] = 'm^2 / m^2' - xr_dataset.nesz_cross_final.attrs['long_name'] = 'Noise Equivalent SigmaNaught' - - # dsig - xr_dataset["dsig_cross"] = windspeed.get_dsig(config["dsig_"+crosspol_gmf+"_NAME"], xr_dataset.incidence, - xr_dataset['sigma0_ocean'].sel(pol=crosspol), xr_dataset.nesz_cross_final) + # dsig + xr_dataset["dsig_cross"] = windspeed.get_dsig(config["dsig_"+crosspol_gmf+"_NAME"], xr_dataset.incidence, + xr_dataset['sigma0_ocean'].sel(pol=crosspol), xr_dataset['sigma0_ocean'].sel(pol=crosspol)) xr_dataset.dsig_cross.attrs['comment'] = 'variable used to ponderate copol and crosspol' xr_dataset.dsig_cross.attrs['formula_used'] = config["dsig_" + @@ -936,8 +928,7 @@ def makeL2(filename, outdir, config_path, overwrite=False, generateCSV=True, add xr_dataset['winddir_dual'] = transform_winddir( wind_dual, xr_dataset.ground_heading, winddir_convention=config["winddir_convention"]) - xr_dataset['winddir_dual'].attrs["model"] = "%s (%s) & %s (%s)" % ( - model_co, copol, model_cross, crosspol) + xr_dataset["winddir_dual"].attrs["model"] = "winddir_dual is a copy of copol wind direction" xr_dataset = xr_dataset.assign( windspeed_cross=(['line', 'sample'], windspeed_cr)) @@ -949,7 +940,7 @@ def makeL2(filename, outdir, config_path, overwrite=False, generateCSV=True, add xr_dataset['winddir_cross'] = xr_dataset['winddir_dual'].copy() xr_dataset['winddir_cross'].attrs = xr_dataset['winddir_dual'].attrs - xr_dataset["winddir_cross"].attrs["model"] = "No model used ; content is a copy of dualpol wind direction" + xr_dataset["winddir_cross"].attrs["model"] = "winddir_cross is a copy of copol wind direction" if config["winddir_convention"] == "oceanographic": attrs = xr_dataset['ancillary_wind_direction'].attrs @@ -960,7 +951,7 @@ def makeL2(filename, outdir, config_path, overwrite=False, generateCSV=True, add "long_name"] = f"{ancillary_name} wind direction in oceanographic convention (clockwise, to), ex: 0°=to north, 90°=to east" xr_dataset, encoding = makeL2asOwi( - xr_dataset, dual_pol, copol, crosspol, add_streaks=add_streaks) + xr_dataset, dual_pol, copol, crosspol, add_streaks=add_streaks, apply_flattening=config["apply_flattening"]) #  add attributes firstMeasurementTime = None